New variables support within stack configs

CHANGES.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.7.3
+- Added support for variables within config files, incl. usual inheritance
+- Set Legacy to False by default, requires templates to check for False explicitly, allows to enabled/disable per stack
+
 ## 0.7.2
 - Add line numbers to easy debugging
 - Fix tests

Makefile

@@ -10,7 +10,7 @@ test:
 	TEST=True pytest --log-cli-level=DEBUG
 
 clean:
-	rm -rf .cache .coverage .eggs cloudbender.egg-info .pytest_cache dist
+	rm -rf .cache build .coverage .eggs cloudbender.egg-info .pytest_cache dist
 
 build: $(PACKAGE_FILE)
 

cloudbender/__init__.py

@@ -2,7 +2,7 @@ import logging
 
 __author__ = "Stefan Reimer"
 __email__ = "stefan@zero-downtimet.net"
-__version__ = "0.7.2"
+__version__ = "0.7.3"
 
 
 # Set up logging to ``/dev/null`` like a library is supposed to.

cloudbender/core.py

@@ -56,11 +56,6 @@ class CloudBender(object):
 
         self.all_stacks = self.sg.get_stacks()
 
-        # If cfn vars config is completely empty set some default for tests to work
-        # if "vars" not in _config:
-        #     _config = { "vars": { 'Azs': {'TestAZ': 'Next'}, 'Segments': {'Testnet': 'internet'}, "Mode": "Piped" } }
-        #     self.vars.update(_config.get('vars'))
-
     def dump_config(self):
         logger.debug("<CloudBender: {}>".format(vars(self)))
         self.sg.dump_config()

cloudbender/jinja.py

@@ -4,6 +4,7 @@ import gzip
 import re
 import base64
 import yaml
+import copy
 
 import jinja2
 from jinja2.utils import missing, object_type_repr
@@ -184,8 +185,14 @@ def JinjaEnv(template_locations=[]):
     return jenv
 
 
-def read_config_file(path, jinja_args=None):
+def read_config_file(path, variables={}):
-    """ reads yaml config file, passes it through jinja and returns data structre """
+    """ reads yaml config file, passes it through jinja and returns data structre
+
+        - OS ENV are available as {{ ENV.<VAR> }}
+        - variables defined in parent configs are available as {{ <VAR> }}
+    """
+    jinja_variables = copy.deepcopy(variables)
+    jinja_variables['ENV'] = os.environ
 
     if os.path.exists(path):
         logger.debug("Reading config file: {}".format(path))
@@ -195,9 +202,7 @@ def read_config_file(path, jinja_args=None):
                 undefined=jinja2.StrictUndefined,
                 extensions=['jinja2.ext.loopcontrols'])
             template = jenv.get_template(os.path.basename(path))
-            rendered_template = template.render(
+            rendered_template = template.render(jinja_variables)
-                env=os.environ
-            )
             data = yaml.safe_load(rendered_template)
             if data:
                 return data

cloudbender/stack.py

@@ -34,7 +34,7 @@ class StackStatus(object):
 
 
 class Stack(object):
-    def __init__(self, name, template, path, rel_path, sg_config, ctx):
+    def __init__(self, name, template, path, rel_path, ctx):
         self.stackname = name
         self.template = template
         self.path = path
@@ -43,23 +43,12 @@ class Stack(object):
 
         self.tags = {}
         self.parameters = {}
-        self.options = {}
+        self.options = {'Legacy': False}
         self.region = 'global'
         self.profile = ''
         self.onfailure = 'DELETE'
         self.notfication_sns = []
 
-        self.tags.update(sg_config.get('tags', {}))
-        self.parameters.update(sg_config.get('parameters', {}))
-        self.options.update(sg_config.get('options', {}))
-
-        if 'region' in sg_config:
-            self.region = sg_config['region']
-        if 'profile' in sg_config:
-            self.profile = sg_config['profile']
-        if 'notfication_sns' in sg_config:
-            self.notfication_sns = sg_config['notfication_sns']
-
         self.id = (self.profile, self.region, self.stackname)
 
         self.md5 = None
@@ -77,8 +66,22 @@ class Stack(object):
     def dump_config(self):
         logger.debug("<Stack {}: {}>".format(self.id, pprint.pformat(vars(self))))
 
-    def read_config(self):
+    def read_config(self, sg_config={}):
-        _config = read_config_file(self.path)
+        """ reads stack config """
+
+        # First set various attributes based on parent stackgroup config
+        self.tags.update(sg_config.get('tags', {}))
+        self.parameters.update(sg_config.get('parameters', {}))
+        self.options.update(sg_config.get('options', {}))
+
+        if 'region' in sg_config:
+            self.region = sg_config['region']
+        if 'profile' in sg_config:
+            self.profile = sg_config['profile']
+        if 'notfication_sns' in sg_config:
+            self.notfication_sns = sg_config['notfication_sns']
+
+        _config = read_config_file(self.path, sg_config.get('variables', {}))
         for p in ["region", "stackname", "template", "default_lock", "multi_delete", "provides", "onfailure", "notification_sns"]:
             if p in _config:
                 setattr(self, p, _config[p])
@@ -93,6 +96,7 @@ class Stack(object):
 
         # backwards comp
         if 'vars' in _config:
+            logger.warn("vars: in config is deprecated, please use options: instead")
             self.options = dict_merge(self.options, _config['vars'])
 
         if 'options' in _config:
@@ -154,7 +158,7 @@ class Stack(object):
         # Add Legacy FortyTwo resource to prevent AWS from replacing existing resources for NO reason ;-(
         include = []
         search_refs(self.cfn_data, include, self.mode)
-        if self.mode != "Piped" and len(include) and 'Legacy' in self.options:
+        if self.mode != "Piped" and len(include) and self.options['Legacy']:
             _res = """
   FortyTwo:
     Type: Custom::FortyTwo

cloudbender/stackgroup.py

@@ -33,12 +33,11 @@ class StackGroup(object):
             s.dump_config()
 
     def read_config(self, parent_config={}):
-
         if not os.path.isdir(self.path):
             return None
 
         # First read config.yaml if present
-        _config = read_config_file(os.path.join(self.path, 'config.yaml'))
+        _config = read_config_file(os.path.join(self.path, 'config.yaml'), parent_config.get('variables', {}))
 
         # Stack Group name if not explicit via config is derived from subfolder, or in case of root object the parent folder
         if "stackgroupname" in _config:
@@ -48,7 +47,6 @@ class StackGroup(object):
 
         # Merge config with parent config
         self.config = dict_merge(parent_config, _config)
-
         stackname_prefix = self.config.get('stacknameprefix', '')
 
         logger.debug("StackGroup {} added.".format(self.name))
@@ -61,8 +59,8 @@ class StackGroup(object):
             if stackname_prefix:
                 stackname = stackname_prefix + stackname
 
-            new_stack = Stack(name=stackname, template=template, path=stack_path, rel_path=str(self.rel_path), sg_config=self.config, ctx=self.ctx)
+            new_stack = Stack(name=stackname, template=template, path=stack_path, rel_path=str(self.rel_path), ctx=self.ctx)
-            new_stack.read_config()
+            new_stack.read_config(self.config)
             self.stacks.append(new_stack)
 
         # Create StackGroups recursively