OS environment variable support in config files, new jinja functions, cleanup

CHANGES.md

@@ -0,0 +1,8 @@
+# Changelog
+
+## 0.4.0
+- support for environment variables in any config file  
+  Example: `profile: {{ env.AWS_DEFAULT_PROFILE }}`
+- support for jinja `{% do %}` extension
+- support for inline yaml style complex data definitions, via custom jinja filter `yaml`
+- missing variables now cause warnings, but rendering continues with ''

cloudbender/__init__.py

@@ -2,7 +2,7 @@ import logging
 
 __author__ = 'Stefan Reimer'
 __email__ = 'stefan@zero-downtimet.net'
-__version__ = '0.3.3'
+__version__ = '0.4.0'
 
 
 # Set up logging to ``/dev/null`` like a library is supposed to.

cloudbender/cli.py

@@ -187,3 +187,6 @@ cli.add_command(provision)
 cli.add_command(delete)
 cli.add_command(clean)
 cli.add_command(create_change_set)
+
+if __name__ == '__main__':
+    cli(obj={})

cloudbender/core.py

@@ -1,8 +1,9 @@
 import os
 import logging
 
-from .utils import read_yaml_file, ensure_dir
+from .utils import ensure_dir
 from .stackgroup import StackGroup
+from .jinja import read_config_file
 
 logger = logging.getLogger(__name__)
 
@@ -20,7 +21,7 @@ class CloudBender(object):
             "artifact_paths": [os.path.join(self.root, "artifacts")]
         }
         self.default_settings = {
-            'vars': {'Mode': 'FortyTwo'}
+            'vars': {'Mode': 'CloudBender'}
         }
 
         if not os.path.isdir(self.root):
@@ -30,7 +31,7 @@ class CloudBender(object):
         """Load the <path>/config.yaml, <path>/*.yaml as stacks, sub-folders are child groups """
 
         # Read top level config.yaml and extract CloudBender CTX
-        _config = read_yaml_file(os.path.join(self.ctx['config_path'], 'config.yaml'))
+        _config = read_config_file(os.path.join(self.ctx['config_path'], 'config.yaml'))
         if _config and _config.get('CloudBender'):
             self.ctx.update(_config.get('CloudBender'))
 

cloudbender/jinja.py

@@ -1,8 +1,13 @@
+import os
 import io
 import gzip
-import jinja2
 import re
 import base64
+import yaml
+
+import jinja2
+from jinja2.utils import missing, object_type_repr
+from jinja2._compat import string_types
 
 import pyminifier.token_utils
 import pyminifier.minification
@@ -26,6 +31,9 @@ def cloudbender_ctx(context, cb_ctx={}, reset=False, command=None, args={}):
     if 'dependencies' not in cb_ctx:
         cb_ctx['dependencies'] = set()
 
+    if 'mandatory_parameters' not in cb_ctx:
+        cb_ctx['mandatory_parameters'] = set()
+
     if command == 'get_dependencies':
         _deps = sorted(list(cb_ctx['dependencies']))
         if _deps:
@@ -179,11 +187,40 @@ def pyminify(source, obfuscate=False, minify=True):
     return minified_source
 
 
+def parse_yaml(block):
+    return yaml.safe_load(block)
+
+
+class SilentUndefined(jinja2.Undefined):
+    '''
+    Log warning for undefiend but continue
+    '''
+    def _fail_with_undefined_error(self, *args, **kwargs):
+        if self._undefined_hint is None:
+            if self._undefined_obj is missing:
+                hint = '%r is undefined' % self._undefined_name
+            elif not isinstance(self._undefined_name, string_types):
+                hint = '%s has no element %r' % (
+                    object_type_repr(self._undefined_obj),
+                    self._undefined_name
+                )
+            else:
+                hint = '%r has no attribute %r' % (
+                    object_type_repr(self._undefined_obj),
+                    self._undefined_name
+                )
+        else:
+            hint = self._undefined_hint
+
+        logger.warning("Undefined variable: {}".format(hint))
+        return ''
+
+
 def JinjaEnv(template_locations=[]):
     jenv = jinja2.Environment(trim_blocks=True,
                               lstrip_blocks=True,
-                              undefined=jinja2.Undefined,
+                              undefined=SilentUndefined,
-                              extensions=['jinja2.ext.loopcontrols'])
+                              extensions=['jinja2.ext.loopcontrols', 'jinja2.ext.do'])
 
     jinja_loaders = []
     for _dir in template_locations:
@@ -198,9 +235,34 @@ def JinjaEnv(template_locations=[]):
 
     jenv.filters['regex_replace'] = regex_replace
     jenv.filters['pyminify'] = pyminify
+    jenv.filters['yaml'] = parse_yaml
 
     jenv.tests['match'] = match
     jenv.tests['regex'] = regex
     jenv.tests['search'] = search
 
     return jenv
+
+
+def read_config_file(path, jinja_args=None):
+    """ reads yaml config file, passes it through jinja and returns data structre """
+
+    if os.path.exists(path):
+        logger.debug("Reading config file: {}".format(path))
+        try:
+            jenv = jinja2.Environment(
+                loader=jinja2.FileSystemLoader(os.path.dirname(path)),
+                undefined=jinja2.StrictUndefined,
+                extensions=['jinja2.ext.loopcontrols'])
+            template = jenv.get_template(os.path.basename(path))
+            rendered_template = template.render(
+                env=os.environ
+            )
+            data = yaml.safe_load(rendered_template)
+            if data:
+                return data
+
+        except Exception as e:
+            logger.exception("Error reading config file: {} ({})".format(path, e))
+
+    return {}

cloudbender/stack.py

@@ -1,6 +1,5 @@
 import os
 import re
-import semver
 import hashlib
 import oyaml as yaml
 import json
@@ -10,12 +9,11 @@ import subprocess
 from datetime import datetime, timedelta
 from dateutil.tz import tzutc
 
-import botocore
 from botocore.exceptions import ClientError
 
-from .utils import read_yaml_file, dict_merge
+from .utils import dict_merge
 from .connection import BotoConnection
-from .jinja import JinjaEnv
+from .jinja import JinjaEnv, read_config_file
 from . import __version__
 
 import cfnlint.core
@@ -61,7 +59,7 @@ class Stack(object):
         logger.debug("<Stack {}: {}>".format(self.id, vars(self)))
 
     def read_config(self):
-        _config = read_yaml_file(self.path)
+        _config = read_config_file(self.path)
         for p in ["region", "stackname", "template", "default_lock", "multi_delete", "provides"]:
             if p in _config:
                 setattr(self, p, _config[p])
@@ -82,27 +80,6 @@ class Stack(object):
 
         logger.debug("Stack {} added.".format(self.id))
 
-    def check_fortytwo(self, template):
-        # Fail early if 42 is enabled but not available
-        if self.cfn['Mode'] == "FortyTwo" and self.template != 'FortyTwo':
-            try:
-                response = self.connection_manager.call(
-                    'lambda', 'get_function', {'FunctionName': 'FortyTwo'},
-                    profile=self.profile, region=self.region)
-
-                # Also verify version in case specified in the template's metadata
-                try:
-                    req_ver = template['Metadata']['FortyTwo']['RequiredVersion']
-                    if 'Release' not in response['Tags']:
-                        raise("Lambda FortyTwo has no Release Tag! Required: {}".format(req_ver))
-                    elif semver.compare(req_ver, re.sub("-.*$", '', response['Tags']['Release'])) > 0:
-                        raise("Lambda FortyTwo version is not recent enough! Required: {} vs. Found: {}".format(req_ver, response['Tags']['Release']))
-                except KeyError:
-                    pass
-
-            except botocore.exceptions.ClientError:
-                raise("No Lambda FortyTwo found in your account")
-
     def render(self):
         """Renders the cfn jinja template for this stack"""
 
@@ -266,7 +243,7 @@ class Stack(object):
                 try:
                     value = str(self.parameters[p])
                     self.cfn_parameters.append({'ParameterKey': p, 'ParameterValue': value})
-                    logger.info('Got {} = {}'.format(p, value))
+                    logger.info('{} {} Parameter {}={}'.format(self.region, self.stackname, p, value))
                 except KeyError:
                     # If we have a Default defined in the CFN skip, as AWS will use it
                     if 'Default' in self.data['Parameters'][p]:
@@ -311,7 +288,7 @@ class Stack(object):
                 'TemplateBody': self.cfn_template,
                 'Parameters': self.cfn_parameters,
                 'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()],
-                'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
+                'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND']},
             profile=self.profile, region=self.region)
 
         return self._wait_for_completion()
@@ -332,7 +309,7 @@ class Stack(object):
                     'TemplateBody': self.cfn_template,
                     'Parameters': self.cfn_parameters,
                     'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()],
-                    'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
+                    'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND']},
                 profile=self.profile, region=self.region)
 
         except ClientError as e:

cloudbender/stackgroup.py

@@ -2,7 +2,8 @@ import os
 import glob
 import logging
 
-from .utils import read_yaml_file, dict_merge
+from .utils import dict_merge
+from .jinja import read_config_file
 from .stack import Stack
 
 logger = logging.getLogger(__name__)
@@ -36,7 +37,7 @@ class StackGroup(object):
             return None
 
         # First read config.yaml if present
-        _config = read_yaml_file(os.path.join(self.path, 'config.yaml'))
+        _config = read_config_file(os.path.join(self.path, 'config.yaml'))
 
         # Stack Group name if not explicit via config is derived from subfolder, or in case of root object the parent folder
         if "stackgroupname" in _config:
@@ -119,40 +120,3 @@ class StackGroup(object):
                     return s
 
         return None
-
-    # TODO: Integrate properly into stackgroup class, broken for now
-    # stackoutput inspection
-    def BROKEN_inspect_stacks(self, conglomerate):
-        # Get all stacks of the conglomertate
-        response = self.connection_manager.call('cloudformation', 'decribe_stacks')
-
-        stacks = []
-        for stack in response['Stacks']:
-            for tag in stack['Tags']:
-                if tag['Key'] == 'Conglomerate' and tag['Value'] == conglomerate:
-                    stacks.append(stack)
-                    break
-
-        # Gather stack outputs, use Tag['Artifact'] as name space: Artifact.OutputName, same as FortyTwo
-        stack_outputs = {}
-        for stack in stacks:
-            # If stack has an Artifact Tag put resources into the namespace Artifact.Resource
-            artifact = None
-            for tag in stack['Tags']:
-                if tag['Key'] == 'Artifact':
-                    artifact = tag['Value']
-
-            if artifact:
-                key_prefix = "{}.".format(artifact)
-            else:
-                key_prefix = ""
-
-            try:
-                for output in stack['Outputs']:
-                    # Gather all outputs of the stack into one dimensional key=value structure
-                    stack_outputs[key_prefix + output['OutputKey']] = output['OutputValue']
-            except KeyError:
-                pass
-
-        # Add outputs from stacks into the data for jinja under StackOutput
-        return stack_outputs

cloudbender/utils.py

@@ -1,5 +1,4 @@
 import os
-import yaml
 import copy
 import logging
 import boto3
@@ -7,21 +6,6 @@ import boto3
 logger = logging.getLogger(__name__)
 
 
-def read_yaml_file(path):
-    data = {}
-    if os.path.exists(path):
-        with open(path, 'r') as config_file_contents:
-            logger.debug("Reading config file: {}".format(path))
-            try:
-                _data = yaml.load(config_file_contents.read())
-                if _data:
-                    data.update(_data)
-            except Exception as e:
-                logger.warning("Error reading config file: {} ({})".format(path, e))
-
-    return data
-
-
 def dict_merge(a, b):
     """ Deep merge to allow proper inheritance for config files"""
     if not a: