Release 0.5.0

This commit is contained in:
Stefan Reimer 2019-04-18 16:30:50 +00:00
parent 55b67909e4
commit badb7b02c8
7 changed files with 132 additions and 142 deletions

View File

@ -1,5 +1,13 @@
# Changelog # Changelog
## 0.5.0
- new custom Jinja function `sub`, works the same as re.sub
- added possibility to use custom Jinja function `inline_yaml` to set data as yaml
- disabled SilentUndefined
- added Jinja2 extension `do` and `loopcontrols`
- new custom Jinja function `option` to access options at render time incl. default support for nested objects
- removed custom Jinja functions around old remote Ref handling
## 0.4.2 ## 0.4.2
- silence warnings by latest PyYaml 5.1 - silence warnings by latest PyYaml 5.1

View File

@ -4,7 +4,7 @@ test:
tox tox
clean: clean:
rm -rf .tox .cache dist rm -rf .tox .cache .coverage .eggs cloudbender.egg-info .pytest_cache dist
dist: dist:
python setup.py bdist_wheel --universal python setup.py bdist_wheel --universal

View File

@ -2,7 +2,7 @@ import logging
__author__ = 'Stefan Reimer' __author__ = 'Stefan Reimer'
__email__ = 'stefan@zero-downtimet.net' __email__ = 'stefan@zero-downtimet.net'
__version__ = '0.4.2' __version__ = '0.5.0'
# Set up logging to ``/dev/null`` like a library is supposed to. # Set up logging to ``/dev/null`` like a library is supposed to.

View File

@ -20,9 +20,6 @@ class CloudBender(object):
"parameter_path": os.path.join(self.root, "parameters"), "parameter_path": os.path.join(self.root, "parameters"),
"artifact_paths": [os.path.join(self.root, "artifacts")] "artifact_paths": [os.path.join(self.root, "artifacts")]
} }
self.default_settings = {
'vars': {'Mode': 'CloudBender'}
}
if not os.path.isdir(self.root): if not os.path.isdir(self.root):
raise "Check '{0}' exists and is a valid project folder.".format(root_path) raise "Check '{0}' exists and is a valid project folder.".format(root_path)
@ -55,7 +52,7 @@ class CloudBender(object):
ensure_dir(self.ctx[k]) ensure_dir(self.ctx[k])
self.sg = StackGroup(self.ctx['config_path'], self.ctx) self.sg = StackGroup(self.ctx['config_path'], self.ctx)
self.sg.read_config(self.default_settings) self.sg.read_config()
self.all_stacks = self.sg.get_stacks() self.all_stacks = self.sg.get_stacks()

View File

@ -8,6 +8,8 @@ import yaml
import jinja2 import jinja2
from jinja2.utils import missing, object_type_repr from jinja2.utils import missing, object_type_repr
from jinja2._compat import string_types from jinja2._compat import string_types
from jinja2.filters import make_attrgetter
from jinja2.runtime import Undefined
import pyminifier.token_utils import pyminifier.token_utils
import pyminifier.minification import pyminifier.minification
@ -15,76 +17,31 @@ import pyminifier.compression
import pyminifier.obfuscate import pyminifier.obfuscate
import types import types
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@jinja2.contextfunction @jinja2.contextfunction
def cloudbender_ctx(context, cb_ctx={}, reset=False, command=None, args={}): def option(context, attribute, default_value=u'', source='options'):
""" Get attribute from options data structure, default_value otherwise """
environment = context.environment
options = environment.globals['_config'][source]
# Reset state if not attribute:
if reset: return default_value
cb_ctx.clear()
return
if 'dependencies' not in cb_ctx:
cb_ctx['dependencies'] = set()
if 'mandatory_parameters' not in cb_ctx:
cb_ctx['mandatory_parameters'] = set()
if command == 'get_dependencies':
_deps = sorted(list(cb_ctx['dependencies']))
if _deps:
logger.debug("Stack depencies: {}".format(','.join(_deps)))
return _deps
elif command == 'add_dependency':
try: try:
cb_ctx['dependencies'].add(args['dep']) getter = make_attrgetter(environment, attribute)
logger.debug("Adding stack depency to {}".format(args['dep'])) value = getter(options)
except KeyError:
pass
else: if isinstance(value, Undefined):
raise("Unknown command") return default_value
return value
@jinja2.contextfunction except (jinja2.exceptions.UndefinedError):
def get_custom_att(context, att=None, ResourceName="FortyTwo", attributes={}, reset=False, dump=False): return default_value
""" Returns the rendered required fragement and also collects all foreign
attributes for the specified CustomResource to include them later in
the actual CustomResource include property """
# Reset state
if reset:
attributes.clear()
return
# return all registered attributes
if dump:
return attributes
# If called with an attribute, return fragement and register dependency
if att:
config = context.get_all()['_config']
if ResourceName not in attributes:
attributes[ResourceName] = set()
attributes[ResourceName].add(att)
if ResourceName == 'FortyTwo':
cloudbender_ctx(context, command='add_dependency', args={'dep': att.split('.')[0]})
if config['cfn']['Mode'] == "FortyTwo":
return('{{ "Fn::GetAtt": ["{0}", "{1}"] }}'.format(ResourceName, att))
elif config['cfn']['Mode'] == "AWSImport" and ResourceName == "FortyTwo":
# AWS only allows - and :, so replace '.' with ":"
return('{{ "Fn::ImportValue": {{ "Fn::Sub": "${{Conglomerate}}:{0}" }} }}'.format(att.replace('.', ':')))
else:
# We need to replace . with some PureAlphaNumeric thx AWS ...
return('{{ Ref: {0} }}'.format(att.replace('.', 'DoT')))
@jinja2.contextfunction @jinja2.contextfunction
@ -107,21 +64,6 @@ def include_raw_gz(context, files=None, gz=True):
return base64.b64encode(buf.getvalue()).decode('utf-8') return base64.b64encode(buf.getvalue()).decode('utf-8')
@jinja2.contextfunction
def render_once(context, name=None, resources=set(), reset=False):
""" Utility function returning True only once per name """
if reset:
resources.clear()
return
if name and name not in resources:
resources.add(name)
return True
return False
@jinja2.contextfunction @jinja2.contextfunction
def raise_helper(context, msg): def raise_helper(context, msg):
raise Exception(msg) raise Exception(msg)
@ -154,7 +96,7 @@ def search(value, pattern='', ignorecase=False):
# Custom filters # Custom filters
def regex_replace(value='', pattern='', replace='', ignorecase=False): def sub(value='', pattern='', replace='', ignorecase=False):
if ignorecase: if ignorecase:
flags = re.I flags = re.I
else: else:
@ -183,11 +125,11 @@ def pyminify(source, obfuscate=False, minify=True):
source = pyminifier.token_utils.untokenize(tokens) source = pyminifier.token_utils.untokenize(tokens)
# logger.info(source) # logger.info(source)
minified_source = pyminifier.compression.gz_pack(source) minified_source = pyminifier.compression.gz_pack(source)
logger.info("Compressed python code to {}".format(len(minified_source))) logger.info("Compressed python code from {} to {}".format(len(source), len(minified_source)))
return minified_source return minified_source
def parse_yaml(block): def inline_yaml(block):
return yaml.safe_load(block) return yaml.safe_load(block)
@ -219,8 +161,8 @@ class SilentUndefined(jinja2.Undefined):
def JinjaEnv(template_locations=[]): def JinjaEnv(template_locations=[]):
jenv = jinja2.Environment(trim_blocks=True, jenv = jinja2.Environment(trim_blocks=True,
lstrip_blocks=True, lstrip_blocks=True,
undefined=SilentUndefined,
extensions=['jinja2.ext.loopcontrols', 'jinja2.ext.do']) extensions=['jinja2.ext.loopcontrols', 'jinja2.ext.do'])
# undefined=SilentUndefined,
jinja_loaders = [] jinja_loaders = []
for _dir in template_locations: for _dir in template_locations:
@ -228,14 +170,12 @@ def JinjaEnv(template_locations=[]):
jenv.loader = jinja2.ChoiceLoader(jinja_loaders) jenv.loader = jinja2.ChoiceLoader(jinja_loaders)
jenv.globals['include_raw'] = include_raw_gz jenv.globals['include_raw'] = include_raw_gz
jenv.globals['get_custom_att'] = get_custom_att
jenv.globals['cloudbender_ctx'] = cloudbender_ctx
jenv.globals['render_once'] = render_once
jenv.globals['raise'] = raise_helper jenv.globals['raise'] = raise_helper
jenv.globals['option'] = option
jenv.filters['regex_replace'] = regex_replace jenv.filters['sub'] = sub
jenv.filters['pyminify'] = pyminify jenv.filters['pyminify'] = pyminify
jenv.filters['yaml'] = parse_yaml jenv.filters['inline_yaml'] = inline_yaml
jenv.tests['match'] = match jenv.tests['match'] = match
jenv.tests['regex'] = regex jenv.tests['regex'] = regex

View File

@ -4,7 +4,6 @@ import hashlib
import oyaml as yaml import oyaml as yaml
import json import json
import time import time
import subprocess
from datetime import datetime, timedelta from datetime import datetime, timedelta
from dateutil.tz import tzutc from dateutil.tz import tzutc
@ -33,17 +32,19 @@ class StackStatus(object):
class Stack(object): class Stack(object):
def __init__(self, name, path, rel_path, tags=None, parameters=None, template_vars=None, region='global', profile=None, template=None, ctx={}): def __init__(self, name, path, rel_path, tags=None, parameters=None, options=None, region='global', profile=None, template=None, ctx={}):
self.id = (profile, region, name) self.id = (profile, region, name)
self.stackname = name self.stackname = name
self.path = path self.path = path
self.rel_path = rel_path self.rel_path = rel_path
self.tags = tags self.tags = tags
self.parameters = parameters self.parameters = parameters
self.template_vars = template_vars self.options = options
self.region = region self.region = region
self.profile = profile self.profile = profile
self.template = template self.template = template
self.md5 = None
self.mode = 'CloudBender'
self.provides = template self.provides = template
self.cfn_template = None self.cfn_template = None
self.cfn_parameters = [] self.cfn_parameters = []
@ -68,11 +69,20 @@ class Stack(object):
if p in _config: if p in _config:
setattr(self, p, dict_merge(getattr(self, p), _config[p])) setattr(self, p, dict_merge(getattr(self, p), _config[p]))
# Inject Artifact for now hard coded # Inject Artifact if not explicitly set
if 'Artifact' not in self.tags:
self.tags['Artifact'] = self.provides self.tags['Artifact'] = self.provides
# backwards comp
if 'vars' in _config: if 'vars' in _config:
self.template_vars = dict_merge(self.template_vars, _config['vars']) self.options = dict_merge(self.options, _config['vars'])
if 'Mode' in self.options:
self.mode = self.options['Mode']
if 'options' in _config:
self.options = dict_merge(self.options, _config['options'])
if 'Mode' in self.options:
self.mode = self.options['Mode']
if 'dependencies' in _config: if 'dependencies' in _config:
for dep in _config['dependencies']: for dep in _config['dependencies']:
@ -83,62 +93,73 @@ class Stack(object):
def render(self): def render(self):
"""Renders the cfn jinja template for this stack""" """Renders the cfn jinja template for this stack"""
jenv = JinjaEnv(self.ctx['artifact_paths'])
template = jenv.get_template('{0}{1}'.format(self.template, '.yaml.jinja'))
template_metadata = { template_metadata = {
'Template.Name': self.template, 'Template.Name': self.template,
'Template.Hash': 'tbd', 'Template.Hash': "__HASH__",
'CloudBender.Version': __version__ 'CloudBender.Version': __version__
} }
cb = False # cfn is provided for old configs
if self.template_vars['Mode'] == "CloudBender": _config = {'mode': self.mode, 'options': self.options, 'metadata': template_metadata, 'cfn': self.options}
cb = True
_config = {'cb': cb, 'cfn': self.template_vars, 'Metadata': template_metadata}
jenv = JinjaEnv(self.ctx['artifact_paths'])
jenv.globals['_config'] = _config jenv.globals['_config'] = _config
# First render pass to calculate a md5 checksum template = jenv.get_template('{0}{1}'.format(self.template, '.yaml.jinja'))
template_metadata['Template.Hash'] = hashlib.md5(template.render(_config).encode('utf-8')).hexdigest()
# Reset and set Metadata for final render pass
jenv.globals['get_custom_att'](context={'_config': self.template_vars}, reset=True)
jenv.globals['render_once'](context={'_config': self.template_vars}, reset=True)
jenv.globals['cloudbender_ctx'](context={'_config': self.template_vars}, reset=True)
# Try to add latest tag/commit for the template source, skip if not in git tree
try:
_comment = subprocess.check_output('git log -1 --pretty=%B {}'.format(template.filename).split(' ')).decode('utf-8').strip().replace('"', '').replace('#', '').replace('\n', '').replace(':', ' ')
if _comment:
template_metadata['Template.LastGitComment'] = _comment
except subprocess.CalledProcessError:
pass
logger.info('Rendering %s', template.filename) logger.info('Rendering %s', template.filename)
rendered = template.render(_config)
try: try:
self.data = yaml.safe_load(rendered) self.cfn_template = template.render(_config)
self.cfn_data = yaml.safe_load(self.cfn_template)
except Exception as e: except Exception as e:
# In case we rendered invalid yaml this helps to debug # In case we rendered invalid yaml this helps to debug
logger.error(rendered) if self.cfn_template:
logger.error(self.cfn_template)
raise e raise e
# Some sanity checks and final cosmetics # Some sanity checks and final cosmetics
# Check for empty top level Parameters, Outputs and Conditions and remove # Check for empty top level Parameters, Outputs and Conditions and remove
for key in ['Parameters', 'Outputs', 'Conditions']: for key in ['Parameters', 'Outputs', 'Conditions']:
if key in self.data and self.data[key] is None: if key in self.cfn_data and self.cfn_data[key] is None:
# Delete from data structure which also takes care of json # Delete from data structure which also takes care of json
del self.data[key] del self.cfn_data[key]
# but also remove from rendered for the yaml file
rendered = rendered.replace('\n' + key + ":", '')
# Condense multiple empty lines to one # but also remove from rendered for the yaml file
self.cfn_template = re.sub(r'\n\s*\n', '\n\n', rendered) self.cfn_template = self.cfn_template.replace('\n' + key + ":", '')
if not re.search('CloudBender::', self.cfn_template):
logger.info("CloudBender not required -> removing Transform")
del self.cfn_data['Transform']
self.cfn_template = self.cfn_template.replace('Transform: [CloudBender]', '')
# Remove and condense multiple empty lines
self.cfn_template = re.sub(r'\n\s*\n', '\n\n', self.cfn_template)
self.cfn_template = re.sub(r'^\s*', '', self.cfn_template)
self.cfn_template = re.sub(r'\s*$', '', self.cfn_template)
# set md5 last
self.md5 = hashlib.md5(self.cfn_template.encode('utf-8')).hexdigest()
self.cfn_data['Metadata']['Hash'] = self.md5
# Add Legacy FortyTwo if needed to prevent AWS from replacing existing resources for NO reason ;-(
include = []
search_attributes(self.cfn_data, include)
if len(include):
_res = """
FortyTwo:
Type: Custom::FortyTwo
Properties:
ServiceToken:
Fn::Sub: "arn:aws:lambda:${{AWS::Region}}:${{AWS::AccountId}}:function:FortyTwo"
UpdateToken: {}
Include: {}""".format(self.md5, sorted(set(include)))
self.cfn_data['Resources'].update(yaml.load(_res))
self.cfn_template = re.sub(r'Resources:', r'Resources:' + _res + '\n', self.cfn_template)
logger.info("Legacy Mode -> added Custom::FortyTwo")
self.cfn_template = self.cfn_template.replace('__HASH__', self.md5)
# Update internal data structures # Update internal data structures
self._parse_metadata() self._parse_metadata()
@ -146,7 +167,7 @@ class Stack(object):
def _parse_metadata(self): def _parse_metadata(self):
# Extract dependencies if present # Extract dependencies if present
try: try:
for dep in self.data['Metadata']['CloudBender']['Dependencies']: for dep in self.cfn_data['Metadata']['CloudBender']['Dependencies']:
self.dependencies.add(dep) self.dependencies.add(dep)
except KeyError: except KeyError:
pass pass
@ -178,7 +199,7 @@ class Stack(object):
self.cfn_template = yaml_contents.read() self.cfn_template = yaml_contents.read()
logger.debug('Read cfn template %s.', yaml_file) logger.debug('Read cfn template %s.', yaml_file)
self.data = yaml.safe_load(self.cfn_template) self.cfn_data = yaml.safe_load(self.cfn_template)
self._parse_metadata() self._parse_metadata()
else: else:
@ -189,14 +210,18 @@ class Stack(object):
self.read_template_file() self.read_template_file()
try: try:
ignore_checks = self.data['Metadata']['cfnlint_ignore'] ignore_checks = self.cfn_data['Metadata']['cfnlint_ignore']
except KeyError: except KeyError:
ignore_checks = [] ignore_checks = []
# Ignore some more checks around injected parameters as we generate these # Ignore some more checks around injected parameters as we generate these
if self.template_vars['Mode'] == "Piped": if self.mode == "Piped":
ignore_checks = ignore_checks + ['W2505', 'W2509', 'W2507'] ignore_checks = ignore_checks + ['W2505', 'W2509', 'W2507']
# Ignore checks regarding overloaded properties
if self.mode == "CloudBender":
ignore_checks = ignore_checks + ['E3035', 'E3002', 'E3012', 'W2001', 'E3001']
filename = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml") filename = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml")
logger.info('Validating {0}'.format(filename)) logger.info('Validating {0}'.format(filename))
@ -223,18 +248,18 @@ class Stack(object):
# Inspect all outputs of the running Conglomerate members # Inspect all outputs of the running Conglomerate members
# if we run in Piped Mode # if we run in Piped Mode
# if self.template_vars['Mode'] == "Piped": # if self.mode == "Piped":
# try: # try:
# stack_outputs = inspect_stacks(config['tags']['Conglomerate']) # stack_outputs = inspect_stacks(config['tags']['Conglomerate'])
# logger.info(pprint.pformat(stack_outputs)) # logger.info(pprint.pformat(stack_outputs))
# except KeyError: # except KeyError:
# pass # pass
if 'Parameters' in self.data: if 'Parameters' in self.cfn_data:
self.cfn_parameters = [] self.cfn_parameters = []
for p in self.data['Parameters']: for p in self.cfn_data['Parameters']:
# In Piped mode we try to resolve all Paramters first via stack_outputs # In Piped mode we try to resolve all Paramters first via stack_outputs
# if config['cfn']['Mode'] == "Piped": # if self.mode == "Piped":
# try: # try:
# # first reverse the rename due to AWS alphanumeric restriction for parameter names # # first reverse the rename due to AWS alphanumeric restriction for parameter names
# _p = p.replace('DoT','.') # _p = p.replace('DoT','.')
@ -252,7 +277,7 @@ class Stack(object):
logger.info('{} {} Parameter {}={}'.format(self.region, self.stackname, p, value)) logger.info('{} {} Parameter {}={}'.format(self.region, self.stackname, p, value))
except KeyError: except KeyError:
# If we have a Default defined in the CFN skip, as AWS will use it # If we have a Default defined in the CFN skip, as AWS will use it
if 'Default' in self.data['Parameters'][p]: if 'Default' in self.cfn_data['Parameters'][p]:
continue continue
else: else:
logger.error('Cannot find value for parameter {0}'.format(p)) logger.error('Cannot find value for parameter {0}'.format(p))
@ -470,3 +495,23 @@ class Stack(object):
# Ensure output dirs exist # Ensure output dirs exist
if not os.path.exists(os.path.join(self.ctx[path], self.rel_path)): if not os.path.exists(os.path.join(self.ctx[path], self.rel_path)):
os.makedirs(os.path.join(self.ctx[path], self.rel_path)) os.makedirs(os.path.join(self.ctx[path], self.rel_path))
def search_attributes(template, attributes):
""" Traverses a template and searches for all Fn::GetAtt calls to FortyTwo
adding them to the passed in attributes set
"""
if isinstance(template, dict):
for k, v in template.items():
# Look for Fn::GetAtt
if k == "Fn::GetAtt" and isinstance(v, list):
if v[0] == "FortyTwo":
attributes.append(v[1])
if isinstance(v, dict) or isinstance(v, list):
search_attributes(v, attributes)
elif isinstance(template, list):
for k in template:
if isinstance(k, dict) or isinstance(k, list):
search_attributes(k, attributes)

View File

@ -50,7 +50,7 @@ class StackGroup(object):
tags = _config.get('tags', {}) tags = _config.get('tags', {})
parameters = _config.get('parameters', {}) parameters = _config.get('parameters', {})
template_vars = _config.get('vars', {}) options = _config.get('options', {})
region = _config.get('region', 'global') region = _config.get('region', 'global')
profile = _config.get('profile', '') profile = _config.get('profile', '')
stackname_prefix = _config.get('stacknameprefix', '') stackname_prefix = _config.get('stacknameprefix', '')
@ -67,7 +67,7 @@ class StackGroup(object):
new_stack = Stack( new_stack = Stack(
name=stackname, template=template, path=stack_path, rel_path=str(self.rel_path), name=stackname, template=template, path=stack_path, rel_path=str(self.rel_path),
tags=dict(tags), parameters=dict(parameters), template_vars=dict(template_vars), tags=dict(tags), parameters=dict(parameters), options=dict(options),
region=str(region), profile=str(profile), ctx=self.ctx) region=str(region), profile=str(profile), ctx=self.ctx)
new_stack.read_config() new_stack.read_config()
self.stacks.append(new_stack) self.stacks.append(new_stack)