Code clean up PEP8, some fixes along the way

This commit is contained in:
Stefan Reimer 2019-02-07 15:36:16 +00:00
parent 3587451170
commit 47b492a65e
7 changed files with 119 additions and 172 deletions

View File

@ -11,13 +11,14 @@ from .utils import setup_logging
import logging
logger = logging.getLogger(__name__)
@click.group()
@click.version_option(version=__version__, prog_name="CloudBender")
@click.option("--debug", is_flag=True, help="Turn on debug logging.")
@click.option("--dir", "directory", help="Specify cloudbender project directory.")
@click.pass_context
def cli(ctx, debug, directory):
logger = setup_logging(debug)
setup_logging(debug)
# Read global config
cb = CloudBender(directory if directory else os.getcwd())
@ -86,7 +87,7 @@ def provision(ctx, stack_names, multi):
futures.append(group.submit(stack.update))
for future in as_completed(futures):
result = future.result()
future.result()
@click.command()
@ -109,7 +110,7 @@ def delete(ctx, stack_names, multi):
futures.append(group.submit(stack.delete))
for future in as_completed(futures):
result = future.result()
future.result()
@click.command()
@ -140,13 +141,14 @@ def sort_stacks(ctx, stacks):
data[s.id] = set(deps)
logger.debug("Stack {} depends on {}".format(s.id, deps))
# Ignore self dependencies
for k, v in data.items():
v.discard(k) # Ignore self dependencies
v.discard(k)
extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys())
data.update({item:set() for item in extra_items_in_deps})
data.update({item: set() for item in extra_items_in_deps})
while True:
ordered = set(item for item,dep in data.items() if not dep)
ordered = set(item for item, dep in data.items() if not dep)
if not ordered:
break
@ -154,10 +156,11 @@ def sort_stacks(ctx, stacks):
result = []
for o in ordered:
for s in stacks:
if s.id == o: result.append(s)
if s.id == o:
result.append(s)
yield result
data = {item: (dep - ordered) for item,dep in data.items()
data = {item: (dep - ordered) for item, dep in data.items()
if item not in ordered}
assert not data, "A cyclic dependency exists amongst %r" % data
@ -167,7 +170,7 @@ def _find_stacks(ctx, stack_names, multi=False):
stacks = []
for s in stack_names:
stacks = stacks+cb.resolve_stacks(s)
stacks = stacks + cb.resolve_stacks(s)
if not multi and len(stacks) > 1:
logger.error('Found more than one stack matching name ({}). Please set --multi if that is what you want.'.format(', '.join(stack_names)))

View File

@ -9,48 +9,46 @@ import logging
logger = logging.getLogger(__name__)
class BotoConnection():
_sessions= {}
_sessions = {}
_clients = {}
def __init__(self, profile=None, region=None):
self.region = region
self.profile = profile
def _get_session(self, profile=None, region=None):
if self._sessions.get((profile,region)):
return self._sessions[(profile,region)]
if self._sessions.get((profile, region)):
return self._sessions[(profile, region)]
# Construct botocore session with cache
# Setup boto to cache STS tokens for MFA
# Change the cache path from the default of ~/.aws/boto/cache to the one used by awscli
session_vars = {}
if profile:
session_vars['profile'] = (None,None,profile,None)
session_vars['profile'] = (None, None, profile, None)
if region and region != 'global':
session_vars['region'] = (None,None,region,None)
session_vars['region'] = (None, None, region, None)
session = botocore.session.Session(session_vars=session_vars)
cli_cache = os.path.join(os.path.expanduser('~'),'.aws/cli/cache')
cli_cache = os.path.join(os.path.expanduser('~'), '.aws/cli/cache')
session.get_component('credential_provider').get_provider('assume-role').cache = credentials.JSONFileCache(cli_cache)
self._sessions[(profile,region)] = session
self._sessions[(profile, region)] = session
return session
def _get_client(self, service, profile=None, region=None):
if self._clients.get((profile,region,service)):
return self._clients[(profile,region,service)]
if self._clients.get((profile, region, service)):
return self._clients[(profile, region, service)]
session = self._get_session(profile,region)
session = self._get_session(profile, region)
client = boto3.Session(botocore_session=session).client(service)
self._clients[(profile,region,service)] = client
self._clients[(profile, region, service)] = client
return client
def call(self, service, command, kwargs={}, profile=None, region=None):
while True:
try:
@ -64,4 +62,3 @@ class BotoConnection():
pass
else:
raise e

View File

@ -1,14 +1,12 @@
import os
import glob
import logging
from .utils import read_yaml_file, ensure_dir
from .stack import Stack
from .stackgroup import StackGroup
from .connection import BotoConnection
logger = logging.getLogger(__name__)
class CloudBender(object):
""" Config Class to handle recursive conf/* config tree """
def __init__(self, root_path):
@ -20,15 +18,14 @@ class CloudBender(object):
"template_path": os.path.join(self.root, "cloudformation"),
"parameter_path": os.path.join(self.root, "parameters"),
"artifact_paths": [os.path.join(self.root, "artifacts")]
}
}
self.default_settings = {
'vars': { 'Mode': 'FortyTwo' }
'vars': {'Mode': 'FortyTwo'}
}
if not os.path.isdir(self.root):
raise "Check '{0}' exists and is a valid project folder.".format(root_path)
def read_config(self):
"""Load the <path>/config.yaml, <path>/*.yaml as stacks, sub-folders are child groups """
@ -39,7 +36,7 @@ class CloudBender(object):
# Make sure all paths are abs
for k, v in self.ctx.items():
if k in ['config_path','template_path','parameter_path','artifact_paths']:
if k in ['config_path', 'template_path', 'parameter_path', 'artifact_paths']:
if isinstance(v, list):
new_list = []
for path in v:
@ -51,9 +48,9 @@ class CloudBender(object):
elif isinstance(v, str):
if not os.path.isabs(v):
self.ctx[k]=os.path.normpath(os.path.join(self.root, v))
self.ctx[k] = os.path.normpath(os.path.join(self.root, v))
if k in ['template_path','parameter_path']:
if k in ['template_path', 'parameter_path']:
ensure_dir(self.ctx[k])
self.sg = StackGroup(self.ctx['config_path'], self.ctx)
@ -66,18 +63,15 @@ class CloudBender(object):
# _config = { "vars": { 'Azs': {'TestAZ': 'Next'}, 'Segments': {'Testnet': 'internet'}, "Mode": "Piped" } }
# self.vars.update(_config.get('vars'))
def dump_config(self):
logger.debug("<CloudBender: {}>".format(vars(self)))
self.sg.dump_config()
def clean(self):
for s in self.all_stacks:
s.delete_template_file()
s.delete_parameter_file()
def resolve_stacks(self, token):
stacks = []
@ -100,7 +94,6 @@ class CloudBender(object):
return stacks
def filter_stacks(self, filter_by, stacks=None):
# filter_by is a dict { property, value }
@ -112,7 +105,7 @@ class CloudBender(object):
for s in stacks:
match = True
for p,v in filter_by.items():
for p, v in filter_by.items():
if not (hasattr(s, p) and getattr(s, p) == v):
match = False
break

View File

@ -1,8 +1,6 @@
import os
import io
import gzip
import jinja2
import oyaml as yaml
import re
import base64
@ -75,10 +73,10 @@ def get_custom_att(context, att=None, ResourceName="FortyTwo", attributes={}, re
return('{{ "Fn::GetAtt": ["{0}", "{1}"] }}'.format(ResourceName, att))
elif config['cfn']['Mode'] == "AWSImport" and ResourceName == "FortyTwo":
# AWS only allows - and :, so replace '.' with ":"
return('{{ "Fn::ImportValue": {{ "Fn::Sub": "${{Conglomerate}}:{0}" }} }}'.format(att.replace('.',':')))
return('{{ "Fn::ImportValue": {{ "Fn::Sub": "${{Conglomerate}}:{0}" }} }}'.format(att.replace('.', ':')))
else:
# We need to replace . with some PureAlphaNumeric thx AWS ...
return('{{ Ref: {0} }}'.format(att.replace('.','DoT')))
return('{{ Ref: {0} }}'.format(att.replace('.', 'DoT')))
@jinja2.contextfunction
@ -133,7 +131,7 @@ def regex(value='', pattern='', ignorecase=False, match_type='search'):
flags = 0
_re = re.compile(pattern, flags=flags)
if getattr(_re, match_type, 'search')(value) is not None:
return True
return True
return False
@ -153,13 +151,15 @@ def regex_replace(value='', pattern='', replace='', ignorecase=False):
flags = re.I
else:
flags = 0
return re.sub(pattern,replace,value,flags=flags)
return re.sub(pattern, replace, value, flags=flags)
def pyminify(source, obfuscate=False, minify=True):
# pyminifier options
options = types.SimpleNamespace(tabs=False,replacement_length=1,use_nonlatin=0,
obfuscate=0,obf_variables=1,obf_classes=0,obf_functions=0,obf_import_methods=0,obf_builtins=0)
options = types.SimpleNamespace(
tabs=False, replacement_length=1, use_nonlatin=0,
obfuscate=0, obf_variables=1, obf_classes=0, obf_functions=0,
obf_import_methods=0, obf_builtins=0)
tokens = pyminifier.token_utils.listified_tokenizer(source)
@ -170,10 +170,10 @@ def pyminify(source, obfuscate=False, minify=True):
if obfuscate:
name_generator = pyminifier.obfuscate.obfuscation_machine(use_unicode=False)
pyminifier.obfuscate.obfuscate("__main__", tokens, options, name_generator=name_generator)
#source = pyminifier.obfuscate.apply_obfuscation(source)
# source = pyminifier.obfuscate.apply_obfuscation(source)
source = pyminifier.token_utils.untokenize(tokens)
#logger.info(source)
# logger.info(source)
minified_source = pyminifier.compression.gz_pack(source)
logger.info("Compressed python code to {}".format(len(minified_source)))
return minified_source

View File

@ -5,6 +5,7 @@ import hashlib
import oyaml as yaml
import json
import time
import subprocess
from datetime import datetime, timedelta
from dateutil.tz import tzutc
@ -56,11 +57,9 @@ class Stack(object):
self.default_lock = None
self.multi_delete = True
def dump_config(self):
logger.debug("<Stack {}: {}>".format(self.id, vars(self)))
def read_config(self):
_config = read_yaml_file(self.path)
for p in ["region", "stackname", "template", "default_lock", "multi_delete", "provides"]:
@ -83,27 +82,26 @@ class Stack(object):
logger.debug("Stack {} added.".format(self.id))
def check_fortytwo(self, template):
# Fail early if 42 is enabled but not available
if self.cfn['Mode'] == "FortyTwo" and self.template != 'FortyTwo':
try:
response = self.connection_manager.call('lambda', 'get_function', {'FunctionName': 'FortyTwo'},
profile=self.profile, region=self.region)
response = self.connection_manager.call(
'lambda', 'get_function', {'FunctionName': 'FortyTwo'},
profile=self.profile, region=self.region)
# Also verify version in case specified in the template's metadata
try:
req_ver = template['Metadata']['FortyTwo']['RequiredVersion']
if 'Release' not in response['Tags']:
abort("Lambda FortyTwo has no Release Tag! Required: {}".format(req_ver))
elif semver.compare(req_ver, re.sub("-.*$",'', response['Tags']['Release'])) > 0:
abort("Lambda FortyTwo version is not recent enough! Required: {} vs. Found: {}".format(req_ver, response['Tags']['Release']))
raise("Lambda FortyTwo has no Release Tag! Required: {}".format(req_ver))
elif semver.compare(req_ver, re.sub("-.*$", '', response['Tags']['Release'])) > 0:
raise("Lambda FortyTwo version is not recent enough! Required: {} vs. Found: {}".format(req_ver, response['Tags']['Release']))
except KeyError:
pass
except botocore.exceptions.ClientError:
abort("No Lambda FortyTwo found in your account")
raise("No Lambda FortyTwo found in your account")
def render(self):
"""Renders the cfn jinja template for this stack"""
@ -114,56 +112,38 @@ class Stack(object):
template_metadata = {
'Template.Name': self.template,
'Template.Hash': 'unknown',
'Template.GitComment': 'unknown',
'Template.Hash': 'tbd',
'CloudBender.Version': __version__
}
jenv.globals['_config'] = { 'cfn': self.template_vars, 'Metadata': template_metadata }
jenv.globals['_config'] = {'cfn': self.template_vars, 'Metadata': template_metadata}
# First render pass to calculate a md5 checksum
template_metadata['Template.Hash'] = hashlib.md5(template.render({ 'cfn': self.template_vars, 'Metadata': template_metadata }).encode('utf-8')).hexdigest()
template_metadata['Template.Hash'] = hashlib.md5(template.render({'cfn': self.template_vars, 'Metadata': template_metadata}).encode('utf-8')).hexdigest()
# Reset and set Metadata for final render pass
jenv.globals['get_custom_att'](context={'_config': self.template_vars}, reset=True)
jenv.globals['render_once'](context={'_config': self.template_vars}, reset=True)
jenv.globals['cloudbender_ctx'](context={'_config': self.template_vars}, reset=True)
# try to get local git info
# Try to add latest tag/commit for the template source, skip if not in git tree
try:
self.template_vars['Metadata']['{}.Version'.format(PROJECT_NAME)] = subprocess.check_output('git describe --tags'.split(' '), universal_newlines=True)[:-1]
except:
pass
# Add latest tag/commit
try:
os.chdir(ROOT_DIR)
_version = subprocess.check_output('git describe --tags'.split(' '), universal_newlines=True)[:-1]
if _version:
self.template_vars['Metadata']['CloudBender.Version'] = _version
os.chdir(os.path.dirname(template.filename))
_comment = subprocess.check_output('git log -1 --pretty=%B {0}{1}'
.format(input_file, TEMPLATE_EXT).split(' ')).decode('utf-8').strip() \
.replace('"', '').replace('#', '').replace('\n', '').replace(':', ' ')
_comment = subprocess.check_output('git log -1 --pretty=%B {}'.format(template.filename).split(' ')).decode('utf-8').strip().replace('"', '').replace('#', '').replace('\n', '').replace(':', ' ')
if _comment:
self.template_vars['Metadata']['Template.GitComment'] = _comment
template_metadata['Template.LastGitComment'] = _comment
os.chdir(PROJECT_DIR)
except:
except subprocess.CalledProcessError:
pass
logger.info('Rendering %s', template.filename)
rendered = template.render({ 'cfn': self.template_vars, 'Metadata': template_metadata })
rendered = template.render({'cfn': self.template_vars, 'Metadata': template_metadata})
try:
self.data = yaml.load(rendered)
except:
except Exception as e:
# In case we rendered invalid yaml this helps to debug
logger.error(rendered)
raise
raise e
# Some sanity checks and final cosmetics
# Check for empty top level Parameters, Outputs and Conditions and remove
@ -172,7 +152,7 @@ class Stack(object):
# Delete from data structure which also takes care of json
del self.data[key]
# but also remove from rendered for the yaml file
rendered = rendered.replace('\n'+key+":",'')
rendered = rendered.replace('\n' + key + ":", '')
# Condense multiple empty lines to one
self.cfn_template = re.sub(r'\n\s*\n', '\n\n', rendered)
@ -180,7 +160,6 @@ class Stack(object):
# Update internal data structures
self._parse_metadata()
def _parse_metadata(self):
# Extract dependencies if present
try:
@ -189,10 +168,9 @@ class Stack(object):
except KeyError:
pass
def write_template_file(self):
if self.cfn_template:
yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml")
yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml")
self._ensure_dirs('template_path')
with open(yaml_file, 'w') as yaml_contents:
yaml_contents.write(self.cfn_template)
@ -201,20 +179,18 @@ class Stack(object):
else:
logger.error('No cfn template rendered yet for stack {}.'.format(self.stackname))
def delete_template_file(self):
yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml")
yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml")
try:
os.remove(yaml_file)
logger.debug('Deleted cfn template %s.', yaml_file)
except OSError:
pass
def read_template_file(self):
""" Reads rendered yaml template from disk and extracts metadata """
if not self.cfn_template:
yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml")
yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml")
with open(yaml_file, 'r') as yaml_contents:
self.cfn_template = yaml_contents.read()
logger.debug('Read cfn template %s.', yaml_file)
@ -225,7 +201,6 @@ class Stack(object):
else:
logger.debug('Using cached cfn template %s.', self.stackname)
def validate(self):
"""Validates the rendered template via cfn-lint"""
self.read_template_file()
@ -237,15 +212,15 @@ class Stack(object):
# Ignore some more checks around injected parameters as we generate these
if self.template_vars['Mode'] == "Piped":
ignore_checks = ignore_checks+['W2505','W2509','W2507']
ignore_checks = ignore_checks + ['W2505', 'W2509', 'W2507']
filename = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml")
filename = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml")
logger.info('Validating {0}'.format(filename))
lint_args = ['--template', filename]
if ignore_checks:
lint_args.append('--ignore-checks')
lint_args = lint_args+ignore_checks
lint_args = lint_args + ignore_checks
logger.info('Ignoring checks: {}'.format(','.join(ignore_checks)))
(args, filenames, formatter) = cfnlint.core.get_args_filenames(lint_args)
@ -258,7 +233,6 @@ class Stack(object):
else:
logger.info("Passed.")
def resolve_parameters(self):
""" Renders parameters for the stack based on the source template and the environment configuration """
@ -271,13 +245,13 @@ class Stack(object):
# stack_outputs = inspect_stacks(config['tags']['Conglomerate'])
# logger.info(pprint.pformat(stack_outputs))
# except KeyError:
# pass
# pass
if 'Parameters' in self.data:
self.cfn_parameters = []
for p in self.data['Parameters']:
# In Piped mode we try to resolve all Paramters first via stack_outputs
#if config['cfn']['Mode'] == "Piped":
# if config['cfn']['Mode'] == "Piped":
# try:
# # first reverse the rename due to AWS alphanumeric restriction for parameter names
# _p = p.replace('DoT','.')
@ -291,18 +265,17 @@ class Stack(object):
# Key name in config tree is: stacks.<self.stackname>.parameters.<parameter>
try:
value = str(self.parameters[p])
self.cfn_parameters.append({'ParameterKey': p, 'ParameterValue': value })
logger.info('Got {} = {}'.format(p,value))
except KeyError as e:
self.cfn_parameters.append({'ParameterKey': p, 'ParameterValue': value})
logger.info('Got {} = {}'.format(p, value))
except KeyError:
# If we have a Default defined in the CFN skip, as AWS will use it
if 'Default' in self.data['Parameters'][p]:
continue
else:
logger.error('Cannot find value for parameter {0}'.format(p))
def write_parameter_file(self):
parameter_file = os.path.join(self.ctx['parameter_path'], self.rel_path, self.stackname+".yaml")
parameter_file = os.path.join(self.ctx['parameter_path'], self.rel_path, self.stackname + ".yaml")
# Render parameters as json for AWS CFN
self._ensure_dirs('parameter_path')
@ -315,16 +288,14 @@ class Stack(object):
if os.path.isfile(parameter_file):
os.remove(parameter_file)
def delete_parameter_file(self):
parameter_file = os.path.join(self.ctx['parameter_path'], self.rel_path, self.stackname+".yaml")
parameter_file = os.path.join(self.ctx['parameter_path'], self.rel_path, self.stackname + ".yaml")
try:
os.remove(parameter_file)
logger.debug('Deleted parameter %s.', parameter_file)
except OSError:
pass
def create(self):
"""Creates a stack """
@ -334,17 +305,17 @@ class Stack(object):
self.read_template_file()
logger.info('Creating {0} {1}'.format(self.region, self.stackname))
response = self.connection_manager.call('cloudformation', 'create_stack',
{'StackName':self.stackname,
'TemplateBody':self.cfn_template,
'Parameters':self.cfn_parameters,
'Tags':[ {"Key": str(k), "Value": str(v)} for k, v in self.tags.items() ],
'Capabilities':['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
profile=self.profile, region=self.region)
self.connection_manager.call(
'cloudformation', 'create_stack',
{'StackName': self.stackname,
'TemplateBody': self.cfn_template,
'Parameters': self.cfn_parameters,
'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()],
'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
profile=self.profile, region=self.region)
return self._wait_for_completion()
def update(self):
"""Updates an existing stack """
@ -355,13 +326,14 @@ class Stack(object):
logger.info('Updating {0} {1}'.format(self.region, self.stackname))
try:
response = self.connection_manager.call('cloudformation', 'update_stack',
{'StackName':self.stackname,
'TemplateBody':self.cfn_template,
'Parameters':self.cfn_parameters,
'Tags':[ {"Key": str(k), "Value": str(v)} for k, v in self.tags.items() ],
'Capabilities':['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
profile=self.profile, region=self.region)
self.connection_manager.call(
'cloudformation', 'update_stack',
{'StackName': self.stackname,
'TemplateBody': self.cfn_template,
'Parameters': self.cfn_parameters,
'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()],
'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
profile=self.profile, region=self.region)
except ClientError as e:
if 'No updates are to be performed' in e.response['Error']['Message']:
@ -372,17 +344,16 @@ class Stack(object):
return self._wait_for_completion()
def delete(self):
"""Deletes a stack """
logger.info('Deleting {0} {1}'.format(self.region, self.stackname))
response = self.connection_manager.call('cloudformation', 'delete_stack',
{'StackName':self.stackname}, profile=self.profile, region=self.region)
self.connection_manager.call(
'cloudformation', 'delete_stack', {'StackName': self.stackname},
profile=self.profile, region=self.region)
return self._wait_for_completion()
def create_change_set(self, change_set_name):
""" Creates a Change Set with the name ``change_set_name``. """
@ -392,17 +363,17 @@ class Stack(object):
self.read_template_file()
logger.info('Creating change set {0} for stack {1}'.format(change_set_name, self.stackname))
response = self.connection_manager.call('cloudformation', 'create_change_set',
{'StackName':self.stackname,
'ChangeSetName': change_set_name,
'TemplateBody':self.cfn_template,
'Parameters':self.cfn_parameters,
'Tags':[ {"Key": str(k), "Value": str(v)} for k, v in self.tags.items() ],
'Capabilities':['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
profile=self.profile, region=self.region)
self.connection_manager.call(
'cloudformation', 'create_change_set',
{'StackName': self.stackname,
'ChangeSetName': change_set_name,
'TemplateBody': self.cfn_template,
'Parameters': self.cfn_parameters,
'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()],
'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
profile=self.profile, region=self.region)
return self._wait_for_completion()
def describe(self):
"""
Returns the a description of the stack.
@ -414,7 +385,6 @@ class Stack(object):
{"StackName": self.stackname},
profile=self.profile, region=self.region)
def get_status(self):
"""
Returns the stack's status.
@ -429,7 +399,6 @@ class Stack(object):
raise e
return status
def describe_events(self):
"""
Returns a dictionary contianing the stack events.
@ -449,7 +418,6 @@ class Stack(object):
return status
def _wait_for_completion(self, timeout=0):
"""
Waits for a stack operation to finish. Prints CloudFormation events while it waits.
@ -477,7 +445,6 @@ class Stack(object):
return status
@staticmethod
def _get_simplified_status(status):
""" Returns the simplified Stack Status. """
@ -493,7 +460,6 @@ class Stack(object):
else:
return 'Unknown'
def _log_new_events(self):
"""
Log the latest stack events while the stack is being built.
@ -517,7 +483,6 @@ class Stack(object):
]))
self.most_recent_event_datetime = event["Timestamp"]
def _ensure_dirs(self, path):
# Ensure output dirs exist
if not os.path.exists(os.path.join(self.ctx[path], self.rel_path)):

View File

@ -13,7 +13,7 @@ class StackGroup(object):
self.name = None
self.ctx = ctx
self.path = path
self.rel_path = os.path.relpath(path ,ctx['config_path'])
self.rel_path = os.path.relpath(path, ctx['config_path'])
self.config = {}
self.sgs = []
self.stacks = []
@ -21,7 +21,6 @@ class StackGroup(object):
if self.rel_path == '.':
self.rel_path = ''
def dump_config(self):
for sg in self.sgs:
sg.dump_config()
@ -31,7 +30,6 @@ class StackGroup(object):
for s in self.stacks:
s.dump_config()
def read_config(self, parent_config={}):
if not os.path.isdir(self.path):
@ -66,18 +64,15 @@ class StackGroup(object):
if stackname_prefix:
stackname = stackname_prefix + stackname
new_stack = Stack(name=stackname, template=template,
path=stack_path, rel_path=str(self.rel_path),
tags=dict(tags), parameters=dict(parameters),
template_vars=dict(template_vars),
region=str(region), profile=str(profile),
ctx=self.ctx
)
new_stack = Stack(
name=stackname, template=template, path=stack_path, rel_path=str(self.rel_path),
tags=dict(tags), parameters=dict(parameters), template_vars=dict(template_vars),
region=str(region), profile=str(profile), ctx=self.ctx)
new_stack.read_config()
self.stacks.append(new_stack)
# Create StackGroups recursively
for sub_group in [f.path for f in os.scandir(self.path) if f.is_dir() ]:
for sub_group in [f.path for f in os.scandir(self.path) if f.is_dir()]:
sg = StackGroup(sub_group, self.ctx)
sg.read_config(_config)
@ -86,7 +81,6 @@ class StackGroup(object):
# Return raw, merged config to parent
return _config
def get_stacks(self, name=None, recursive=True, match_by='name'):
""" Returns [stack] matching stack_name or [all] """
stacks = []
@ -105,11 +99,10 @@ class StackGroup(object):
for sg in self.sgs:
s = sg.get_stacks(name, recursive, match_by)
if s:
stacks = stacks+s
stacks = stacks + s
return stacks
def get_stackgroup(self, name=None, recursive=True, match_by='name'):
""" Returns stack group matching stackgroup_name or all if None """
if not name or (self.name == name and match_by == 'name') or (self.path.endswith(name) and match_by == 'path'):
@ -127,22 +120,19 @@ class StackGroup(object):
return None
# TODO: Integrate properly into stackgroup class, borken for now
# TODO: Integrate properly into stackgroup class, broken for now
# stackoutput inspection
def BROKEN_inspect_stacks(conglomerate):
def BROKEN_inspect_stacks(self, conglomerate):
# Get all stacks of the conglomertate
client = Connection.get_connection('cloudformation')
running_stacks=client.describe_stacks()
response = self.connection_manager.call('cloudformation', 'decribe_stacks')
stacks = []
for stack in running_stacks['Stacks']:
for stack in response['Stacks']:
for tag in stack['Tags']:
if tag['Key'] == 'Conglomerate' and tag['Value'] == conglomerate:
stacks.append(stack)
break
# Gather stack outputs, use Tag['Artifact'] as name space: Artifact.OutputName, same as FortyTwo
stack_outputs = {}
for stack in stacks:
@ -160,10 +150,9 @@ class StackGroup(object):
try:
for output in stack['Outputs']:
# Gather all outputs of the stack into one dimensional key=value structure
stack_outputs[key_prefix+output['OutputKey']]=output['OutputValue']
stack_outputs[key_prefix + output['OutputKey']] = output['OutputValue']
except KeyError:
pass
# Add outputs from stacks into the data for jinja under StackOutput
return stack_outputs

View File

@ -6,6 +6,7 @@ import boto3
logger = logging.getLogger(__name__)
def read_yaml_file(path):
data = {}
if os.path.exists(path):
@ -16,7 +17,7 @@ def read_yaml_file(path):
if _data:
data.update(_data)
except Exception as e:
logger.warning("Error reading config file: {} ({})".format(path,e))
logger.warning("Error reading config file: {} ({})".format(path, e))
return data
@ -29,13 +30,13 @@ def dict_merge(a, b):
if not b:
return a
if not isinstance(a, dict) or not isinstance(b, dict):
if not isinstance(a, dict) or not isinstance(b, dict):
raise TypeError
result = copy.deepcopy(a)
for k, v in b.items():
if k in result and isinstance(result[k], dict):
result[k] = dict_merge(result[k], v)
result[k] = dict_merge(result[k], v)
else:
result[k] = copy.deepcopy(v)
return result
@ -67,7 +68,6 @@ def setup_logging(debug):
datefmt="%Y-%m-%d %H:%M:%S"
)
log_handler = logging.StreamHandler()
log_handler.setFormatter(formatter)
logger = logging.getLogger("cloudbender")