diff --git a/cloudbender/cli.py b/cloudbender/cli.py index fbcbf0c..d111c9e 100644 --- a/cloudbender/cli.py +++ b/cloudbender/cli.py @@ -11,13 +11,14 @@ from .utils import setup_logging import logging logger = logging.getLogger(__name__) + @click.group() @click.version_option(version=__version__, prog_name="CloudBender") @click.option("--debug", is_flag=True, help="Turn on debug logging.") @click.option("--dir", "directory", help="Specify cloudbender project directory.") @click.pass_context def cli(ctx, debug, directory): - logger = setup_logging(debug) + setup_logging(debug) # Read global config cb = CloudBender(directory if directory else os.getcwd()) @@ -86,7 +87,7 @@ def provision(ctx, stack_names, multi): futures.append(group.submit(stack.update)) for future in as_completed(futures): - result = future.result() + future.result() @click.command() @@ -109,7 +110,7 @@ def delete(ctx, stack_names, multi): futures.append(group.submit(stack.delete)) for future in as_completed(futures): - result = future.result() + future.result() @click.command() @@ -140,13 +141,14 @@ def sort_stacks(ctx, stacks): data[s.id] = set(deps) logger.debug("Stack {} depends on {}".format(s.id, deps)) + # Ignore self dependencies for k, v in data.items(): - v.discard(k) # Ignore self dependencies + v.discard(k) extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys()) - data.update({item:set() for item in extra_items_in_deps}) + data.update({item: set() for item in extra_items_in_deps}) while True: - ordered = set(item for item,dep in data.items() if not dep) + ordered = set(item for item, dep in data.items() if not dep) if not ordered: break @@ -154,10 +156,11 @@ def sort_stacks(ctx, stacks): result = [] for o in ordered: for s in stacks: - if s.id == o: result.append(s) + if s.id == o: + result.append(s) yield result - data = {item: (dep - ordered) for item,dep in data.items() + data = {item: (dep - ordered) for item, dep in data.items() if item not in ordered} assert not data, "A cyclic dependency exists amongst %r" % data @@ -167,7 +170,7 @@ def _find_stacks(ctx, stack_names, multi=False): stacks = [] for s in stack_names: - stacks = stacks+cb.resolve_stacks(s) + stacks = stacks + cb.resolve_stacks(s) if not multi and len(stacks) > 1: logger.error('Found more than one stack matching name ({}). Please set --multi if that is what you want.'.format(', '.join(stack_names))) diff --git a/cloudbender/connection.py b/cloudbender/connection.py index ba39a2a..ec6a835 100644 --- a/cloudbender/connection.py +++ b/cloudbender/connection.py @@ -9,48 +9,46 @@ import logging logger = logging.getLogger(__name__) + class BotoConnection(): - _sessions= {} + _sessions = {} _clients = {} def __init__(self, profile=None, region=None): self.region = region self.profile = profile - def _get_session(self, profile=None, region=None): - if self._sessions.get((profile,region)): - return self._sessions[(profile,region)] + if self._sessions.get((profile, region)): + return self._sessions[(profile, region)] # Construct botocore session with cache # Setup boto to cache STS tokens for MFA # Change the cache path from the default of ~/.aws/boto/cache to the one used by awscli session_vars = {} if profile: - session_vars['profile'] = (None,None,profile,None) + session_vars['profile'] = (None, None, profile, None) if region and region != 'global': - session_vars['region'] = (None,None,region,None) + session_vars['region'] = (None, None, region, None) session = botocore.session.Session(session_vars=session_vars) - cli_cache = os.path.join(os.path.expanduser('~'),'.aws/cli/cache') + cli_cache = os.path.join(os.path.expanduser('~'), '.aws/cli/cache') session.get_component('credential_provider').get_provider('assume-role').cache = credentials.JSONFileCache(cli_cache) - self._sessions[(profile,region)] = session + self._sessions[(profile, region)] = session return session - def _get_client(self, service, profile=None, region=None): - if self._clients.get((profile,region,service)): - return self._clients[(profile,region,service)] + if self._clients.get((profile, region, service)): + return self._clients[(profile, region, service)] - session = self._get_session(profile,region) + session = self._get_session(profile, region) client = boto3.Session(botocore_session=session).client(service) - self._clients[(profile,region,service)] = client + self._clients[(profile, region, service)] = client return client - def call(self, service, command, kwargs={}, profile=None, region=None): while True: try: @@ -64,4 +62,3 @@ class BotoConnection(): pass else: raise e - diff --git a/cloudbender/core.py b/cloudbender/core.py index 5251677..870052c 100644 --- a/cloudbender/core.py +++ b/cloudbender/core.py @@ -1,14 +1,12 @@ import os -import glob import logging from .utils import read_yaml_file, ensure_dir -from .stack import Stack from .stackgroup import StackGroup -from .connection import BotoConnection logger = logging.getLogger(__name__) + class CloudBender(object): """ Config Class to handle recursive conf/* config tree """ def __init__(self, root_path): @@ -20,15 +18,14 @@ class CloudBender(object): "template_path": os.path.join(self.root, "cloudformation"), "parameter_path": os.path.join(self.root, "parameters"), "artifact_paths": [os.path.join(self.root, "artifacts")] - } + } self.default_settings = { - 'vars': { 'Mode': 'FortyTwo' } + 'vars': {'Mode': 'FortyTwo'} } if not os.path.isdir(self.root): raise "Check '{0}' exists and is a valid project folder.".format(root_path) - def read_config(self): """Load the /config.yaml, /*.yaml as stacks, sub-folders are child groups """ @@ -39,7 +36,7 @@ class CloudBender(object): # Make sure all paths are abs for k, v in self.ctx.items(): - if k in ['config_path','template_path','parameter_path','artifact_paths']: + if k in ['config_path', 'template_path', 'parameter_path', 'artifact_paths']: if isinstance(v, list): new_list = [] for path in v: @@ -51,9 +48,9 @@ class CloudBender(object): elif isinstance(v, str): if not os.path.isabs(v): - self.ctx[k]=os.path.normpath(os.path.join(self.root, v)) + self.ctx[k] = os.path.normpath(os.path.join(self.root, v)) - if k in ['template_path','parameter_path']: + if k in ['template_path', 'parameter_path']: ensure_dir(self.ctx[k]) self.sg = StackGroup(self.ctx['config_path'], self.ctx) @@ -66,22 +63,19 @@ class CloudBender(object): # _config = { "vars": { 'Azs': {'TestAZ': 'Next'}, 'Segments': {'Testnet': 'internet'}, "Mode": "Piped" } } # self.vars.update(_config.get('vars')) - def dump_config(self): logger.debug("".format(vars(self))) self.sg.dump_config() - def clean(self): for s in self.all_stacks: s.delete_template_file() s.delete_parameter_file() - def resolve_stacks(self, token): stacks = [] - # remove optional leading "config/" to allow bash path expansions + # remove optional leading "config/" to allow bash path expansions if token.startswith("config/"): token = token[7:] @@ -100,7 +94,6 @@ class CloudBender(object): return stacks - def filter_stacks(self, filter_by, stacks=None): # filter_by is a dict { property, value } @@ -112,12 +105,12 @@ class CloudBender(object): for s in stacks: match = True - for p,v in filter_by.items(): + for p, v in filter_by.items(): if not (hasattr(s, p) and getattr(s, p) == v): match = False break if match: matching_stacks.append(s) - + return matching_stacks diff --git a/cloudbender/jinja.py b/cloudbender/jinja.py index 39bb7d4..22a3bc9 100644 --- a/cloudbender/jinja.py +++ b/cloudbender/jinja.py @@ -1,8 +1,6 @@ -import os import io import gzip import jinja2 -import oyaml as yaml import re import base64 @@ -75,10 +73,10 @@ def get_custom_att(context, att=None, ResourceName="FortyTwo", attributes={}, re return('{{ "Fn::GetAtt": ["{0}", "{1}"] }}'.format(ResourceName, att)) elif config['cfn']['Mode'] == "AWSImport" and ResourceName == "FortyTwo": # AWS only allows - and :, so replace '.' with ":" - return('{{ "Fn::ImportValue": {{ "Fn::Sub": "${{Conglomerate}}:{0}" }} }}'.format(att.replace('.',':'))) + return('{{ "Fn::ImportValue": {{ "Fn::Sub": "${{Conglomerate}}:{0}" }} }}'.format(att.replace('.', ':'))) else: # We need to replace . with some PureAlphaNumeric thx AWS ... - return('{{ Ref: {0} }}'.format(att.replace('.','DoT'))) + return('{{ Ref: {0} }}'.format(att.replace('.', 'DoT'))) @jinja2.contextfunction @@ -133,7 +131,7 @@ def regex(value='', pattern='', ignorecase=False, match_type='search'): flags = 0 _re = re.compile(pattern, flags=flags) if getattr(_re, match_type, 'search')(value) is not None: - return True + return True return False @@ -153,13 +151,15 @@ def regex_replace(value='', pattern='', replace='', ignorecase=False): flags = re.I else: flags = 0 - return re.sub(pattern,replace,value,flags=flags) + return re.sub(pattern, replace, value, flags=flags) def pyminify(source, obfuscate=False, minify=True): # pyminifier options - options = types.SimpleNamespace(tabs=False,replacement_length=1,use_nonlatin=0, - obfuscate=0,obf_variables=1,obf_classes=0,obf_functions=0,obf_import_methods=0,obf_builtins=0) + options = types.SimpleNamespace( + tabs=False, replacement_length=1, use_nonlatin=0, + obfuscate=0, obf_variables=1, obf_classes=0, obf_functions=0, + obf_import_methods=0, obf_builtins=0) tokens = pyminifier.token_utils.listified_tokenizer(source) @@ -170,10 +170,10 @@ def pyminify(source, obfuscate=False, minify=True): if obfuscate: name_generator = pyminifier.obfuscate.obfuscation_machine(use_unicode=False) pyminifier.obfuscate.obfuscate("__main__", tokens, options, name_generator=name_generator) - #source = pyminifier.obfuscate.apply_obfuscation(source) + # source = pyminifier.obfuscate.apply_obfuscation(source) source = pyminifier.token_utils.untokenize(tokens) - #logger.info(source) + # logger.info(source) minified_source = pyminifier.compression.gz_pack(source) logger.info("Compressed python code to {}".format(len(minified_source))) return minified_source diff --git a/cloudbender/stack.py b/cloudbender/stack.py index 04faef8..60fccd1 100644 --- a/cloudbender/stack.py +++ b/cloudbender/stack.py @@ -5,6 +5,7 @@ import hashlib import oyaml as yaml import json import time +import subprocess from datetime import datetime, timedelta from dateutil.tz import tzutc @@ -56,11 +57,9 @@ class Stack(object): self.default_lock = None self.multi_delete = True - def dump_config(self): logger.debug("".format(self.id, vars(self))) - def read_config(self): _config = read_yaml_file(self.path) for p in ["region", "stackname", "template", "default_lock", "multi_delete", "provides"]: @@ -83,27 +82,26 @@ class Stack(object): logger.debug("Stack {} added.".format(self.id)) - def check_fortytwo(self, template): # Fail early if 42 is enabled but not available if self.cfn['Mode'] == "FortyTwo" and self.template != 'FortyTwo': try: - response = self.connection_manager.call('lambda', 'get_function', {'FunctionName': 'FortyTwo'}, - profile=self.profile, region=self.region) + response = self.connection_manager.call( + 'lambda', 'get_function', {'FunctionName': 'FortyTwo'}, + profile=self.profile, region=self.region) # Also verify version in case specified in the template's metadata try: req_ver = template['Metadata']['FortyTwo']['RequiredVersion'] if 'Release' not in response['Tags']: - abort("Lambda FortyTwo has no Release Tag! Required: {}".format(req_ver)) - elif semver.compare(req_ver, re.sub("-.*$",'', response['Tags']['Release'])) > 0: - abort("Lambda FortyTwo version is not recent enough! Required: {} vs. Found: {}".format(req_ver, response['Tags']['Release'])) + raise("Lambda FortyTwo has no Release Tag! Required: {}".format(req_ver)) + elif semver.compare(req_ver, re.sub("-.*$", '', response['Tags']['Release'])) > 0: + raise("Lambda FortyTwo version is not recent enough! Required: {} vs. Found: {}".format(req_ver, response['Tags']['Release'])) except KeyError: pass except botocore.exceptions.ClientError: - abort("No Lambda FortyTwo found in your account") - + raise("No Lambda FortyTwo found in your account") def render(self): """Renders the cfn jinja template for this stack""" @@ -114,56 +112,38 @@ class Stack(object): template_metadata = { 'Template.Name': self.template, - 'Template.Hash': 'unknown', - 'Template.GitComment': 'unknown', + 'Template.Hash': 'tbd', 'CloudBender.Version': __version__ } - jenv.globals['_config'] = { 'cfn': self.template_vars, 'Metadata': template_metadata } + jenv.globals['_config'] = {'cfn': self.template_vars, 'Metadata': template_metadata} # First render pass to calculate a md5 checksum - template_metadata['Template.Hash'] = hashlib.md5(template.render({ 'cfn': self.template_vars, 'Metadata': template_metadata }).encode('utf-8')).hexdigest() + template_metadata['Template.Hash'] = hashlib.md5(template.render({'cfn': self.template_vars, 'Metadata': template_metadata}).encode('utf-8')).hexdigest() # Reset and set Metadata for final render pass jenv.globals['get_custom_att'](context={'_config': self.template_vars}, reset=True) jenv.globals['render_once'](context={'_config': self.template_vars}, reset=True) jenv.globals['cloudbender_ctx'](context={'_config': self.template_vars}, reset=True) - # try to get local git info + # Try to add latest tag/commit for the template source, skip if not in git tree try: - self.template_vars['Metadata']['{}.Version'.format(PROJECT_NAME)] = subprocess.check_output('git describe --tags'.split(' '), universal_newlines=True)[:-1] - - except: - pass - - # Add latest tag/commit - try: - os.chdir(ROOT_DIR) - _version = subprocess.check_output('git describe --tags'.split(' '), universal_newlines=True)[:-1] - if _version: - self.template_vars['Metadata']['CloudBender.Version'] = _version - - os.chdir(os.path.dirname(template.filename)) - _comment = subprocess.check_output('git log -1 --pretty=%B {0}{1}' - .format(input_file, TEMPLATE_EXT).split(' ')).decode('utf-8').strip() \ - .replace('"', '').replace('#', '').replace('\n', '').replace(':', ' ') + _comment = subprocess.check_output('git log -1 --pretty=%B {}'.format(template.filename).split(' ')).decode('utf-8').strip().replace('"', '').replace('#', '').replace('\n', '').replace(':', ' ') if _comment: - self.template_vars['Metadata']['Template.GitComment'] = _comment + template_metadata['Template.LastGitComment'] = _comment - os.chdir(PROJECT_DIR) - - except: + except subprocess.CalledProcessError: pass logger.info('Rendering %s', template.filename) - rendered = template.render({ 'cfn': self.template_vars, 'Metadata': template_metadata }) + rendered = template.render({'cfn': self.template_vars, 'Metadata': template_metadata}) try: self.data = yaml.load(rendered) - except: + except Exception as e: # In case we rendered invalid yaml this helps to debug logger.error(rendered) - raise + raise e # Some sanity checks and final cosmetics # Check for empty top level Parameters, Outputs and Conditions and remove @@ -172,7 +152,7 @@ class Stack(object): # Delete from data structure which also takes care of json del self.data[key] # but also remove from rendered for the yaml file - rendered = rendered.replace('\n'+key+":",'') + rendered = rendered.replace('\n' + key + ":", '') # Condense multiple empty lines to one self.cfn_template = re.sub(r'\n\s*\n', '\n\n', rendered) @@ -180,7 +160,6 @@ class Stack(object): # Update internal data structures self._parse_metadata() - def _parse_metadata(self): # Extract dependencies if present try: @@ -189,10 +168,9 @@ class Stack(object): except KeyError: pass - def write_template_file(self): if self.cfn_template: - yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml") + yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml") self._ensure_dirs('template_path') with open(yaml_file, 'w') as yaml_contents: yaml_contents.write(self.cfn_template) @@ -201,20 +179,18 @@ class Stack(object): else: logger.error('No cfn template rendered yet for stack {}.'.format(self.stackname)) - def delete_template_file(self): - yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml") + yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml") try: os.remove(yaml_file) logger.debug('Deleted cfn template %s.', yaml_file) except OSError: pass - def read_template_file(self): """ Reads rendered yaml template from disk and extracts metadata """ if not self.cfn_template: - yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml") + yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml") with open(yaml_file, 'r') as yaml_contents: self.cfn_template = yaml_contents.read() logger.debug('Read cfn template %s.', yaml_file) @@ -225,7 +201,6 @@ class Stack(object): else: logger.debug('Using cached cfn template %s.', self.stackname) - def validate(self): """Validates the rendered template via cfn-lint""" self.read_template_file() @@ -237,15 +212,15 @@ class Stack(object): # Ignore some more checks around injected parameters as we generate these if self.template_vars['Mode'] == "Piped": - ignore_checks = ignore_checks+['W2505','W2509','W2507'] + ignore_checks = ignore_checks + ['W2505', 'W2509', 'W2507'] - filename = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml") + filename = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml") logger.info('Validating {0}'.format(filename)) lint_args = ['--template', filename] if ignore_checks: lint_args.append('--ignore-checks') - lint_args = lint_args+ignore_checks + lint_args = lint_args + ignore_checks logger.info('Ignoring checks: {}'.format(','.join(ignore_checks))) (args, filenames, formatter) = cfnlint.core.get_args_filenames(lint_args) @@ -258,7 +233,6 @@ class Stack(object): else: logger.info("Passed.") - def resolve_parameters(self): """ Renders parameters for the stack based on the source template and the environment configuration """ @@ -271,13 +245,13 @@ class Stack(object): # stack_outputs = inspect_stacks(config['tags']['Conglomerate']) # logger.info(pprint.pformat(stack_outputs)) # except KeyError: - # pass + # pass if 'Parameters' in self.data: self.cfn_parameters = [] for p in self.data['Parameters']: # In Piped mode we try to resolve all Paramters first via stack_outputs - #if config['cfn']['Mode'] == "Piped": + # if config['cfn']['Mode'] == "Piped": # try: # # first reverse the rename due to AWS alphanumeric restriction for parameter names # _p = p.replace('DoT','.') @@ -291,18 +265,17 @@ class Stack(object): # Key name in config tree is: stacks..parameters. try: value = str(self.parameters[p]) - self.cfn_parameters.append({'ParameterKey': p, 'ParameterValue': value }) - logger.info('Got {} = {}'.format(p,value)) - except KeyError as e: + self.cfn_parameters.append({'ParameterKey': p, 'ParameterValue': value}) + logger.info('Got {} = {}'.format(p, value)) + except KeyError: # If we have a Default defined in the CFN skip, as AWS will use it if 'Default' in self.data['Parameters'][p]: continue else: logger.error('Cannot find value for parameter {0}'.format(p)) - def write_parameter_file(self): - parameter_file = os.path.join(self.ctx['parameter_path'], self.rel_path, self.stackname+".yaml") + parameter_file = os.path.join(self.ctx['parameter_path'], self.rel_path, self.stackname + ".yaml") # Render parameters as json for AWS CFN self._ensure_dirs('parameter_path') @@ -315,16 +288,14 @@ class Stack(object): if os.path.isfile(parameter_file): os.remove(parameter_file) - def delete_parameter_file(self): - parameter_file = os.path.join(self.ctx['parameter_path'], self.rel_path, self.stackname+".yaml") + parameter_file = os.path.join(self.ctx['parameter_path'], self.rel_path, self.stackname + ".yaml") try: os.remove(parameter_file) logger.debug('Deleted parameter %s.', parameter_file) except OSError: pass - def create(self): """Creates a stack """ @@ -334,17 +305,17 @@ class Stack(object): self.read_template_file() logger.info('Creating {0} {1}'.format(self.region, self.stackname)) - response = self.connection_manager.call('cloudformation', 'create_stack', - {'StackName':self.stackname, - 'TemplateBody':self.cfn_template, - 'Parameters':self.cfn_parameters, - 'Tags':[ {"Key": str(k), "Value": str(v)} for k, v in self.tags.items() ], - 'Capabilities':['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']}, - profile=self.profile, region=self.region) + self.connection_manager.call( + 'cloudformation', 'create_stack', + {'StackName': self.stackname, + 'TemplateBody': self.cfn_template, + 'Parameters': self.cfn_parameters, + 'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()], + 'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']}, + profile=self.profile, region=self.region) return self._wait_for_completion() - def update(self): """Updates an existing stack """ @@ -355,13 +326,14 @@ class Stack(object): logger.info('Updating {0} {1}'.format(self.region, self.stackname)) try: - response = self.connection_manager.call('cloudformation', 'update_stack', - {'StackName':self.stackname, - 'TemplateBody':self.cfn_template, - 'Parameters':self.cfn_parameters, - 'Tags':[ {"Key": str(k), "Value": str(v)} for k, v in self.tags.items() ], - 'Capabilities':['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']}, - profile=self.profile, region=self.region) + self.connection_manager.call( + 'cloudformation', 'update_stack', + {'StackName': self.stackname, + 'TemplateBody': self.cfn_template, + 'Parameters': self.cfn_parameters, + 'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()], + 'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']}, + profile=self.profile, region=self.region) except ClientError as e: if 'No updates are to be performed' in e.response['Error']['Message']: @@ -372,17 +344,16 @@ class Stack(object): return self._wait_for_completion() - def delete(self): """Deletes a stack """ logger.info('Deleting {0} {1}'.format(self.region, self.stackname)) - response = self.connection_manager.call('cloudformation', 'delete_stack', - {'StackName':self.stackname}, profile=self.profile, region=self.region) + self.connection_manager.call( + 'cloudformation', 'delete_stack', {'StackName': self.stackname}, + profile=self.profile, region=self.region) return self._wait_for_completion() - def create_change_set(self, change_set_name): """ Creates a Change Set with the name ``change_set_name``. """ @@ -392,17 +363,17 @@ class Stack(object): self.read_template_file() logger.info('Creating change set {0} for stack {1}'.format(change_set_name, self.stackname)) - response = self.connection_manager.call('cloudformation', 'create_change_set', - {'StackName':self.stackname, - 'ChangeSetName': change_set_name, - 'TemplateBody':self.cfn_template, - 'Parameters':self.cfn_parameters, - 'Tags':[ {"Key": str(k), "Value": str(v)} for k, v in self.tags.items() ], - 'Capabilities':['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']}, - profile=self.profile, region=self.region) + self.connection_manager.call( + 'cloudformation', 'create_change_set', + {'StackName': self.stackname, + 'ChangeSetName': change_set_name, + 'TemplateBody': self.cfn_template, + 'Parameters': self.cfn_parameters, + 'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()], + 'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']}, + profile=self.profile, region=self.region) return self._wait_for_completion() - def describe(self): """ Returns the a description of the stack. @@ -414,7 +385,6 @@ class Stack(object): {"StackName": self.stackname}, profile=self.profile, region=self.region) - def get_status(self): """ Returns the stack's status. @@ -429,7 +399,6 @@ class Stack(object): raise e return status - def describe_events(self): """ Returns a dictionary contianing the stack events. @@ -449,7 +418,6 @@ class Stack(object): return status - def _wait_for_completion(self, timeout=0): """ Waits for a stack operation to finish. Prints CloudFormation events while it waits. @@ -477,7 +445,6 @@ class Stack(object): return status - @staticmethod def _get_simplified_status(status): """ Returns the simplified Stack Status. """ @@ -493,7 +460,6 @@ class Stack(object): else: return 'Unknown' - def _log_new_events(self): """ Log the latest stack events while the stack is being built. @@ -517,7 +483,6 @@ class Stack(object): ])) self.most_recent_event_datetime = event["Timestamp"] - def _ensure_dirs(self, path): # Ensure output dirs exist if not os.path.exists(os.path.join(self.ctx[path], self.rel_path)): diff --git a/cloudbender/stackgroup.py b/cloudbender/stackgroup.py index 8b9ce2d..2960390 100644 --- a/cloudbender/stackgroup.py +++ b/cloudbender/stackgroup.py @@ -13,7 +13,7 @@ class StackGroup(object): self.name = None self.ctx = ctx self.path = path - self.rel_path = os.path.relpath(path ,ctx['config_path']) + self.rel_path = os.path.relpath(path, ctx['config_path']) self.config = {} self.sgs = [] self.stacks = [] @@ -21,7 +21,6 @@ class StackGroup(object): if self.rel_path == '.': self.rel_path = '' - def dump_config(self): for sg in self.sgs: sg.dump_config() @@ -31,7 +30,6 @@ class StackGroup(object): for s in self.stacks: s.dump_config() - def read_config(self, parent_config={}): if not os.path.isdir(self.path): @@ -66,18 +64,15 @@ class StackGroup(object): if stackname_prefix: stackname = stackname_prefix + stackname - new_stack = Stack(name=stackname, template=template, - path=stack_path, rel_path=str(self.rel_path), - tags=dict(tags), parameters=dict(parameters), - template_vars=dict(template_vars), - region=str(region), profile=str(profile), - ctx=self.ctx - ) + new_stack = Stack( + name=stackname, template=template, path=stack_path, rel_path=str(self.rel_path), + tags=dict(tags), parameters=dict(parameters), template_vars=dict(template_vars), + region=str(region), profile=str(profile), ctx=self.ctx) new_stack.read_config() self.stacks.append(new_stack) # Create StackGroups recursively - for sub_group in [f.path for f in os.scandir(self.path) if f.is_dir() ]: + for sub_group in [f.path for f in os.scandir(self.path) if f.is_dir()]: sg = StackGroup(sub_group, self.ctx) sg.read_config(_config) @@ -86,7 +81,6 @@ class StackGroup(object): # Return raw, merged config to parent return _config - def get_stacks(self, name=None, recursive=True, match_by='name'): """ Returns [stack] matching stack_name or [all] """ stacks = [] @@ -105,11 +99,10 @@ class StackGroup(object): for sg in self.sgs: s = sg.get_stacks(name, recursive, match_by) if s: - stacks = stacks+s + stacks = stacks + s return stacks - def get_stackgroup(self, name=None, recursive=True, match_by='name'): """ Returns stack group matching stackgroup_name or all if None """ if not name or (self.name == name and match_by == 'name') or (self.path.endswith(name) and match_by == 'path'): @@ -127,22 +120,19 @@ class StackGroup(object): return None - - # TODO: Integrate properly into stackgroup class, borken for now + # TODO: Integrate properly into stackgroup class, broken for now # stackoutput inspection - def BROKEN_inspect_stacks(conglomerate): + def BROKEN_inspect_stacks(self, conglomerate): # Get all stacks of the conglomertate - client = Connection.get_connection('cloudformation') - running_stacks=client.describe_stacks() + response = self.connection_manager.call('cloudformation', 'decribe_stacks') stacks = [] - for stack in running_stacks['Stacks']: + for stack in response['Stacks']: for tag in stack['Tags']: if tag['Key'] == 'Conglomerate' and tag['Value'] == conglomerate: stacks.append(stack) break - # Gather stack outputs, use Tag['Artifact'] as name space: Artifact.OutputName, same as FortyTwo stack_outputs = {} for stack in stacks: @@ -160,10 +150,9 @@ class StackGroup(object): try: for output in stack['Outputs']: # Gather all outputs of the stack into one dimensional key=value structure - stack_outputs[key_prefix+output['OutputKey']]=output['OutputValue'] + stack_outputs[key_prefix + output['OutputKey']] = output['OutputValue'] except KeyError: pass # Add outputs from stacks into the data for jinja under StackOutput return stack_outputs - diff --git a/cloudbender/utils.py b/cloudbender/utils.py index e0c4b4c..49a6d33 100644 --- a/cloudbender/utils.py +++ b/cloudbender/utils.py @@ -6,6 +6,7 @@ import boto3 logger = logging.getLogger(__name__) + def read_yaml_file(path): data = {} if os.path.exists(path): @@ -16,7 +17,7 @@ def read_yaml_file(path): if _data: data.update(_data) except Exception as e: - logger.warning("Error reading config file: {} ({})".format(path,e)) + logger.warning("Error reading config file: {} ({})".format(path, e)) return data @@ -29,13 +30,13 @@ def dict_merge(a, b): if not b: return a - if not isinstance(a, dict) or not isinstance(b, dict): + if not isinstance(a, dict) or not isinstance(b, dict): raise TypeError result = copy.deepcopy(a) for k, v in b.items(): if k in result and isinstance(result[k], dict): - result[k] = dict_merge(result[k], v) + result[k] = dict_merge(result[k], v) else: result[k] = copy.deepcopy(v) return result @@ -67,7 +68,6 @@ def setup_logging(debug): datefmt="%Y-%m-%d %H:%M:%S" ) - log_handler = logging.StreamHandler() log_handler.setFormatter(formatter) logger = logging.getLogger("cloudbender")