From 54df65564872c4af1f63674f2125ceab995579b4 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 4 Feb 2019 15:43:34 +0000 Subject: [PATCH] Allow dependencies to be resolved automatically and merged with stack configs, resolve global deps, code cleanup --- cloudbender/cli.py | 7 ++++- cloudbender/jinja.py | 18 ++++++++---- cloudbender/stack.py | 70 ++++++++++++++++++++++++++------------------ 3 files changed, 60 insertions(+), 35 deletions(-) diff --git a/cloudbender/cli.py b/cloudbender/cli.py index f038da7..fbcbf0c 100644 --- a/cloudbender/cli.py +++ b/cloudbender/cli.py @@ -126,14 +126,19 @@ def sort_stacks(ctx, stacks): data = {} for s in stacks: - # Resolve dependencies + # To resolve dependencies we have to read each template + s.read_template_file() deps = [] for d in s.dependencies: # For now we assume deps are artifacts so we prepend them with our local profile and region to match stack.id for dep_stack in cb.filter_stacks({'region': s.region, 'profile': s.profile, 'provides': d}): deps.append(dep_stack.id) + # also look for global services + for dep_stack in cb.filter_stacks({'region': 'global', 'profile': s.profile, 'provides': d}): + deps.append(dep_stack.id) data[s.id] = set(deps) + logger.debug("Stack {} depends on {}".format(s.id, deps)) for k, v in data.items(): v.discard(k) # Ignore self dependencies diff --git a/cloudbender/jinja.py b/cloudbender/jinja.py index a440694..96e71bf 100644 --- a/cloudbender/jinja.py +++ b/cloudbender/jinja.py @@ -22,6 +22,9 @@ def get_custom_att(context, att=None, ResourceName="FortyTwo", attributes={}, fl attributes for the specified CustomResource to include them later in the actual CustomResource include property """ + if ResourceName not in attributes: + attributes[ResourceName] = set() + # If flush is set all we do is empty our state dict if flush: attributes.clear() @@ -32,21 +35,24 @@ def get_custom_att(context, att=None, ResourceName="FortyTwo", attributes={}, fl return attributes # If dependencies, return all Artifacts this stack depends on, which are the attr of FortyTwo + config = context.get_all()['_config'] if dependencies: deps = set() - if ResourceName in attributes: - for att in attributes[ResourceName]: + try: + for att in attributes['FortyTwo']: deps.add(att.split('.')[0]) + except KeyError: + pass + + # Incl. FortyTwo itself if any FortyTwo function is used + if config['cfn']['Mode'] == "FortyTwo" and attributes: + deps.add('FortyTwo') return list(deps) # If call with an attribute, return fragement and register if att: - if ResourceName not in attributes: - attributes[ResourceName] = set() - attributes[ResourceName].add(att) - config = context.get_all()['_config'] if config['cfn']['Mode'] == "FortyTwo": return('{{ "Fn::GetAtt": ["{0}", "{1}"] }}'.format(ResourceName, att)) elif config['cfn']['Mode'] == "AWSImport" and ResourceName == "FortyTwo": diff --git a/cloudbender/stack.py b/cloudbender/stack.py index b332bf5..965f542 100644 --- a/cloudbender/stack.py +++ b/cloudbender/stack.py @@ -48,6 +48,7 @@ class Stack(object): self.provides = template self.cfn_template = None self.cfn_parameters = [] + self.cfn_data = None self.connection_manager = BotoConnection(self.profile, self.region) self.ctx = ctx self.status = None @@ -62,7 +63,7 @@ class Stack(object): def read_config(self): _config = read_yaml_file(self.path) - for p in ["region", "stackname", "template", "dependencies", "default_lock", "multi_delete", "provides"]: + for p in ["region", "stackname", "template", "default_lock", "multi_delete", "provides"]: if p in _config: setattr(self, p, _config[p]) @@ -76,6 +77,10 @@ class Stack(object): if 'vars' in _config: self.template_vars = dict_merge(self.template_vars, _config['vars']) + if 'dependencies' in _config: + for dep in _config['dependencies']: + self.dependencies.add(dep) + logger.debug("Stack {} added.".format(self.id)) @@ -153,7 +158,7 @@ class Stack(object): rendered = template.render({ 'cfn': self.template_vars, 'Metadata': template_metadata }) try: - data = yaml.load(rendered) + self.data = yaml.load(rendered) except: # In case we rendered invalid yaml this helps to debug logger.error(rendered) @@ -162,15 +167,27 @@ class Stack(object): # Some sanity checks and final cosmetics # Check for empty top level Parameters, Outputs and Conditions and remove for key in ['Parameters', 'Outputs', 'Conditions']: - if key in data and data[key] is None: + if key in self.data and self.data[key] is None: # Delete from data structure which also takes care of json - del data[key] + del self.data[key] # but also remove from rendered for the yaml file rendered = rendered.replace('\n'+key+":",'') # Condense multiple empty lines to one self.cfn_template = re.sub(r'\n\s*\n', '\n\n', rendered) + # Update internal data structures + self._parse_metadata() + + + def _parse_metadata(self): + # Extract dependencies if present + try: + for dep in self.data['Metadata']['CloudBender']['Dependencies']: + self.dependencies.add(dep) + except KeyError: + pass + def write_template_file(self): if self.cfn_template: @@ -194,20 +211,26 @@ class Stack(object): def read_template_file(self): - yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml") - with open(yaml_file, 'r') as yaml_contents: - self.cfn_template = yaml_contents.read() - logger.debug('Read cfn template %s.', yaml_file) + """ Reads rendered yaml template from disk and extracts metadata """ + if not self.cfn_template: + yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml") + with open(yaml_file, 'r') as yaml_contents: + self.cfn_template = yaml_contents.read() + logger.debug('Read cfn template %s.', yaml_file) + + self.data = yaml.load(self.cfn_template) + self._parse_metadata() + + else: + logger.debug('Using cached cfn template %s.', yaml_file) def validate(self): """Validates the rendered template via cfn-lint""" - if not self.cfn_template: - self.read_template_file() + self.read_template_file() - data = yaml.load(self.cfn_template) try: - ignore_checks = data['Metadata']['cfnlint_ignore'] + ignore_checks = self.data['Metadata']['cfnlint_ignore'] except KeyError: ignore_checks = [] @@ -238,10 +261,7 @@ class Stack(object): def resolve_parameters(self): """ Renders parameters for the stack based on the source template and the environment configuration """ - if not self.cfn_template: - self.read_template_file() - - data = yaml.load(self.cfn_template) + self.read_template_file() # Inspect all outputs of the running Conglomerate members # if we run in Piped Mode @@ -252,9 +272,9 @@ class Stack(object): # except KeyError: # pass - if 'Parameters' in data: + if 'Parameters' in self.data: self.cfn_parameters = [] - for p in data['Parameters']: + for p in self.data['Parameters']: # In Piped mode we try to resolve all Paramters first via stack_outputs #if config['cfn']['Mode'] == "Piped": # try: @@ -274,7 +294,7 @@ class Stack(object): logger.info('Got {} = {}'.format(p,value)) except KeyError as e: # If we have a Default defined in the CFN skip, as AWS will use it - if 'Default' in data['Parameters'][p]: + if 'Default' in self.data['Parameters'][p]: continue else: logger.error('Cannot find value for parameter {0}'.format(p)) @@ -310,9 +330,7 @@ class Stack(object): # Prepare parameters self.resolve_parameters() self.write_parameter_file() - - if not self.cfn_template: - self.read_template_file() + self.read_template_file() logger.info('Creating {0} {1}'.format(self.region, self.stackname)) response = self.connection_manager.call('cloudformation', 'create_stack', @@ -332,9 +350,7 @@ class Stack(object): # Prepare parameters self.resolve_parameters() self.write_parameter_file() - - if not self.cfn_template: - self.read_template_file() + self.read_template_file() logger.info('Updating {0} {1}'.format(self.region, self.stackname)) try: @@ -372,9 +388,7 @@ class Stack(object): # Prepare parameters self.resolve_parameters() self.write_parameter_file() - - if not self.cfn_template: - self.read_template_file() + self.read_template_file() logger.info('Creating change set {0} for stack {1}'.format(change_set_name, self.stackname)) response = self.connection_manager.call('cloudformation', 'create_change_set',