diff --git a/CHANGES.md b/CHANGES.md index 4528e2d..88dc001 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,12 @@ # Changelog +## 0.9.5 +### New Features +Support for uploading and retrieving rendered templates from S3! + +Enabled by setting `template_bucket_url` to a valid S3 location: ```s3://[/]``` +Templates will still be stored and updated in the local file system to allow tracking via git. + ## 0.9.4 - new option to generate Dot Graph files via `--graph` option for the create-docs command - fix validate command using latest cfn-lint library diff --git a/cloudbender/core.py b/cloudbender/core.py index dde0337..853bb44 100644 --- a/cloudbender/core.py +++ b/cloudbender/core.py @@ -63,7 +63,6 @@ class CloudBender(object): def clean(self): for s in self.all_stacks: s.delete_template_file() - s.delete_parameter_file() def resolve_stacks(self, token): stacks = [] diff --git a/cloudbender/stack.py b/cloudbender/stack.py index fc0118c..45dee8c 100644 --- a/cloudbender/stack.py +++ b/cloudbender/stack.py @@ -11,7 +11,7 @@ from dateutil.tz import tzutc from botocore.exceptions import ClientError -from .utils import dict_merge, search_refs, ensure_dir +from .utils import dict_merge, search_refs, ensure_dir, get_s3_url from .connection import BotoConnection from .jinja import JinjaEnv, read_config_file from . import __version__ @@ -65,6 +65,7 @@ class Stack(object): self.hooks = {'post_create': [], 'post_update': [], 'pre_create': [], 'pre_update': []} self.default_lock = None self.multi_delete = True + self.template_bucket_url = None def dump_config(self): logger.debug("".format(self.id, pprint.pformat(vars(self)))) @@ -77,15 +78,14 @@ class Stack(object): self.parameters.update(sg_config.get('parameters', {})) self.options.update(sg_config.get('options', {})) - if 'region' in sg_config: - self.region = sg_config['region'] - if 'profile' in sg_config: - self.profile = sg_config['profile'] - if 'notfication_sns' in sg_config: - self.notfication_sns = sg_config['notfication_sns'] + # by default inherit parent group settings + for p in ['region', 'profile', 'notfication_sns', 'template_bucket_url']: + if p in sg_config: + setattr(self, p, sg_config[p]) + # now override stack specific settings _config = read_config_file(self.path, sg_config.get('variables', {})) - for p in ["region", "stackname", "template", "default_lock", "multi_delete", "provides", "onfailure", "notification_sns"]: + for p in ["region", "stackname", "template", "default_lock", "multi_delete", "provides", "onfailure", "notification_sns", "template_bucket_url"]: if p in _config: setattr(self, p, _config[p]) @@ -97,18 +97,13 @@ class Stack(object): if 'Artifact' not in self.tags: self.tags['Artifact'] = self.provides - # backwards comp - if 'vars' in _config: - logger.warn("vars: in config is deprecated, please use options: instead") - self.options = dict_merge(self.options, _config['vars']) - if 'options' in _config: self.options = dict_merge(self.options, _config['options']) if 'Mode' in self.options: self.mode = self.options['Mode'] - if 'StoreOutputs' in self.options: + if 'StoreOutputs' in self.options and self.options['StoreOutputs']: self.store_outputs = True if 'dependencies' in _config: @@ -244,9 +239,26 @@ class Stack(object): with open(yaml_file, 'w') as yaml_contents: yaml_contents.write(self.cfn_template) logger.info('Wrote %s to %s', self.template, yaml_file) - if len(self.cfn_template) > 51200: - logger.warning("Rendered template exceeds maximum allowed size of 51200, actual size: {} !".format(len(self.cfn_template))) + # upload template to s3 if set + if self.template_bucket_url: + try: + (bucket, path) = get_s3_url(self.template_bucket_url, self.rel_path, self.stackname + ".yaml") + self.connection_manager.call( + 's3', 'put_object', + {'Bucket': bucket, + 'Key': path, + 'Body': self.cfn_template, + 'ServerSideEncryption': 'AES256'}, + profile=self.profile, region=self.region) + + logger.info("Uploaded template to s3://{}/{}".format(bucket, path)) + except ClientError as e: + logger.error("Error trying to upload template so S3: {}, {}".format(self.template_bucket_url, e)) + + else: + if len(self.cfn_template) > 51200: + logger.warning("template_bucket_url not set and rendered template exceeds maximum allowed size of 51200, actual size: {} !".format(len(self.cfn_template))) else: logger.error('No cfn template rendered yet for stack {}.'.format(self.stackname)) @@ -258,21 +270,55 @@ class Stack(object): except OSError: pass - def read_template_file(self): - """ Reads rendered yaml template from disk and extracts metadata """ - if not self.cfn_template: - yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml") - + if self.template_bucket_url: try: - with open(yaml_file, 'r') as yaml_contents: - self.cfn_template = yaml_contents.read() - logger.debug('Read cfn template %s.', yaml_file) + (bucket, path) = get_s3_url(self.template_bucket_url, self.rel_path, self.stackname + ".yaml") + self.connection_manager.call( + 's3', 'delete_object', + {'Bucket': bucket, + 'Key': path}, + profile=self.profile, region=self.region) - self.cfn_data = yaml.safe_load(self.cfn_template) - self._parse_metadata() - except FileNotFoundError as e: - logger.warn("Could not find template file: {}".format(yaml_file)) - raise e + logger.info("Deleted template from s3://{}/{}".format(bucket, path)) + except ClientError as e: + logger.error("Error trying to delete template from S3: {}, {}".format(self.template_bucket_url, e)) + + def read_template_file(self): + """ Reads rendered yaml template from disk or s3 and extracts metadata """ + if not self.cfn_template: + if self.template_bucket_url: + try: + (bucket, path) = get_s3_url(self.template_bucket_url, self.rel_path, self.stackname + ".yaml") + template = self.connection_manager.call( + 's3', 'get_object', + {'Bucket': bucket, + 'Key': path}, + profile=self.profile, region=self.region) + logger.debug("Got template from s3://{}/{}".format(bucket, path)) + + except ClientError as e: + logger.error("Could not find template file on S3: {}/{}, {}".format(bucket, path, e)) + + self.cfn_template = template['Body'].read().decode('utf-8') + + # Overwrite local copy + yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml") + ensure_dir(os.path.join(self.ctx['template_path'], self.rel_path)) + with open(yaml_file, 'w') as yaml_contents: + yaml_contents.write(self.cfn_template) + else: + yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml") + + try: + with open(yaml_file, 'r') as yaml_contents: + self.cfn_template = yaml_contents.read() + logger.debug('Read cfn template %s.', yaml_file) + except FileNotFoundError as e: + logger.warn("Could not find template file: {}".format(yaml_file)) + raise e + + self.cfn_data = yaml.safe_load(self.cfn_template) + self._parse_metadata() else: logger.debug('Using cached cfn template %s.', self.stackname) diff --git a/cloudbender/utils.py b/cloudbender/utils.py index 9594bc4..2101728 100644 --- a/cloudbender/utils.py +++ b/cloudbender/utils.py @@ -1,6 +1,7 @@ import os import copy import logging +import re logger = logging.getLogger(__name__) @@ -87,3 +88,17 @@ def search_refs(template, attributes, mode): for k in template: if isinstance(k, dict) or isinstance(k, list): search_refs(k, attributes, mode) + + +def get_s3_url(url, *args): + bucket = None + path = None + + m = re.match('^(s3://)?([^/]*)(/.*)?', url) + bucket = m[2] + if m[3]: + path = m[3].lstrip('/') + + path = os.path.join(path, *args) + + return(bucket, path)