Code clean up PEP8, some fixes along the way
This commit is contained in:
Stefan Reimer
parent
3587451170
commit
47b492a65e
@ -11,13 +11,14 @@ from .utils import setup_logging
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@click.group()
|
||||
@click.version_option(version=__version__, prog_name="CloudBender")
|
||||
@click.option("--debug", is_flag=True, help="Turn on debug logging.")
|
||||
@click.option("--dir", "directory", help="Specify cloudbender project directory.")
|
||||
@click.pass_context
|
||||
def cli(ctx, debug, directory):
|
||||
logger = setup_logging(debug)
|
||||
setup_logging(debug)
|
||||
|
||||
# Read global config
|
||||
cb = CloudBender(directory if directory else os.getcwd())
|
||||
@ -86,7 +87,7 @@ def provision(ctx, stack_names, multi):
|
||||
futures.append(group.submit(stack.update))
|
||||
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
future.result()
|
||||
|
||||
|
||||
@click.command()
|
||||
@ -109,7 +110,7 @@ def delete(ctx, stack_names, multi):
|
||||
futures.append(group.submit(stack.delete))
|
||||
|
||||
for future in as_completed(futures):
|
||||
result = future.result()
|
||||
future.result()
|
||||
|
||||
|
||||
@click.command()
|
||||
@ -140,13 +141,14 @@ def sort_stacks(ctx, stacks):
|
||||
data[s.id] = set(deps)
|
||||
logger.debug("Stack {} depends on {}".format(s.id, deps))
|
||||
|
||||
# Ignore self dependencies
|
||||
for k, v in data.items():
|
||||
v.discard(k) # Ignore self dependencies
|
||||
v.discard(k)
|
||||
|
||||
extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys())
|
||||
data.update({item:set() for item in extra_items_in_deps})
|
||||
data.update({item: set() for item in extra_items_in_deps})
|
||||
while True:
|
||||
ordered = set(item for item,dep in data.items() if not dep)
|
||||
ordered = set(item for item, dep in data.items() if not dep)
|
||||
if not ordered:
|
||||
break
|
||||
|
||||
@ -154,10 +156,11 @@ def sort_stacks(ctx, stacks):
|
||||
result = []
|
||||
for o in ordered:
|
||||
for s in stacks:
|
||||
if s.id == o: result.append(s)
|
||||
if s.id == o:
|
||||
result.append(s)
|
||||
yield result
|
||||
|
||||
data = {item: (dep - ordered) for item,dep in data.items()
|
||||
data = {item: (dep - ordered) for item, dep in data.items()
|
||||
if item not in ordered}
|
||||
assert not data, "A cyclic dependency exists amongst %r" % data
|
||||
|
||||
@ -167,7 +170,7 @@ def _find_stacks(ctx, stack_names, multi=False):
|
||||
|
||||
stacks = []
|
||||
for s in stack_names:
|
||||
stacks = stacks+cb.resolve_stacks(s)
|
||||
stacks = stacks + cb.resolve_stacks(s)
|
||||
|
||||
if not multi and len(stacks) > 1:
|
||||
logger.error('Found more than one stack matching name ({}). Please set --multi if that is what you want.'.format(', '.join(stack_names)))
|
||||
|
||||
@ -9,48 +9,46 @@ import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BotoConnection():
|
||||
_sessions= {}
|
||||
_sessions = {}
|
||||
_clients = {}
|
||||
|
||||
def __init__(self, profile=None, region=None):
|
||||
self.region = region
|
||||
self.profile = profile
|
||||
|
||||
|
||||
def _get_session(self, profile=None, region=None):
|
||||
if self._sessions.get((profile,region)):
|
||||
return self._sessions[(profile,region)]
|
||||
if self._sessions.get((profile, region)):
|
||||
return self._sessions[(profile, region)]
|
||||
|
||||
# Construct botocore session with cache
|
||||
# Setup boto to cache STS tokens for MFA
|
||||
# Change the cache path from the default of ~/.aws/boto/cache to the one used by awscli
|
||||
session_vars = {}
|
||||
if profile:
|
||||
session_vars['profile'] = (None,None,profile,None)
|
||||
session_vars['profile'] = (None, None, profile, None)
|
||||
if region and region != 'global':
|
||||
session_vars['region'] = (None,None,region,None)
|
||||
session_vars['region'] = (None, None, region, None)
|
||||
|
||||
session = botocore.session.Session(session_vars=session_vars)
|
||||
cli_cache = os.path.join(os.path.expanduser('~'),'.aws/cli/cache')
|
||||
cli_cache = os.path.join(os.path.expanduser('~'), '.aws/cli/cache')
|
||||
session.get_component('credential_provider').get_provider('assume-role').cache = credentials.JSONFileCache(cli_cache)
|
||||
|
||||
self._sessions[(profile,region)] = session
|
||||
self._sessions[(profile, region)] = session
|
||||
|
||||
return session
|
||||
|
||||
|
||||
def _get_client(self, service, profile=None, region=None):
|
||||
if self._clients.get((profile,region,service)):
|
||||
return self._clients[(profile,region,service)]
|
||||
if self._clients.get((profile, region, service)):
|
||||
return self._clients[(profile, region, service)]
|
||||
|
||||
session = self._get_session(profile,region)
|
||||
session = self._get_session(profile, region)
|
||||
client = boto3.Session(botocore_session=session).client(service)
|
||||
|
||||
self._clients[(profile,region,service)] = client
|
||||
self._clients[(profile, region, service)] = client
|
||||
return client
|
||||
|
||||
|
||||
def call(self, service, command, kwargs={}, profile=None, region=None):
|
||||
while True:
|
||||
try:
|
||||
@ -64,4 +62,3 @@ class BotoConnection():
|
||||
pass
|
||||
else:
|
||||
raise e
|
||||
|
||||
|
||||
@ -1,14 +1,12 @@
|
||||
import os
|
||||
import glob
|
||||
import logging
|
||||
|
||||
from .utils import read_yaml_file, ensure_dir
|
||||
from .stack import Stack
|
||||
from .stackgroup import StackGroup
|
||||
from .connection import BotoConnection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CloudBender(object):
|
||||
""" Config Class to handle recursive conf/* config tree """
|
||||
def __init__(self, root_path):
|
||||
@ -20,15 +18,14 @@ class CloudBender(object):
|
||||
"template_path": os.path.join(self.root, "cloudformation"),
|
||||
"parameter_path": os.path.join(self.root, "parameters"),
|
||||
"artifact_paths": [os.path.join(self.root, "artifacts")]
|
||||
}
|
||||
}
|
||||
self.default_settings = {
|
||||
'vars': { 'Mode': 'FortyTwo' }
|
||||
'vars': {'Mode': 'FortyTwo'}
|
||||
}
|
||||
|
||||
if not os.path.isdir(self.root):
|
||||
raise "Check '{0}' exists and is a valid project folder.".format(root_path)
|
||||
|
||||
|
||||
def read_config(self):
|
||||
"""Load the <path>/config.yaml, <path>/*.yaml as stacks, sub-folders are child groups """
|
||||
|
||||
@ -39,7 +36,7 @@ class CloudBender(object):
|
||||
|
||||
# Make sure all paths are abs
|
||||
for k, v in self.ctx.items():
|
||||
if k in ['config_path','template_path','parameter_path','artifact_paths']:
|
||||
if k in ['config_path', 'template_path', 'parameter_path', 'artifact_paths']:
|
||||
if isinstance(v, list):
|
||||
new_list = []
|
||||
for path in v:
|
||||
@ -51,9 +48,9 @@ class CloudBender(object):
|
||||
|
||||
elif isinstance(v, str):
|
||||
if not os.path.isabs(v):
|
||||
self.ctx[k]=os.path.normpath(os.path.join(self.root, v))
|
||||
self.ctx[k] = os.path.normpath(os.path.join(self.root, v))
|
||||
|
||||
if k in ['template_path','parameter_path']:
|
||||
if k in ['template_path', 'parameter_path']:
|
||||
ensure_dir(self.ctx[k])
|
||||
|
||||
self.sg = StackGroup(self.ctx['config_path'], self.ctx)
|
||||
@ -66,22 +63,19 @@ class CloudBender(object):
|
||||
# _config = { "vars": { 'Azs': {'TestAZ': 'Next'}, 'Segments': {'Testnet': 'internet'}, "Mode": "Piped" } }
|
||||
# self.vars.update(_config.get('vars'))
|
||||
|
||||
|
||||
def dump_config(self):
|
||||
logger.debug("<CloudBender: {}>".format(vars(self)))
|
||||
self.sg.dump_config()
|
||||
|
||||
|
||||
def clean(self):
|
||||
for s in self.all_stacks:
|
||||
s.delete_template_file()
|
||||
s.delete_parameter_file()
|
||||
|
||||
|
||||
def resolve_stacks(self, token):
|
||||
stacks = []
|
||||
|
||||
# remove optional leading "config/" to allow bash path expansions
|
||||
# remove optional leading "config/" to allow bash path expansions
|
||||
if token.startswith("config/"):
|
||||
token = token[7:]
|
||||
|
||||
@ -100,7 +94,6 @@ class CloudBender(object):
|
||||
|
||||
return stacks
|
||||
|
||||
|
||||
def filter_stacks(self, filter_by, stacks=None):
|
||||
# filter_by is a dict { property, value }
|
||||
|
||||
@ -112,12 +105,12 @@ class CloudBender(object):
|
||||
for s in stacks:
|
||||
match = True
|
||||
|
||||
for p,v in filter_by.items():
|
||||
for p, v in filter_by.items():
|
||||
if not (hasattr(s, p) and getattr(s, p) == v):
|
||||
match = False
|
||||
break
|
||||
|
||||
if match:
|
||||
matching_stacks.append(s)
|
||||
|
||||
|
||||
return matching_stacks
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
import os
|
||||
import io
|
||||
import gzip
|
||||
import jinja2
|
||||
import oyaml as yaml
|
||||
import re
|
||||
import base64
|
||||
|
||||
@ -75,10 +73,10 @@ def get_custom_att(context, att=None, ResourceName="FortyTwo", attributes={}, re
|
||||
return('{{ "Fn::GetAtt": ["{0}", "{1}"] }}'.format(ResourceName, att))
|
||||
elif config['cfn']['Mode'] == "AWSImport" and ResourceName == "FortyTwo":
|
||||
# AWS only allows - and :, so replace '.' with ":"
|
||||
return('{{ "Fn::ImportValue": {{ "Fn::Sub": "${{Conglomerate}}:{0}" }} }}'.format(att.replace('.',':')))
|
||||
return('{{ "Fn::ImportValue": {{ "Fn::Sub": "${{Conglomerate}}:{0}" }} }}'.format(att.replace('.', ':')))
|
||||
else:
|
||||
# We need to replace . with some PureAlphaNumeric thx AWS ...
|
||||
return('{{ Ref: {0} }}'.format(att.replace('.','DoT')))
|
||||
return('{{ Ref: {0} }}'.format(att.replace('.', 'DoT')))
|
||||
|
||||
|
||||
@jinja2.contextfunction
|
||||
@ -133,7 +131,7 @@ def regex(value='', pattern='', ignorecase=False, match_type='search'):
|
||||
flags = 0
|
||||
_re = re.compile(pattern, flags=flags)
|
||||
if getattr(_re, match_type, 'search')(value) is not None:
|
||||
return True
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@ -153,13 +151,15 @@ def regex_replace(value='', pattern='', replace='', ignorecase=False):
|
||||
flags = re.I
|
||||
else:
|
||||
flags = 0
|
||||
return re.sub(pattern,replace,value,flags=flags)
|
||||
return re.sub(pattern, replace, value, flags=flags)
|
||||
|
||||
|
||||
def pyminify(source, obfuscate=False, minify=True):
|
||||
# pyminifier options
|
||||
options = types.SimpleNamespace(tabs=False,replacement_length=1,use_nonlatin=0,
|
||||
obfuscate=0,obf_variables=1,obf_classes=0,obf_functions=0,obf_import_methods=0,obf_builtins=0)
|
||||
options = types.SimpleNamespace(
|
||||
tabs=False, replacement_length=1, use_nonlatin=0,
|
||||
obfuscate=0, obf_variables=1, obf_classes=0, obf_functions=0,
|
||||
obf_import_methods=0, obf_builtins=0)
|
||||
|
||||
tokens = pyminifier.token_utils.listified_tokenizer(source)
|
||||
|
||||
@ -170,10 +170,10 @@ def pyminify(source, obfuscate=False, minify=True):
|
||||
if obfuscate:
|
||||
name_generator = pyminifier.obfuscate.obfuscation_machine(use_unicode=False)
|
||||
pyminifier.obfuscate.obfuscate("__main__", tokens, options, name_generator=name_generator)
|
||||
#source = pyminifier.obfuscate.apply_obfuscation(source)
|
||||
# source = pyminifier.obfuscate.apply_obfuscation(source)
|
||||
|
||||
source = pyminifier.token_utils.untokenize(tokens)
|
||||
#logger.info(source)
|
||||
# logger.info(source)
|
||||
minified_source = pyminifier.compression.gz_pack(source)
|
||||
logger.info("Compressed python code to {}".format(len(minified_source)))
|
||||
return minified_source
|
||||
|
||||
@ -5,6 +5,7 @@ import hashlib
|
||||
import oyaml as yaml
|
||||
import json
|
||||
import time
|
||||
import subprocess
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from dateutil.tz import tzutc
|
||||
@ -56,11 +57,9 @@ class Stack(object):
|
||||
self.default_lock = None
|
||||
self.multi_delete = True
|
||||
|
||||
|
||||
def dump_config(self):
|
||||
logger.debug("<Stack {}: {}>".format(self.id, vars(self)))
|
||||
|
||||
|
||||
def read_config(self):
|
||||
_config = read_yaml_file(self.path)
|
||||
for p in ["region", "stackname", "template", "default_lock", "multi_delete", "provides"]:
|
||||
@ -83,27 +82,26 @@ class Stack(object):
|
||||
|
||||
logger.debug("Stack {} added.".format(self.id))
|
||||
|
||||
|
||||
def check_fortytwo(self, template):
|
||||
# Fail early if 42 is enabled but not available
|
||||
if self.cfn['Mode'] == "FortyTwo" and self.template != 'FortyTwo':
|
||||
try:
|
||||
response = self.connection_manager.call('lambda', 'get_function', {'FunctionName': 'FortyTwo'},
|
||||
profile=self.profile, region=self.region)
|
||||
response = self.connection_manager.call(
|
||||
'lambda', 'get_function', {'FunctionName': 'FortyTwo'},
|
||||
profile=self.profile, region=self.region)
|
||||
|
||||
# Also verify version in case specified in the template's metadata
|
||||
try:
|
||||
req_ver = template['Metadata']['FortyTwo']['RequiredVersion']
|
||||
if 'Release' not in response['Tags']:
|
||||
abort("Lambda FortyTwo has no Release Tag! Required: {}".format(req_ver))
|
||||
elif semver.compare(req_ver, re.sub("-.*$",'', response['Tags']['Release'])) > 0:
|
||||
abort("Lambda FortyTwo version is not recent enough! Required: {} vs. Found: {}".format(req_ver, response['Tags']['Release']))
|
||||
raise("Lambda FortyTwo has no Release Tag! Required: {}".format(req_ver))
|
||||
elif semver.compare(req_ver, re.sub("-.*$", '', response['Tags']['Release'])) > 0:
|
||||
raise("Lambda FortyTwo version is not recent enough! Required: {} vs. Found: {}".format(req_ver, response['Tags']['Release']))
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
except botocore.exceptions.ClientError:
|
||||
abort("No Lambda FortyTwo found in your account")
|
||||
|
||||
raise("No Lambda FortyTwo found in your account")
|
||||
|
||||
def render(self):
|
||||
"""Renders the cfn jinja template for this stack"""
|
||||
@ -114,56 +112,38 @@ class Stack(object):
|
||||
|
||||
template_metadata = {
|
||||
'Template.Name': self.template,
|
||||
'Template.Hash': 'unknown',
|
||||
'Template.GitComment': 'unknown',
|
||||
'Template.Hash': 'tbd',
|
||||
'CloudBender.Version': __version__
|
||||
}
|
||||
|
||||
jenv.globals['_config'] = { 'cfn': self.template_vars, 'Metadata': template_metadata }
|
||||
jenv.globals['_config'] = {'cfn': self.template_vars, 'Metadata': template_metadata}
|
||||
|
||||
# First render pass to calculate a md5 checksum
|
||||
template_metadata['Template.Hash'] = hashlib.md5(template.render({ 'cfn': self.template_vars, 'Metadata': template_metadata }).encode('utf-8')).hexdigest()
|
||||
template_metadata['Template.Hash'] = hashlib.md5(template.render({'cfn': self.template_vars, 'Metadata': template_metadata}).encode('utf-8')).hexdigest()
|
||||
|
||||
# Reset and set Metadata for final render pass
|
||||
jenv.globals['get_custom_att'](context={'_config': self.template_vars}, reset=True)
|
||||
jenv.globals['render_once'](context={'_config': self.template_vars}, reset=True)
|
||||
jenv.globals['cloudbender_ctx'](context={'_config': self.template_vars}, reset=True)
|
||||
|
||||
# try to get local git info
|
||||
# Try to add latest tag/commit for the template source, skip if not in git tree
|
||||
try:
|
||||
self.template_vars['Metadata']['{}.Version'.format(PROJECT_NAME)] = subprocess.check_output('git describe --tags'.split(' '), universal_newlines=True)[:-1]
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
# Add latest tag/commit
|
||||
try:
|
||||
os.chdir(ROOT_DIR)
|
||||
_version = subprocess.check_output('git describe --tags'.split(' '), universal_newlines=True)[:-1]
|
||||
if _version:
|
||||
self.template_vars['Metadata']['CloudBender.Version'] = _version
|
||||
|
||||
os.chdir(os.path.dirname(template.filename))
|
||||
_comment = subprocess.check_output('git log -1 --pretty=%B {0}{1}'
|
||||
.format(input_file, TEMPLATE_EXT).split(' ')).decode('utf-8').strip() \
|
||||
.replace('"', '').replace('#', '').replace('\n', '').replace(':', ' ')
|
||||
_comment = subprocess.check_output('git log -1 --pretty=%B {}'.format(template.filename).split(' ')).decode('utf-8').strip().replace('"', '').replace('#', '').replace('\n', '').replace(':', ' ')
|
||||
if _comment:
|
||||
self.template_vars['Metadata']['Template.GitComment'] = _comment
|
||||
template_metadata['Template.LastGitComment'] = _comment
|
||||
|
||||
os.chdir(PROJECT_DIR)
|
||||
|
||||
except:
|
||||
except subprocess.CalledProcessError:
|
||||
pass
|
||||
|
||||
logger.info('Rendering %s', template.filename)
|
||||
rendered = template.render({ 'cfn': self.template_vars, 'Metadata': template_metadata })
|
||||
rendered = template.render({'cfn': self.template_vars, 'Metadata': template_metadata})
|
||||
|
||||
try:
|
||||
self.data = yaml.load(rendered)
|
||||
except:
|
||||
except Exception as e:
|
||||
# In case we rendered invalid yaml this helps to debug
|
||||
logger.error(rendered)
|
||||
raise
|
||||
raise e
|
||||
|
||||
# Some sanity checks and final cosmetics
|
||||
# Check for empty top level Parameters, Outputs and Conditions and remove
|
||||
@ -172,7 +152,7 @@ class Stack(object):
|
||||
# Delete from data structure which also takes care of json
|
||||
del self.data[key]
|
||||
# but also remove from rendered for the yaml file
|
||||
rendered = rendered.replace('\n'+key+":",'')
|
||||
rendered = rendered.replace('\n' + key + ":", '')
|
||||
|
||||
# Condense multiple empty lines to one
|
||||
self.cfn_template = re.sub(r'\n\s*\n', '\n\n', rendered)
|
||||
@ -180,7 +160,6 @@ class Stack(object):
|
||||
# Update internal data structures
|
||||
self._parse_metadata()
|
||||
|
||||
|
||||
def _parse_metadata(self):
|
||||
# Extract dependencies if present
|
||||
try:
|
||||
@ -189,10 +168,9 @@ class Stack(object):
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
|
||||
def write_template_file(self):
|
||||
if self.cfn_template:
|
||||
yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml")
|
||||
yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml")
|
||||
self._ensure_dirs('template_path')
|
||||
with open(yaml_file, 'w') as yaml_contents:
|
||||
yaml_contents.write(self.cfn_template)
|
||||
@ -201,20 +179,18 @@ class Stack(object):
|
||||
else:
|
||||
logger.error('No cfn template rendered yet for stack {}.'.format(self.stackname))
|
||||
|
||||
|
||||
def delete_template_file(self):
|
||||
yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml")
|
||||
yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml")
|
||||
try:
|
||||
os.remove(yaml_file)
|
||||
logger.debug('Deleted cfn template %s.', yaml_file)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def read_template_file(self):
|
||||
""" Reads rendered yaml template from disk and extracts metadata """
|
||||
if not self.cfn_template:
|
||||
yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml")
|
||||
yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml")
|
||||
with open(yaml_file, 'r') as yaml_contents:
|
||||
self.cfn_template = yaml_contents.read()
|
||||
logger.debug('Read cfn template %s.', yaml_file)
|
||||
@ -225,7 +201,6 @@ class Stack(object):
|
||||
else:
|
||||
logger.debug('Using cached cfn template %s.', self.stackname)
|
||||
|
||||
|
||||
def validate(self):
|
||||
"""Validates the rendered template via cfn-lint"""
|
||||
self.read_template_file()
|
||||
@ -237,15 +212,15 @@ class Stack(object):
|
||||
|
||||
# Ignore some more checks around injected parameters as we generate these
|
||||
if self.template_vars['Mode'] == "Piped":
|
||||
ignore_checks = ignore_checks+['W2505','W2509','W2507']
|
||||
ignore_checks = ignore_checks + ['W2505', 'W2509', 'W2507']
|
||||
|
||||
filename = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname+".yaml")
|
||||
filename = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml")
|
||||
logger.info('Validating {0}'.format(filename))
|
||||
|
||||
lint_args = ['--template', filename]
|
||||
if ignore_checks:
|
||||
lint_args.append('--ignore-checks')
|
||||
lint_args = lint_args+ignore_checks
|
||||
lint_args = lint_args + ignore_checks
|
||||
logger.info('Ignoring checks: {}'.format(','.join(ignore_checks)))
|
||||
|
||||
(args, filenames, formatter) = cfnlint.core.get_args_filenames(lint_args)
|
||||
@ -258,7 +233,6 @@ class Stack(object):
|
||||
else:
|
||||
logger.info("Passed.")
|
||||
|
||||
|
||||
def resolve_parameters(self):
|
||||
""" Renders parameters for the stack based on the source template and the environment configuration """
|
||||
|
||||
@ -271,13 +245,13 @@ class Stack(object):
|
||||
# stack_outputs = inspect_stacks(config['tags']['Conglomerate'])
|
||||
# logger.info(pprint.pformat(stack_outputs))
|
||||
# except KeyError:
|
||||
# pass
|
||||
# pass
|
||||
|
||||
if 'Parameters' in self.data:
|
||||
self.cfn_parameters = []
|
||||
for p in self.data['Parameters']:
|
||||
# In Piped mode we try to resolve all Paramters first via stack_outputs
|
||||
#if config['cfn']['Mode'] == "Piped":
|
||||
# if config['cfn']['Mode'] == "Piped":
|
||||
# try:
|
||||
# # first reverse the rename due to AWS alphanumeric restriction for parameter names
|
||||
# _p = p.replace('DoT','.')
|
||||
@ -291,18 +265,17 @@ class Stack(object):
|
||||
# Key name in config tree is: stacks.<self.stackname>.parameters.<parameter>
|
||||
try:
|
||||
value = str(self.parameters[p])
|
||||
self.cfn_parameters.append({'ParameterKey': p, 'ParameterValue': value })
|
||||
logger.info('Got {} = {}'.format(p,value))
|
||||
except KeyError as e:
|
||||
self.cfn_parameters.append({'ParameterKey': p, 'ParameterValue': value})
|
||||
logger.info('Got {} = {}'.format(p, value))
|
||||
except KeyError:
|
||||
# If we have a Default defined in the CFN skip, as AWS will use it
|
||||
if 'Default' in self.data['Parameters'][p]:
|
||||
continue
|
||||
else:
|
||||
logger.error('Cannot find value for parameter {0}'.format(p))
|
||||
|
||||
|
||||
def write_parameter_file(self):
|
||||
parameter_file = os.path.join(self.ctx['parameter_path'], self.rel_path, self.stackname+".yaml")
|
||||
parameter_file = os.path.join(self.ctx['parameter_path'], self.rel_path, self.stackname + ".yaml")
|
||||
|
||||
# Render parameters as json for AWS CFN
|
||||
self._ensure_dirs('parameter_path')
|
||||
@ -315,16 +288,14 @@ class Stack(object):
|
||||
if os.path.isfile(parameter_file):
|
||||
os.remove(parameter_file)
|
||||
|
||||
|
||||
def delete_parameter_file(self):
|
||||
parameter_file = os.path.join(self.ctx['parameter_path'], self.rel_path, self.stackname+".yaml")
|
||||
parameter_file = os.path.join(self.ctx['parameter_path'], self.rel_path, self.stackname + ".yaml")
|
||||
try:
|
||||
os.remove(parameter_file)
|
||||
logger.debug('Deleted parameter %s.', parameter_file)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def create(self):
|
||||
"""Creates a stack """
|
||||
|
||||
@ -334,17 +305,17 @@ class Stack(object):
|
||||
self.read_template_file()
|
||||
|
||||
logger.info('Creating {0} {1}'.format(self.region, self.stackname))
|
||||
response = self.connection_manager.call('cloudformation', 'create_stack',
|
||||
{'StackName':self.stackname,
|
||||
'TemplateBody':self.cfn_template,
|
||||
'Parameters':self.cfn_parameters,
|
||||
'Tags':[ {"Key": str(k), "Value": str(v)} for k, v in self.tags.items() ],
|
||||
'Capabilities':['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
|
||||
profile=self.profile, region=self.region)
|
||||
self.connection_manager.call(
|
||||
'cloudformation', 'create_stack',
|
||||
{'StackName': self.stackname,
|
||||
'TemplateBody': self.cfn_template,
|
||||
'Parameters': self.cfn_parameters,
|
||||
'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()],
|
||||
'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
|
||||
profile=self.profile, region=self.region)
|
||||
|
||||
return self._wait_for_completion()
|
||||
|
||||
|
||||
def update(self):
|
||||
"""Updates an existing stack """
|
||||
|
||||
@ -355,13 +326,14 @@ class Stack(object):
|
||||
|
||||
logger.info('Updating {0} {1}'.format(self.region, self.stackname))
|
||||
try:
|
||||
response = self.connection_manager.call('cloudformation', 'update_stack',
|
||||
{'StackName':self.stackname,
|
||||
'TemplateBody':self.cfn_template,
|
||||
'Parameters':self.cfn_parameters,
|
||||
'Tags':[ {"Key": str(k), "Value": str(v)} for k, v in self.tags.items() ],
|
||||
'Capabilities':['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
|
||||
profile=self.profile, region=self.region)
|
||||
self.connection_manager.call(
|
||||
'cloudformation', 'update_stack',
|
||||
{'StackName': self.stackname,
|
||||
'TemplateBody': self.cfn_template,
|
||||
'Parameters': self.cfn_parameters,
|
||||
'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()],
|
||||
'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
|
||||
profile=self.profile, region=self.region)
|
||||
|
||||
except ClientError as e:
|
||||
if 'No updates are to be performed' in e.response['Error']['Message']:
|
||||
@ -372,17 +344,16 @@ class Stack(object):
|
||||
|
||||
return self._wait_for_completion()
|
||||
|
||||
|
||||
def delete(self):
|
||||
"""Deletes a stack """
|
||||
|
||||
logger.info('Deleting {0} {1}'.format(self.region, self.stackname))
|
||||
response = self.connection_manager.call('cloudformation', 'delete_stack',
|
||||
{'StackName':self.stackname}, profile=self.profile, region=self.region)
|
||||
self.connection_manager.call(
|
||||
'cloudformation', 'delete_stack', {'StackName': self.stackname},
|
||||
profile=self.profile, region=self.region)
|
||||
|
||||
return self._wait_for_completion()
|
||||
|
||||
|
||||
def create_change_set(self, change_set_name):
|
||||
""" Creates a Change Set with the name ``change_set_name``. """
|
||||
|
||||
@ -392,17 +363,17 @@ class Stack(object):
|
||||
self.read_template_file()
|
||||
|
||||
logger.info('Creating change set {0} for stack {1}'.format(change_set_name, self.stackname))
|
||||
response = self.connection_manager.call('cloudformation', 'create_change_set',
|
||||
{'StackName':self.stackname,
|
||||
'ChangeSetName': change_set_name,
|
||||
'TemplateBody':self.cfn_template,
|
||||
'Parameters':self.cfn_parameters,
|
||||
'Tags':[ {"Key": str(k), "Value": str(v)} for k, v in self.tags.items() ],
|
||||
'Capabilities':['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
|
||||
profile=self.profile, region=self.region)
|
||||
self.connection_manager.call(
|
||||
'cloudformation', 'create_change_set',
|
||||
{'StackName': self.stackname,
|
||||
'ChangeSetName': change_set_name,
|
||||
'TemplateBody': self.cfn_template,
|
||||
'Parameters': self.cfn_parameters,
|
||||
'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()],
|
||||
'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM']},
|
||||
profile=self.profile, region=self.region)
|
||||
return self._wait_for_completion()
|
||||
|
||||
|
||||
def describe(self):
|
||||
"""
|
||||
Returns the a description of the stack.
|
||||
@ -414,7 +385,6 @@ class Stack(object):
|
||||
{"StackName": self.stackname},
|
||||
profile=self.profile, region=self.region)
|
||||
|
||||
|
||||
def get_status(self):
|
||||
"""
|
||||
Returns the stack's status.
|
||||
@ -429,7 +399,6 @@ class Stack(object):
|
||||
raise e
|
||||
return status
|
||||
|
||||
|
||||
def describe_events(self):
|
||||
"""
|
||||
Returns a dictionary contianing the stack events.
|
||||
@ -449,7 +418,6 @@ class Stack(object):
|
||||
|
||||
return status
|
||||
|
||||
|
||||
def _wait_for_completion(self, timeout=0):
|
||||
"""
|
||||
Waits for a stack operation to finish. Prints CloudFormation events while it waits.
|
||||
@ -477,7 +445,6 @@ class Stack(object):
|
||||
|
||||
return status
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _get_simplified_status(status):
|
||||
""" Returns the simplified Stack Status. """
|
||||
@ -493,7 +460,6 @@ class Stack(object):
|
||||
else:
|
||||
return 'Unknown'
|
||||
|
||||
|
||||
def _log_new_events(self):
|
||||
"""
|
||||
Log the latest stack events while the stack is being built.
|
||||
@ -517,7 +483,6 @@ class Stack(object):
|
||||
]))
|
||||
self.most_recent_event_datetime = event["Timestamp"]
|
||||
|
||||
|
||||
def _ensure_dirs(self, path):
|
||||
# Ensure output dirs exist
|
||||
if not os.path.exists(os.path.join(self.ctx[path], self.rel_path)):
|
||||
|
||||
@ -13,7 +13,7 @@ class StackGroup(object):
|
||||
self.name = None
|
||||
self.ctx = ctx
|
||||
self.path = path
|
||||
self.rel_path = os.path.relpath(path ,ctx['config_path'])
|
||||
self.rel_path = os.path.relpath(path, ctx['config_path'])
|
||||
self.config = {}
|
||||
self.sgs = []
|
||||
self.stacks = []
|
||||
@ -21,7 +21,6 @@ class StackGroup(object):
|
||||
if self.rel_path == '.':
|
||||
self.rel_path = ''
|
||||
|
||||
|
||||
def dump_config(self):
|
||||
for sg in self.sgs:
|
||||
sg.dump_config()
|
||||
@ -31,7 +30,6 @@ class StackGroup(object):
|
||||
for s in self.stacks:
|
||||
s.dump_config()
|
||||
|
||||
|
||||
def read_config(self, parent_config={}):
|
||||
|
||||
if not os.path.isdir(self.path):
|
||||
@ -66,18 +64,15 @@ class StackGroup(object):
|
||||
if stackname_prefix:
|
||||
stackname = stackname_prefix + stackname
|
||||
|
||||
new_stack = Stack(name=stackname, template=template,
|
||||
path=stack_path, rel_path=str(self.rel_path),
|
||||
tags=dict(tags), parameters=dict(parameters),
|
||||
template_vars=dict(template_vars),
|
||||
region=str(region), profile=str(profile),
|
||||
ctx=self.ctx
|
||||
)
|
||||
new_stack = Stack(
|
||||
name=stackname, template=template, path=stack_path, rel_path=str(self.rel_path),
|
||||
tags=dict(tags), parameters=dict(parameters), template_vars=dict(template_vars),
|
||||
region=str(region), profile=str(profile), ctx=self.ctx)
|
||||
new_stack.read_config()
|
||||
self.stacks.append(new_stack)
|
||||
|
||||
# Create StackGroups recursively
|
||||
for sub_group in [f.path for f in os.scandir(self.path) if f.is_dir() ]:
|
||||
for sub_group in [f.path for f in os.scandir(self.path) if f.is_dir()]:
|
||||
sg = StackGroup(sub_group, self.ctx)
|
||||
sg.read_config(_config)
|
||||
|
||||
@ -86,7 +81,6 @@ class StackGroup(object):
|
||||
# Return raw, merged config to parent
|
||||
return _config
|
||||
|
||||
|
||||
def get_stacks(self, name=None, recursive=True, match_by='name'):
|
||||
""" Returns [stack] matching stack_name or [all] """
|
||||
stacks = []
|
||||
@ -105,11 +99,10 @@ class StackGroup(object):
|
||||
for sg in self.sgs:
|
||||
s = sg.get_stacks(name, recursive, match_by)
|
||||
if s:
|
||||
stacks = stacks+s
|
||||
stacks = stacks + s
|
||||
|
||||
return stacks
|
||||
|
||||
|
||||
def get_stackgroup(self, name=None, recursive=True, match_by='name'):
|
||||
""" Returns stack group matching stackgroup_name or all if None """
|
||||
if not name or (self.name == name and match_by == 'name') or (self.path.endswith(name) and match_by == 'path'):
|
||||
@ -127,22 +120,19 @@ class StackGroup(object):
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# TODO: Integrate properly into stackgroup class, borken for now
|
||||
# TODO: Integrate properly into stackgroup class, broken for now
|
||||
# stackoutput inspection
|
||||
def BROKEN_inspect_stacks(conglomerate):
|
||||
def BROKEN_inspect_stacks(self, conglomerate):
|
||||
# Get all stacks of the conglomertate
|
||||
client = Connection.get_connection('cloudformation')
|
||||
running_stacks=client.describe_stacks()
|
||||
response = self.connection_manager.call('cloudformation', 'decribe_stacks')
|
||||
|
||||
stacks = []
|
||||
for stack in running_stacks['Stacks']:
|
||||
for stack in response['Stacks']:
|
||||
for tag in stack['Tags']:
|
||||
if tag['Key'] == 'Conglomerate' and tag['Value'] == conglomerate:
|
||||
stacks.append(stack)
|
||||
break
|
||||
|
||||
|
||||
# Gather stack outputs, use Tag['Artifact'] as name space: Artifact.OutputName, same as FortyTwo
|
||||
stack_outputs = {}
|
||||
for stack in stacks:
|
||||
@ -160,10 +150,9 @@ class StackGroup(object):
|
||||
try:
|
||||
for output in stack['Outputs']:
|
||||
# Gather all outputs of the stack into one dimensional key=value structure
|
||||
stack_outputs[key_prefix+output['OutputKey']]=output['OutputValue']
|
||||
stack_outputs[key_prefix + output['OutputKey']] = output['OutputValue']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# Add outputs from stacks into the data for jinja under StackOutput
|
||||
return stack_outputs
|
||||
|
||||
|
||||
@ -6,6 +6,7 @@ import boto3
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def read_yaml_file(path):
|
||||
data = {}
|
||||
if os.path.exists(path):
|
||||
@ -16,7 +17,7 @@ def read_yaml_file(path):
|
||||
if _data:
|
||||
data.update(_data)
|
||||
except Exception as e:
|
||||
logger.warning("Error reading config file: {} ({})".format(path,e))
|
||||
logger.warning("Error reading config file: {} ({})".format(path, e))
|
||||
|
||||
return data
|
||||
|
||||
@ -29,13 +30,13 @@ def dict_merge(a, b):
|
||||
if not b:
|
||||
return a
|
||||
|
||||
if not isinstance(a, dict) or not isinstance(b, dict):
|
||||
if not isinstance(a, dict) or not isinstance(b, dict):
|
||||
raise TypeError
|
||||
|
||||
result = copy.deepcopy(a)
|
||||
for k, v in b.items():
|
||||
if k in result and isinstance(result[k], dict):
|
||||
result[k] = dict_merge(result[k], v)
|
||||
result[k] = dict_merge(result[k], v)
|
||||
else:
|
||||
result[k] = copy.deepcopy(v)
|
||||
return result
|
||||
@ -67,7 +68,6 @@ def setup_logging(debug):
|
||||
datefmt="%Y-%m-%d %H:%M:%S"
|
||||
)
|
||||
|
||||
|
||||
log_handler = logging.StreamHandler()
|
||||
log_handler.setFormatter(formatter)
|
||||
logger = logging.getLogger("cloudbender")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user