fix: code style / flake8 automation
This commit is contained in:
parent
129d287ae5
commit
7d6135e099
3
.flake8
Normal file
3
.flake8
Normal file
@ -0,0 +1,3 @@
|
||||
[flake8]
|
||||
extend-ignore = E501
|
||||
exclude = .git,__pycache__,build,dist,report
|
||||
2
Makefile
2
Makefile
@ -20,7 +20,7 @@ dev_setup:
|
||||
pip install -r dev-requirements.txt --user
|
||||
|
||||
pytest:
|
||||
flake8 --ignore=E501 cloudbender tests
|
||||
flake8 cloudbender tests
|
||||
TEST=True pytest --log-cli-level=DEBUG
|
||||
|
||||
clean:
|
||||
|
||||
@ -17,4 +17,4 @@ class NullHandler(logging.Handler): # pragma: no cover
|
||||
pass
|
||||
|
||||
|
||||
logging.getLogger('cloudbender').addHandler(NullHandler())
|
||||
logging.getLogger("cloudbender").addHandler(NullHandler())
|
||||
|
||||
@ -12,6 +12,7 @@ from .utils import setup_logging
|
||||
from .exceptions import InvalidProjectDir
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@ -27,8 +28,8 @@ def cli(ctx, debug, directory):
|
||||
if directory:
|
||||
if not os.path.isabs(directory):
|
||||
directory = os.path.normpath(os.path.join(os.getcwd(), directory))
|
||||
elif os.getenv('CLOUDBENDER_PROJECT_ROOT'):
|
||||
directory = os.getenv('CLOUDBENDER_PROJECT_ROOT')
|
||||
elif os.getenv("CLOUDBENDER_PROJECT_ROOT"):
|
||||
directory = os.getenv("CLOUDBENDER_PROJECT_ROOT")
|
||||
else:
|
||||
directory = os.getcwd()
|
||||
|
||||
@ -50,7 +51,7 @@ def cli(ctx, debug, directory):
|
||||
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
|
||||
@click.pass_obj
|
||||
def render(cb, stack_names, multi):
|
||||
""" Renders template and its parameters - CFN only"""
|
||||
"""Renders template and its parameters - CFN only"""
|
||||
|
||||
stacks = _find_stacks(cb, stack_names, multi)
|
||||
_render(stacks)
|
||||
@ -61,7 +62,7 @@ def render(cb, stack_names, multi):
|
||||
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
|
||||
@click.pass_obj
|
||||
def sync(cb, stack_names, multi):
|
||||
""" Renders template and provisions it right away """
|
||||
"""Renders template and provisions it right away"""
|
||||
|
||||
stacks = _find_stacks(cb, stack_names, multi)
|
||||
|
||||
@ -74,7 +75,7 @@ def sync(cb, stack_names, multi):
|
||||
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
|
||||
@click.pass_obj
|
||||
def validate(cb, stack_names, multi):
|
||||
""" Validates already rendered templates using cfn-lint - CFN only"""
|
||||
"""Validates already rendered templates using cfn-lint - CFN only"""
|
||||
stacks = _find_stacks(cb, stack_names, multi)
|
||||
|
||||
for s in stacks:
|
||||
@ -86,11 +87,17 @@ def validate(cb, stack_names, multi):
|
||||
@click.command()
|
||||
@click.argument("stack_names", nargs=-1)
|
||||
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
|
||||
@click.option("--include", default='.*', help="regex matching wanted outputs, default '.*'")
|
||||
@click.option("--values", is_flag=True, help="Only output values, most useful if only one outputs is returned")
|
||||
@click.option(
|
||||
"--include", default=".*", help="regex matching wanted outputs, default '.*'"
|
||||
)
|
||||
@click.option(
|
||||
"--values",
|
||||
is_flag=True,
|
||||
help="Only output values, most useful if only one outputs is returned",
|
||||
)
|
||||
@click.pass_obj
|
||||
def outputs(cb, stack_names, multi, include, values):
|
||||
""" Prints all stack outputs """
|
||||
"""Prints all stack outputs"""
|
||||
|
||||
stacks = _find_stacks(cb, stack_names, multi)
|
||||
for s in stacks:
|
||||
@ -110,7 +117,7 @@ def outputs(cb, stack_names, multi, include, values):
|
||||
@click.option("--graph", is_flag=True, help="Create Dot Graph file")
|
||||
@click.pass_obj
|
||||
def create_docs(cb, stack_names, multi, graph):
|
||||
""" Parses all documentation fragments out of rendered templates creating docs/*.md file """
|
||||
"""Parses all documentation fragments out of rendered templates creating docs/*.md file"""
|
||||
|
||||
stacks = _find_stacks(cb, stack_names, multi)
|
||||
for s in stacks:
|
||||
@ -122,7 +129,7 @@ def create_docs(cb, stack_names, multi, graph):
|
||||
@click.argument("change_set_name")
|
||||
@click.pass_obj
|
||||
def create_change_set(cb, stack_name, change_set_name):
|
||||
""" Creates a change set for an existing stack - CFN only"""
|
||||
"""Creates a change set for an existing stack - CFN only"""
|
||||
stacks = _find_stacks(cb, [stack_name])
|
||||
|
||||
for s in stacks:
|
||||
@ -133,29 +140,33 @@ def create_change_set(cb, stack_name, change_set_name):
|
||||
@click.argument("stack_name")
|
||||
@click.pass_obj
|
||||
def refresh(cb, stack_name):
|
||||
""" Refreshes Pulumi stack / Drift detection """
|
||||
"""Refreshes Pulumi stack / Drift detection"""
|
||||
stacks = _find_stacks(cb, [stack_name])
|
||||
|
||||
for s in stacks:
|
||||
if s.mode == 'pulumi':
|
||||
if s.mode == "pulumi":
|
||||
s.refresh()
|
||||
else:
|
||||
logger.info('{} uses Cloudformation, refresh skipped.'.format(s.stackname))
|
||||
logger.info("{} uses Cloudformation, refresh skipped.".format(s.stackname))
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument("stack_name")
|
||||
@click.option("--reset", is_flag=True, help="All pending stack operations are removed and the stack will be re-imported")
|
||||
@click.option(
|
||||
"--reset",
|
||||
is_flag=True,
|
||||
help="All pending stack operations are removed and the stack will be re-imported",
|
||||
)
|
||||
@click.pass_obj
|
||||
def export(cb, stack_name, reset=False):
|
||||
""" Exports a Pulumi stack to repair state """
|
||||
"""Exports a Pulumi stack to repair state"""
|
||||
stacks = _find_stacks(cb, [stack_name])
|
||||
|
||||
for s in stacks:
|
||||
if s.mode == 'pulumi':
|
||||
if s.mode == "pulumi":
|
||||
s.export(reset)
|
||||
else:
|
||||
logger.info('{} uses Cloudformation, export skipped.'.format(s.stackname))
|
||||
logger.info("{} uses Cloudformation, export skipped.".format(s.stackname))
|
||||
|
||||
|
||||
@click.command()
|
||||
@ -165,7 +176,7 @@ def export(cb, stack_name, reset=False):
|
||||
@click.option("--secret", is_flag=True, help="Value is a secret")
|
||||
@click.pass_obj
|
||||
def set_config(cb, stack_name, key, value, secret=False):
|
||||
""" Sets a config value, encrypts with stack key if secret """
|
||||
"""Sets a config value, encrypts with stack key if secret"""
|
||||
stacks = _find_stacks(cb, [stack_name])
|
||||
|
||||
for s in stacks:
|
||||
@ -177,7 +188,7 @@ def set_config(cb, stack_name, key, value, secret=False):
|
||||
@click.argument("key")
|
||||
@click.pass_obj
|
||||
def get_config(cb, stack_name, key):
|
||||
""" Get a config value, decrypted if secret """
|
||||
"""Get a config value, decrypted if secret"""
|
||||
stacks = _find_stacks(cb, [stack_name])
|
||||
|
||||
for s in stacks:
|
||||
@ -188,14 +199,18 @@ def get_config(cb, stack_name, key):
|
||||
@click.argument("stack_name")
|
||||
@click.pass_obj
|
||||
def preview(cb, stack_name):
|
||||
""" Preview of Pulumi stack up operation """
|
||||
"""Preview of Pulumi stack up operation"""
|
||||
stacks = _find_stacks(cb, [stack_name])
|
||||
|
||||
for s in stacks:
|
||||
if s.mode == 'pulumi':
|
||||
if s.mode == "pulumi":
|
||||
s.preview()
|
||||
else:
|
||||
logger.warning('{} uses Cloudformation, use create-change-set for previews.'.format(s.stackname))
|
||||
logger.warning(
|
||||
"{} uses Cloudformation, use create-change-set for previews.".format(
|
||||
s.stackname
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@click.command()
|
||||
@ -203,7 +218,7 @@ def preview(cb, stack_name):
|
||||
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
|
||||
@click.pass_obj
|
||||
def provision(cb, stack_names, multi):
|
||||
""" Creates or updates stacks or stack groups """
|
||||
"""Creates or updates stacks or stack groups"""
|
||||
|
||||
stacks = _find_stacks(cb, stack_names, multi)
|
||||
_provision(cb, stacks)
|
||||
@ -214,7 +229,7 @@ def provision(cb, stack_names, multi):
|
||||
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
|
||||
@click.pass_obj
|
||||
def delete(cb, stack_names, multi):
|
||||
""" Deletes stacks or stack groups """
|
||||
"""Deletes stacks or stack groups"""
|
||||
stacks = _find_stacks(cb, stack_names, multi)
|
||||
|
||||
# Reverse steps
|
||||
@ -235,16 +250,16 @@ def delete(cb, stack_names, multi):
|
||||
@click.command()
|
||||
@click.pass_obj
|
||||
def clean(cb):
|
||||
""" Deletes all previously rendered files locally """
|
||||
"""Deletes all previously rendered files locally"""
|
||||
cb.clean()
|
||||
|
||||
|
||||
def sort_stacks(cb, stacks):
|
||||
""" Sort stacks by dependencies """
|
||||
"""Sort stacks by dependencies"""
|
||||
|
||||
data = {}
|
||||
for s in stacks:
|
||||
if s.mode == 'pulumi':
|
||||
if s.mode == "pulumi":
|
||||
data[s.id] = set()
|
||||
continue
|
||||
|
||||
@ -253,10 +268,14 @@ def sort_stacks(cb, stacks):
|
||||
deps = []
|
||||
for d in s.dependencies:
|
||||
# For now we assume deps are artifacts so we prepend them with our local profile and region to match stack.id
|
||||
for dep_stack in cb.filter_stacks({'region': s.region, 'profile': s.profile, 'provides': d}):
|
||||
for dep_stack in cb.filter_stacks(
|
||||
{"region": s.region, "profile": s.profile, "provides": d}
|
||||
):
|
||||
deps.append(dep_stack.id)
|
||||
# also look for global services
|
||||
for dep_stack in cb.filter_stacks({'region': 'global', 'profile': s.profile, 'provides': d}):
|
||||
for dep_stack in cb.filter_stacks(
|
||||
{"region": "global", "profile": s.profile, "provides": d}
|
||||
):
|
||||
deps.append(dep_stack.id)
|
||||
|
||||
data[s.id] = set(deps)
|
||||
@ -267,7 +286,9 @@ def sort_stacks(cb, stacks):
|
||||
v.discard(k)
|
||||
|
||||
if data:
|
||||
extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys())
|
||||
extra_items_in_deps = functools.reduce(set.union, data.values()) - set(
|
||||
data.keys()
|
||||
)
|
||||
data.update({item: set() for item in extra_items_in_deps})
|
||||
|
||||
while True:
|
||||
@ -283,41 +304,46 @@ def sort_stacks(cb, stacks):
|
||||
result.append(s)
|
||||
yield result
|
||||
|
||||
data = {item: (dep - ordered) for item, dep in data.items()
|
||||
if item not in ordered}
|
||||
data = {
|
||||
item: (dep - ordered) for item, dep in data.items() if item not in ordered
|
||||
}
|
||||
assert not data, "A cyclic dependency exists amongst %r" % data
|
||||
|
||||
|
||||
def _find_stacks(cb, stack_names, multi=False):
|
||||
""" search stacks by name """
|
||||
"""search stacks by name"""
|
||||
|
||||
stacks = []
|
||||
for s in stack_names:
|
||||
stacks = stacks + cb.resolve_stacks(s)
|
||||
|
||||
if not multi and len(stacks) > 1:
|
||||
logger.error('Found more than one stack matching name ({}). Please set --multi if that is what you want.'.format(', '.join(stack_names)))
|
||||
logger.error(
|
||||
"Found more than one stack matching name ({}). Please set --multi if that is what you want.".format(
|
||||
", ".join(stack_names)
|
||||
)
|
||||
)
|
||||
raise click.Abort()
|
||||
|
||||
if not stacks:
|
||||
logger.error('Cannot find stack matching: {}'.format(', '.join(stack_names)))
|
||||
logger.error("Cannot find stack matching: {}".format(", ".join(stack_names)))
|
||||
raise click.Abort()
|
||||
|
||||
return stacks
|
||||
|
||||
|
||||
def _render(stacks):
|
||||
""" Utility function to reuse code between tasks """
|
||||
"""Utility function to reuse code between tasks"""
|
||||
for s in stacks:
|
||||
if s.mode != 'pulumi':
|
||||
if s.mode != "pulumi":
|
||||
s.render()
|
||||
s.write_template_file()
|
||||
else:
|
||||
logger.info('{} uses Pulumi, render skipped.'.format(s.stackname))
|
||||
logger.info("{} uses Pulumi, render skipped.".format(s.stackname))
|
||||
|
||||
|
||||
def _provision(cb, stacks):
|
||||
""" Utility function to reuse code between tasks """
|
||||
"""Utility function to reuse code between tasks"""
|
||||
for step in sort_stacks(cb, stacks):
|
||||
if step:
|
||||
with ThreadPoolExecutor(max_workers=len(step)) as group:
|
||||
@ -348,5 +374,5 @@ cli.add_command(set_config)
|
||||
cli.add_command(get_config)
|
||||
cli.add_command(export)
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
cli(obj={})
|
||||
|
||||
@ -10,7 +10,7 @@ import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BotoConnection():
|
||||
class BotoConnection:
|
||||
_sessions = {}
|
||||
_clients = {}
|
||||
|
||||
@ -27,13 +27,15 @@ class BotoConnection():
|
||||
# Change the cache path from the default of ~/.aws/boto/cache to the one used by awscli
|
||||
session_vars = {}
|
||||
if profile:
|
||||
session_vars['profile'] = (None, None, profile, None)
|
||||
if region and region != 'global':
|
||||
session_vars['region'] = (None, None, region, None)
|
||||
session_vars["profile"] = (None, None, profile, None)
|
||||
if region and region != "global":
|
||||
session_vars["region"] = (None, None, region, None)
|
||||
|
||||
session = botocore.session.Session(session_vars=session_vars)
|
||||
cli_cache = os.path.join(os.path.expanduser('~'), '.aws/cli/cache')
|
||||
session.get_component('credential_provider').get_provider('assume-role').cache = credentials.JSONFileCache(cli_cache)
|
||||
cli_cache = os.path.join(os.path.expanduser("~"), ".aws/cli/cache")
|
||||
session.get_component("credential_provider").get_provider(
|
||||
"assume-role"
|
||||
).cache = credentials.JSONFileCache(cli_cache)
|
||||
|
||||
self._sessions[(profile, region)] = session
|
||||
|
||||
@ -41,7 +43,9 @@ class BotoConnection():
|
||||
|
||||
def _get_client(self, service, profile=None, region=None):
|
||||
if self._clients.get((profile, region, service)):
|
||||
logger.debug("Reusing boto session for {} {} {}".format(profile, region, service))
|
||||
logger.debug(
|
||||
"Reusing boto session for {} {} {}".format(profile, region, service)
|
||||
)
|
||||
return self._clients[(profile, region, service)]
|
||||
|
||||
session = self._get_session(profile, region)
|
||||
@ -59,8 +63,12 @@ class BotoConnection():
|
||||
return getattr(client, command)(**kwargs)
|
||||
|
||||
except botocore.exceptions.ClientError as e:
|
||||
if e.response['Error']['Code'] == 'Throttling':
|
||||
logger.warning("Throttling exception occured during {} - retry after 3s".format(command))
|
||||
if e.response["Error"]["Code"] == "Throttling":
|
||||
logger.warning(
|
||||
"Throttling exception occured during {} - retry after 3s".format(
|
||||
command
|
||||
)
|
||||
)
|
||||
time.sleep(3)
|
||||
pass
|
||||
else:
|
||||
|
||||
@ -9,7 +9,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CloudBender(object):
|
||||
""" Config Class to handle recursive conf/* config tree """
|
||||
"""Config Class to handle recursive conf/* config tree"""
|
||||
|
||||
def __init__(self, root_path):
|
||||
self.root = pathlib.Path(root_path)
|
||||
self.sg = None
|
||||
@ -20,28 +21,39 @@ class CloudBender(object):
|
||||
"hooks_path": self.root.joinpath("hooks"),
|
||||
"docs_path": self.root.joinpath("docs"),
|
||||
"outputs_path": self.root.joinpath("outputs"),
|
||||
"artifact_paths": [self.root.joinpath("artifacts")]
|
||||
"artifact_paths": [self.root.joinpath("artifacts")],
|
||||
}
|
||||
|
||||
if not self.ctx['config_path'].is_dir():
|
||||
raise InvalidProjectDir("Check '{0}' exists and is a valid CloudBender project folder.".format(self.ctx['config_path']))
|
||||
if not self.ctx["config_path"].is_dir():
|
||||
raise InvalidProjectDir(
|
||||
"Check '{0}' exists and is a valid CloudBender project folder.".format(
|
||||
self.ctx["config_path"]
|
||||
)
|
||||
)
|
||||
|
||||
def read_config(self):
|
||||
"""Load the <path>/config.yaml, <path>/*.yaml as stacks, sub-folders are sub-groups """
|
||||
"""Load the <path>/config.yaml, <path>/*.yaml as stacks, sub-folders are sub-groups"""
|
||||
|
||||
# Read top level config.yaml and extract CloudBender CTX
|
||||
_config = read_config_file(self.ctx['config_path'].joinpath('config.yaml'))
|
||||
_config = read_config_file(self.ctx["config_path"].joinpath("config.yaml"))
|
||||
|
||||
# Legacy naming
|
||||
if _config and _config.get('CloudBender'):
|
||||
self.ctx.update(_config.get('CloudBender'))
|
||||
if _config and _config.get("CloudBender"):
|
||||
self.ctx.update(_config.get("CloudBender"))
|
||||
|
||||
if _config and _config.get('cloudbender'):
|
||||
self.ctx.update(_config.get('cloudbender'))
|
||||
if _config and _config.get("cloudbender"):
|
||||
self.ctx.update(_config.get("cloudbender"))
|
||||
|
||||
# Make sure all paths are abs
|
||||
for k, v in self.ctx.items():
|
||||
if k in ['config_path', 'template_path', 'hooks_path', 'docs_path', 'artifact_paths', 'outputs_path']:
|
||||
if k in [
|
||||
"config_path",
|
||||
"template_path",
|
||||
"hooks_path",
|
||||
"docs_path",
|
||||
"artifact_paths",
|
||||
"outputs_path",
|
||||
]:
|
||||
if isinstance(v, list):
|
||||
new_list = []
|
||||
for p in v:
|
||||
@ -56,7 +68,7 @@ class CloudBender(object):
|
||||
if not v.is_absolute():
|
||||
self.ctx[k] = self.root.joinpath(v)
|
||||
|
||||
self.sg = StackGroup(self.ctx['config_path'], self.ctx)
|
||||
self.sg = StackGroup(self.ctx["config_path"], self.ctx)
|
||||
self.sg.read_config()
|
||||
|
||||
self.all_stacks = self.sg.get_stacks()
|
||||
@ -77,15 +89,15 @@ class CloudBender(object):
|
||||
token = token[7:]
|
||||
|
||||
# If path ends with yaml we look for stacks
|
||||
if token.endswith('.yaml'):
|
||||
stacks = self.sg.get_stacks(token, match_by='path')
|
||||
if token.endswith(".yaml"):
|
||||
stacks = self.sg.get_stacks(token, match_by="path")
|
||||
|
||||
# otherwise assume we look for a group, if we find a group return all stacks below
|
||||
else:
|
||||
# Strip potential trailing slash
|
||||
token = token.rstrip('/')
|
||||
token = token.rstrip("/")
|
||||
|
||||
sg = self.sg.get_stackgroup(token, match_by='path')
|
||||
sg = self.sg.get_stackgroup(token, match_by="path")
|
||||
if sg:
|
||||
stacks = sg.get_stacks()
|
||||
|
||||
|
||||
@ -8,6 +8,7 @@ from functools import wraps
|
||||
from .exceptions import InvalidHook
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@ -40,7 +41,9 @@ def pulumi_ws(func):
|
||||
@wraps(func)
|
||||
def decorated(self, *args, **kwargs):
|
||||
# setup temp workspace
|
||||
self.work_dir = tempfile.mkdtemp(dir=tempfile.gettempdir(), prefix="cloudbender-")
|
||||
self.work_dir = tempfile.mkdtemp(
|
||||
dir=tempfile.gettempdir(), prefix="cloudbender-"
|
||||
)
|
||||
|
||||
response = func(self, *args, **kwargs)
|
||||
|
||||
@ -63,4 +66,4 @@ def cmd(stack, arguments):
|
||||
hook = subprocess.run(arguments, stdout=subprocess.PIPE)
|
||||
logger.info(hook.stdout.decode("utf-8"))
|
||||
except TypeError:
|
||||
raise InvalidHook('Invalid argument {}'.format(arguments))
|
||||
raise InvalidHook("Invalid argument {}".format(arguments))
|
||||
|
||||
@ -24,10 +24,10 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@jinja2.contextfunction
|
||||
def option(context, attribute, default_value=u'', source='options'):
|
||||
""" Get attribute from options data structure, default_value otherwise """
|
||||
def option(context, attribute, default_value="", source="options"):
|
||||
"""Get attribute from options data structure, default_value otherwise"""
|
||||
environment = context.environment
|
||||
options = environment.globals['_config'][source]
|
||||
options = environment.globals["_config"][source]
|
||||
|
||||
if not attribute:
|
||||
return default_value
|
||||
@ -48,7 +48,7 @@ def option(context, attribute, default_value=u'', source='options'):
|
||||
@jinja2.contextfunction
|
||||
def include_raw_gz(context, files=None, gz=True, remove_comments=False):
|
||||
jenv = context.environment
|
||||
output = ''
|
||||
output = ""
|
||||
|
||||
# For shell script we can even remove whitespaces so treat them individually
|
||||
# sed -e '2,$ {/^ *$/d ; /^ *#/d ; /^[ \t] *#/d ; /*^/d ; s/^[ \t]*// ; s/*[ \t]$// ; s/ $//}'
|
||||
@ -57,33 +57,35 @@ def include_raw_gz(context, files=None, gz=True, remove_comments=False):
|
||||
|
||||
if remove_comments:
|
||||
# Remove full line comments but not shebang
|
||||
_re_comment = re.compile(r'^\s*#[^!]')
|
||||
_re_blank = re.compile(r'^\s*$')
|
||||
_re_keep = re.compile(r'^## template: jinja$')
|
||||
stripped_output = ''
|
||||
_re_comment = re.compile(r"^\s*#[^!]")
|
||||
_re_blank = re.compile(r"^\s*$")
|
||||
_re_keep = re.compile(r"^## template: jinja$")
|
||||
stripped_output = ""
|
||||
for curline in output.splitlines():
|
||||
if re.match(_re_blank, curline):
|
||||
continue
|
||||
elif re.match(_re_keep, curline):
|
||||
stripped_output = stripped_output + curline + '\n'
|
||||
stripped_output = stripped_output + curline + "\n"
|
||||
elif re.match(_re_comment, curline):
|
||||
logger.debug("Removed {}".format(curline))
|
||||
else:
|
||||
stripped_output = stripped_output + curline + '\n'
|
||||
stripped_output = stripped_output + curline + "\n"
|
||||
|
||||
output = stripped_output
|
||||
|
||||
if not gz:
|
||||
return(output)
|
||||
return output
|
||||
|
||||
buf = io.BytesIO()
|
||||
f = gzip.GzipFile(mode='w', fileobj=buf, mtime=0)
|
||||
f = gzip.GzipFile(mode="w", fileobj=buf, mtime=0)
|
||||
f.write(output.encode())
|
||||
f.close()
|
||||
|
||||
# MaxSize is 21847
|
||||
logger.info("Compressed user-data from {} to {}".format(len(output), len(buf.getvalue())))
|
||||
return base64.b64encode(buf.getvalue()).decode('utf-8')
|
||||
logger.info(
|
||||
"Compressed user-data from {} to {}".format(len(output), len(buf.getvalue()))
|
||||
)
|
||||
return base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
|
||||
|
||||
@jinja2.contextfunction
|
||||
@ -92,33 +94,33 @@ def raise_helper(context, msg):
|
||||
|
||||
|
||||
# Custom tests
|
||||
def regex(value='', pattern='', ignorecase=False, match_type='search'):
|
||||
''' Expose `re` as a boolean filter using the `search` method by default.
|
||||
This is likely only useful for `search` and `match` which already
|
||||
have their own filters.
|
||||
'''
|
||||
def regex(value="", pattern="", ignorecase=False, match_type="search"):
|
||||
"""Expose `re` as a boolean filter using the `search` method by default.
|
||||
This is likely only useful for `search` and `match` which already
|
||||
have their own filters.
|
||||
"""
|
||||
if ignorecase:
|
||||
flags = re.I
|
||||
else:
|
||||
flags = 0
|
||||
_re = re.compile(pattern, flags=flags)
|
||||
if getattr(_re, match_type, 'search')(value) is not None:
|
||||
if getattr(_re, match_type, "search")(value) is not None:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def match(value, pattern='', ignorecase=False):
|
||||
''' Perform a `re.match` returning a boolean '''
|
||||
return regex(value, pattern, ignorecase, 'match')
|
||||
def match(value, pattern="", ignorecase=False):
|
||||
"""Perform a `re.match` returning a boolean"""
|
||||
return regex(value, pattern, ignorecase, "match")
|
||||
|
||||
|
||||
def search(value, pattern='', ignorecase=False):
|
||||
''' Perform a `re.search` returning a boolean '''
|
||||
return regex(value, pattern, ignorecase, 'search')
|
||||
def search(value, pattern="", ignorecase=False):
|
||||
"""Perform a `re.search` returning a boolean"""
|
||||
return regex(value, pattern, ignorecase, "search")
|
||||
|
||||
|
||||
# Custom filters
|
||||
def sub(value='', pattern='', replace='', ignorecase=False):
|
||||
def sub(value="", pattern="", replace="", ignorecase=False):
|
||||
if ignorecase:
|
||||
flags = re.I
|
||||
else:
|
||||
@ -129,9 +131,16 @@ def sub(value='', pattern='', replace='', ignorecase=False):
|
||||
def pyminify(source, obfuscate=False, minify=True):
|
||||
# pyminifier options
|
||||
options = types.SimpleNamespace(
|
||||
tabs=False, replacement_length=1, use_nonlatin=0,
|
||||
obfuscate=0, obf_variables=1, obf_classes=0, obf_functions=0,
|
||||
obf_import_methods=0, obf_builtins=0)
|
||||
tabs=False,
|
||||
replacement_length=1,
|
||||
use_nonlatin=0,
|
||||
obfuscate=0,
|
||||
obf_variables=1,
|
||||
obf_classes=0,
|
||||
obf_functions=0,
|
||||
obf_import_methods=0,
|
||||
obf_builtins=0,
|
||||
)
|
||||
|
||||
tokens = pyminifier.token_utils.listified_tokenizer(source)
|
||||
|
||||
@ -141,13 +150,17 @@ def pyminify(source, obfuscate=False, minify=True):
|
||||
|
||||
if obfuscate:
|
||||
name_generator = pyminifier.obfuscate.obfuscation_machine(use_unicode=False)
|
||||
pyminifier.obfuscate.obfuscate("__main__", tokens, options, name_generator=name_generator)
|
||||
pyminifier.obfuscate.obfuscate(
|
||||
"__main__", tokens, options, name_generator=name_generator
|
||||
)
|
||||
# source = pyminifier.obfuscate.apply_obfuscation(source)
|
||||
|
||||
source = pyminifier.token_utils.untokenize(tokens)
|
||||
# logger.info(source)
|
||||
minified_source = pyminifier.compression.gz_pack(source)
|
||||
logger.info("Compressed python code from {} to {}".format(len(source), len(minified_source)))
|
||||
logger.info(
|
||||
"Compressed python code from {} to {}".format(len(source), len(minified_source))
|
||||
)
|
||||
return minified_source
|
||||
|
||||
|
||||
@ -157,10 +170,12 @@ def inline_yaml(block):
|
||||
|
||||
def JinjaEnv(template_locations=[]):
|
||||
LoggingUndefined = jinja2.make_logging_undefined(logger=logger, base=Undefined)
|
||||
jenv = jinja2.Environment(trim_blocks=True,
|
||||
lstrip_blocks=True,
|
||||
undefined=LoggingUndefined,
|
||||
extensions=['jinja2.ext.loopcontrols', 'jinja2.ext.do'])
|
||||
jenv = jinja2.Environment(
|
||||
trim_blocks=True,
|
||||
lstrip_blocks=True,
|
||||
undefined=LoggingUndefined,
|
||||
extensions=["jinja2.ext.loopcontrols", "jinja2.ext.do"],
|
||||
)
|
||||
|
||||
if template_locations:
|
||||
jinja_loaders = []
|
||||
@ -171,29 +186,29 @@ def JinjaEnv(template_locations=[]):
|
||||
else:
|
||||
jenv.loader = jinja2.BaseLoader()
|
||||
|
||||
jenv.globals['include_raw'] = include_raw_gz
|
||||
jenv.globals['raise'] = raise_helper
|
||||
jenv.globals['option'] = option
|
||||
jenv.globals["include_raw"] = include_raw_gz
|
||||
jenv.globals["raise"] = raise_helper
|
||||
jenv.globals["option"] = option
|
||||
|
||||
jenv.filters['sub'] = sub
|
||||
jenv.filters['pyminify'] = pyminify
|
||||
jenv.filters['inline_yaml'] = inline_yaml
|
||||
jenv.filters["sub"] = sub
|
||||
jenv.filters["pyminify"] = pyminify
|
||||
jenv.filters["inline_yaml"] = inline_yaml
|
||||
|
||||
jenv.tests['match'] = match
|
||||
jenv.tests['regex'] = regex
|
||||
jenv.tests['search'] = search
|
||||
jenv.tests["match"] = match
|
||||
jenv.tests["regex"] = regex
|
||||
jenv.tests["search"] = search
|
||||
|
||||
return jenv
|
||||
|
||||
|
||||
def read_config_file(path, variables={}):
|
||||
""" reads yaml config file, passes it through jinja and returns data structre
|
||||
"""reads yaml config file, passes it through jinja and returns data structre
|
||||
|
||||
- OS ENV are available as {{ ENV.<VAR> }}
|
||||
- variables defined in parent configs are available as {{ <VAR> }}
|
||||
- OS ENV are available as {{ ENV.<VAR> }}
|
||||
- variables defined in parent configs are available as {{ <VAR> }}
|
||||
"""
|
||||
jinja_variables = copy.deepcopy(variables)
|
||||
jinja_variables['ENV'] = os.environ
|
||||
jinja_variables["ENV"] = os.environ
|
||||
|
||||
if path.exists():
|
||||
logger.debug("Reading config file: {}".format(path))
|
||||
@ -205,7 +220,8 @@ def read_config_file(path, variables={}):
|
||||
auto_reload=False,
|
||||
loader=jinja2.FunctionLoader(_sops_loader),
|
||||
undefined=jinja2.StrictUndefined,
|
||||
extensions=['jinja2.ext.loopcontrols'])
|
||||
extensions=["jinja2.ext.loopcontrols"],
|
||||
)
|
||||
template = jenv.get_template(str(path))
|
||||
rendered_template = template.render(jinja_variables)
|
||||
data = yaml.safe_load(rendered_template)
|
||||
@ -220,26 +236,37 @@ def read_config_file(path, variables={}):
|
||||
|
||||
|
||||
def _sops_loader(path):
|
||||
""" Tries to loads yaml file
|
||||
If "sops" key is detected the file is piped through sops before returned
|
||||
"""Tries to loads yaml file
|
||||
If "sops" key is detected the file is piped through sops before returned
|
||||
"""
|
||||
with open(path, 'r') as f:
|
||||
with open(path, "r") as f:
|
||||
config_raw = f.read()
|
||||
data = yaml.safe_load(config_raw)
|
||||
|
||||
if data and 'sops' in data and 'DISABLE_SOPS' not in os.environ:
|
||||
if data and "sops" in data and "DISABLE_SOPS" not in os.environ:
|
||||
try:
|
||||
result = subprocess.run([
|
||||
'sops',
|
||||
'--input-type', 'yaml',
|
||||
'--output-type', 'yaml',
|
||||
'--decrypt', '/dev/stdin'
|
||||
], stdout=subprocess.PIPE, input=config_raw.encode('utf-8'),
|
||||
env=dict(os.environ, **{"AWS_SDK_LOAD_CONFIG": "1"}))
|
||||
result = subprocess.run(
|
||||
[
|
||||
"sops",
|
||||
"--input-type",
|
||||
"yaml",
|
||||
"--output-type",
|
||||
"yaml",
|
||||
"--decrypt",
|
||||
"/dev/stdin",
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
input=config_raw.encode("utf-8"),
|
||||
env=dict(os.environ, **{"AWS_SDK_LOAD_CONFIG": "1"}),
|
||||
)
|
||||
except FileNotFoundError:
|
||||
logger.exception("SOPS encrypted config {}, but unable to find sops binary! Try eg: https://github.com/mozilla/sops/releases/download/v3.5.0/sops-v3.5.0.linux".format(path))
|
||||
logger.exception(
|
||||
"SOPS encrypted config {}, but unable to find sops binary! Try eg: https://github.com/mozilla/sops/releases/download/v3.5.0/sops-v3.5.0.linux".format(
|
||||
path
|
||||
)
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
return result.stdout.decode('utf-8')
|
||||
return result.stdout.decode("utf-8")
|
||||
else:
|
||||
return config_raw
|
||||
|
||||
@ -7,30 +7,38 @@ import pkg_resources
|
||||
import pulumi
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def pulumi_init(stack):
|
||||
|
||||
# Fail early if pulumi binaries are not available
|
||||
if not shutil.which('pulumi'):
|
||||
raise FileNotFoundError("Cannot find pulumi binary, see https://www.pulumi.com/docs/get-started/install/")
|
||||
if not shutil.which("pulumi"):
|
||||
raise FileNotFoundError(
|
||||
"Cannot find pulumi binary, see https://www.pulumi.com/docs/get-started/install/"
|
||||
)
|
||||
|
||||
# add all artifact_paths/pulumi to the search path for easier imports in the pulumi code
|
||||
for artifacts_path in stack.ctx['artifact_paths']:
|
||||
_path = '{}/pulumi'.format(artifacts_path.resolve())
|
||||
for artifacts_path in stack.ctx["artifact_paths"]:
|
||||
_path = "{}/pulumi".format(artifacts_path.resolve())
|
||||
sys.path.append(_path)
|
||||
|
||||
# Try local implementation first, similar to Jinja2 mode
|
||||
_found = False
|
||||
try:
|
||||
_stack = importlib.import_module('config.{}.{}'.format(stack.rel_path, stack.template).replace('/', '.'))
|
||||
_stack = importlib.import_module(
|
||||
"config.{}.{}".format(stack.rel_path, stack.template).replace("/", ".")
|
||||
)
|
||||
_found = True
|
||||
|
||||
except ImportError:
|
||||
for artifacts_path in stack.ctx['artifact_paths']:
|
||||
for artifacts_path in stack.ctx["artifact_paths"]:
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location("_stack", '{}/pulumi/{}.py'.format(artifacts_path.resolve(), stack.template))
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"_stack",
|
||||
"{}/pulumi/{}.py".format(artifacts_path.resolve(), stack.template),
|
||||
)
|
||||
_stack = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(_stack)
|
||||
_found = True
|
||||
@ -39,36 +47,61 @@ def pulumi_init(stack):
|
||||
pass
|
||||
|
||||
if not _found:
|
||||
raise FileNotFoundError("Cannot find Pulumi implementation for {}".format(stack.stackname))
|
||||
raise FileNotFoundError(
|
||||
"Cannot find Pulumi implementation for {}".format(stack.stackname)
|
||||
)
|
||||
|
||||
project_name = stack.parameters['Conglomerate']
|
||||
project_name = stack.parameters["Conglomerate"]
|
||||
|
||||
# Remove stacknameprefix if equals Conglomerate as Pulumi implicitly prefixes project_name
|
||||
pulumi_stackname = re.sub(r'^' + project_name + '-?', '', stack.stackname)
|
||||
pulumi_stackname = re.sub(r"^" + project_name + "-?", "", stack.stackname)
|
||||
try:
|
||||
pulumi_backend = '{}/{}/{}'.format(stack.pulumi['backend'], project_name, stack.region)
|
||||
pulumi_backend = "{}/{}/{}".format(
|
||||
stack.pulumi["backend"], project_name, stack.region
|
||||
)
|
||||
|
||||
except KeyError:
|
||||
raise KeyError('Missing pulumi.backend setting !')
|
||||
raise KeyError("Missing pulumi.backend setting !")
|
||||
|
||||
account_id = stack.connection_manager.call('sts', 'get_caller_identity', profile=stack.profile, region=stack.region)['Account']
|
||||
account_id = stack.connection_manager.call(
|
||||
"sts", "get_caller_identity", profile=stack.profile, region=stack.region
|
||||
)["Account"]
|
||||
# Ugly hack as Pulumi currently doesnt support MFA_TOKENs during role assumptions
|
||||
# Do NOT set them via 'aws:secretKey' as they end up in the stack.json in plain text !!!
|
||||
if stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().token:
|
||||
os.environ['AWS_SESSION_TOKEN'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().token
|
||||
if (
|
||||
stack.connection_manager._sessions[(stack.profile, stack.region)]
|
||||
.get_credentials()
|
||||
.token
|
||||
):
|
||||
os.environ["AWS_SESSION_TOKEN"] = (
|
||||
stack.connection_manager._sessions[(stack.profile, stack.region)]
|
||||
.get_credentials()
|
||||
.token
|
||||
)
|
||||
|
||||
os.environ['AWS_ACCESS_KEY_ID'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().access_key
|
||||
os.environ['AWS_SECRET_ACCESS_KEY'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().secret_key
|
||||
os.environ['AWS_DEFAULT_REGION'] = stack.region
|
||||
os.environ["AWS_ACCESS_KEY_ID"] = (
|
||||
stack.connection_manager._sessions[(stack.profile, stack.region)]
|
||||
.get_credentials()
|
||||
.access_key
|
||||
)
|
||||
os.environ["AWS_SECRET_ACCESS_KEY"] = (
|
||||
stack.connection_manager._sessions[(stack.profile, stack.region)]
|
||||
.get_credentials()
|
||||
.secret_key
|
||||
)
|
||||
os.environ["AWS_DEFAULT_REGION"] = stack.region
|
||||
|
||||
# Secrets provider
|
||||
try:
|
||||
secrets_provider = stack.pulumi['secretsProvider']
|
||||
if secrets_provider == 'passphrase' and 'PULUMI_CONFIG_PASSPHRASE' not in os.environ:
|
||||
raise ValueError('Missing PULUMI_CONFIG_PASSPHRASE environment variable!')
|
||||
secrets_provider = stack.pulumi["secretsProvider"]
|
||||
if (
|
||||
secrets_provider == "passphrase"
|
||||
and "PULUMI_CONFIG_PASSPHRASE" not in os.environ
|
||||
):
|
||||
raise ValueError("Missing PULUMI_CONFIG_PASSPHRASE environment variable!")
|
||||
|
||||
except KeyError:
|
||||
logger.warning('Missing pulumi.secretsProvider setting, secrets disabled !')
|
||||
logger.warning("Missing pulumi.secretsProvider setting, secrets disabled !")
|
||||
secrets_provider = None
|
||||
|
||||
# Set tag for stack file name and version
|
||||
@ -76,9 +109,11 @@ def pulumi_init(stack):
|
||||
try:
|
||||
_version = _stack.VERSION
|
||||
except AttributeError:
|
||||
_version = 'undefined'
|
||||
_version = "undefined"
|
||||
|
||||
_tags['zero-downtime.net/cloudbender'] = '{}:{}'.format(os.path.basename(_stack.__file__), _version)
|
||||
_tags["zero-downtime.net/cloudbender"] = "{}:{}".format(
|
||||
os.path.basename(_stack.__file__), _version
|
||||
)
|
||||
|
||||
_config = {
|
||||
"aws:region": stack.region,
|
||||
@ -90,27 +125,36 @@ def pulumi_init(stack):
|
||||
|
||||
# inject all parameters as config in the <Conglomerate> namespace
|
||||
for p in stack.parameters:
|
||||
_config['{}:{}'.format(stack.parameters['Conglomerate'], p)] = stack.parameters[p]
|
||||
_config["{}:{}".format(stack.parameters["Conglomerate"], p)] = stack.parameters[
|
||||
p
|
||||
]
|
||||
|
||||
stack_settings = pulumi.automation.StackSettings(
|
||||
config=_config,
|
||||
secrets_provider=secrets_provider,
|
||||
encryption_salt=stack.pulumi.get('encryptionsalt', None),
|
||||
encrypted_key=stack.pulumi.get('encryptedkey', None)
|
||||
encryption_salt=stack.pulumi.get("encryptionsalt", None),
|
||||
encrypted_key=stack.pulumi.get("encryptedkey", None),
|
||||
)
|
||||
|
||||
project_settings = pulumi.automation.ProjectSettings(
|
||||
name=project_name,
|
||||
runtime="python",
|
||||
backend={"url": pulumi_backend})
|
||||
name=project_name, runtime="python", backend={"url": pulumi_backend}
|
||||
)
|
||||
|
||||
ws_opts = pulumi.automation.LocalWorkspaceOptions(
|
||||
work_dir=stack.work_dir,
|
||||
project_settings=project_settings,
|
||||
stack_settings={pulumi_stackname: stack_settings},
|
||||
secrets_provider=secrets_provider)
|
||||
secrets_provider=secrets_provider,
|
||||
)
|
||||
|
||||
stack = pulumi.automation.create_or_select_stack(stack_name=pulumi_stackname, project_name=project_name, program=_stack.pulumi_program, opts=ws_opts)
|
||||
stack.workspace.install_plugin("aws", pkg_resources.get_distribution("pulumi_aws").version)
|
||||
stack = pulumi.automation.create_or_select_stack(
|
||||
stack_name=pulumi_stackname,
|
||||
project_name=project_name,
|
||||
program=_stack.pulumi_program,
|
||||
opts=ws_opts,
|
||||
)
|
||||
stack.workspace.install_plugin(
|
||||
"aws", pkg_resources.get_distribution("pulumi_aws").version
|
||||
)
|
||||
|
||||
return stack
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -13,19 +13,21 @@ class StackGroup(object):
|
||||
self.name = None
|
||||
self.ctx = ctx
|
||||
self.path = path
|
||||
self.rel_path = path.relative_to(ctx['config_path'])
|
||||
self.rel_path = path.relative_to(ctx["config_path"])
|
||||
self.config = {}
|
||||
self.sgs = []
|
||||
self.stacks = []
|
||||
|
||||
if self.rel_path == '.':
|
||||
self.rel_path = ''
|
||||
if self.rel_path == ".":
|
||||
self.rel_path = ""
|
||||
|
||||
def dump_config(self):
|
||||
for sg in self.sgs:
|
||||
sg.dump_config()
|
||||
|
||||
logger.debug("StackGroup {}: {}".format(self.rel_path, pprint.pformat(self.config)))
|
||||
logger.debug(
|
||||
"StackGroup {}: {}".format(self.rel_path, pprint.pformat(self.config))
|
||||
)
|
||||
|
||||
for s in self.stacks:
|
||||
s.dump_config()
|
||||
@ -35,7 +37,9 @@ class StackGroup(object):
|
||||
return None
|
||||
|
||||
# First read config.yaml if present
|
||||
_config = read_config_file(self.path.joinpath('config.yaml'), parent_config.get('variables', {}))
|
||||
_config = read_config_file(
|
||||
self.path.joinpath("config.yaml"), parent_config.get("variables", {})
|
||||
)
|
||||
|
||||
# Stack Group name if not explicit via config is derived from subfolder, or in case of root object the parent folder
|
||||
if "stackgroupname" in _config:
|
||||
@ -45,19 +49,25 @@ class StackGroup(object):
|
||||
|
||||
# Merge config with parent config
|
||||
self.config = dict_merge(parent_config, _config)
|
||||
stackname_prefix = self.config.get('stacknameprefix', '')
|
||||
stackname_prefix = self.config.get("stacknameprefix", "")
|
||||
|
||||
logger.debug("StackGroup {} added.".format(self.name))
|
||||
|
||||
# Add stacks
|
||||
stacks = [s for s in self.path.glob('*.yaml') if not s.name == "config.yaml"]
|
||||
stacks = [s for s in self.path.glob("*.yaml") if not s.name == "config.yaml"]
|
||||
for stack_path in stacks:
|
||||
stackname = stack_path.name.split('.')[0]
|
||||
stackname = stack_path.name.split(".")[0]
|
||||
template = stackname
|
||||
if stackname_prefix:
|
||||
stackname = stackname_prefix + stackname
|
||||
|
||||
new_stack = Stack(name=stackname, template=template, path=stack_path, rel_path=str(self.rel_path), ctx=self.ctx)
|
||||
new_stack = Stack(
|
||||
name=stackname,
|
||||
template=template,
|
||||
path=stack_path,
|
||||
rel_path=str(self.rel_path),
|
||||
ctx=self.ctx,
|
||||
)
|
||||
new_stack.read_config(self.config)
|
||||
self.stacks.append(new_stack)
|
||||
|
||||
@ -68,22 +78,24 @@ class StackGroup(object):
|
||||
|
||||
self.sgs.append(sg)
|
||||
|
||||
def get_stacks(self, name=None, recursive=True, match_by='name'):
|
||||
""" Returns [stack] matching stack_name or [all] """
|
||||
def get_stacks(self, name=None, recursive=True, match_by="name"):
|
||||
"""Returns [stack] matching stack_name or [all]"""
|
||||
stacks = []
|
||||
if name:
|
||||
logger.debug("Looking for stack {} in group {}".format(name, self.name))
|
||||
|
||||
for s in self.stacks:
|
||||
if name:
|
||||
if match_by == 'name' and s.stackname != name:
|
||||
if match_by == "name" and s.stackname != name:
|
||||
continue
|
||||
|
||||
if match_by == 'path' and not s.path.match(name):
|
||||
if match_by == "path" and not s.path.match(name):
|
||||
continue
|
||||
|
||||
if self.rel_path:
|
||||
logger.debug("Found stack {} in group {}".format(s.stackname, self.rel_path))
|
||||
logger.debug(
|
||||
"Found stack {} in group {}".format(s.stackname, self.rel_path)
|
||||
)
|
||||
else:
|
||||
logger.debug("Found stack {}".format(s.stackname))
|
||||
stacks.append(s)
|
||||
@ -96,14 +108,20 @@ class StackGroup(object):
|
||||
|
||||
return stacks
|
||||
|
||||
def get_stackgroup(self, name=None, recursive=True, match_by='name'):
|
||||
""" Returns stack group matching stackgroup_name or all if None """
|
||||
if not name or (self.name == name and match_by == 'name') or (self.path.match(name) and match_by == 'path'):
|
||||
def get_stackgroup(self, name=None, recursive=True, match_by="name"):
|
||||
"""Returns stack group matching stackgroup_name or all if None"""
|
||||
if (
|
||||
not name
|
||||
or (self.name == name and match_by == "name")
|
||||
or (self.path.match(name) and match_by == "path")
|
||||
):
|
||||
logger.debug("Found stack_group {}".format(self.name))
|
||||
return self
|
||||
|
||||
if name and self.name != 'config':
|
||||
logger.debug("Looking for stack_group {} in group {}".format(name, self.name))
|
||||
if name and self.name != "config":
|
||||
logger.debug(
|
||||
"Looking for stack_group {} in group {}".format(name, self.name)
|
||||
)
|
||||
|
||||
if recursive:
|
||||
for sg in self.sgs:
|
||||
|
||||
@ -5,7 +5,7 @@ import re
|
||||
|
||||
|
||||
def dict_merge(a, b):
|
||||
""" Deep merge to allow proper inheritance for config files"""
|
||||
"""Deep merge to allow proper inheritance for config files"""
|
||||
if not a:
|
||||
return b
|
||||
|
||||
@ -36,16 +36,14 @@ def setup_logging(debug):
|
||||
logging.getLogger("botocore").setLevel(logging.INFO)
|
||||
|
||||
formatter = logging.Formatter(
|
||||
fmt="[%(asctime)s] %(name)s %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S"
|
||||
fmt="[%(asctime)s] %(name)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
|
||||
)
|
||||
else:
|
||||
our_level = logging.INFO
|
||||
logging.getLogger("botocore").setLevel(logging.CRITICAL)
|
||||
|
||||
formatter = logging.Formatter(
|
||||
fmt="[%(asctime)s] %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S"
|
||||
fmt="[%(asctime)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
|
||||
)
|
||||
|
||||
log_handler = logging.StreamHandler()
|
||||
@ -57,8 +55,8 @@ def setup_logging(debug):
|
||||
|
||||
|
||||
def search_refs(template, attributes, mode):
|
||||
""" Traverses a template and searches for any remote references and
|
||||
adds them to the attributes set
|
||||
"""Traverses a template and searches for any remote references and
|
||||
adds them to the attributes set
|
||||
"""
|
||||
if isinstance(template, dict):
|
||||
for k, v in template.items():
|
||||
@ -70,7 +68,7 @@ def search_refs(template, attributes, mode):
|
||||
# CloudBender::StackRef
|
||||
if k == "CloudBender::StackRef":
|
||||
try:
|
||||
attributes.append(v['StackTags']['Artifact'])
|
||||
attributes.append(v["StackTags"]["Artifact"])
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
@ -91,11 +89,11 @@ def get_s3_url(url, *args):
|
||||
bucket = None
|
||||
path = None
|
||||
|
||||
m = re.match('^(s3://)?([^/]*)(/.*)?', url)
|
||||
m = re.match("^(s3://)?([^/]*)(/.*)?", url)
|
||||
bucket = m[2]
|
||||
if m[3]:
|
||||
path = m[3].lstrip('/')
|
||||
path = m[3].lstrip("/")
|
||||
|
||||
path = os.path.join(path, *args)
|
||||
|
||||
return(bucket, path)
|
||||
return (bucket, path)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user