Implemented intial support for hooks and outputs

This commit is contained in:
Stefan Reimer 2020-06-22 12:30:54 +01:00
parent 478f8a4bfa
commit c6c34b5dc1
7 changed files with 133 additions and 17 deletions

View File

@ -1,5 +1,14 @@
# Changelog # Changelog
## 0.9.0
New Features:
- *Hooks* can now be defined as artifact metadata and are executed at the specified step.
Current supported hooks are: `pre_create, pre_update, post_create, post_update`
- Stack *Outputs* are now written into a yaml file under `outputs` if enabled. Enabled via `options.StoreOutputs`
- Removed deprecated support for storing parameters as these can be constructed any time from existing and tracked configs
- some code cleanups and minor changes for cli outputs
## 0.8.4 ## 0.8.4
- New Feature: `create-docs` command - New Feature: `create-docs` command
Renders a markdown documentation next to the rendered stack templated by parsing parameters and other relvant metadata Renders a markdown documentation next to the rendered stack templated by parsing parameters and other relvant metadata

View File

@ -2,7 +2,7 @@ import logging
__author__ = "Stefan Reimer" __author__ = "Stefan Reimer"
__email__ = "stefan@zero-downtimet.net" __email__ = "stefan@zero-downtimet.net"
__version__ = "0.8.5" __version__ = "0.9.0"
# Set up logging to ``/dev/null`` like a library is supposed to. # Set up logging to ``/dev/null`` like a library is supposed to.

View File

@ -2,6 +2,7 @@ import os
import sys import sys
import click import click
import functools import functools
import re
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
@ -91,7 +92,14 @@ def outputs(cb, stack_names, multi, include, values):
stacks = _find_stacks(cb, stack_names, multi) stacks = _find_stacks(cb, stack_names, multi)
for s in stacks: for s in stacks:
s.get_outputs(include, values) s.get_outputs()
for output in s.outputs.keys():
if re.search(include, output):
if values:
print("{}".format(output))
else:
print("{}={}".format(output, s.outputs[output]))
@click.command() @click.command()

View File

@ -19,6 +19,7 @@ class CloudBender(object):
"template_path": self.root.joinpath("cloudformation"), "template_path": self.root.joinpath("cloudformation"),
"hooks_path": self.root.joinpath("hooks"), "hooks_path": self.root.joinpath("hooks"),
"docs_path": self.root.joinpath("docs"), "docs_path": self.root.joinpath("docs"),
"outputs_path": self.root.joinpath("outputs"),
"artifact_paths": [self.root.joinpath("artifacts")] "artifact_paths": [self.root.joinpath("artifacts")]
} }
@ -35,7 +36,7 @@ class CloudBender(object):
# Make sure all paths are abs # Make sure all paths are abs
for k, v in self.ctx.items(): for k, v in self.ctx.items():
if k in ['config_path', 'template_path', 'hooks_path', 'docs_path', 'artifact_paths']: if k in ['config_path', 'template_path', 'hooks_path', 'docs_path', 'artifact_paths', 'outputs_path']:
if isinstance(v, list): if isinstance(v, list):
new_list = [] new_list = []
for p in v: for p in v:

View File

@ -8,3 +8,7 @@ class ParameterIllegalValue(Exception):
class InvalidProjectDir(Exception): class InvalidProjectDir(Exception):
"""My documentation""" """My documentation"""
class InvalidHook(Exception):
"""My documentation"""

49
cloudbender/hooks.py Normal file
View File

@ -0,0 +1,49 @@
import sys
import subprocess
from functools import wraps
from .exceptions import InvalidHook
import logging
logger = logging.getLogger(__name__)
def execute_hooks(hooks, stack):
for hook in hooks:
tokens = hook.split()
if tokens[0] in dir(sys.modules[__name__]):
logger.info("Executing hook: {}".format(hook))
globals()[tokens[0]](arguments=tokens[1:], stack=stack)
else:
logger.warning("Unknown hook: {}".format(hook))
def exec_hooks(func):
@wraps(func)
def decorated(self, *args, **kwargs):
execute_hooks(self.hooks.get("pre_" + func.__name__, []), self)
response = func(self, *args, **kwargs)
execute_hooks(self.hooks.get("post_" + func.__name__, []), self)
return response
return decorated
# Various hooks
def cmd(stack, arguments):
"""
Generic command via subprocess
"""
try:
hook = subprocess.run(arguments, stdout=subprocess.PIPE)
logger.info(hook.stdout.decode("utf-8"))
except TypeError:
raise InvalidHook('Invalid argument {}'.format(arguments))
def export_outputs_kubezero(stack, arguments):
""" Write outputs in yaml for kubezero helm chart """
logger.info(stack.outputs)

View File

@ -16,6 +16,7 @@ from .connection import BotoConnection
from .jinja import JinjaEnv, read_config_file from .jinja import JinjaEnv, read_config_file
from . import __version__ from . import __version__
from .exceptions import ParameterNotFound, ParameterIllegalValue from .exceptions import ParameterNotFound, ParameterIllegalValue
from .hooks import exec_hooks
import cfnlint.core import cfnlint.core
@ -49,6 +50,7 @@ class Stack(object):
self.tags = {} self.tags = {}
self.parameters = {} self.parameters = {}
self.outputs = {}
self.options = {'Legacy': False} self.options = {'Legacy': False}
self.region = 'global' self.region = 'global'
self.profile = '' self.profile = ''
@ -56,6 +58,7 @@ class Stack(object):
self.notfication_sns = [] self.notfication_sns = []
self.id = (self.profile, self.region, self.stackname) self.id = (self.profile, self.region, self.stackname)
self.aws_stackid = None
self.md5 = None self.md5 = None
self.mode = 'CloudBender' self.mode = 'CloudBender'
@ -65,7 +68,9 @@ class Stack(object):
self.cfn_data = None self.cfn_data = None
self.connection_manager = BotoConnection(self.profile, self.region) self.connection_manager = BotoConnection(self.profile, self.region)
self.status = None self.status = None
self.store_outputs = False
self.dependencies = set() self.dependencies = set()
self.hooks = {'post_create': [], 'post_update': [], 'pre_create': [], 'pre_update': []}
self.default_lock = None self.default_lock = None
self.multi_delete = True self.multi_delete = True
@ -111,6 +116,9 @@ class Stack(object):
if 'Mode' in self.options: if 'Mode' in self.options:
self.mode = self.options['Mode'] self.mode = self.options['Mode']
if 'StoreOutputs' in self.options:
self.store_outputs = True
if 'dependencies' in _config: if 'dependencies' in _config:
for dep in _config['dependencies']: for dep in _config['dependencies']:
self.dependencies.add(dep) self.dependencies.add(dep)
@ -226,6 +234,17 @@ class Stack(object):
else: else:
self.dependencies.add(ref.split('DoT')[0]) self.dependencies.add(ref.split('DoT')[0])
# Extract hooks
try:
for hook, func in self.cfn_data['Metadata']['Hooks'].items():
if hook in ['post_update', 'post_create', 'pre_create', 'pre_update']:
if isinstance(func, list):
self.hooks[hook].extend(func)
else:
self.hooks[hook].append(func)
except KeyError:
pass
def write_template_file(self): def write_template_file(self):
if self.cfn_template: if self.cfn_template:
yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml") yaml_file = os.path.join(self.ctx['template_path'], self.rel_path, self.stackname + ".yaml")
@ -298,7 +317,7 @@ class Stack(object):
logger.info("Passed.") logger.info("Passed.")
def get_outputs(self, include='.*', values=False): def get_outputs(self, include='.*', values=False):
""" Returns outputs of the stack as key=value """ """ gets outputs of the stack """
try: try:
stacks = self.connection_manager.call( stacks = self.connection_manager.call(
@ -308,19 +327,26 @@ class Stack(object):
profile=self.profile, region=self.region)['Stacks'] profile=self.profile, region=self.region)['Stacks']
try: try:
logger.debug("Stack outputs for {} in {}:".format(self.stackname, self.region))
for output in stacks[0]['Outputs']: for output in stacks[0]['Outputs']:
if re.search(include, output['OutputKey']): self.outputs[output['OutputKey']] = output['OutputValue']
if values: logger.debug("Stack outputs for {} in {}: {}".format(self.stackname, self.region, self.outputs))
print("{}".format(output['OutputValue']))
else:
print("{}={}".format(output['OutputKey'], output['OutputValue']))
except KeyError: except KeyError:
pass pass
except ClientError as e: except ClientError as e:
raise e raise e
logger.info('{} {} Outputs:\n{}'.format(self.region, self.stackname, pprint.pformat(self.outputs, indent=2)))
def write_outputs_file(self):
output_file = os.path.join(self.ctx['outputs_path'], self.rel_path, self.stackname + ".yaml")
ensure_dir(os.path.join(self.ctx['outputs_path'], self.rel_path))
# Render outputs as yaml under top level key "Outputs"
with open(output_file, 'w') as output_contents:
output_contents.write(yaml.dump({'Outputs': self.outputs}))
logger.info('Wrote outputs for %s to %s', self.stackname, output_file)
def create_docs(self, template=False): def create_docs(self, template=False):
""" Read rendered template, parse documentation fragments, eg. parameter description """ Read rendered template, parse documentation fragments, eg. parameter description
and create a mardown doc file for the stack and create a mardown doc file for the stack
@ -369,6 +395,7 @@ class Stack(object):
if 'Parameters' in self.cfn_data: if 'Parameters' in self.cfn_data:
_errors = [] _errors = []
_found = {}
self.cfn_parameters = [] self.cfn_parameters = []
for p in self.cfn_data['Parameters']: for p in self.cfn_data['Parameters']:
# In Piped mode we try to resolve all Paramters first via stack_outputs # In Piped mode we try to resolve all Paramters first via stack_outputs
@ -392,7 +419,7 @@ class Stack(object):
if 'NoEcho' in self.cfn_data['Parameters'][p] and self.cfn_data['Parameters'][p]['NoEcho']: if 'NoEcho' in self.cfn_data['Parameters'][p] and self.cfn_data['Parameters'][p]['NoEcho']:
value = '****' value = '****'
logger.info('{} {} Parameter {}={}'.format(self.region, self.stackname, p, value)) _found[p] = value
else: else:
# If we have a Default defined in the CFN skip, as AWS will use it # If we have a Default defined in the CFN skip, as AWS will use it
if 'Default' not in self.cfn_data['Parameters'][p]: if 'Default' not in self.cfn_data['Parameters'][p]:
@ -401,6 +428,9 @@ class Stack(object):
if _errors: if _errors:
raise ParameterNotFound('Cannot find value for parameters: {0}'.format(_errors)) raise ParameterNotFound('Cannot find value for parameters: {0}'.format(_errors))
logger.info('{} {} Parameters:\n{}'.format(self.region, self.stackname, pprint.pformat(_found, indent=2)))
@exec_hooks
def create(self): def create(self):
"""Creates a stack """ """Creates a stack """
@ -410,7 +440,7 @@ class Stack(object):
self.read_template_file() self.read_template_file()
logger.info('Creating {0} {1}'.format(self.region, self.stackname)) logger.info('Creating {0} {1}'.format(self.region, self.stackname))
self.connection_manager.call( self.aws_stackid = self.connection_manager.call(
'cloudformation', 'create_stack', 'cloudformation', 'create_stack',
{'StackName': self.stackname, {'StackName': self.stackname,
'TemplateBody': self.cfn_template, 'TemplateBody': self.cfn_template,
@ -421,8 +451,15 @@ class Stack(object):
'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND']}, 'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND']},
profile=self.profile, region=self.region) profile=self.profile, region=self.region)
return self._wait_for_completion() status = self._wait_for_completion()
self.get_outputs()
if self.store_outputs:
self.write_outputs_file()
return status
@exec_hooks
def update(self): def update(self):
"""Updates an existing stack """ """Updates an existing stack """
@ -433,7 +470,7 @@ class Stack(object):
logger.info('Updating {0} {1}'.format(self.region, self.stackname)) logger.info('Updating {0} {1}'.format(self.region, self.stackname))
try: try:
self.connection_manager.call( self.aws_stackid = self.connection_manager.call(
'cloudformation', 'update_stack', 'cloudformation', 'update_stack',
{'StackName': self.stackname, {'StackName': self.stackname,
'TemplateBody': self.cfn_template, 'TemplateBody': self.cfn_template,
@ -450,17 +487,25 @@ class Stack(object):
else: else:
raise e raise e
return self._wait_for_completion() status = self._wait_for_completion()
self.get_outputs()
if self.store_outputs:
self.write_outputs_file()
return status
@exec_hooks
def delete(self): def delete(self):
"""Deletes a stack """ """Deletes a stack """
logger.info('Deleting {0} {1}'.format(self.region, self.stackname)) logger.info('Deleting {0} {1}'.format(self.region, self.stackname))
self.connection_manager.call( self.aws_stackid = self.connection_manager.call(
'cloudformation', 'delete_stack', {'StackName': self.stackname}, 'cloudformation', 'delete_stack', {'StackName': self.stackname},
profile=self.profile, region=self.region) profile=self.profile, region=self.region)
return self._wait_for_completion() status = self._wait_for_completion()
return status
def create_change_set(self, change_set_name): def create_change_set(self, change_set_name):
""" Creates a Change Set with the name ``change_set_name``. """ """ Creates a Change Set with the name ``change_set_name``. """