diff --git a/Makefile b/Makefile index 2576008..d759fae 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ VERSION ?= $(shell grep '__version__' cloudbender/__init__.py | cut -d' ' -f3 | cut -d'-' -f1 | sed -e 's/"//g') -PACKAGE_FILE := dist/cloudbender-$(VERSION)-py2.py3-none-any.whl +PACKAGE_FILE := dist/cloudbender-$(VERSION).py3-none-any.whl .PHONY: test build test_upload upload all diff --git a/cloudbender/__init__.py b/cloudbender/__init__.py index 93065f4..246d479 100644 --- a/cloudbender/__init__.py +++ b/cloudbender/__init__.py @@ -2,7 +2,7 @@ import logging __author__ = "Stefan Reimer" __email__ = "stefan@zero-downtimet.net" -__version__ = "0.9.9" +__version__ = "0.10.0" # Set up logging to ``/dev/null`` like a library is supposed to. diff --git a/cloudbender/cli.py b/cloudbender/cli.py index c45d3e0..eca8649 100644 --- a/cloudbender/cli.py +++ b/cloudbender/cli.py @@ -36,7 +36,7 @@ def cli(ctx, debug, directory): try: cb = CloudBender(directory) except InvalidProjectDir as e: - print(e) + logger.error(e) sys.exit(1) cb.read_config() @@ -50,7 +50,7 @@ def cli(ctx, debug, directory): @click.option("--multi", is_flag=True, help="Allow more than one stack to match") @click.pass_obj def render(cb, stack_names, multi): - """ Renders template and its parameters """ + """ Renders template and its parameters - CFN only""" stacks = _find_stacks(cb, stack_names, multi) _render(stacks) @@ -74,7 +74,7 @@ def sync(cb, stack_names, multi): @click.option("--multi", is_flag=True, help="Allow more than one stack to match") @click.pass_obj def validate(cb, stack_names, multi): - """ Validates already rendered templates using cfn-lint """ + """ Validates already rendered templates using cfn-lint - CFN only""" stacks = _find_stacks(cb, stack_names, multi) for s in stacks: @@ -122,13 +122,67 @@ def create_docs(cb, stack_names, multi, graph): @click.argument("change_set_name") @click.pass_obj def create_change_set(cb, stack_name, change_set_name): - """ Creates a change set for an existing stack """ + """ Creates a change set for an existing stack - CFN only""" stacks = _find_stacks(cb, [stack_name]) for s in stacks: s.create_change_set(change_set_name) +@click.command() +@click.argument("stack_name") +@click.pass_obj +def refresh(cb, stack_name): + """ Refreshes Pulumi stack / Drift detection """ + stacks = _find_stacks(cb, [stack_name]) + + for s in stacks: + if s.mode == 'pulumi': + s.refresh() + else: + logger.info('{} uses Cloudformation, refresh skipped.'.format(s.stackname)) + + +@click.command() +@click.argument("stack_name") +@click.argument("key") +@click.argument("value") +@click.option("--secret", is_flag=True, help="Value is a secret") +@click.pass_obj +def set_config(cb, stack_name, key, value, secret=False): + """ Sets a config value, encrypts with stack key if secret """ + stacks = _find_stacks(cb, [stack_name]) + + for s in stacks: + s.set_config(key, value, secret) + + +@click.command() +@click.argument("stack_name") +@click.argument("key") +@click.pass_obj +def get_config(cb, stack_name, key): + """ Get a config value, decrypted if secret """ + stacks = _find_stacks(cb, [stack_name]) + + for s in stacks: + s.get_config(key) + + +@click.command() +@click.argument("stack_name") +@click.pass_obj +def preview(cb, stack_name): + """ Preview of Pulumi stack up operation """ + stacks = _find_stacks(cb, [stack_name]) + + for s in stacks: + if s.mode == 'pulumi': + s.preview() + else: + logger.warning('{} uses Cloudformation, use create-change-set for previews.'.format(s.stackname)) + + @click.command() @click.argument("stack_names", nargs=-1) @click.option("--multi", is_flag=True, help="Allow more than one stack to match") @@ -175,6 +229,10 @@ def sort_stacks(cb, stacks): data = {} for s in stacks: + if s.mode == 'pulumi': + data[s.id] = set() + continue + # To resolve dependencies we have to read each template s.read_template_file() deps = [] @@ -193,8 +251,10 @@ def sort_stacks(cb, stacks): for k, v in data.items(): v.discard(k) - extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys()) - data.update({item: set() for item in extra_items_in_deps}) + if data: + extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys()) + data.update({item: set() for item in extra_items_in_deps}) + while True: ordered = set(item for item, dep in data.items() if not dep) if not ordered: @@ -234,8 +294,11 @@ def _find_stacks(cb, stack_names, multi=False): def _render(stacks): """ Utility function to reuse code between tasks """ for s in stacks: - s.render() - s.write_template_file() + if s.mode != 'pulumi': + s.render() + s.write_template_file() + else: + logger.info('{} uses Pulumi, render skipped.'.format(s.stackname)) def _provision(cb, stacks): @@ -264,6 +327,10 @@ cli.add_command(clean) cli.add_command(create_change_set) cli.add_command(outputs) cli.add_command(create_docs) +cli.add_command(refresh) +cli.add_command(preview) +cli.add_command(set_config) +cli.add_command(get_config) if __name__ == '__main__': cli(obj={}) diff --git a/cloudbender/core.py b/cloudbender/core.py index 853bb44..cefead6 100644 --- a/cloudbender/core.py +++ b/cloudbender/core.py @@ -31,9 +31,14 @@ class CloudBender(object): # Read top level config.yaml and extract CloudBender CTX _config = read_config_file(self.ctx['config_path'].joinpath('config.yaml')) + + # Legacy naming if _config and _config.get('CloudBender'): self.ctx.update(_config.get('CloudBender')) + if _config and _config.get('cloudbender'): + self.ctx.update(_config.get('cloudbender')) + # Make sure all paths are abs for k, v in self.ctx.items(): if k in ['config_path', 'template_path', 'hooks_path', 'docs_path', 'artifact_paths', 'outputs_path']: diff --git a/cloudbender/hooks.py b/cloudbender/hooks.py index af5d78e..3969da7 100644 --- a/cloudbender/hooks.py +++ b/cloudbender/hooks.py @@ -1,5 +1,8 @@ +import os import sys import subprocess +import tempfile +import shutil from functools import wraps from .exceptions import InvalidHook @@ -33,8 +36,24 @@ def exec_hooks(func): return decorated -# Various hooks +def pulumi_ws(func): + @wraps(func) + def decorated(self, *args, **kwargs): + # setup temp workspace + self.work_dir = tempfile.mkdtemp(dir=tempfile.gettempdir(), prefix="cloudbender-") + response = func(self, *args, **kwargs) + + # Cleanup temp workspace + if os.path.exists(self.work_dir): + shutil.rmtree(self.work_dir) + + return response + + return decorated + + +# Various hooks def cmd(stack, arguments): """ Generic command via subprocess diff --git a/cloudbender/jinja.py b/cloudbender/jinja.py index 8d0d08b..58d4ee6 100644 --- a/cloudbender/jinja.py +++ b/cloudbender/jinja.py @@ -9,8 +9,6 @@ import subprocess import sys import jinja2 -from jinja2.utils import missing, object_type_repr -from jinja2._compat import string_types from jinja2.filters import make_attrgetter from jinja2.runtime import Undefined @@ -157,36 +155,10 @@ def inline_yaml(block): return yaml.safe_load(block) -class SilentUndefined(jinja2.Undefined): - ''' - Log warning for undefiend but continue - ''' - def _fail_with_undefined_error(self, *args, **kwargs): - if self._undefined_hint is None: - if self._undefined_obj is missing: - hint = '%r is undefined' % self._undefined_name - elif not isinstance(self._undefined_name, string_types): - hint = '%s has no element %r' % ( - object_type_repr(self._undefined_obj), - self._undefined_name - ) - else: - hint = '%r has no attribute %r' % ( - object_type_repr(self._undefined_obj), - self._undefined_name - ) - else: - hint = self._undefined_hint - - logger.warning("Undefined variable: {}".format(hint)) - return '' - - def JinjaEnv(template_locations=[]): jenv = jinja2.Environment(trim_blocks=True, lstrip_blocks=True, extensions=['jinja2.ext.loopcontrols', 'jinja2.ext.do']) -# undefined=SilentUndefined, if template_locations: jinja_loaders = [] diff --git a/cloudbender/pulumi.py b/cloudbender/pulumi.py new file mode 100644 index 0000000..fb0e035 --- /dev/null +++ b/cloudbender/pulumi.py @@ -0,0 +1,100 @@ +import sys +import os +import re +import shutil +import importlib +import pulumi + +import logging +logger = logging.getLogger(__name__) + + +def pulumi_init(stack): + + # Fail early if pulumi binaries are not available + if not shutil.which('pulumi'): + raise FileNotFoundError("Cannot find pulumi binary, see https://www.pulumi.com/docs/get-started/install/") + + # add all artifact_paths/pulumi to the search path for easier imports in the pulumi code + for artifacts_path in stack.ctx['artifact_paths']: + _path = '{}/pulumi'.format(artifacts_path.resolve()) + sys.path.append(_path) + + # Try local implementation first, similar to Jinja2 mode + _found = False + try: + _stack = importlib.import_module('config.{}.{}'.format(stack.rel_path, stack.template).replace('/', '.')) + _found = True + + except ImportError: + for artifacts_path in stack.ctx['artifact_paths']: + try: + spec = importlib.util.spec_from_file_location("_stack", '{}/pulumi/{}.py'.format(artifacts_path.resolve(), stack.template)) + _stack = importlib.util.module_from_spec(spec) + spec.loader.exec_module(_stack) + _found = True + + except FileNotFoundError: + pass + + if not _found: + raise FileNotFoundError("Cannot find Pulumi implementation for {}".format(stack.stackname)) + + project_name = stack.parameters['Conglomerate'] + + # Remove stacknameprefix if equals Conglomerate as Pulumi implicitly prefixes project_name + pulumi_stackname = re.sub(r'^' + project_name + '-?', '', stack.stackname) + pulumi_backend = '{}/{}/{}'.format(stack.pulumi['backend'], project_name, stack.region) + + account_id = stack.connection_manager.call('sts', 'get_caller_identity', profile=stack.profile, region=stack.region)['Account'] + # Ugly hack as Pulumi currently doesnt support MFA_TOKENs during role assumptions + # Do NOT set them via 'aws:secretKey' as they end up in the stack.json in plain text !!! + if stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().token: + os.environ['AWS_SESSION_TOKEN'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().token + + os.environ['AWS_ACCESS_KEY_ID'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().access_key + os.environ['AWS_SECRET_ACCESS_KEY'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().secret_key + + # Secrets provider + try: + secrets_provider = stack.pulumi['secretsProvider'] + if secrets_provider == 'passphrase' and 'PULUMI_CONFIG_PASSPHRASE' not in os.environ: + raise ValueError('Missing PULUMI_CONFIG_PASSPHRASE environment variable!') + + except KeyError: + raise KeyError('Missing Pulumi securityProvider setting !') + + _config = { + "aws:region": stack.region, + "aws:profile": stack.profile, + "aws:defaultTags": {"tags": stack.tags}, + "zdt:region": stack.region, + "zdt:awsAccount": account_id, + } + + # inject all parameters as config in the namespace + for p in stack.parameters: + _config['{}:{}'.format(stack.parameters['Conglomerate'], p)] = stack.parameters[p] + + stack_settings = pulumi.automation.StackSettings( + config=_config, + secrets_provider=secrets_provider, + encryption_salt=stack.pulumi.get('encryptionsalt', None), + encrypted_key=stack.pulumi.get('encryptedkey', None) + ) + + project_settings = pulumi.automation.ProjectSettings( + name=project_name, + runtime="python", + backend={"url": pulumi_backend}) + + ws_opts = pulumi.automation.LocalWorkspaceOptions( + work_dir=stack.work_dir, + project_settings=project_settings, + stack_settings={pulumi_stackname: stack_settings}, + secrets_provider=secrets_provider) + + stack = pulumi.automation.create_or_select_stack(stack_name=pulumi_stackname, project_name=project_name, program=_stack.pulumi_program, opts=ws_opts) + stack.workspace.install_plugin("aws", "4.19.0") + + return stack diff --git a/cloudbender/stack.py b/cloudbender/stack.py index d9e27e1..94bf095 100644 --- a/cloudbender/stack.py +++ b/cloudbender/stack.py @@ -5,6 +5,7 @@ import yaml import time import pathlib import pprint +import pulumi from datetime import datetime, timedelta from dateutil.tz import tzutc @@ -16,16 +17,14 @@ from .connection import BotoConnection from .jinja import JinjaEnv, read_config_file from . import __version__ from .exceptions import ParameterNotFound, ParameterIllegalValue, ChecksumError -from .hooks import exec_hooks +from .hooks import exec_hooks, pulumi_ws +from .pulumi import pulumi_init import cfnlint.core import cfnlint.template import cfnlint.graph -try: - import importlib.resources as pkg_resources -except ImportError: - import importlib_resources as pkg_resources +import importlib.resources as pkg_resources from . import templates import logging @@ -54,7 +53,7 @@ class Stack(object): self.outputs = {} self.options = {} self.region = 'global' - self.profile = '' + self.profile = 'default' self.onfailure = 'DELETE' self.notfication_sns = [] @@ -75,6 +74,8 @@ class Stack(object): self.default_lock = None self.multi_delete = True self.template_bucket_url = None + self.work_dir = None + self.pulumi = {} def dump_config(self): logger.debug("".format(self.id, pprint.pformat(vars(self)))) @@ -86,6 +87,7 @@ class Stack(object): self.tags.update(sg_config.get('tags', {})) self.parameters.update(sg_config.get('parameters', {})) self.options.update(sg_config.get('options', {})) + self.pulumi.update(sg_config.get('pulumi', {})) # by default inherit parent group settings for p in ['region', 'profile', 'notfication_sns', 'template_bucket_url']: @@ -98,7 +100,7 @@ class Stack(object): if p in _config: setattr(self, p, _config[p]) - for p in ["parameters", "tags"]: + for p in ["parameters", "tags", "pulumi"]: if p in _config: setattr(self, p, dict_merge(getattr(self, p), _config[p])) @@ -109,7 +111,7 @@ class Stack(object): if 'options' in _config: self.options = dict_merge(self.options, _config['options']) - if 'Mode' in self.options and self.options['Mode'] == 'Piped': + if 'Mode' in self.options: self.mode = self.options['Mode'] if 'StoreOutputs' in self.options and self.options['StoreOutputs']: @@ -379,24 +381,29 @@ class Stack(object): def get_outputs(self, include='.*', values=False): """ gets outputs of the stack """ - self.read_template_file() - try: - stacks = self.connection_manager.call( - "cloudformation", - "describe_stacks", - {'StackName': self.stackname}, - profile=self.profile, region=self.region)['Stacks'] + if self.mode == 'pulumi': + stack = pulumi_init(self) + self.outputs = stack.outputs() + else: + self.read_template_file() try: - for output in stacks[0]['Outputs']: - self.outputs[output['OutputKey']] = output['OutputValue'] - logger.debug("Stack outputs for {} in {}: {}".format(self.stackname, self.region, self.outputs)) - except KeyError: - pass + stacks = self.connection_manager.call( + "cloudformation", + "describe_stacks", + {'StackName': self.stackname}, + profile=self.profile, region=self.region)['Stacks'] - except ClientError: - logger.warn("Could not get outputs of {}".format(self.stackname)) - pass + try: + for output in stacks[0]['Outputs']: + self.outputs[output['OutputKey']] = output['OutputValue'] + logger.debug("Stack outputs for {} in {}: {}".format(self.stackname, self.region, self.outputs)) + except KeyError: + pass + + except ClientError: + logger.warn("Could not get outputs of {}".format(self.stackname)) + pass if self.outputs: logger.info('{} {} Outputs:\n{}'.format(self.region, self.stackname, pprint.pformat(self.outputs, indent=2))) @@ -550,34 +557,46 @@ class Stack(object): # Return dict of explicitly set parameters return _found + @pulumi_ws @exec_hooks def create(self): """Creates a stack """ - # Prepare parameters - self.resolve_parameters() + if self.mode == 'pulumi': + stack = pulumi_init(self) + stack.up(on_output=self._log_pulumi) - logger.info('Creating {0} {1}'.format(self.region, self.stackname)) - kwargs = {'StackName': self.stackname, - 'Parameters': self.cfn_parameters, - 'OnFailure': self.onfailure, - 'NotificationARNs': self.notfication_sns, - 'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()], - 'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND']} - kwargs = self._add_template_arg(kwargs) + else: + # Prepare parameters + self.resolve_parameters() - self.aws_stackid = self.connection_manager.call( - 'cloudformation', 'create_stack', kwargs, profile=self.profile, region=self.region) + logger.info('Creating {0} {1}'.format(self.region, self.stackname)) + kwargs = {'StackName': self.stackname, + 'Parameters': self.cfn_parameters, + 'OnFailure': self.onfailure, + 'NotificationARNs': self.notfication_sns, + 'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()], + 'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND']} + kwargs = self._add_template_arg(kwargs) - status = self._wait_for_completion() - self.get_outputs() + self.aws_stackid = self.connection_manager.call( + 'cloudformation', 'create_stack', kwargs, profile=self.profile, region=self.region) - return status + status = self._wait_for_completion() + self.get_outputs() + return status + + @pulumi_ws @exec_hooks def update(self): """Updates an existing stack """ + # We cannot migrate directly so bail out if CFN stack still exists + if self.mode == 'pulumi': + logger.error("Cloudformation stack {} still exists, cannot use Pulumi!".format(self.stackname)) + return + # Prepare parameters self.resolve_parameters() @@ -605,11 +624,19 @@ class Stack(object): return status + @pulumi_ws @exec_hooks def delete(self): - """Deletes a stack """ + """ Deletes a stack """ logger.info('Deleting {0} {1}'.format(self.region, self.stackname)) + + if self.mode == 'pulumi': + stack = pulumi_init(self) + stack.destroy(on_output=self._log_pulumi) + + return + self.aws_stackid = self.connection_manager.call( 'cloudformation', 'delete_stack', {'StackName': self.stackname}, profile=self.profile, region=self.region) @@ -617,6 +644,62 @@ class Stack(object): status = self._wait_for_completion() return status + @pulumi_ws + def refresh(self): + """ Refreshes a Pulumi stack """ + + stack = pulumi_init(self) + stack.refresh(on_output=self._log_pulumi) + + return + + @pulumi_ws + def preview(self): + """ Preview a Pulumi stack up operation""" + + stack = pulumi_init(self) + stack.preview(on_output=self._log_pulumi) + + return + + @pulumi_ws + def set_config(self, key, value, secret): + """ Set a config or secret """ + + stack = pulumi_init(self) + stack.set_config(key, pulumi.automation.ConfigValue(value, secret)) + + # Store salt or key and encrypted value in CloudBender stack config + settings = None + pulumi_settings = stack.workspace.stack_settings(stack.name)._serialize() + + with open(self.path, "r") as file: + settings = yaml.safe_load(file) + + try: + if 'pulumi' not in settings: + settings['pulumi'] = {} + settings['pulumi']['encryptionsalt'] = pulumi_settings['encryptionsalt'] + settings['pulumi']['encryptedkey'] = pulumi_settings['encryptedkey'] + except KeyError: + pass + + if 'parameters' not in settings: + settings['parameters'] = {} + settings['parameters'][key] = pulumi_settings['config']['{}:{}'.format(self.parameters['Conglomerate'], key)] + + with open(self.path, "w") as file: + yaml.dump(settings, stream=file) + + return + + @pulumi_ws + def get_config(self, key): + """ Get a config or secret """ + + stack = pulumi_init(self) + print(stack.get_config(key).value) + def create_change_set(self, change_set_name): """ Creates a Change Set with the name ``change_set_name``. """ @@ -792,3 +875,6 @@ class Stack(object): kwargs['TemplateBody'] = self.cfn_template return kwargs + + def _log_pulumi(self, text): + logger.info(" ".join([self.region, self.stackname, text])) diff --git a/cloudbender/utils.py b/cloudbender/utils.py index 2101728..4b41d8e 100644 --- a/cloudbender/utils.py +++ b/cloudbender/utils.py @@ -3,8 +3,6 @@ import copy import logging import re -logger = logging.getLogger(__name__) - def dict_merge(a, b): """ Deep merge to allow proper inheritance for config files""" @@ -30,7 +28,6 @@ def ensure_dir(path): """Creates dir if it does not already exist.""" if not os.path.exists(path): os.makedirs(path) - logger.debug('Created directory: %s', path) def setup_logging(debug): diff --git a/requirements.txt b/requirements.txt index 3c1550d..4a6e3c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ boto3 -Jinja2 +Jinja2<3 click pyminifier cfn-lint>=0.34 +pulumi diff --git a/setup.py b/setup.py index cf53429..075e7cc 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( package_data={ 'cloudbender': ['templates/*.md'], }, include_package_data=True, entry_points={'console_scripts': [ "cloudbender = cloudbender.cli:cli" ]}, - install_requires=['boto3', 'Jinja2', 'click', 'cfn-lint>=0.34', 'pyminifier'], + install_requires=['boto3', 'Jinja2', 'click', 'cfn-lint>=0.34', 'pyminifier', 'pulumi'], tests_require=["pytest-cov", "moto", "mock", 'pytest'], cmdclass={"test": PyTest}, classifiers=[