feat: Add support for Pulumi

Makefile

@@ -1,5 +1,5 @@
 VERSION ?= $(shell grep '__version__' cloudbender/__init__.py | cut -d' ' -f3 | cut -d'-' -f1 | sed -e 's/"//g')
-PACKAGE_FILE := dist/cloudbender-$(VERSION)-py2.py3-none-any.whl
+PACKAGE_FILE := dist/cloudbender-$(VERSION).py3-none-any.whl
 
 .PHONY: test build test_upload upload all
 

cloudbender/__init__.py

@@ -2,7 +2,7 @@ import logging
 
 __author__ = "Stefan Reimer"
 __email__ = "stefan@zero-downtimet.net"
-__version__ = "0.9.9"
+__version__ = "0.10.0"
 
 
 # Set up logging to ``/dev/null`` like a library is supposed to.

cloudbender/cli.py

@@ -36,7 +36,7 @@ def cli(ctx, debug, directory):
     try:
         cb = CloudBender(directory)
     except InvalidProjectDir as e:
-        print(e)
+        logger.error(e)
         sys.exit(1)
 
     cb.read_config()
@@ -50,7 +50,7 @@ def cli(ctx, debug, directory):
 @click.option("--multi", is_flag=True, help="Allow more than one stack to match")
 @click.pass_obj
 def render(cb, stack_names, multi):
-    """ Renders template and its parameters """
+    """ Renders template and its parameters - CFN only"""
 
     stacks = _find_stacks(cb, stack_names, multi)
     _render(stacks)
@@ -74,7 +74,7 @@ def sync(cb, stack_names, multi):
 @click.option("--multi", is_flag=True, help="Allow more than one stack to match")
 @click.pass_obj
 def validate(cb, stack_names, multi):
-    """ Validates already rendered templates using cfn-lint """
+    """ Validates already rendered templates using cfn-lint - CFN only"""
     stacks = _find_stacks(cb, stack_names, multi)
 
     for s in stacks:
@@ -122,13 +122,67 @@ def create_docs(cb, stack_names, multi, graph):
 @click.argument("change_set_name")
 @click.pass_obj
 def create_change_set(cb, stack_name, change_set_name):
-    """ Creates a change set for an existing stack """
+    """ Creates a change set for an existing stack - CFN only"""
     stacks = _find_stacks(cb, [stack_name])
 
     for s in stacks:
         s.create_change_set(change_set_name)
 
 
+@click.command()
+@click.argument("stack_name")
+@click.pass_obj
+def refresh(cb, stack_name):
+    """ Refreshes Pulumi stack / Drift detection """
+    stacks = _find_stacks(cb, [stack_name])
+
+    for s in stacks:
+        if s.mode == 'pulumi':
+            s.refresh()
+        else:
+            logger.info('{} uses Cloudformation, refresh skipped.'.format(s.stackname))
+
+
+@click.command()
+@click.argument("stack_name")
+@click.argument("key")
+@click.argument("value")
+@click.option("--secret", is_flag=True, help="Value is a secret")
+@click.pass_obj
+def set_config(cb, stack_name, key, value, secret=False):
+    """ Sets a config value, encrypts with stack key if secret """
+    stacks = _find_stacks(cb, [stack_name])
+
+    for s in stacks:
+        s.set_config(key, value, secret)
+
+
+@click.command()
+@click.argument("stack_name")
+@click.argument("key")
+@click.pass_obj
+def get_config(cb, stack_name, key):
+    """ Get a config value, decrypted if secret """
+    stacks = _find_stacks(cb, [stack_name])
+
+    for s in stacks:
+        s.get_config(key)
+
+
+@click.command()
+@click.argument("stack_name")
+@click.pass_obj
+def preview(cb, stack_name):
+    """ Preview of Pulumi stack up operation """
+    stacks = _find_stacks(cb, [stack_name])
+
+    for s in stacks:
+        if s.mode == 'pulumi':
+            s.preview()
+        else:
+            logger.warning('{} uses Cloudformation, use create-change-set for previews.'.format(s.stackname))
+
+
 @click.command()
 @click.argument("stack_names", nargs=-1)
 @click.option("--multi", is_flag=True, help="Allow more than one stack to match")
@@ -175,6 +229,10 @@ def sort_stacks(cb, stacks):
 
     data = {}
     for s in stacks:
+        if s.mode == 'pulumi':
+            data[s.id] = set()
+            continue
+
         # To resolve dependencies we have to read each template
         s.read_template_file()
         deps = []
@@ -193,8 +251,10 @@ def sort_stacks(cb, stacks):
     for k, v in data.items():
         v.discard(k)
 
-    extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys())
-    data.update({item: set() for item in extra_items_in_deps})
+    if data:
+        extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys())
+        data.update({item: set() for item in extra_items_in_deps})
+
     while True:
         ordered = set(item for item, dep in data.items() if not dep)
         if not ordered:
@@ -234,8 +294,11 @@ def _find_stacks(cb, stack_names, multi=False):
 def _render(stacks):
     """ Utility function to reuse code between tasks """
     for s in stacks:
-        s.render()
-        s.write_template_file()
+        if s.mode != 'pulumi':
+            s.render()
+            s.write_template_file()
+        else:
+            logger.info('{} uses Pulumi, render skipped.'.format(s.stackname))
 
 
 def _provision(cb, stacks):
@@ -264,6 +327,10 @@ cli.add_command(clean)
 cli.add_command(create_change_set)
 cli.add_command(outputs)
 cli.add_command(create_docs)
+cli.add_command(refresh)
+cli.add_command(preview)
+cli.add_command(set_config)
+cli.add_command(get_config)
 
 if __name__ == '__main__':
     cli(obj={})

cloudbender/core.py

@@ -31,9 +31,14 @@ class CloudBender(object):
 
         # Read top level config.yaml and extract CloudBender CTX
         _config = read_config_file(self.ctx['config_path'].joinpath('config.yaml'))
+
+        # Legacy naming
         if _config and _config.get('CloudBender'):
             self.ctx.update(_config.get('CloudBender'))
 
+        if _config and _config.get('cloudbender'):
+            self.ctx.update(_config.get('cloudbender'))
+
         # Make sure all paths are abs
         for k, v in self.ctx.items():
             if k in ['config_path', 'template_path', 'hooks_path', 'docs_path', 'artifact_paths', 'outputs_path']:

cloudbender/hooks.py

@@ -1,5 +1,8 @@
+import os
 import sys
 import subprocess
+import tempfile
+import shutil
 from functools import wraps
 
 from .exceptions import InvalidHook
@@ -33,8 +36,24 @@ def exec_hooks(func):
     return decorated
 
 
-# Various hooks
+def pulumi_ws(func):
+    @wraps(func)
+    def decorated(self, *args, **kwargs):
+        # setup temp workspace
+        self.work_dir = tempfile.mkdtemp(dir=tempfile.gettempdir(), prefix="cloudbender-")
+
+        response = func(self, *args, **kwargs)
+
+        # Cleanup temp workspace
+        if os.path.exists(self.work_dir):
+            shutil.rmtree(self.work_dir)
 
+        return response
+
+    return decorated
+
+
+# Various hooks
 def cmd(stack, arguments):
     """
     Generic command via subprocess

cloudbender/jinja.py

@@ -9,8 +9,6 @@ import subprocess
 import sys
 
 import jinja2
-from jinja2.utils import missing, object_type_repr
-from jinja2._compat import string_types
 from jinja2.filters import make_attrgetter
 from jinja2.runtime import Undefined
 
@@ -157,36 +155,10 @@ def inline_yaml(block):
     return yaml.safe_load(block)
 
 
-class SilentUndefined(jinja2.Undefined):
-    '''
-    Log warning for undefiend but continue
-    '''
-    def _fail_with_undefined_error(self, *args, **kwargs):
-        if self._undefined_hint is None:
-            if self._undefined_obj is missing:
-                hint = '%r is undefined' % self._undefined_name
-            elif not isinstance(self._undefined_name, string_types):
-                hint = '%s has no element %r' % (
-                    object_type_repr(self._undefined_obj),
-                    self._undefined_name
-                )
-            else:
-                hint = '%r has no attribute %r' % (
-                    object_type_repr(self._undefined_obj),
-                    self._undefined_name
-                )
-        else:
-            hint = self._undefined_hint
-
-        logger.warning("Undefined variable: {}".format(hint))
-        return ''
-
-
 def JinjaEnv(template_locations=[]):
     jenv = jinja2.Environment(trim_blocks=True,
                               lstrip_blocks=True,
                               extensions=['jinja2.ext.loopcontrols', 'jinja2.ext.do'])
-#                              undefined=SilentUndefined,
 
     if template_locations:
         jinja_loaders = []

cloudbender/pulumi.py

@@ -0,0 +1,100 @@
+import sys
+import os
+import re
+import shutil
+import importlib
+import pulumi
+
+import logging
+logger = logging.getLogger(__name__)
+
+
+def pulumi_init(stack):
+
+    # Fail early if pulumi binaries are not available
+    if not shutil.which('pulumi'):
+        raise FileNotFoundError("Cannot find pulumi binary, see https://www.pulumi.com/docs/get-started/install/")
+
+    # add all artifact_paths/pulumi to the search path for easier imports in the pulumi code
+    for artifacts_path in stack.ctx['artifact_paths']:
+        _path = '{}/pulumi'.format(artifacts_path.resolve())
+        sys.path.append(_path)
+
+    # Try local implementation first, similar to Jinja2 mode
+    _found = False
+    try:
+        _stack = importlib.import_module('config.{}.{}'.format(stack.rel_path, stack.template).replace('/', '.'))
+        _found = True
+
+    except ImportError:
+        for artifacts_path in stack.ctx['artifact_paths']:
+            try:
+                spec = importlib.util.spec_from_file_location("_stack", '{}/pulumi/{}.py'.format(artifacts_path.resolve(), stack.template))
+                _stack = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(_stack)
+                _found = True
+
+            except FileNotFoundError:
+                pass
+
+    if not _found:
+        raise FileNotFoundError("Cannot find Pulumi implementation for {}".format(stack.stackname))
+
+    project_name = stack.parameters['Conglomerate']
+
+    # Remove stacknameprefix if equals Conglomerate as Pulumi implicitly prefixes project_name
+    pulumi_stackname = re.sub(r'^' + project_name + '-?', '', stack.stackname)
+    pulumi_backend = '{}/{}/{}'.format(stack.pulumi['backend'], project_name, stack.region)
+
+    account_id = stack.connection_manager.call('sts', 'get_caller_identity', profile=stack.profile, region=stack.region)['Account']
+    # Ugly hack as Pulumi currently doesnt support MFA_TOKENs during role assumptions
+    # Do NOT set them via 'aws:secretKey' as they end up in the stack.json in plain text !!!
+    if stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().token:
+        os.environ['AWS_SESSION_TOKEN'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().token
+
+    os.environ['AWS_ACCESS_KEY_ID'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().access_key
+    os.environ['AWS_SECRET_ACCESS_KEY'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().secret_key
+
+    # Secrets provider
+    try:
+        secrets_provider = stack.pulumi['secretsProvider']
+        if secrets_provider == 'passphrase' and 'PULUMI_CONFIG_PASSPHRASE' not in os.environ:
+            raise ValueError('Missing PULUMI_CONFIG_PASSPHRASE environment variable!')
+
+    except KeyError:
+        raise KeyError('Missing Pulumi securityProvider setting !')
+
+    _config = {
+        "aws:region": stack.region,
+        "aws:profile": stack.profile,
+        "aws:defaultTags": {"tags": stack.tags},
+        "zdt:region": stack.region,
+        "zdt:awsAccount": account_id,
+    }
+
+    # inject all parameters as config in the <Conglomerate> namespace
+    for p in stack.parameters:
+        _config['{}:{}'.format(stack.parameters['Conglomerate'], p)] = stack.parameters[p]
+
+    stack_settings = pulumi.automation.StackSettings(
+        config=_config,
+        secrets_provider=secrets_provider,
+        encryption_salt=stack.pulumi.get('encryptionsalt', None),
+        encrypted_key=stack.pulumi.get('encryptedkey', None)
+    )
+
+    project_settings = pulumi.automation.ProjectSettings(
+        name=project_name,
+        runtime="python",
+        backend={"url": pulumi_backend})
+
+    ws_opts = pulumi.automation.LocalWorkspaceOptions(
+        work_dir=stack.work_dir,
+        project_settings=project_settings,
+        stack_settings={pulumi_stackname: stack_settings},
+        secrets_provider=secrets_provider)
+
+    stack = pulumi.automation.create_or_select_stack(stack_name=pulumi_stackname, project_name=project_name, program=_stack.pulumi_program, opts=ws_opts)
+    stack.workspace.install_plugin("aws", "4.19.0")
+
+    return stack

cloudbender/stack.py

@@ -5,6 +5,7 @@ import yaml
 import time
 import pathlib
 import pprint
+import pulumi
 
 from datetime import datetime, timedelta
 from dateutil.tz import tzutc
@@ -16,16 +17,14 @@ from .connection import BotoConnection
 from .jinja import JinjaEnv, read_config_file
 from . import __version__
 from .exceptions import ParameterNotFound, ParameterIllegalValue, ChecksumError
-from .hooks import exec_hooks
+from .hooks import exec_hooks, pulumi_ws
+from .pulumi import pulumi_init
 
 import cfnlint.core
 import cfnlint.template
 import cfnlint.graph
 
-try:
-    import importlib.resources as pkg_resources
-except ImportError:
-    import importlib_resources as pkg_resources
+import importlib.resources as pkg_resources
 from . import templates
 
 import logging
@@ -54,7 +53,7 @@ class Stack(object):
         self.outputs = {}
         self.options = {}
         self.region = 'global'
-        self.profile = ''
+        self.profile = 'default'
         self.onfailure = 'DELETE'
         self.notfication_sns = []
 
@@ -75,6 +74,8 @@ class Stack(object):
         self.default_lock = None
         self.multi_delete = True
         self.template_bucket_url = None
+        self.work_dir = None
+        self.pulumi = {}
 
     def dump_config(self):
         logger.debug("<Stack {}: {}>".format(self.id, pprint.pformat(vars(self))))
@@ -86,6 +87,7 @@ class Stack(object):
         self.tags.update(sg_config.get('tags', {}))
         self.parameters.update(sg_config.get('parameters', {}))
         self.options.update(sg_config.get('options', {}))
+        self.pulumi.update(sg_config.get('pulumi', {}))
 
         # by default inherit parent group settings
         for p in ['region', 'profile', 'notfication_sns', 'template_bucket_url']:
@@ -98,7 +100,7 @@ class Stack(object):
             if p in _config:
                 setattr(self, p, _config[p])
 
-        for p in ["parameters", "tags"]:
+        for p in ["parameters", "tags", "pulumi"]:
             if p in _config:
                 setattr(self, p, dict_merge(getattr(self, p), _config[p]))
 
@@ -109,7 +111,7 @@ class Stack(object):
         if 'options' in _config:
             self.options = dict_merge(self.options, _config['options'])
 
-        if 'Mode' in self.options and self.options['Mode'] == 'Piped':
+        if 'Mode' in self.options:
             self.mode = self.options['Mode']
 
         if 'StoreOutputs' in self.options and self.options['StoreOutputs']:
@@ -379,24 +381,29 @@ class Stack(object):
     def get_outputs(self, include='.*', values=False):
         """ gets outputs of the stack """
 
-        self.read_template_file()
-        try:
-            stacks = self.connection_manager.call(
-                "cloudformation",
-                "describe_stacks",
-                {'StackName': self.stackname},
-                profile=self.profile, region=self.region)['Stacks']
+        if self.mode == 'pulumi':
+            stack = pulumi_init(self)
+            self.outputs = stack.outputs()
 
+        else:
+            self.read_template_file()
             try:
-                for output in stacks[0]['Outputs']:
-                    self.outputs[output['OutputKey']] = output['OutputValue']
-                logger.debug("Stack outputs for {} in {}: {}".format(self.stackname, self.region, self.outputs))
-            except KeyError:
-                pass
+                stacks = self.connection_manager.call(
+                    "cloudformation",
+                    "describe_stacks",
+                    {'StackName': self.stackname},
+                    profile=self.profile, region=self.region)['Stacks']
 
-        except ClientError:
-            logger.warn("Could not get outputs of {}".format(self.stackname))
-            pass
+                try:
+                    for output in stacks[0]['Outputs']:
+                        self.outputs[output['OutputKey']] = output['OutputValue']
+                    logger.debug("Stack outputs for {} in {}: {}".format(self.stackname, self.region, self.outputs))
+                except KeyError:
+                    pass
+
+            except ClientError:
+                logger.warn("Could not get outputs of {}".format(self.stackname))
+                pass
 
         if self.outputs:
             logger.info('{} {} Outputs:\n{}'.format(self.region, self.stackname, pprint.pformat(self.outputs, indent=2)))
@@ -550,34 +557,46 @@ class Stack(object):
         # Return dict of explicitly set parameters
         return _found
 
+    @pulumi_ws
     @exec_hooks
     def create(self):
         """Creates a stack """
 
-        # Prepare parameters
-        self.resolve_parameters()
+        if self.mode == 'pulumi':
+            stack = pulumi_init(self)
+            stack.up(on_output=self._log_pulumi)
 
-        logger.info('Creating {0} {1}'.format(self.region, self.stackname))
-        kwargs = {'StackName': self.stackname,
-                  'Parameters': self.cfn_parameters,
-                  'OnFailure': self.onfailure,
-                  'NotificationARNs': self.notfication_sns,
-                  'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()],
-                  'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND']}
-        kwargs = self._add_template_arg(kwargs)
+        else:
+            # Prepare parameters
+            self.resolve_parameters()
 
-        self.aws_stackid = self.connection_manager.call(
-            'cloudformation', 'create_stack', kwargs, profile=self.profile, region=self.region)
+            logger.info('Creating {0} {1}'.format(self.region, self.stackname))
+            kwargs = {'StackName': self.stackname,
+                      'Parameters': self.cfn_parameters,
+                      'OnFailure': self.onfailure,
+                      'NotificationARNs': self.notfication_sns,
+                      'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()],
+                      'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND']}
+            kwargs = self._add_template_arg(kwargs)
 
-        status = self._wait_for_completion()
-        self.get_outputs()
+            self.aws_stackid = self.connection_manager.call(
+                'cloudformation', 'create_stack', kwargs, profile=self.profile, region=self.region)
 
-        return status
+            status = self._wait_for_completion()
+            self.get_outputs()
+
+            return status
 
+    @pulumi_ws
     @exec_hooks
     def update(self):
         """Updates an existing stack """
 
+        # We cannot migrate directly so bail out if CFN stack still exists
+        if self.mode == 'pulumi':
+            logger.error("Cloudformation stack {} still exists, cannot use Pulumi!".format(self.stackname))
+            return
+
         # Prepare parameters
         self.resolve_parameters()
 
@@ -605,11 +624,19 @@ class Stack(object):
 
         return status
 
+    @pulumi_ws
     @exec_hooks
     def delete(self):
-        """Deletes a stack """
+        """ Deletes a stack """
 
         logger.info('Deleting {0} {1}'.format(self.region, self.stackname))
+
+        if self.mode == 'pulumi':
+            stack = pulumi_init(self)
+            stack.destroy(on_output=self._log_pulumi)
+
+            return
+
         self.aws_stackid = self.connection_manager.call(
             'cloudformation', 'delete_stack', {'StackName': self.stackname},
             profile=self.profile, region=self.region)
@@ -617,6 +644,62 @@ class Stack(object):
         status = self._wait_for_completion()
         return status
 
+    @pulumi_ws
+    def refresh(self):
+        """ Refreshes a Pulumi stack """
+
+        stack = pulumi_init(self)
+        stack.refresh(on_output=self._log_pulumi)
+
+        return
+
+    @pulumi_ws
+    def preview(self):
+        """ Preview a Pulumi stack up operation"""
+
+        stack = pulumi_init(self)
+        stack.preview(on_output=self._log_pulumi)
+
+        return
+
+    @pulumi_ws
+    def set_config(self, key, value, secret):
+        """ Set a config or secret """
+
+        stack = pulumi_init(self)
+        stack.set_config(key, pulumi.automation.ConfigValue(value, secret))
+
+        # Store salt or key and encrypted value in CloudBender stack config
+        settings = None
+        pulumi_settings = stack.workspace.stack_settings(stack.name)._serialize()
+
+        with open(self.path, "r") as file:
+            settings = yaml.safe_load(file)
+
+            try:
+                if 'pulumi' not in settings:
+                    settings['pulumi'] = {}
+                settings['pulumi']['encryptionsalt'] = pulumi_settings['encryptionsalt']
+                settings['pulumi']['encryptedkey'] = pulumi_settings['encryptedkey']
+            except KeyError:
+                pass
+
+            if 'parameters' not in settings:
+                settings['parameters'] = {}
+            settings['parameters'][key] = pulumi_settings['config']['{}:{}'.format(self.parameters['Conglomerate'], key)]
+
+        with open(self.path, "w") as file:
+            yaml.dump(settings, stream=file)
+
+        return
+
+    @pulumi_ws
+    def get_config(self, key):
+        """ Get a config or secret """
+
+        stack = pulumi_init(self)
+        print(stack.get_config(key).value)
+
     def create_change_set(self, change_set_name):
         """ Creates a Change Set with the name ``change_set_name``.  """
 
@@ -792,3 +875,6 @@ class Stack(object):
             kwargs['TemplateBody'] = self.cfn_template
 
         return kwargs
+
+    def _log_pulumi(self, text):
+        logger.info(" ".join([self.region, self.stackname, text]))

cloudbender/utils.py

@@ -3,8 +3,6 @@ import copy
 import logging
 import re
 
-logger = logging.getLogger(__name__)
-
 
 def dict_merge(a, b):
     """ Deep merge to allow proper inheritance for config files"""
@@ -30,7 +28,6 @@ def ensure_dir(path):
     """Creates dir if it does not already exist."""
     if not os.path.exists(path):
         os.makedirs(path)
-        logger.debug('Created directory: %s', path)
 
 
 def setup_logging(debug):

requirements.txt

@@ -1,5 +1,6 @@
 boto3
-Jinja2
+Jinja2<3
 click
 pyminifier
 cfn-lint>=0.34
+pulumi

setup.py

@@ -57,7 +57,7 @@ setup(
     package_data={ 'cloudbender': ['templates/*.md'], },
     include_package_data=True,
     entry_points={'console_scripts': [ "cloudbender = cloudbender.cli:cli" ]},
-    install_requires=['boto3', 'Jinja2', 'click', 'cfn-lint>=0.34', 'pyminifier'],
+    install_requires=['boto3', 'Jinja2', 'click', 'cfn-lint>=0.34', 'pyminifier', 'pulumi'],
     tests_require=["pytest-cov", "moto", "mock", 'pytest'],
     cmdclass={"test": PyTest},
     classifiers=[