feat: Add support for Pulumi

This commit is contained in:
Stefan Reimer 2021-09-20 16:19:14 +02:00
parent 7a66cf3ec5
commit c2c0a2d077
11 changed files with 330 additions and 83 deletions

View File

@ -1,5 +1,5 @@
VERSION ?= $(shell grep '__version__' cloudbender/__init__.py | cut -d' ' -f3 | cut -d'-' -f1 | sed -e 's/"//g') VERSION ?= $(shell grep '__version__' cloudbender/__init__.py | cut -d' ' -f3 | cut -d'-' -f1 | sed -e 's/"//g')
PACKAGE_FILE := dist/cloudbender-$(VERSION)-py2.py3-none-any.whl PACKAGE_FILE := dist/cloudbender-$(VERSION).py3-none-any.whl
.PHONY: test build test_upload upload all .PHONY: test build test_upload upload all

View File

@ -2,7 +2,7 @@ import logging
__author__ = "Stefan Reimer" __author__ = "Stefan Reimer"
__email__ = "stefan@zero-downtimet.net" __email__ = "stefan@zero-downtimet.net"
__version__ = "0.9.9" __version__ = "0.10.0"
# Set up logging to ``/dev/null`` like a library is supposed to. # Set up logging to ``/dev/null`` like a library is supposed to.

View File

@ -36,7 +36,7 @@ def cli(ctx, debug, directory):
try: try:
cb = CloudBender(directory) cb = CloudBender(directory)
except InvalidProjectDir as e: except InvalidProjectDir as e:
print(e) logger.error(e)
sys.exit(1) sys.exit(1)
cb.read_config() cb.read_config()
@ -50,7 +50,7 @@ def cli(ctx, debug, directory):
@click.option("--multi", is_flag=True, help="Allow more than one stack to match") @click.option("--multi", is_flag=True, help="Allow more than one stack to match")
@click.pass_obj @click.pass_obj
def render(cb, stack_names, multi): def render(cb, stack_names, multi):
""" Renders template and its parameters """ """ Renders template and its parameters - CFN only"""
stacks = _find_stacks(cb, stack_names, multi) stacks = _find_stacks(cb, stack_names, multi)
_render(stacks) _render(stacks)
@ -74,7 +74,7 @@ def sync(cb, stack_names, multi):
@click.option("--multi", is_flag=True, help="Allow more than one stack to match") @click.option("--multi", is_flag=True, help="Allow more than one stack to match")
@click.pass_obj @click.pass_obj
def validate(cb, stack_names, multi): def validate(cb, stack_names, multi):
""" Validates already rendered templates using cfn-lint """ """ Validates already rendered templates using cfn-lint - CFN only"""
stacks = _find_stacks(cb, stack_names, multi) stacks = _find_stacks(cb, stack_names, multi)
for s in stacks: for s in stacks:
@ -122,13 +122,67 @@ def create_docs(cb, stack_names, multi, graph):
@click.argument("change_set_name") @click.argument("change_set_name")
@click.pass_obj @click.pass_obj
def create_change_set(cb, stack_name, change_set_name): def create_change_set(cb, stack_name, change_set_name):
""" Creates a change set for an existing stack """ """ Creates a change set for an existing stack - CFN only"""
stacks = _find_stacks(cb, [stack_name]) stacks = _find_stacks(cb, [stack_name])
for s in stacks: for s in stacks:
s.create_change_set(change_set_name) s.create_change_set(change_set_name)
@click.command()
@click.argument("stack_name")
@click.pass_obj
def refresh(cb, stack_name):
""" Refreshes Pulumi stack / Drift detection """
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
if s.mode == 'pulumi':
s.refresh()
else:
logger.info('{} uses Cloudformation, refresh skipped.'.format(s.stackname))
@click.command()
@click.argument("stack_name")
@click.argument("key")
@click.argument("value")
@click.option("--secret", is_flag=True, help="Value is a secret")
@click.pass_obj
def set_config(cb, stack_name, key, value, secret=False):
""" Sets a config value, encrypts with stack key if secret """
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
s.set_config(key, value, secret)
@click.command()
@click.argument("stack_name")
@click.argument("key")
@click.pass_obj
def get_config(cb, stack_name, key):
""" Get a config value, decrypted if secret """
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
s.get_config(key)
@click.command()
@click.argument("stack_name")
@click.pass_obj
def preview(cb, stack_name):
""" Preview of Pulumi stack up operation """
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
if s.mode == 'pulumi':
s.preview()
else:
logger.warning('{} uses Cloudformation, use create-change-set for previews.'.format(s.stackname))
@click.command() @click.command()
@click.argument("stack_names", nargs=-1) @click.argument("stack_names", nargs=-1)
@click.option("--multi", is_flag=True, help="Allow more than one stack to match") @click.option("--multi", is_flag=True, help="Allow more than one stack to match")
@ -175,6 +229,10 @@ def sort_stacks(cb, stacks):
data = {} data = {}
for s in stacks: for s in stacks:
if s.mode == 'pulumi':
data[s.id] = set()
continue
# To resolve dependencies we have to read each template # To resolve dependencies we have to read each template
s.read_template_file() s.read_template_file()
deps = [] deps = []
@ -193,8 +251,10 @@ def sort_stacks(cb, stacks):
for k, v in data.items(): for k, v in data.items():
v.discard(k) v.discard(k)
if data:
extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys()) extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys())
data.update({item: set() for item in extra_items_in_deps}) data.update({item: set() for item in extra_items_in_deps})
while True: while True:
ordered = set(item for item, dep in data.items() if not dep) ordered = set(item for item, dep in data.items() if not dep)
if not ordered: if not ordered:
@ -234,8 +294,11 @@ def _find_stacks(cb, stack_names, multi=False):
def _render(stacks): def _render(stacks):
""" Utility function to reuse code between tasks """ """ Utility function to reuse code between tasks """
for s in stacks: for s in stacks:
if s.mode != 'pulumi':
s.render() s.render()
s.write_template_file() s.write_template_file()
else:
logger.info('{} uses Pulumi, render skipped.'.format(s.stackname))
def _provision(cb, stacks): def _provision(cb, stacks):
@ -264,6 +327,10 @@ cli.add_command(clean)
cli.add_command(create_change_set) cli.add_command(create_change_set)
cli.add_command(outputs) cli.add_command(outputs)
cli.add_command(create_docs) cli.add_command(create_docs)
cli.add_command(refresh)
cli.add_command(preview)
cli.add_command(set_config)
cli.add_command(get_config)
if __name__ == '__main__': if __name__ == '__main__':
cli(obj={}) cli(obj={})

View File

@ -31,9 +31,14 @@ class CloudBender(object):
# Read top level config.yaml and extract CloudBender CTX # Read top level config.yaml and extract CloudBender CTX
_config = read_config_file(self.ctx['config_path'].joinpath('config.yaml')) _config = read_config_file(self.ctx['config_path'].joinpath('config.yaml'))
# Legacy naming
if _config and _config.get('CloudBender'): if _config and _config.get('CloudBender'):
self.ctx.update(_config.get('CloudBender')) self.ctx.update(_config.get('CloudBender'))
if _config and _config.get('cloudbender'):
self.ctx.update(_config.get('cloudbender'))
# Make sure all paths are abs # Make sure all paths are abs
for k, v in self.ctx.items(): for k, v in self.ctx.items():
if k in ['config_path', 'template_path', 'hooks_path', 'docs_path', 'artifact_paths', 'outputs_path']: if k in ['config_path', 'template_path', 'hooks_path', 'docs_path', 'artifact_paths', 'outputs_path']:

View File

@ -1,5 +1,8 @@
import os
import sys import sys
import subprocess import subprocess
import tempfile
import shutil
from functools import wraps from functools import wraps
from .exceptions import InvalidHook from .exceptions import InvalidHook
@ -33,8 +36,24 @@ def exec_hooks(func):
return decorated return decorated
# Various hooks def pulumi_ws(func):
@wraps(func)
def decorated(self, *args, **kwargs):
# setup temp workspace
self.work_dir = tempfile.mkdtemp(dir=tempfile.gettempdir(), prefix="cloudbender-")
response = func(self, *args, **kwargs)
# Cleanup temp workspace
if os.path.exists(self.work_dir):
shutil.rmtree(self.work_dir)
return response
return decorated
# Various hooks
def cmd(stack, arguments): def cmd(stack, arguments):
""" """
Generic command via subprocess Generic command via subprocess

View File

@ -9,8 +9,6 @@ import subprocess
import sys import sys
import jinja2 import jinja2
from jinja2.utils import missing, object_type_repr
from jinja2._compat import string_types
from jinja2.filters import make_attrgetter from jinja2.filters import make_attrgetter
from jinja2.runtime import Undefined from jinja2.runtime import Undefined
@ -157,36 +155,10 @@ def inline_yaml(block):
return yaml.safe_load(block) return yaml.safe_load(block)
class SilentUndefined(jinja2.Undefined):
'''
Log warning for undefiend but continue
'''
def _fail_with_undefined_error(self, *args, **kwargs):
if self._undefined_hint is None:
if self._undefined_obj is missing:
hint = '%r is undefined' % self._undefined_name
elif not isinstance(self._undefined_name, string_types):
hint = '%s has no element %r' % (
object_type_repr(self._undefined_obj),
self._undefined_name
)
else:
hint = '%r has no attribute %r' % (
object_type_repr(self._undefined_obj),
self._undefined_name
)
else:
hint = self._undefined_hint
logger.warning("Undefined variable: {}".format(hint))
return ''
def JinjaEnv(template_locations=[]): def JinjaEnv(template_locations=[]):
jenv = jinja2.Environment(trim_blocks=True, jenv = jinja2.Environment(trim_blocks=True,
lstrip_blocks=True, lstrip_blocks=True,
extensions=['jinja2.ext.loopcontrols', 'jinja2.ext.do']) extensions=['jinja2.ext.loopcontrols', 'jinja2.ext.do'])
# undefined=SilentUndefined,
if template_locations: if template_locations:
jinja_loaders = [] jinja_loaders = []

100
cloudbender/pulumi.py Normal file
View File

@ -0,0 +1,100 @@
import sys
import os
import re
import shutil
import importlib
import pulumi
import logging
logger = logging.getLogger(__name__)
def pulumi_init(stack):
# Fail early if pulumi binaries are not available
if not shutil.which('pulumi'):
raise FileNotFoundError("Cannot find pulumi binary, see https://www.pulumi.com/docs/get-started/install/")
# add all artifact_paths/pulumi to the search path for easier imports in the pulumi code
for artifacts_path in stack.ctx['artifact_paths']:
_path = '{}/pulumi'.format(artifacts_path.resolve())
sys.path.append(_path)
# Try local implementation first, similar to Jinja2 mode
_found = False
try:
_stack = importlib.import_module('config.{}.{}'.format(stack.rel_path, stack.template).replace('/', '.'))
_found = True
except ImportError:
for artifacts_path in stack.ctx['artifact_paths']:
try:
spec = importlib.util.spec_from_file_location("_stack", '{}/pulumi/{}.py'.format(artifacts_path.resolve(), stack.template))
_stack = importlib.util.module_from_spec(spec)
spec.loader.exec_module(_stack)
_found = True
except FileNotFoundError:
pass
if not _found:
raise FileNotFoundError("Cannot find Pulumi implementation for {}".format(stack.stackname))
project_name = stack.parameters['Conglomerate']
# Remove stacknameprefix if equals Conglomerate as Pulumi implicitly prefixes project_name
pulumi_stackname = re.sub(r'^' + project_name + '-?', '', stack.stackname)
pulumi_backend = '{}/{}/{}'.format(stack.pulumi['backend'], project_name, stack.region)
account_id = stack.connection_manager.call('sts', 'get_caller_identity', profile=stack.profile, region=stack.region)['Account']
# Ugly hack as Pulumi currently doesnt support MFA_TOKENs during role assumptions
# Do NOT set them via 'aws:secretKey' as they end up in the stack.json in plain text !!!
if stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().token:
os.environ['AWS_SESSION_TOKEN'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().token
os.environ['AWS_ACCESS_KEY_ID'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().access_key
os.environ['AWS_SECRET_ACCESS_KEY'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().secret_key
# Secrets provider
try:
secrets_provider = stack.pulumi['secretsProvider']
if secrets_provider == 'passphrase' and 'PULUMI_CONFIG_PASSPHRASE' not in os.environ:
raise ValueError('Missing PULUMI_CONFIG_PASSPHRASE environment variable!')
except KeyError:
raise KeyError('Missing Pulumi securityProvider setting !')
_config = {
"aws:region": stack.region,
"aws:profile": stack.profile,
"aws:defaultTags": {"tags": stack.tags},
"zdt:region": stack.region,
"zdt:awsAccount": account_id,
}
# inject all parameters as config in the <Conglomerate> namespace
for p in stack.parameters:
_config['{}:{}'.format(stack.parameters['Conglomerate'], p)] = stack.parameters[p]
stack_settings = pulumi.automation.StackSettings(
config=_config,
secrets_provider=secrets_provider,
encryption_salt=stack.pulumi.get('encryptionsalt', None),
encrypted_key=stack.pulumi.get('encryptedkey', None)
)
project_settings = pulumi.automation.ProjectSettings(
name=project_name,
runtime="python",
backend={"url": pulumi_backend})
ws_opts = pulumi.automation.LocalWorkspaceOptions(
work_dir=stack.work_dir,
project_settings=project_settings,
stack_settings={pulumi_stackname: stack_settings},
secrets_provider=secrets_provider)
stack = pulumi.automation.create_or_select_stack(stack_name=pulumi_stackname, project_name=project_name, program=_stack.pulumi_program, opts=ws_opts)
stack.workspace.install_plugin("aws", "4.19.0")
return stack

View File

@ -5,6 +5,7 @@ import yaml
import time import time
import pathlib import pathlib
import pprint import pprint
import pulumi
from datetime import datetime, timedelta from datetime import datetime, timedelta
from dateutil.tz import tzutc from dateutil.tz import tzutc
@ -16,16 +17,14 @@ from .connection import BotoConnection
from .jinja import JinjaEnv, read_config_file from .jinja import JinjaEnv, read_config_file
from . import __version__ from . import __version__
from .exceptions import ParameterNotFound, ParameterIllegalValue, ChecksumError from .exceptions import ParameterNotFound, ParameterIllegalValue, ChecksumError
from .hooks import exec_hooks from .hooks import exec_hooks, pulumi_ws
from .pulumi import pulumi_init
import cfnlint.core import cfnlint.core
import cfnlint.template import cfnlint.template
import cfnlint.graph import cfnlint.graph
try:
import importlib.resources as pkg_resources import importlib.resources as pkg_resources
except ImportError:
import importlib_resources as pkg_resources
from . import templates from . import templates
import logging import logging
@ -54,7 +53,7 @@ class Stack(object):
self.outputs = {} self.outputs = {}
self.options = {} self.options = {}
self.region = 'global' self.region = 'global'
self.profile = '' self.profile = 'default'
self.onfailure = 'DELETE' self.onfailure = 'DELETE'
self.notfication_sns = [] self.notfication_sns = []
@ -75,6 +74,8 @@ class Stack(object):
self.default_lock = None self.default_lock = None
self.multi_delete = True self.multi_delete = True
self.template_bucket_url = None self.template_bucket_url = None
self.work_dir = None
self.pulumi = {}
def dump_config(self): def dump_config(self):
logger.debug("<Stack {}: {}>".format(self.id, pprint.pformat(vars(self)))) logger.debug("<Stack {}: {}>".format(self.id, pprint.pformat(vars(self))))
@ -86,6 +87,7 @@ class Stack(object):
self.tags.update(sg_config.get('tags', {})) self.tags.update(sg_config.get('tags', {}))
self.parameters.update(sg_config.get('parameters', {})) self.parameters.update(sg_config.get('parameters', {}))
self.options.update(sg_config.get('options', {})) self.options.update(sg_config.get('options', {}))
self.pulumi.update(sg_config.get('pulumi', {}))
# by default inherit parent group settings # by default inherit parent group settings
for p in ['region', 'profile', 'notfication_sns', 'template_bucket_url']: for p in ['region', 'profile', 'notfication_sns', 'template_bucket_url']:
@ -98,7 +100,7 @@ class Stack(object):
if p in _config: if p in _config:
setattr(self, p, _config[p]) setattr(self, p, _config[p])
for p in ["parameters", "tags"]: for p in ["parameters", "tags", "pulumi"]:
if p in _config: if p in _config:
setattr(self, p, dict_merge(getattr(self, p), _config[p])) setattr(self, p, dict_merge(getattr(self, p), _config[p]))
@ -109,7 +111,7 @@ class Stack(object):
if 'options' in _config: if 'options' in _config:
self.options = dict_merge(self.options, _config['options']) self.options = dict_merge(self.options, _config['options'])
if 'Mode' in self.options and self.options['Mode'] == 'Piped': if 'Mode' in self.options:
self.mode = self.options['Mode'] self.mode = self.options['Mode']
if 'StoreOutputs' in self.options and self.options['StoreOutputs']: if 'StoreOutputs' in self.options and self.options['StoreOutputs']:
@ -379,6 +381,11 @@ class Stack(object):
def get_outputs(self, include='.*', values=False): def get_outputs(self, include='.*', values=False):
""" gets outputs of the stack """ """ gets outputs of the stack """
if self.mode == 'pulumi':
stack = pulumi_init(self)
self.outputs = stack.outputs()
else:
self.read_template_file() self.read_template_file()
try: try:
stacks = self.connection_manager.call( stacks = self.connection_manager.call(
@ -550,10 +557,16 @@ class Stack(object):
# Return dict of explicitly set parameters # Return dict of explicitly set parameters
return _found return _found
@pulumi_ws
@exec_hooks @exec_hooks
def create(self): def create(self):
"""Creates a stack """ """Creates a stack """
if self.mode == 'pulumi':
stack = pulumi_init(self)
stack.up(on_output=self._log_pulumi)
else:
# Prepare parameters # Prepare parameters
self.resolve_parameters() self.resolve_parameters()
@ -574,10 +587,16 @@ class Stack(object):
return status return status
@pulumi_ws
@exec_hooks @exec_hooks
def update(self): def update(self):
"""Updates an existing stack """ """Updates an existing stack """
# We cannot migrate directly so bail out if CFN stack still exists
if self.mode == 'pulumi':
logger.error("Cloudformation stack {} still exists, cannot use Pulumi!".format(self.stackname))
return
# Prepare parameters # Prepare parameters
self.resolve_parameters() self.resolve_parameters()
@ -605,11 +624,19 @@ class Stack(object):
return status return status
@pulumi_ws
@exec_hooks @exec_hooks
def delete(self): def delete(self):
""" Deletes a stack """ """ Deletes a stack """
logger.info('Deleting {0} {1}'.format(self.region, self.stackname)) logger.info('Deleting {0} {1}'.format(self.region, self.stackname))
if self.mode == 'pulumi':
stack = pulumi_init(self)
stack.destroy(on_output=self._log_pulumi)
return
self.aws_stackid = self.connection_manager.call( self.aws_stackid = self.connection_manager.call(
'cloudformation', 'delete_stack', {'StackName': self.stackname}, 'cloudformation', 'delete_stack', {'StackName': self.stackname},
profile=self.profile, region=self.region) profile=self.profile, region=self.region)
@ -617,6 +644,62 @@ class Stack(object):
status = self._wait_for_completion() status = self._wait_for_completion()
return status return status
@pulumi_ws
def refresh(self):
""" Refreshes a Pulumi stack """
stack = pulumi_init(self)
stack.refresh(on_output=self._log_pulumi)
return
@pulumi_ws
def preview(self):
""" Preview a Pulumi stack up operation"""
stack = pulumi_init(self)
stack.preview(on_output=self._log_pulumi)
return
@pulumi_ws
def set_config(self, key, value, secret):
""" Set a config or secret """
stack = pulumi_init(self)
stack.set_config(key, pulumi.automation.ConfigValue(value, secret))
# Store salt or key and encrypted value in CloudBender stack config
settings = None
pulumi_settings = stack.workspace.stack_settings(stack.name)._serialize()
with open(self.path, "r") as file:
settings = yaml.safe_load(file)
try:
if 'pulumi' not in settings:
settings['pulumi'] = {}
settings['pulumi']['encryptionsalt'] = pulumi_settings['encryptionsalt']
settings['pulumi']['encryptedkey'] = pulumi_settings['encryptedkey']
except KeyError:
pass
if 'parameters' not in settings:
settings['parameters'] = {}
settings['parameters'][key] = pulumi_settings['config']['{}:{}'.format(self.parameters['Conglomerate'], key)]
with open(self.path, "w") as file:
yaml.dump(settings, stream=file)
return
@pulumi_ws
def get_config(self, key):
""" Get a config or secret """
stack = pulumi_init(self)
print(stack.get_config(key).value)
def create_change_set(self, change_set_name): def create_change_set(self, change_set_name):
""" Creates a Change Set with the name ``change_set_name``. """ """ Creates a Change Set with the name ``change_set_name``. """
@ -792,3 +875,6 @@ class Stack(object):
kwargs['TemplateBody'] = self.cfn_template kwargs['TemplateBody'] = self.cfn_template
return kwargs return kwargs
def _log_pulumi(self, text):
logger.info(" ".join([self.region, self.stackname, text]))

View File

@ -3,8 +3,6 @@ import copy
import logging import logging
import re import re
logger = logging.getLogger(__name__)
def dict_merge(a, b): def dict_merge(a, b):
""" Deep merge to allow proper inheritance for config files""" """ Deep merge to allow proper inheritance for config files"""
@ -30,7 +28,6 @@ def ensure_dir(path):
"""Creates dir if it does not already exist.""" """Creates dir if it does not already exist."""
if not os.path.exists(path): if not os.path.exists(path):
os.makedirs(path) os.makedirs(path)
logger.debug('Created directory: %s', path)
def setup_logging(debug): def setup_logging(debug):

View File

@ -1,5 +1,6 @@
boto3 boto3
Jinja2 Jinja2<3
click click
pyminifier pyminifier
cfn-lint>=0.34 cfn-lint>=0.34
pulumi

View File

@ -57,7 +57,7 @@ setup(
package_data={ 'cloudbender': ['templates/*.md'], }, package_data={ 'cloudbender': ['templates/*.md'], },
include_package_data=True, include_package_data=True,
entry_points={'console_scripts': [ "cloudbender = cloudbender.cli:cli" ]}, entry_points={'console_scripts': [ "cloudbender = cloudbender.cli:cli" ]},
install_requires=['boto3', 'Jinja2', 'click', 'cfn-lint>=0.34', 'pyminifier'], install_requires=['boto3', 'Jinja2', 'click', 'cfn-lint>=0.34', 'pyminifier', 'pulumi'],
tests_require=["pytest-cov", "moto", "mock", 'pytest'], tests_require=["pytest-cov", "moto", "mock", 'pytest'],
cmdclass={"test": PyTest}, cmdclass={"test": PyTest},
classifiers=[ classifiers=[