ZeroDownTime/CloudBender
3
0
Fork 0
Code Issues 1 Pull Requests 4 Packages Projects Releases Wiki Activity

feat: Add support for Pulumi

Browse Source
This commit is contained in:
Stefan Reimer 2021-09-20 16:19:14 +02:00
parent 7a66cf3ec5
commit c2c0a2d077
11 changed files with 330 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -1,5 +1,5 @@
VERSION ?= $(shell grep '__version__' cloudbender/__init__.py | cut -d' ' -f3 | cut -d'-' -f1 | sed -e 's/"//g')
PACKAGE_FILE := dist/cloudbender-$(VERSION)-py2.py3-none-any.whl
PACKAGE_FILE := dist/cloudbender-$(VERSION).py3-none-any.whl
.PHONY: test build test_upload upload all

2
cloudbender/__init__.py
View File

@ -2,7 +2,7 @@ import logging
__author__ = "Stefan Reimer"
__email__ = "stefan@zero-downtimet.net"
__version__ = "0.9.9"
__version__ = "0.10.0"
# Set up logging to ``/dev/null`` like a library is supposed to.

83
cloudbender/cli.py
View File

@ -36,7 +36,7 @@ def cli(ctx, debug, directory):
try:
cb = CloudBender(directory)
except InvalidProjectDir as e:
print(e)
logger.error(e)
sys.exit(1)
cb.read_config()
@ -50,7 +50,7 @@ def cli(ctx, debug, directory):
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
@click.pass_obj
def render(cb, stack_names, multi):
""" Renders template and its parameters """
""" Renders template and its parameters - CFN only"""
stacks = _find_stacks(cb, stack_names, multi)
_render(stacks)
@ -74,7 +74,7 @@ def sync(cb, stack_names, multi):
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
@click.pass_obj
def validate(cb, stack_names, multi):
""" Validates already rendered templates using cfn-lint """
""" Validates already rendered templates using cfn-lint - CFN only"""
stacks = _find_stacks(cb, stack_names, multi)
for s in stacks:
@ -122,13 +122,67 @@ def create_docs(cb, stack_names, multi, graph):
@click.argument("change_set_name")
@click.pass_obj
def create_change_set(cb, stack_name, change_set_name):
""" Creates a change set for an existing stack """
""" Creates a change set for an existing stack - CFN only"""
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
s.create_change_set(change_set_name)
@click.command()
@click.argument("stack_name")
@click.pass_obj
def refresh(cb, stack_name):
""" Refreshes Pulumi stack / Drift detection """
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
if s.mode == 'pulumi':
s.refresh()
else:
logger.info('{} uses Cloudformation, refresh skipped.'.format(s.stackname))
@click.command()
@click.argument("stack_name")
@click.argument("key")
@click.argument("value")
@click.option("--secret", is_flag=True, help="Value is a secret")
@click.pass_obj
def set_config(cb, stack_name, key, value, secret=False):
""" Sets a config value, encrypts with stack key if secret """
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
s.set_config(key, value, secret)
@click.command()
@click.argument("stack_name")
@click.argument("key")
@click.pass_obj
def get_config(cb, stack_name, key):
""" Get a config value, decrypted if secret """
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
s.get_config(key)
@click.command()
@click.argument("stack_name")
@click.pass_obj
def preview(cb, stack_name):
""" Preview of Pulumi stack up operation """
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
if s.mode == 'pulumi':
s.preview()
else:
logger.warning('{} uses Cloudformation, use create-change-set for previews.'.format(s.stackname))
@click.command()
@click.argument("stack_names", nargs=-1)
@click.option("--multi", is_flag=True, help="Allow more than one stack to match")
@ -175,6 +229,10 @@ def sort_stacks(cb, stacks):
data = {}
for s in stacks:
if s.mode == 'pulumi':
data[s.id] = set()
continue
# To resolve dependencies we have to read each template
s.read_template_file()
deps = []
@ -193,8 +251,10 @@ def sort_stacks(cb, stacks):
for k, v in data.items():
v.discard(k)
extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys())
data.update({item: set() for item in extra_items_in_deps})
if data:
extra_items_in_deps = functools.reduce(set.union, data.values()) - set(data.keys())
data.update({item: set() for item in extra_items_in_deps})
while True:
ordered = set(item for item, dep in data.items() if not dep)
if not ordered:
@ -234,8 +294,11 @@ def _find_stacks(cb, stack_names, multi=False):
def _render(stacks):
""" Utility function to reuse code between tasks """
for s in stacks:
s.render()
s.write_template_file()
if s.mode != 'pulumi':
s.render()
s.write_template_file()
else:
logger.info('{} uses Pulumi, render skipped.'.format(s.stackname))
def _provision(cb, stacks):
@ -264,6 +327,10 @@ cli.add_command(clean)
cli.add_command(create_change_set)
cli.add_command(outputs)
cli.add_command(create_docs)
cli.add_command(refresh)
cli.add_command(preview)
cli.add_command(set_config)
cli.add_command(get_config)
if __name__ == '__main__':
cli(obj={})

5
cloudbender/core.py
View File

@ -31,9 +31,14 @@ class CloudBender(object):
# Read top level config.yaml and extract CloudBender CTX
_config = read_config_file(self.ctx['config_path'].joinpath('config.yaml'))
# Legacy naming
if _config and _config.get('CloudBender'):
self.ctx.update(_config.get('CloudBender'))
if _config and _config.get('cloudbender'):
self.ctx.update(_config.get('cloudbender'))
# Make sure all paths are abs
for k, v in self.ctx.items():
if k in ['config_path', 'template_path', 'hooks_path', 'docs_path', 'artifact_paths', 'outputs_path']:

21
cloudbender/hooks.py
View File

@ -1,5 +1,8 @@
import os
import sys
import subprocess
import tempfile
import shutil
from functools import wraps
from .exceptions import InvalidHook
@ -33,8 +36,24 @@ def exec_hooks(func):
return decorated
# Various hooks
def pulumi_ws(func):
@wraps(func)
def decorated(self, *args, **kwargs):
# setup temp workspace
self.work_dir = tempfile.mkdtemp(dir=tempfile.gettempdir(), prefix="cloudbender-")
response = func(self, *args, **kwargs)
# Cleanup temp workspace
if os.path.exists(self.work_dir):
shutil.rmtree(self.work_dir)
return response
return decorated
# Various hooks
def cmd(stack, arguments):
"""
Generic command via subprocess

28
cloudbender/jinja.py
View File

@ -9,8 +9,6 @@ import subprocess
import sys
import jinja2
from jinja2.utils import missing, object_type_repr
from jinja2._compat import string_types
from jinja2.filters import make_attrgetter
from jinja2.runtime import Undefined
@ -157,36 +155,10 @@ def inline_yaml(block):
return yaml.safe_load(block)
class SilentUndefined(jinja2.Undefined):
'''
Log warning for undefiend but continue
'''
def _fail_with_undefined_error(self, *args, **kwargs):
if self._undefined_hint is None:
if self._undefined_obj is missing:
hint = '%r is undefined' % self._undefined_name
elif not isinstance(self._undefined_name, string_types):
hint = '%s has no element %r' % (
object_type_repr(self._undefined_obj),
self._undefined_name
)
else:
hint = '%r has no attribute %r' % (
object_type_repr(self._undefined_obj),
self._undefined_name
)
else:
hint = self._undefined_hint
logger.warning("Undefined variable: {}".format(hint))
return ''
def JinjaEnv(template_locations=[]):
jenv = jinja2.Environment(trim_blocks=True,
lstrip_blocks=True,
extensions=['jinja2.ext.loopcontrols', 'jinja2.ext.do'])
# undefined=SilentUndefined,
if template_locations:
jinja_loaders = []

100
cloudbender/pulumi.py Normal file
View File

@ -0,0 +1,100 @@
import sys
import os
import re
import shutil
import importlib
import pulumi
import logging
logger = logging.getLogger(__name__)
def pulumi_init(stack):
# Fail early if pulumi binaries are not available
if not shutil.which('pulumi'):
raise FileNotFoundError("Cannot find pulumi binary, see https://www.pulumi.com/docs/get-started/install/")
# add all artifact_paths/pulumi to the search path for easier imports in the pulumi code
for artifacts_path in stack.ctx['artifact_paths']:
_path = '{}/pulumi'.format(artifacts_path.resolve())
sys.path.append(_path)
# Try local implementation first, similar to Jinja2 mode
_found = False
try:
_stack = importlib.import_module('config.{}.{}'.format(stack.rel_path, stack.template).replace('/', '.'))
_found = True
except ImportError:
for artifacts_path in stack.ctx['artifact_paths']:
try:
spec = importlib.util.spec_from_file_location("_stack", '{}/pulumi/{}.py'.format(artifacts_path.resolve(), stack.template))
_stack = importlib.util.module_from_spec(spec)
spec.loader.exec_module(_stack)
_found = True
except FileNotFoundError:
pass
if not _found:
raise FileNotFoundError("Cannot find Pulumi implementation for {}".format(stack.stackname))
project_name = stack.parameters['Conglomerate']
# Remove stacknameprefix if equals Conglomerate as Pulumi implicitly prefixes project_name
pulumi_stackname = re.sub(r'^' + project_name + '-?', '', stack.stackname)
pulumi_backend = '{}/{}/{}'.format(stack.pulumi['backend'], project_name, stack.region)
account_id = stack.connection_manager.call('sts', 'get_caller_identity', profile=stack.profile, region=stack.region)['Account']
# Ugly hack as Pulumi currently doesnt support MFA_TOKENs during role assumptions
# Do NOT set them via 'aws:secretKey' as they end up in the stack.json in plain text !!!
if stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().token:
os.environ['AWS_SESSION_TOKEN'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().token
os.environ['AWS_ACCESS_KEY_ID'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().access_key
os.environ['AWS_SECRET_ACCESS_KEY'] = stack.connection_manager._sessions[(stack.profile, stack.region)].get_credentials().secret_key
# Secrets provider
try:
secrets_provider = stack.pulumi['secretsProvider']
if secrets_provider == 'passphrase' and 'PULUMI_CONFIG_PASSPHRASE' not in os.environ:
raise ValueError('Missing PULUMI_CONFIG_PASSPHRASE environment variable!')
except KeyError:
raise KeyError('Missing Pulumi securityProvider setting !')
_config = {
"aws:region": stack.region,
"aws:profile": stack.profile,
"aws:defaultTags": {"tags": stack.tags},
"zdt:region": stack.region,
"zdt:awsAccount": account_id,
}
# inject all parameters as config in the <Conglomerate> namespace
for p in stack.parameters:
_config['{}:{}'.format(stack.parameters['Conglomerate'], p)] = stack.parameters[p]
stack_settings = pulumi.automation.StackSettings(
config=_config,
secrets_provider=secrets_provider,
encryption_salt=stack.pulumi.get('encryptionsalt', None),
encrypted_key=stack.pulumi.get('encryptedkey', None)
)
project_settings = pulumi.automation.ProjectSettings(
name=project_name,
runtime="python",
backend={"url": pulumi_backend})
ws_opts = pulumi.automation.LocalWorkspaceOptions(
work_dir=stack.work_dir,
project_settings=project_settings,
stack_settings={pulumi_stackname: stack_settings},
secrets_provider=secrets_provider)
stack = pulumi.automation.create_or_select_stack(stack_name=pulumi_stackname, project_name=project_name, program=_stack.pulumi_program, opts=ws_opts)
stack.workspace.install_plugin("aws", "4.19.0")
return stack

164
cloudbender/stack.py
View File

@ -5,6 +5,7 @@ import yaml
import time
import pathlib
import pprint
import pulumi
from datetime import datetime, timedelta
from dateutil.tz import tzutc
@ -16,16 +17,14 @@ from .connection import BotoConnection
from .jinja import JinjaEnv, read_config_file
from . import __version__
from .exceptions import ParameterNotFound, ParameterIllegalValue, ChecksumError
from .hooks import exec_hooks
from .hooks import exec_hooks, pulumi_ws
from .pulumi import pulumi_init
import cfnlint.core
import cfnlint.template
import cfnlint.graph
try:
import importlib.resources as pkg_resources
except ImportError:
import importlib_resources as pkg_resources
import importlib.resources as pkg_resources
from . import templates
import logging
@ -54,7 +53,7 @@ class Stack(object):
self.outputs = {}
self.options = {}
self.region = 'global'
self.profile = ''
self.profile = 'default'
self.onfailure = 'DELETE'
self.notfication_sns = []
@ -75,6 +74,8 @@ class Stack(object):
self.default_lock = None
self.multi_delete = True
self.template_bucket_url = None
self.work_dir = None
self.pulumi = {}
def dump_config(self):
logger.debug("<Stack {}: {}>".format(self.id, pprint.pformat(vars(self))))
@ -86,6 +87,7 @@ class Stack(object):
self.tags.update(sg_config.get('tags', {}))
self.parameters.update(sg_config.get('parameters', {}))
self.options.update(sg_config.get('options', {}))
self.pulumi.update(sg_config.get('pulumi', {}))
# by default inherit parent group settings
for p in ['region', 'profile', 'notfication_sns', 'template_bucket_url']:
@ -98,7 +100,7 @@ class Stack(object):
if p in _config:
setattr(self, p, _config[p])
for p in ["parameters", "tags"]:
for p in ["parameters", "tags", "pulumi"]:
if p in _config:
setattr(self, p, dict_merge(getattr(self, p), _config[p]))
@ -109,7 +111,7 @@ class Stack(object):
if 'options' in _config:
self.options = dict_merge(self.options, _config['options'])
if 'Mode' in self.options and self.options['Mode'] == 'Piped':
if 'Mode' in self.options:
self.mode = self.options['Mode']
if 'StoreOutputs' in self.options and self.options['StoreOutputs']:
@ -379,24 +381,29 @@ class Stack(object):
def get_outputs(self, include='.*', values=False):
""" gets outputs of the stack """
self.read_template_file()
try:
stacks = self.connection_manager.call(
"cloudformation",
"describe_stacks",
{'StackName': self.stackname},
profile=self.profile, region=self.region)['Stacks']
if self.mode == 'pulumi':
stack = pulumi_init(self)
self.outputs = stack.outputs()
else:
self.read_template_file()
try:
for output in stacks[0]['Outputs']:
self.outputs[output['OutputKey']] = output['OutputValue']
logger.debug("Stack outputs for {} in {}: {}".format(self.stackname, self.region, self.outputs))
except KeyError:
pass
stacks = self.connection_manager.call(
"cloudformation",
"describe_stacks",
{'StackName': self.stackname},
profile=self.profile, region=self.region)['Stacks']
except ClientError:
logger.warn("Could not get outputs of {}".format(self.stackname))
pass
try:
for output in stacks[0]['Outputs']:
self.outputs[output['OutputKey']] = output['OutputValue']
logger.debug("Stack outputs for {} in {}: {}".format(self.stackname, self.region, self.outputs))
except KeyError:
pass
except ClientError:
logger.warn("Could not get outputs of {}".format(self.stackname))
pass
if self.outputs:
logger.info('{} {} Outputs:\n{}'.format(self.region, self.stackname, pprint.pformat(self.outputs, indent=2)))
@ -550,34 +557,46 @@ class Stack(object):
# Return dict of explicitly set parameters
return _found
@pulumi_ws
@exec_hooks
def create(self):
"""Creates a stack """
# Prepare parameters
self.resolve_parameters()
if self.mode == 'pulumi':
stack = pulumi_init(self)
stack.up(on_output=self._log_pulumi)
logger.info('Creating {0} {1}'.format(self.region, self.stackname))
kwargs = {'StackName': self.stackname,
'Parameters': self.cfn_parameters,
'OnFailure': self.onfailure,
'NotificationARNs': self.notfication_sns,
'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()],
'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND']}
kwargs = self._add_template_arg(kwargs)
else:
# Prepare parameters
self.resolve_parameters()
self.aws_stackid = self.connection_manager.call(
'cloudformation', 'create_stack', kwargs, profile=self.profile, region=self.region)
logger.info('Creating {0} {1}'.format(self.region, self.stackname))
kwargs = {'StackName': self.stackname,
'Parameters': self.cfn_parameters,
'OnFailure': self.onfailure,
'NotificationARNs': self.notfication_sns,
'Tags': [{"Key": str(k), "Value": str(v)} for k, v in self.tags.items()],
'Capabilities': ['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM', 'CAPABILITY_AUTO_EXPAND']}
kwargs = self._add_template_arg(kwargs)
status = self._wait_for_completion()
self.get_outputs()
self.aws_stackid = self.connection_manager.call(
'cloudformation', 'create_stack', kwargs, profile=self.profile, region=self.region)
return status
status = self._wait_for_completion()
self.get_outputs()
return status
@pulumi_ws
@exec_hooks
def update(self):
"""Updates an existing stack """
# We cannot migrate directly so bail out if CFN stack still exists
if self.mode == 'pulumi':
logger.error("Cloudformation stack {} still exists, cannot use Pulumi!".format(self.stackname))
return
# Prepare parameters
self.resolve_parameters()
@ -605,11 +624,19 @@ class Stack(object):
return status
@pulumi_ws
@exec_hooks
def delete(self):
"""Deletes a stack """
""" Deletes a stack """
logger.info('Deleting {0} {1}'.format(self.region, self.stackname))
if self.mode == 'pulumi':
stack = pulumi_init(self)
stack.destroy(on_output=self._log_pulumi)
return
self.aws_stackid = self.connection_manager.call(
'cloudformation', 'delete_stack', {'StackName': self.stackname},
profile=self.profile, region=self.region)
@ -617,6 +644,62 @@ class Stack(object):
status = self._wait_for_completion()
return status
@pulumi_ws
def refresh(self):
""" Refreshes a Pulumi stack """
stack = pulumi_init(self)
stack.refresh(on_output=self._log_pulumi)
return
@pulumi_ws
def preview(self):
""" Preview a Pulumi stack up operation"""
stack = pulumi_init(self)
stack.preview(on_output=self._log_pulumi)
return
@pulumi_ws
def set_config(self, key, value, secret):
""" Set a config or secret """
stack = pulumi_init(self)
stack.set_config(key, pulumi.automation.ConfigValue(value, secret))
# Store salt or key and encrypted value in CloudBender stack config
settings = None
pulumi_settings = stack.workspace.stack_settings(stack.name)._serialize()
with open(self.path, "r") as file:
settings = yaml.safe_load(file)
try:
if 'pulumi' not in settings:
settings['pulumi'] = {}
settings['pulumi']['encryptionsalt'] = pulumi_settings['encryptionsalt']
settings['pulumi']['encryptedkey'] = pulumi_settings['encryptedkey']
except KeyError:
pass
if 'parameters' not in settings:
settings['parameters'] = {}
settings['parameters'][key] = pulumi_settings['config']['{}:{}'.format(self.parameters['Conglomerate'], key)]
with open(self.path, "w") as file:
yaml.dump(settings, stream=file)
return
@pulumi_ws
def get_config(self, key):
""" Get a config or secret """
stack = pulumi_init(self)
print(stack.get_config(key).value)
def create_change_set(self, change_set_name):
""" Creates a Change Set with the name ``change_set_name``. """
@ -792,3 +875,6 @@ class Stack(object):
kwargs['TemplateBody'] = self.cfn_template
return kwargs
def _log_pulumi(self, text):
logger.info(" ".join([self.region, self.stackname, text]))

3
cloudbender/utils.py
View File

@ -3,8 +3,6 @@ import copy
import logging
import re
logger = logging.getLogger(__name__)
def dict_merge(a, b):
""" Deep merge to allow proper inheritance for config files"""
@ -30,7 +28,6 @@ def ensure_dir(path):
"""Creates dir if it does not already exist."""
if not os.path.exists(path):
os.makedirs(path)
logger.debug('Created directory: %s', path)
def setup_logging(debug):

3
requirements.txt
View File

@ -1,5 +1,6 @@
boto3
Jinja2
Jinja2<3
click
pyminifier
cfn-lint>=0.34
pulumi

2
setup.py
View File

@ -57,7 +57,7 @@ setup(
package_data={ 'cloudbender': ['templates/*.md'], },
include_package_data=True,
entry_points={'console_scripts': [ "cloudbender = cloudbender.cli:cli" ]},
install_requires=['boto3', 'Jinja2', 'click', 'cfn-lint>=0.34', 'pyminifier'],
install_requires=['boto3', 'Jinja2', 'click', 'cfn-lint>=0.34', 'pyminifier', 'pulumi'],
tests_require=["pytest-cov", "moto", "mock", 'pytest'],
cmdclass={"test": PyTest},
classifiers=[