feat: add assimilate task to import resources into Pulumi stacks, various Pulumi related fixes
ZeroDownTime/CloudBender/pipeline/head This commit looks good Details

This commit is contained in:
Stefan Reimer 2022-03-16 16:18:23 +01:00
parent 4196b54a20
commit 5db67920f7
4 changed files with 77 additions and 35 deletions

View File

@ -1,6 +1,6 @@
ARG RUNTIME_VERSION="3.7"
ARG DISTRO_VERSION="3.15"
ARG PULUMI_VERSION="3.24.1"
ARG PULUMI_VERSION="3.26.1"
FROM python:${RUNTIME_VERSION}-alpine${DISTRO_VERSION} AS builder
ARG PULUMI_VERSION

View File

@ -169,6 +169,22 @@ def export(cb, stack_name, reset=False):
logger.info("{} uses Cloudformation, export skipped.".format(s.stackname))
@click.command()
@click.argument("stack_name")
@click.pass_obj
def assimilate(cb, stack_name):
"""Imports potentially existing resources into Pulumi stack"""
stacks = _find_stacks(cb, [stack_name])
for s in stacks:
if s.mode == "pulumi":
s.assimilate()
else:
logger.info(
"{} uses Cloudformation, cannot assimilate.".format(s.stackname)
)
@click.command()
@click.argument("stack_name")
@click.argument("key")
@ -391,6 +407,7 @@ cli.add_command(preview)
cli.add_command(set_config)
cli.add_command(get_config)
cli.add_command(export)
cli.add_command(assimilate)
if __name__ == "__main__":
cli(obj={})

View File

@ -11,7 +11,7 @@ import logging
logger = logging.getLogger(__name__)
def pulumi_init(stack):
def pulumi_init(stack, create=False):
# Fail early if pulumi binaries are not available
if not shutil.which("pulumi"):
@ -51,6 +51,8 @@ def pulumi_init(stack):
"Cannot find Pulumi implementation for {}".format(stack.stackname)
)
# Store internal pulumi code reference
stack._pulumi_code = _stack
project_name = stack.parameters["Conglomerate"]
# Remove stacknameprefix if equals Conglomerate as Pulumi implicitly prefixes project_name
@ -102,8 +104,10 @@ def pulumi_init(stack):
else:
try:
if _stack.IKNOWHATIDO:
logger.warning("Missing pulumi.secretsProvider setting, IKNOWHATIDO enabled ... ")
if stack._pulumi_code.IKNOWHATIDO:
logger.warning(
"Missing pulumi.secretsProvider setting, IKNOWHATIDO enabled ... "
)
secrets_provider = None
except AttributeError:
raise ValueError("Missing pulumi.secretsProvider setting!")
@ -111,12 +115,12 @@ def pulumi_init(stack):
# Set tag for stack file name and version
_tags = stack.tags
try:
_version = _stack.VERSION
_version = stack._pulumi_code.VERSION
except AttributeError:
_version = "undefined"
_tags["zero-downtime.net/cloudbender"] = "{}:{}".format(
os.path.basename(_stack.__file__), _version
os.path.basename(stack._pulumi_code.__file__), _version
)
_config = {
@ -151,14 +155,23 @@ def pulumi_init(stack):
secrets_provider=secrets_provider,
)
stack = pulumi.automation.create_or_select_stack(
stack_name=pulumi_stackname,
project_name=project_name,
program=_stack.pulumi_program,
opts=ws_opts,
)
stack.workspace.install_plugin(
"aws", pkg_resources.get_distribution("pulumi_aws").version
)
if create:
pulumi_stack = pulumi.automation.create_or_select_stack(
stack_name=pulumi_stackname,
project_name=project_name,
program=stack._pulumi_code.pulumi_program,
opts=ws_opts,
)
pulumi_stack.workspace.install_plugin(
"aws", pkg_resources.get_distribution("pulumi_aws").version
)
return stack
else:
pulumi_stack = pulumi.automation.select_stack(
stack_name=pulumi_stackname,
project_name=project_name,
program=stack._pulumi_code.pulumi_program,
opts=ws_opts,
)
return pulumi_stack

View File

@ -83,6 +83,7 @@ class Stack(object):
self.template_bucket_url = None
self.work_dir = None
self.pulumi = {}
self._pulumi_stack = None
def dump_config(self):
logger.debug("<Stack {}: {}>".format(self.id, pprint.pformat(vars(self))))
@ -483,8 +484,7 @@ class Stack(object):
"""gets outputs of the stack"""
if self.mode == "pulumi":
stack = pulumi_init(self)
self.outputs = stack.outputs()
self.outputs = pulumi_init(self).outputs()
else:
self.read_template_file()
@ -545,7 +545,7 @@ class Stack(object):
# If secrets replace with clear values for now, display ONLY
for k in self.outputs.keys():
if hasattr(self.outputs[k], 'secret') and self.outputs[k].secret:
if hasattr(self.outputs[k], "secret") and self.outputs[k].secret:
self.outputs[k] = self.outputs[k].value
logger.info(
@ -713,8 +713,7 @@ class Stack(object):
"""Creates a stack"""
if self.mode == "pulumi":
stack = pulumi_init(self)
stack.up(on_output=self._log_pulumi)
pulumi_init(self, create=True).up(on_output=self._log_pulumi)
else:
# Prepare parameters
@ -812,8 +811,9 @@ class Stack(object):
logger.info("Deleting {0} {1}".format(self.region, self.stackname))
if self.mode == "pulumi":
stack = pulumi_init(self)
stack.destroy(on_output=self._log_pulumi)
pulumi_stack = pulumi_init(self)
pulumi_stack.destroy(on_output=self._log_pulumi)
pulumi_stack.workspace.remove_stack(pulumi_stack.name)
return
@ -832,8 +832,7 @@ class Stack(object):
def refresh(self):
"""Refreshes a Pulumi stack"""
stack = pulumi_init(self)
stack.refresh(on_output=self._log_pulumi)
pulumi_init(self).refresh(on_output=self._log_pulumi)
return
@ -841,8 +840,20 @@ class Stack(object):
def preview(self):
"""Preview a Pulumi stack up operation"""
stack = pulumi_init(self)
stack.preview(on_output=self._log_pulumi)
pulumi_init(self).preview(on_output=self._log_pulumi)
return
@pulumi_ws
def assimilate(self):
"""Import resources into Pulumi stack"""
pulumi_stack = pulumi_init(self, create=True)
# now lets import each defined resource
for r in self._pulumi_code.RESOURCES:
args = ["import", r["type"], r["name"], r["id"], "--yes"]
pulumi_stack._run_pulumi_cmd_sync(args)
return
@ -850,12 +861,12 @@ class Stack(object):
def export(self, reset):
"""Exports a Pulumi stack"""
stack = pulumi_init(self)
deployment = stack.export_stack()
pulumi_stack = pulumi_init(self)
deployment = pulumi_stack.export_stack()
if reset:
deployment.deployment.pop("pending_operations", None)
stack.import_stack(deployment)
pulumi_stack.import_stack(deployment)
logger.info("Removed all pending_operations from %s" % self.stackname)
else:
print(json.dumps(deployment.deployment))
@ -866,12 +877,14 @@ class Stack(object):
def set_config(self, key, value, secret):
"""Set a config or secret"""
stack = pulumi_init(self)
stack.set_config(key, pulumi.automation.ConfigValue(value, secret))
pulumi_stack = pulumi_init(self, create=True)
pulumi_stack.set_config(key, pulumi.automation.ConfigValue(value, secret))
# Store salt or key and encrypted value in CloudBender stack config
settings = None
pulumi_settings = stack.workspace.stack_settings(stack.name)._serialize()
pulumi_settings = pulumi_stack.workspace.stack_settings(
pulumi_stack.name
)._serialize()
with open(self.path, "r") as file:
settings = yaml.safe_load(file)
@ -899,8 +912,7 @@ class Stack(object):
def get_config(self, key):
"""Get a config or secret"""
stack = pulumi_init(self)
print(stack.get_config(key).value)
print(pulumi_init(self).get_config(key).value)
def create_change_set(self, change_set_name):
"""Creates a Change Set with the name ``change_set_name``."""