From 5db67920f7ae4e2d0d6f17ab7336ce0e5fcd3a5a Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 16 Mar 2022 16:18:23 +0100 Subject: [PATCH] feat: add assimilate task to import resources into Pulumi stacks, various Pulumi related fixes --- Dockerfile | 2 +- cloudbender/cli.py | 17 +++++++++++++++ cloudbender/pulumi.py | 43 ++++++++++++++++++++++++------------- cloudbender/stack.py | 50 +++++++++++++++++++++++++++---------------- 4 files changed, 77 insertions(+), 35 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8c73e2c..f972bd6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG RUNTIME_VERSION="3.7" ARG DISTRO_VERSION="3.15" -ARG PULUMI_VERSION="3.24.1" +ARG PULUMI_VERSION="3.26.1" FROM python:${RUNTIME_VERSION}-alpine${DISTRO_VERSION} AS builder ARG PULUMI_VERSION diff --git a/cloudbender/cli.py b/cloudbender/cli.py index 884b2ab..0156b5b 100644 --- a/cloudbender/cli.py +++ b/cloudbender/cli.py @@ -169,6 +169,22 @@ def export(cb, stack_name, reset=False): logger.info("{} uses Cloudformation, export skipped.".format(s.stackname)) +@click.command() +@click.argument("stack_name") +@click.pass_obj +def assimilate(cb, stack_name): + """Imports potentially existing resources into Pulumi stack""" + stacks = _find_stacks(cb, [stack_name]) + + for s in stacks: + if s.mode == "pulumi": + s.assimilate() + else: + logger.info( + "{} uses Cloudformation, cannot assimilate.".format(s.stackname) + ) + + @click.command() @click.argument("stack_name") @click.argument("key") @@ -391,6 +407,7 @@ cli.add_command(preview) cli.add_command(set_config) cli.add_command(get_config) cli.add_command(export) +cli.add_command(assimilate) if __name__ == "__main__": cli(obj={}) diff --git a/cloudbender/pulumi.py b/cloudbender/pulumi.py index 4e7ee82..560184b 100644 --- a/cloudbender/pulumi.py +++ b/cloudbender/pulumi.py @@ -11,7 +11,7 @@ import logging logger = logging.getLogger(__name__) -def pulumi_init(stack): +def pulumi_init(stack, create=False): # Fail early if pulumi binaries are not available if not shutil.which("pulumi"): @@ -51,6 +51,8 @@ def pulumi_init(stack): "Cannot find Pulumi implementation for {}".format(stack.stackname) ) + # Store internal pulumi code reference + stack._pulumi_code = _stack project_name = stack.parameters["Conglomerate"] # Remove stacknameprefix if equals Conglomerate as Pulumi implicitly prefixes project_name @@ -102,8 +104,10 @@ def pulumi_init(stack): else: try: - if _stack.IKNOWHATIDO: - logger.warning("Missing pulumi.secretsProvider setting, IKNOWHATIDO enabled ... ") + if stack._pulumi_code.IKNOWHATIDO: + logger.warning( + "Missing pulumi.secretsProvider setting, IKNOWHATIDO enabled ... " + ) secrets_provider = None except AttributeError: raise ValueError("Missing pulumi.secretsProvider setting!") @@ -111,12 +115,12 @@ def pulumi_init(stack): # Set tag for stack file name and version _tags = stack.tags try: - _version = _stack.VERSION + _version = stack._pulumi_code.VERSION except AttributeError: _version = "undefined" _tags["zero-downtime.net/cloudbender"] = "{}:{}".format( - os.path.basename(_stack.__file__), _version + os.path.basename(stack._pulumi_code.__file__), _version ) _config = { @@ -151,14 +155,23 @@ def pulumi_init(stack): secrets_provider=secrets_provider, ) - stack = pulumi.automation.create_or_select_stack( - stack_name=pulumi_stackname, - project_name=project_name, - program=_stack.pulumi_program, - opts=ws_opts, - ) - stack.workspace.install_plugin( - "aws", pkg_resources.get_distribution("pulumi_aws").version - ) + if create: + pulumi_stack = pulumi.automation.create_or_select_stack( + stack_name=pulumi_stackname, + project_name=project_name, + program=stack._pulumi_code.pulumi_program, + opts=ws_opts, + ) + pulumi_stack.workspace.install_plugin( + "aws", pkg_resources.get_distribution("pulumi_aws").version + ) - return stack + else: + pulumi_stack = pulumi.automation.select_stack( + stack_name=pulumi_stackname, + project_name=project_name, + program=stack._pulumi_code.pulumi_program, + opts=ws_opts, + ) + + return pulumi_stack diff --git a/cloudbender/stack.py b/cloudbender/stack.py index 2312335..fe4d365 100644 --- a/cloudbender/stack.py +++ b/cloudbender/stack.py @@ -83,6 +83,7 @@ class Stack(object): self.template_bucket_url = None self.work_dir = None self.pulumi = {} + self._pulumi_stack = None def dump_config(self): logger.debug("".format(self.id, pprint.pformat(vars(self)))) @@ -483,8 +484,7 @@ class Stack(object): """gets outputs of the stack""" if self.mode == "pulumi": - stack = pulumi_init(self) - self.outputs = stack.outputs() + self.outputs = pulumi_init(self).outputs() else: self.read_template_file() @@ -545,7 +545,7 @@ class Stack(object): # If secrets replace with clear values for now, display ONLY for k in self.outputs.keys(): - if hasattr(self.outputs[k], 'secret') and self.outputs[k].secret: + if hasattr(self.outputs[k], "secret") and self.outputs[k].secret: self.outputs[k] = self.outputs[k].value logger.info( @@ -713,8 +713,7 @@ class Stack(object): """Creates a stack""" if self.mode == "pulumi": - stack = pulumi_init(self) - stack.up(on_output=self._log_pulumi) + pulumi_init(self, create=True).up(on_output=self._log_pulumi) else: # Prepare parameters @@ -812,8 +811,9 @@ class Stack(object): logger.info("Deleting {0} {1}".format(self.region, self.stackname)) if self.mode == "pulumi": - stack = pulumi_init(self) - stack.destroy(on_output=self._log_pulumi) + pulumi_stack = pulumi_init(self) + pulumi_stack.destroy(on_output=self._log_pulumi) + pulumi_stack.workspace.remove_stack(pulumi_stack.name) return @@ -832,8 +832,7 @@ class Stack(object): def refresh(self): """Refreshes a Pulumi stack""" - stack = pulumi_init(self) - stack.refresh(on_output=self._log_pulumi) + pulumi_init(self).refresh(on_output=self._log_pulumi) return @@ -841,8 +840,20 @@ class Stack(object): def preview(self): """Preview a Pulumi stack up operation""" - stack = pulumi_init(self) - stack.preview(on_output=self._log_pulumi) + pulumi_init(self).preview(on_output=self._log_pulumi) + + return + + @pulumi_ws + def assimilate(self): + """Import resources into Pulumi stack""" + + pulumi_stack = pulumi_init(self, create=True) + + # now lets import each defined resource + for r in self._pulumi_code.RESOURCES: + args = ["import", r["type"], r["name"], r["id"], "--yes"] + pulumi_stack._run_pulumi_cmd_sync(args) return @@ -850,12 +861,12 @@ class Stack(object): def export(self, reset): """Exports a Pulumi stack""" - stack = pulumi_init(self) - deployment = stack.export_stack() + pulumi_stack = pulumi_init(self) + deployment = pulumi_stack.export_stack() if reset: deployment.deployment.pop("pending_operations", None) - stack.import_stack(deployment) + pulumi_stack.import_stack(deployment) logger.info("Removed all pending_operations from %s" % self.stackname) else: print(json.dumps(deployment.deployment)) @@ -866,12 +877,14 @@ class Stack(object): def set_config(self, key, value, secret): """Set a config or secret""" - stack = pulumi_init(self) - stack.set_config(key, pulumi.automation.ConfigValue(value, secret)) + pulumi_stack = pulumi_init(self, create=True) + pulumi_stack.set_config(key, pulumi.automation.ConfigValue(value, secret)) # Store salt or key and encrypted value in CloudBender stack config settings = None - pulumi_settings = stack.workspace.stack_settings(stack.name)._serialize() + pulumi_settings = pulumi_stack.workspace.stack_settings( + pulumi_stack.name + )._serialize() with open(self.path, "r") as file: settings = yaml.safe_load(file) @@ -899,8 +912,7 @@ class Stack(object): def get_config(self, key): """Get a config or secret""" - stack = pulumi_init(self) - print(stack.get_config(key).value) + print(pulumi_init(self).get_config(key).value) def create_change_set(self, change_set_name): """Creates a Change Set with the name ``change_set_name``."""