feat: add assimilate task to import resources into Pulumi stacks, various Pulumi related fixes

Dockerfile

@@ -1,6 +1,6 @@
 ARG RUNTIME_VERSION="3.7"
 ARG DISTRO_VERSION="3.15"
-ARG PULUMI_VERSION="3.24.1"
+ARG PULUMI_VERSION="3.26.1"
 
 FROM python:${RUNTIME_VERSION}-alpine${DISTRO_VERSION} AS builder
 ARG PULUMI_VERSION

cloudbender/cli.py

@@ -169,6 +169,22 @@ def export(cb, stack_name, reset=False):
             logger.info("{} uses Cloudformation, export skipped.".format(s.stackname))
 
 
+@click.command()
+@click.argument("stack_name")
+@click.pass_obj
+def assimilate(cb, stack_name):
+    """Imports potentially existing resources into Pulumi stack"""
+    stacks = _find_stacks(cb, [stack_name])
+
+    for s in stacks:
+        if s.mode == "pulumi":
+            s.assimilate()
+        else:
+            logger.info(
+                "{} uses Cloudformation, cannot assimilate.".format(s.stackname)
+            )
+
+
 @click.command()
 @click.argument("stack_name")
 @click.argument("key")
@@ -391,6 +407,7 @@ cli.add_command(preview)
 cli.add_command(set_config)
 cli.add_command(get_config)
 cli.add_command(export)
+cli.add_command(assimilate)
 
 if __name__ == "__main__":
     cli(obj={})

cloudbender/pulumi.py

@@ -11,7 +11,7 @@ import logging
 logger = logging.getLogger(__name__)
 
 
-def pulumi_init(stack):
+def pulumi_init(stack, create=False):
 
     # Fail early if pulumi binaries are not available
     if not shutil.which("pulumi"):
@@ -51,6 +51,8 @@ def pulumi_init(stack):
             "Cannot find Pulumi implementation for {}".format(stack.stackname)
         )
 
+    # Store internal pulumi code reference
+    stack._pulumi_code = _stack
     project_name = stack.parameters["Conglomerate"]
 
     # Remove stacknameprefix if equals Conglomerate as Pulumi implicitly prefixes project_name
@@ -102,8 +104,10 @@ def pulumi_init(stack):
 
     else:
         try:
-            if _stack.IKNOWHATIDO:
+            if stack._pulumi_code.IKNOWHATIDO:
-                logger.warning("Missing pulumi.secretsProvider setting, IKNOWHATIDO enabled ... ")
+                logger.warning(
+                    "Missing pulumi.secretsProvider setting, IKNOWHATIDO enabled ... "
+                )
                 secrets_provider = None
         except AttributeError:
             raise ValueError("Missing pulumi.secretsProvider setting!")
@@ -111,12 +115,12 @@ def pulumi_init(stack):
     # Set tag for stack file name and version
     _tags = stack.tags
     try:
-        _version = _stack.VERSION
+        _version = stack._pulumi_code.VERSION
     except AttributeError:
         _version = "undefined"
 
     _tags["zero-downtime.net/cloudbender"] = "{}:{}".format(
-        os.path.basename(_stack.__file__), _version
+        os.path.basename(stack._pulumi_code.__file__), _version
     )
 
     _config = {
@@ -151,14 +155,23 @@ def pulumi_init(stack):
         secrets_provider=secrets_provider,
     )
 
-    stack = pulumi.automation.create_or_select_stack(
+    if create:
-        stack_name=pulumi_stackname,
+        pulumi_stack = pulumi.automation.create_or_select_stack(
-        project_name=project_name,
+            stack_name=pulumi_stackname,
-        program=_stack.pulumi_program,
+            project_name=project_name,
-        opts=ws_opts,
+            program=stack._pulumi_code.pulumi_program,
-    )
+            opts=ws_opts,
-    stack.workspace.install_plugin(
+        )
-        "aws", pkg_resources.get_distribution("pulumi_aws").version
+        pulumi_stack.workspace.install_plugin(
-    )
+            "aws", pkg_resources.get_distribution("pulumi_aws").version
+        )
 
-    return stack
+    else:
+        pulumi_stack = pulumi.automation.select_stack(
+            stack_name=pulumi_stackname,
+            project_name=project_name,
+            program=stack._pulumi_code.pulumi_program,
+            opts=ws_opts,
+        )
+
+    return pulumi_stack

cloudbender/stack.py

@@ -83,6 +83,7 @@ class Stack(object):
         self.template_bucket_url = None
         self.work_dir = None
         self.pulumi = {}
+        self._pulumi_stack = None
 
     def dump_config(self):
         logger.debug("<Stack {}: {}>".format(self.id, pprint.pformat(vars(self))))
@@ -483,8 +484,7 @@ class Stack(object):
         """gets outputs of the stack"""
 
         if self.mode == "pulumi":
-            stack = pulumi_init(self)
+            self.outputs = pulumi_init(self).outputs()
-            self.outputs = stack.outputs()
 
         else:
             self.read_template_file()
@@ -545,7 +545,7 @@ class Stack(object):
 
             # If secrets replace with clear values for now, display ONLY
             for k in self.outputs.keys():
-                if hasattr(self.outputs[k], 'secret') and self.outputs[k].secret:
+                if hasattr(self.outputs[k], "secret") and self.outputs[k].secret:
                     self.outputs[k] = self.outputs[k].value
 
             logger.info(
@@ -713,8 +713,7 @@ class Stack(object):
         """Creates a stack"""
 
         if self.mode == "pulumi":
-            stack = pulumi_init(self)
+            pulumi_init(self, create=True).up(on_output=self._log_pulumi)
-            stack.up(on_output=self._log_pulumi)
 
         else:
             # Prepare parameters
@@ -812,8 +811,9 @@ class Stack(object):
         logger.info("Deleting {0} {1}".format(self.region, self.stackname))
 
         if self.mode == "pulumi":
-            stack = pulumi_init(self)
+            pulumi_stack = pulumi_init(self)
-            stack.destroy(on_output=self._log_pulumi)
+            pulumi_stack.destroy(on_output=self._log_pulumi)
+            pulumi_stack.workspace.remove_stack(pulumi_stack.name)
 
             return
 
@@ -832,8 +832,7 @@ class Stack(object):
     def refresh(self):
         """Refreshes a Pulumi stack"""
 
-        stack = pulumi_init(self)
+        pulumi_init(self).refresh(on_output=self._log_pulumi)
-        stack.refresh(on_output=self._log_pulumi)
 
         return
 
@@ -841,8 +840,20 @@ class Stack(object):
     def preview(self):
         """Preview a Pulumi stack up operation"""
 
-        stack = pulumi_init(self)
+        pulumi_init(self).preview(on_output=self._log_pulumi)
-        stack.preview(on_output=self._log_pulumi)
+
+        return
+
+    @pulumi_ws
+    def assimilate(self):
+        """Import resources into Pulumi stack"""
+
+        pulumi_stack = pulumi_init(self, create=True)
+
+        # now lets import each defined resource
+        for r in self._pulumi_code.RESOURCES:
+            args = ["import", r["type"], r["name"], r["id"], "--yes"]
+            pulumi_stack._run_pulumi_cmd_sync(args)
 
         return
 
@@ -850,12 +861,12 @@ class Stack(object):
     def export(self, reset):
         """Exports a Pulumi stack"""
 
-        stack = pulumi_init(self)
+        pulumi_stack = pulumi_init(self)
-        deployment = stack.export_stack()
+        deployment = pulumi_stack.export_stack()
 
         if reset:
             deployment.deployment.pop("pending_operations", None)
-            stack.import_stack(deployment)
+            pulumi_stack.import_stack(deployment)
             logger.info("Removed all pending_operations from %s" % self.stackname)
         else:
             print(json.dumps(deployment.deployment))
@@ -866,12 +877,14 @@ class Stack(object):
     def set_config(self, key, value, secret):
         """Set a config or secret"""
 
-        stack = pulumi_init(self)
+        pulumi_stack = pulumi_init(self, create=True)
-        stack.set_config(key, pulumi.automation.ConfigValue(value, secret))
+        pulumi_stack.set_config(key, pulumi.automation.ConfigValue(value, secret))
 
         # Store salt or key and encrypted value in CloudBender stack config
         settings = None
-        pulumi_settings = stack.workspace.stack_settings(stack.name)._serialize()
+        pulumi_settings = pulumi_stack.workspace.stack_settings(
+            pulumi_stack.name
+        )._serialize()
 
         with open(self.path, "r") as file:
             settings = yaml.safe_load(file)
@@ -899,8 +912,7 @@ class Stack(object):
     def get_config(self, key):
         """Get a config or secret"""
 
-        stack = pulumi_init(self)
+        print(pulumi_init(self).get_config(key).value)
-        print(stack.get_config(key).value)
 
     def create_change_set(self, change_set_name):
         """Creates a Change Set with the name ``change_set_name``."""