fix: make Pulumi work again with python 3.12

06fcff5 feat: improve image cleanup to incl. all tags and repositories

git-subtree-dir: .ci
git-subtree-split: 06fcff501dd9dbff3a97a95176f6d7a8590289a7

.ci/podman.mk

@@ -46,7 +46,7 @@ test:: ## test built artificats
 
 scan: ## Scan image using trivy
 	echo "Scanning $(IMAGE):$(TAG)-$(_ARCH) using Trivy $(TRIVY_REMOTE)"
-	trivy image $(TRIVY_OPTS) --quiet --no-progress localhost/$(IMAGE):$(TAG)-$(_ARCH)
+	trivy image $(TRIVY_OPTS) --quiet --no-progress --ignorefile ./.trivyignore.yaml localhost/$(IMAGE):$(TAG)-$(_ARCH)
 
 # first tag and push all actual images
 # create new manifest for each tag and add all available TAG-ARCH before pushing
@@ -73,8 +73,10 @@ rm-remote-untagged: ## delete all remote untagged and in-dev images, keep 10 tag
 clean:: ## clean up source folder
 
 rm-image:
-	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH) > /dev/null
-	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || echo "Error: Removing image failed"
+	for t in $(TAG) latest $(EXTRA_TAGS); do \
+		test -z "$$(podman image ls -q $(IMAGE):$${t}-$(_ARCH))" || podman image rm -f $(IMAGE):$${t}-$(_ARCH); \
+		test -z "$$(podman image ls -q $(IMAGE):$${t})" || podman image rm -f $(IMAGE):$${t}; \
+	done
 
 ## some useful tasks during development
 ci-pull-upstream: ## pull latest shared .ci subtree

.trivyignore.yaml

@@ -0,0 +1,7 @@
+secrets:
+  - id: gcp-service-account
+    paths:
+      - "/venv/lib/python*/site-packages/pulumi_aws/glue/connection.py"
+  - id: private-key
+    paths:
+      - "/venv/lib/python*/site-packages/pulumi_aws/glue/connection.py"

Dockerfile

@@ -1,8 +1,8 @@
-ARG RUNTIME_VERSION="3.11"
-ARG DISTRO_VERSION="3.19"
+ARG RUNTIME_VERSION="3.12"
+ARG DISTRO_VERSION="3.20"
 
 FROM python:${RUNTIME_VERSION}-alpine${DISTRO_VERSION} AS builder
-ARG RUNTIME_VERSION="3.11"
+ARG RUNTIME_VERSION="3.12"
 
 RUN apk add --no-cache \
     autoconf \
@@ -33,9 +33,6 @@ RUN curl -fsSL https://get.pulumi.com/ | sh -s -- --version $(pip show pulumi --
 # minimal pulumi
 RUN cd /root/.pulumi/bin && rm -f *dotnet *yaml *go *java && strip pulumi* || true
 
-# Remove AWS keys from docstring to prevent trivy alerts later
-RUN sed -i -e 's/AKIA.*//' /venv/lib/python${RUNTIME_VERSION}/site-packages/pulumi_aws/lightsail/bucket_access_key.py
-
 # Now build the final runtime, incl. running rootless containers
 FROM python:${RUNTIME_VERSION}-alpine${DISTRO_VERSION}
 

cloudbender/cli.py

@@ -447,10 +447,10 @@ def _provision(cb, stacks):
             # Pulumi is still not thread safe
             if _anyPulumi(step):
                 _threads = 1
-            else
+            else:
                 _threads = len(step)
 
-            with ThreadPoolExecutor(max_workers=_threads)) as group:
+            with ThreadPoolExecutor(max_workers=_threads) as group:
                 futures = []
                 for stack in step:
                     if stack.mode != "pulumi":

cloudbender/pulumi.py

@@ -52,11 +52,14 @@ def resolve_outputs(outputs):
 def pulumi_ws(func):
     @wraps(func)
     def decorated(self, *args, **kwargs):
+        cwd = None
         # setup temp workspace
         if self.mode == "pulumi":
             self.work_dir = tempfile.mkdtemp(
                 dir=tempfile.gettempdir(), prefix="cloudbender-"
             )
+            cwd = os.getcwd()
+            os.chdir(self.work_dir)
 
             # add all artifact_paths/pulumi to the search path for easier
             # imports in the pulumi code
@@ -146,7 +149,8 @@ def pulumi_ws(func):
             try:
                 _min_version = self._pulumi_code.MIN_CLOUDBENDER_VERSION
                 if semver.compare(
-                        semver.Version.parse(__version__.strip("v")).finalize_version(),
+                        semver.Version.parse(
+                            __version__.strip("v")).finalize_version(),
                         _min_version.strip("v")) < 0:
                     raise ValueError(
                         f"Minimal required CloudBender version is {_min_version}, but we are {__version__}!"
@@ -189,7 +193,7 @@ def pulumi_ws(func):
             )
 
             project_settings = pulumi.automation.ProjectSettings(
-                name=project_name, runtime="python", backend={"url": pulumi_backend}
+                name=project_name, runtime="python", backend=pulumi.automation.ProjectBackend(url=pulumi_backend)
             )
 
             self.pulumi_ws_opts = pulumi.automation.LocalWorkspaceOptions(
@@ -202,6 +206,9 @@ def pulumi_ws(func):
         response = func(self, *args, **kwargs)
 
         # Cleanup temp workspace
+        if cwd:
+            os.chdir(cwd)
+
         if self.work_dir and os.path.exists(self.work_dir):
             shutil.rmtree(self.work_dir)
 

cloudbender/stack.py

@@ -8,7 +8,6 @@ import pathlib
 import pprint
 import pulumi
 import importlib
-import pkg_resources
 
 from datetime import datetime, timedelta
 from dateutil.tz import tzutc
@@ -535,7 +534,6 @@ class Stack(object):
             logger.info("Passed.")
             return 0
 
-    @pulumi_ws
     def get_outputs(self, include=".*", values=False):
         """gets outputs of the stack"""
 
@@ -852,6 +850,7 @@ class Stack(object):
 
         return status
 
+    @pulumi_ws
     @exec_hooks
     def update(self):
         """Updates an existing stack"""
@@ -1305,7 +1304,6 @@ class Stack(object):
             logger.info(" ".join([self.region, self.stackname, text]))
 
     def _get_pulumi_stack(self, create=False):
-
         if create:
             pulumi_stack = pulumi.automation.create_or_select_stack(
                 stack_name=self.pulumi_stackname,
@@ -1314,7 +1312,7 @@ class Stack(object):
                 opts=self.pulumi_ws_opts,
             )
             pulumi_stack.workspace.install_plugin(
-                "aws", pkg_resources.get_distribution("pulumi_aws").version
+                "aws", importlib.metadata.version("pulumi_aws")
             )
 
         else:

pyproject.toml

@@ -11,21 +11,22 @@ authors = [
 description = "Deploy and maintain infrastructure in automated and trackable manner"
 readme = "README.md"
 license = "AGPL-3.0-or-later"
-requires-python = ">=3.9"
+requires-python = ">=3.12"
 dependencies = [
-        "boto3==1.34.89",
+        "boto3==1.35.46",
         "mock==5.1.0",
-        "Jinja2==3.1.3",
+        "Jinja2==3.1.4",
         "click==8.1.7",
         "pexpect==4.9.0",
-        "python-minifier==2.9.0",
-        "cfn-lint==0.86.4",
+        "python-minifier==2.11.2",
+        "cfn-lint==1.12.4",
         "ruamel.yaml==0.18.6",
-        "pulumi==3.113.3",
-        "pulumi-aws==6.32.0",
-        "pulumi-aws-native==0.103.0",
-        "pulumi-policy==1.11.0",
-        "pulumi-command==0.10.0",
+        "pulumi==3.137.0",
+        "pulumi-aws==6.56.1",
+        "pulumi-aws-native==1.3.0",
+        "pulumi-policy==1.13.0",
+        "pulumi-command==1.0.1",
+        "pulumi_random==4.16.6",
         ]
 
 classifiers = [