Merge pull request 'chore(deps): update jenkins/inbound-agent:alpine-jdk17 docker digest to 38484c0' (#7) from renovate/jenkins-inbound-agent-alpine-jdk17 into master

Reviewed-on: #7

.ci/ecr_public_lifecycle.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+import argparse
+import boto3
+
+parser = argparse.ArgumentParser(
+    description='Implement basic public ECR lifecycle policy')
+parser.add_argument('--repo', dest='repositoryName', action='store', required=True,
+                    help='Name of the public ECR repository')
+parser.add_argument('--keep', dest='keep', action='store', default=10, type=int,
+                    help='number of tagged images to keep, default 10')
+parser.add_argument('--dev', dest='delete_dev', action='store_true',
+                    help='also delete in-development images only having tags like v0.1.1-commitNr-githash')
+
+args = parser.parse_args()
+
+client = boto3.client('ecr-public', region_name='us-east-1')
+
+images = client.describe_images(repositoryName=args.repositoryName)[
+    "imageDetails"]
+
+untagged = []
+kept = 0
+
+# actual Image
+# imageManifestMediaType: 'application/vnd.oci.image.manifest.v1+json'
+# image Index
+# imageManifestMediaType: 'application/vnd.oci.image.index.v1+json'
+
+# Sort by date uploaded
+for image in sorted(images, key=lambda d: d['imagePushedAt'], reverse=True):
+    # Remove all untagged
+    # if registry uses image index all actual images will be untagged anyways
+    if 'imageTags' not in image:
+        untagged.append({"imageDigest": image['imageDigest']})
+        # print("Delete untagged image {}".format(image["imageDigest"]))
+        continue
+
+    # check for dev tags
+    if args.delete_dev:
+        _delete = True
+        for tag in image["imageTags"]:
+            # Look for at least one tag NOT beign a SemVer dev tag
+            if "-" not in tag:
+                _delete = False
+        if _delete:
+            print("Deleting development image {}".format(image["imageTags"]))
+            untagged.append({"imageDigest": image['imageDigest']})
+            continue
+
+    if kept < args.keep:
+        kept = kept+1
+        print("Keeping tagged image {}".format(image["imageTags"]))
+        continue
+    else:
+        print("Deleting tagged image {}".format(image["imageTags"]))
+        untagged.append({"imageDigest": image['imageDigest']})
+
+deleted_images = client.batch_delete_image(
+    repositoryName=args.repositoryName, imageIds=untagged)
+
+if deleted_images["imageIds"]:
+    print("Deleted images: {}".format(deleted_images["imageIds"]))

.ci/podman.mk

@@ -1,25 +1,26 @@
 # Parse version from latest git semver tag
-GIT_BRANCH ?= $(shell git name-rev --name-only HEAD 2>/dev/null | sed -e 's,remotes/origin/,,' -e 's/[^a-zA-Z0-9]/-/g')
-GIT_TAG := $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)
+GIT_TAG ?= $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)
+GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
 
-TAG := $(GIT_TAG)
+TAG ::= $(GIT_TAG)
 # append branch name to tag if NOT main nor master
 ifeq (,$(filter main master, $(GIT_BRANCH)))
 	# If branch is substring of tag, omit branch name
 	ifeq ($(findstring $(GIT_BRANCH), $(GIT_TAG)),)
 		# only append branch name if not equal tag
 		ifneq ($(GIT_TAG), $(GIT_BRANCH))
-			TAG = $(GIT_TAG)-$(GIT_BRANCH)
+			# Sanitize GIT_BRANCH to allowed Docker tag character set
+			TAG = $(GIT_TAG)-$(shell echo $$GIT_BRANCH | sed -e 's/[^a-zA-Z0-9]/-/g')
 		endif
 	endif
 endif
 
-ARCH := amd64
-ALL_ARCHS := amd64 arm64
+ARCH ::= amd64
+ALL_ARCHS ::= amd64 arm64
 _ARCH = $(or $(filter $(ARCH),$(ALL_ARCHS)),$(error $$ARCH [$(ARCH)] must be exactly one of "$(ALL_ARCHS)"))
 
 ifneq ($(TRIVY_REMOTE),)
-	TRIVY_OPTS := --server $(TRIVY_REMOTE)
+	TRIVY_OPTS ::= --server $(TRIVY_REMOTE)
 endif
 
 .SILENT: ; # no need for @
@@ -32,18 +33,20 @@ endif
 help: ## Show Help
 	grep -E '^[a-zA-Z_-]+:.*?## .*$$' .ci/podman.mk | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
+prepare:: ## custom step on the build agent before building
+
+fmt:: ## auto format source
+
+lint:: ## Lint source
+
 build: ## Build the app
 	buildah build --rm --layers -t $(IMAGE):$(TAG)-$(_ARCH) --build-arg TAG=$(TAG) --build-arg ARCH=$(_ARCH) --platform linux/$(_ARCH) .
 
-test: rm-test-image ## Execute Dockerfile.test
-	test -f Dockerfile.test && \
-		{ buildah build --rm --layers -t $(REGISTRY)/$(IMAGE):$(TAG)-$(_ARCH)-test --from=$(REGISTRY)/$(IMAGE):$(TAG) -f Dockerfile.test --platform linux/$(_ARCH) . && \
-			podman run --rm --env-host -t $(REGISTRY)/$(IMAGE):$(TAG)-$(_ARCH)-test; } || \
-		echo "No Dockerfile.test found, skipping test"
+test:: ## test built artificats
 
 scan: ## Scan image using trivy
 	echo "Scanning $(IMAGE):$(TAG)-$(_ARCH) using Trivy $(TRIVY_REMOTE)"
-	trivy image $(TRIVY_OPTS) localhost/$(IMAGE):$(TAG)-$(_ARCH)
+	trivy image $(TRIVY_OPTS) --quiet --no-progress localhost/$(IMAGE):$(TAG)-$(_ARCH)
 
 # first tag and push all actual images
 # create new manifest for each tag and add all available TAG-ARCH before pushing
@@ -63,24 +66,19 @@ push: ecr-login ## push images to registry
 ecr-login: ## log into AWS ECR public
 	aws ecr-public get-login-password --region $(REGION) | podman login --username AWS --password-stdin $(REGISTRY)
 
-clean: rm-test-image rm-image ## delete local built container and test images
+rm-remote-untagged: ## delete all remote untagged and in-dev images, keep 10 tagged
+	echo "Removing all untagged and in-dev images from $(IMAGE) in $(REGION)"
+	.ci/ecr_public_lifecycle.py --repo $(IMAGE) --dev
 
-rm-remote-untagged: ## delete all remote untagged images
-	echo "Removing all untagged images from $(IMAGE) in $(REGION)"
-	IMAGE_IDS=$$(for image in $$(aws ecr-public describe-images --repository-name $(IMAGE) --region $(REGION) --output json | jq -r '.imageDetails[] | select(.imageTags | not ).imageDigest'); do echo -n "imageDigest=$$image "; done) ; \
-		[ -n "$$IMAGE_IDS" ] && aws ecr-public batch-delete-image --repository-name $(IMAGE) --region $(REGION) --image-ids $$IMAGE_IDS | jq -r '.imageIds[]' || echo "No image to remove"
+clean:: ## clean up source folder
 
 rm-image:
 	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH) > /dev/null
 	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || echo "Error: Removing image failed"
 
-# Ensure we run the tests by removing any previous runs
-rm-test-image:
-	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH)-test > /dev/null
-	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || echo "Error: Removing test image failed"
-
+## some useful tasks during development
 ci-pull-upstream: ## pull latest shared .ci subtree
-	git stash && git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash && git stash pop
+	git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash -m "Merge latest ci-tools-lib"
 
 create-repo: ## create new AWS ECR public repository
 	aws ecr-public create-repository --repository-name $(IMAGE) --region $(REGION)

.ci/vars/buildPodman.groovy

@@ -2,6 +2,9 @@
 
 def call(Map config=[:]) {
     pipeline {
+      options {
+        disableConcurrentBuilds()
+      }
       agent {
         node {
           label 'podman-aws-trivy'
@@ -10,18 +13,22 @@ def call(Map config=[:]) {
       stages {
         stage('Prepare') {
           steps {
+            sh 'mkdir -p reports'
+
+            // we set pull tags as project adv. options
             // pull tags
-            withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) {
-              sh 'git fetch -q --tags ${GIT_URL}'
-            }
-            sh 'make prepare || true'
+            //withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) {
+            //  sh 'git fetch -q --tags ${GIT_URL}'
+            //}
+            // Optional project specific preparations
+            sh 'make prepare'
           }
         }
 
         // Build using rootless podman
         stage('Build') {
           steps {
-            sh 'make build'
+            sh 'make build GIT_BRANCH=$GIT_BRANCH'
           }
         }
 
@@ -33,12 +40,13 @@ def call(Map config=[:]) {
 
         // Scan via trivy
         stage('Scan') {
-          environment {
-            TRIVY_FORMAT = "template"
-            TRIVY_OUTPUT = "reports/trivy.html"
-          }
           steps {
-            sh 'mkdir -p reports && make scan'
+            // we always scan and create the full json report
+            sh 'TRIVY_FORMAT=json TRIVY_OUTPUT="reports/trivy.json" make scan'
+
+            // render custom full html report
+            sh 'trivy convert -f template -t @/home/jenkins/html.tpl -o reports/trivy.html reports/trivy.json'
+
             publishHTML target: [
               allowMissing: true,
               alwaysLinkToLastBuild: true,
@@ -48,31 +56,31 @@ def call(Map config=[:]) {
               reportName: 'TrivyScan',
               reportTitles: 'TrivyScan'
             ]
+            sh 'echo "Trivy report at: $BUILD_URL/TrivyScan"'
 
-            // Scan again and fail on CRITICAL vulns, if not overridden
+            // fail build if issues found above trivy threshold
             script {
-              if (config.trivyFail == 'NONE') {
-                echo 'trivyFail == NONE, review Trivy report manually. Proceeding ...'
-              } else {
-                sh "TRIVY_EXIT_CODE=1 TRIVY_SEVERITY=${config.trivyFail} make scan"
+              if ( config.trivyFail ) {
+                sh "TRIVY_SEVERITY=${config.trivyFail} trivy convert --report summary --exit-code 1 reports/trivy.json"
               }
             }
           }
         }
 
-        // Push to container registry, skip if PR
+        // Push to container registry if not PR
+        // incl. basic registry retention removing any untagged images
         stage('Push') {
           when { not { changeRequest() } }
           steps {
             sh 'make push'
+            sh 'make rm-remote-untagged'
           }
         }
 
-        // Basic registry retention removing untagged images
+        // generic clean
         stage('cleanup') {
-          when { not { changeRequest() } }
           steps {
-            sh 'make rm-remote-untagged'
+            sh 'make clean'
           }
         }
       }

Dockerfile

@@ -2,7 +2,7 @@
 # https://hub.docker.com/r/jenkins/inbound-agent/tags
 
 
-FROM jenkins/inbound-agent:alpine-jdk17@sha256:9ffaf2ae447d87b274be28958432070a7b980eca99287bb8bb4f58125364071b
+FROM jenkins/inbound-agent:alpine-jdk17@sha256:38484c0f035a57b3cab5838468fb6c5c2b3f3051e7a2c8b39c99a30823f393ce
 
 ARG BUILDUSER=jenkins
 
@@ -15,6 +15,7 @@ RUN echo "@edge-testing http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /e
       jq \
       yq \
       strace \
+      fuse-overlayfs \
       podman \
       buildah \
       py-boto3 \
@@ -31,8 +32,8 @@ ADD entrypoint.sh /usr/local/bin/entrypoint.sh
 # conf/registries.conf will be mounted RO at runtime to inherit worker settings incl. caching proxies
 ADD --chown=$BUILDUSER:$BUILDUSER conf/containers.conf conf/storage.conf /home/$BUILDUSER/.config/containers
 
-RUN echo -e "$BUILDUSER:1:999\n$BUILDUSER:1001:64535" > /etc/subuid && \
-    echo -e "$BUILDUSER:1:999\n$BUILDUSER:1001:64535" > /etc/subgid && \
+RUN echo -e "$BUILDUSER:100000:65535" > /etc/subuid && \
+    echo -e "$BUILDUSER:100000:65535" > /etc/subgid && \
     cd /usr/bin && ln -s podman docker && \
     chown $BUILDUSER:$BUILDUSER -R /home/$BUILDUSER
 
@@ -42,9 +43,12 @@ RUN sed -i -e 's/exec \$JAVA_BIN/podman system service -t0\&\n        exec \$JAV
 ENV XDG_RUNTIME_DIR=/home/$BUILDUSER/agent/xdg-run
 ENV BUILDAH_ISOLATION=chroot
 ENV _CONTAINERS_USERNS_CONFIGURED=""
-ENV TRIVY_TEMPLATE="@/home/jenkins/html.tpl"
 ENV HOME=/home/$BUILDUSER
 
+# Until we setup the logging and metrics pipelines in OTEL
+ENV OTEL_LOGS_EXPORTER=none
+ENV OTEL_METRICS_EXPORTER=none
+
 USER $BUILDUSER
 
 ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/entrypoint.sh"]

html.tpl

@@ -83,11 +83,13 @@
     </script>
   </head>
   <body>
-    <h1><img src="https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png" style="padding-right:10px";>
-    Trivy Report - {{ now | date "2006-01-02 15:04:05 -0700" }}</h1>
     <table>
+    <tr><td colspan="7">
+    <h1><img src="https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png" style="padding:10px;float:left";>
+    {{- escapeXML ( index . 0 ).Target }}<br/>Trivy Report - {{ now | date "2006-01-02 15:04:05 -0700" }}</h1>
+    </td></tr>
     {{- range . }}
-      <tr class="group-header"><th colspan="7">{{ escapeXML .Target }}({{ escapeXML .Type }})</th></tr>
+      <tr class="group-header"><th colspan="7">{{ escapeXML .Target }} ({{ .Type | toString | escapeXML }})</th></tr>
       {{- if (eq (len .Vulnerabilities) 0) }}
       <tr><th colspan="7">No Vulnerabilities found</th></tr>
       {{- else }}
@@ -101,10 +103,10 @@
         <th>Links</th>
       </tr>
       {{- range .Vulnerabilities }}
-      <tr class="severity-{{ escapeXML .Vulnerability.Severity }}">
+      <tr class="severity-{{ escapeXML .Severity }}">
         <td class="pkg-name">{{ escapeXML .PkgName }}</td>
         <td>{{ escapeXML .VulnerabilityID }}</td>
-        <td class="severity">{{ escapeXML .Vulnerability.Severity }}</td>
+        <td class="severity">{{ escapeXML .Severity }}</td>
         <td class="pkg-version">{{ escapeXML .InstalledVersion }}</td>
         <td>{{ escapeXML .FixedVersion }}</td>
         <td>{{ escapeXML .Title }}</td>
@@ -145,6 +147,26 @@
       </tr>
         {{- end }}
       {{- end }}
+      {{- if (eq (len .Secrets) 0) }}
+      <tr><th colspan="7">No Secrets found</th></tr>
+      {{- else }}
+      <tr class="sub-header">
+        <th colspan="2">Rule ID</th>
+        <th>Severity</th>
+        <th>Category</th>
+        <th colspan="2">Title</th>
+        <th>Lines</th>
+      </tr>
+      {{- range .Secrets }}
+      <tr class="severity-{{ escapeXML .Severity }}">
+        <td colspan="2">{{ escapeXML .RuleID }}</td>
+        <td class="severity">{{ escapeXML .Severity }}</td>
+        <td>{{ .Category | toString | escapeXML }}</td>
+        <td colspan="2">{{ escapeXML .Title }}</td>
+        <td>{{ .StartLine | toString }} - {{ .EndLine | toString }}</td>
+      </tr>
+      {{- end }}
+      {{- end }}
     {{- end }}
     </table>
 {{- else }}