Squashed '.ci/' changes from 63421d1..2c44e4f

2c44e4f Disable concurrent builds
7144a42 Improve Trivy scanning logic
c1a48a6 Remove auto stash push / pop as being too dangerous
318c19e Add merge comment for subtree
22ed100 Fix custom branch docker tags
227e39f Allow custom GIT_TAG
38a9cda Debug CI pipeline
3efcc81 Debug CI pipeline
5023473 Make branch detection work for tagged commits
cdc32e0 Improve cleanup flow
8df60af Fix derp
748a4bd Migrate to :: to allow custom make steps, add generic stubs
955afa7 Apply pep8
5819ded Improve ECR public lifecycle handling via python script
5d4e4ad Make rm-remote-untagged less noisy
f00e541 Add cleanup step to remove untagged images by default
0821e91 Ensure tag names are valid for remote branches like PRs
79eebe4 add ARCH support for tests
aea1ccc Only add branch name to tags, if not part of actual tag
a5875db Make EXTRA_TAGS work again

git-subtree-dir: .ci
git-subtree-split: 2c44e4fd8550d30fba503a2bcccec8e0bac1c151

ecr_public_lifecycle.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+import argparse
+import boto3
+
+parser = argparse.ArgumentParser(
+    description='Implement basic public ECR lifecycle policy')
+parser.add_argument('--repo', dest='repositoryName', action='store', required=True,
+                    help='Name of the public ECR repository')
+parser.add_argument('--keep', dest='keep', action='store', default=10, type=int,
+                    help='number of tagged images to keep, default 10')
+parser.add_argument('--dev', dest='delete_dev', action='store_true',
+                    help='also delete in-development images only having tags like v0.1.1-commitNr-githash')
+
+args = parser.parse_args()
+
+client = boto3.client('ecr-public', region_name='us-east-1')
+
+images = client.describe_images(repositoryName=args.repositoryName)[
+    "imageDetails"]
+
+untagged = []
+kept = 0
+
+# actual Image
+# imageManifestMediaType: 'application/vnd.oci.image.manifest.v1+json'
+# image Index
+# imageManifestMediaType: 'application/vnd.oci.image.index.v1+json'
+
+# Sort by date uploaded
+for image in sorted(images, key=lambda d: d['imagePushedAt'], reverse=True):
+    # Remove all untagged
+    # if registry uses image index all actual images will be untagged anyways
+    if 'imageTags' not in image:
+        untagged.append({"imageDigest": image['imageDigest']})
+        # print("Delete untagged image {}".format(image["imageDigest"]))
+        continue
+
+    # check for dev tags
+    if args.delete_dev:
+        _delete = True
+        for tag in image["imageTags"]:
+            # Look for at least one tag NOT beign a SemVer dev tag
+            if "-" not in tag:
+                _delete = False
+        if _delete:
+            print("Deleting development image {}".format(image["imageTags"]))
+            untagged.append({"imageDigest": image['imageDigest']})
+            continue
+
+    if kept < args.keep:
+        kept = kept+1
+        print("Keeping tagged image {}".format(image["imageTags"]))
+        continue
+    else:
+        print("Deleting tagged image {}".format(image["imageTags"]))
+        untagged.append({"imageDigest": image['imageDigest']})
+
+deleted_images = client.batch_delete_image(
+    repositoryName=args.repositoryName, imageIds=untagged)
+
+if deleted_images["imageIds"]:
+    print("Deleted images: {}".format(deleted_images["imageIds"]))

podman.mk

@@ -1,24 +1,26 @@
 # Parse version from latest git semver tag
+GIT_TAG ?= $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)
 GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
-GIT_TAG := $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)
 
+TAG ::= $(GIT_TAG)
 # append branch name to tag if NOT main nor master
-TAG := $(GIT_TAG)
 ifeq (,$(filter main master, $(GIT_BRANCH)))
-	ifneq ($(GIT_TAG), $(GIT_BRANCH))
+	# If branch is substring of tag, omit branch name
-		TAG = $(GIT_TAG)-$(GIT_BRANCH)
+	ifeq ($(findstring $(GIT_BRANCH), $(GIT_TAG)),)
+		# only append branch name if not equal tag
+		ifneq ($(GIT_TAG), $(GIT_BRANCH))
+			# Sanitize GIT_BRANCH to allowed Docker tag character set
+			TAG = $(GIT_TAG)-$(shell echo $$GIT_BRANCH | sed -e 's/[^a-zA-Z0-9]/-/g')
+		endif
 	endif
 endif
 
-# optionally set by the caller
+ARCH ::= amd64
-EXTRA_TAGS :=
+ALL_ARCHS ::= amd64 arm64
-
-ARCH := amd64
-ALL_ARCHS := amd64 arm64
 _ARCH = $(or $(filter $(ARCH),$(ALL_ARCHS)),$(error $$ARCH [$(ARCH)] must be exactly one of "$(ALL_ARCHS)"))
 
 ifneq ($(TRIVY_REMOTE),)
-	TRIVY_OPTS := --server $(TRIVY_REMOTE)
+	TRIVY_OPTS ::= --server $(TRIVY_REMOTE)
 endif
 
 .SILENT: ; # no need for @
@@ -31,18 +33,20 @@ endif
 help: ## Show Help
 	grep -E '^[a-zA-Z_-]+:.*?## .*$$' .ci/podman.mk | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
+prepare:: ## custom step on the build agent before building
+
+fmt:: ## auto format source
+
+lint:: ## Lint source
+
 build: ## Build the app
 	buildah build --rm --layers -t $(IMAGE):$(TAG)-$(_ARCH) --build-arg TAG=$(TAG) --build-arg ARCH=$(_ARCH) --platform linux/$(_ARCH) .
 
-test: rm-test-image ## Execute Dockerfile.test
+test:: ## test built artificats
-	test -f Dockerfile.test && \
-		{ buildah build --rm --layers -t $(REGISTRY)/$(IMAGE):$(TAG)-test --from=$(REGISTRY)/$(IMAGE):$(TAG) -f Dockerfile.test --platform linux/$(_ARCH) . && \
-			podman run --rm --env-host -t $(REGISTRY)/$(IMAGE):$(TAG)-$(_ARCH)-test; } || \
-		echo "No Dockerfile.test found, skipping test"
 
 scan: ## Scan image using trivy
 	echo "Scanning $(IMAGE):$(TAG)-$(_ARCH) using Trivy $(TRIVY_REMOTE)"
-	trivy image $(TRIVY_OPTS) localhost/$(IMAGE):$(TAG)-$(_ARCH)
+	trivy image $(TRIVY_OPTS) --quiet --no-progress localhost/$(IMAGE):$(TAG)-$(_ARCH)
 
 # first tag and push all actual images
 # create new manifest for each tag and add all available TAG-ARCH before pushing
@@ -62,24 +66,19 @@ push: ecr-login ## push images to registry
 ecr-login: ## log into AWS ECR public
 	aws ecr-public get-login-password --region $(REGION) | podman login --username AWS --password-stdin $(REGISTRY)
 
-clean: rm-test-image rm-image ## delete local built container and test images
+rm-remote-untagged: ## delete all remote untagged and in-dev images, keep 10 tagged
+	echo "Removing all untagged and in-dev images from $(IMAGE) in $(REGION)"
+	.ci/ecr_public_lifecycle.py --repo $(IMAGE) --dev
 
-rm-remote-untagged: ## delete all remote untagged images
+clean:: ## clean up source folder
-	echo "Removing all untagged images from $(IMAGE) in $(REGION)"
-	IMAGE_IDS=$$(for image in $$(aws ecr-public describe-images --repository-name $(IMAGE) --region $(REGION) --output json | jq -r '.imageDetails[] | select(.imageTags | not ).imageDigest'); do echo -n "imageDigest=$$image "; done) ; \
-		[ -n "$$IMAGE_IDS" ] && aws ecr-public batch-delete-image --repository-name $(IMAGE) --region $(REGION) --image-ids $$IMAGE_IDS || echo "No image to remove"
 
 rm-image:
 	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH) > /dev/null
 	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || echo "Error: Removing image failed"
 
-# Ensure we run the tests by removing any previous runs
+## some useful tasks during development
-rm-test-image:
-	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH)-test > /dev/null
-	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || echo "Error: Removing test image failed"
-
 ci-pull-upstream: ## pull latest shared .ci subtree
-	git stash && git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash && git stash pop
+	git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash -m "Merge latest ci-tools-lib"
 
 create-repo: ## create new AWS ECR public repository
 	aws ecr-public create-repository --repository-name $(IMAGE) --region $(REGION)

vars/buildPodman.groovy

@@ -2,6 +2,9 @@
 
 def call(Map config=[:]) {
     pipeline {
+      options {
+        disableConcurrentBuilds()
+      }
       agent {
         node {
           label 'podman-aws-trivy'
@@ -10,18 +13,22 @@ def call(Map config=[:]) {
       stages {
         stage('Prepare') {
           steps {
+            sh 'mkdir -p reports'
+
+            // we set pull tags as project adv. options
             // pull tags
-            withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) {
+            //withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) {
-              sh 'git fetch -q --tags ${GIT_URL}'
+            //  sh 'git fetch -q --tags ${GIT_URL}'
-            }
+            //}
-            sh 'make prepare || true'
+            // Optional project specific preparations
+            sh 'make prepare'
           }
         }
 
         // Build using rootless podman
         stage('Build') {
           steps {
-            sh 'make build'
+            sh 'make build GIT_BRANCH=$GIT_BRANCH'
           }
         }
 
@@ -33,12 +40,13 @@ def call(Map config=[:]) {
 
         // Scan via trivy
         stage('Scan') {
-          environment {
-            TRIVY_FORMAT = "template"
-            TRIVY_OUTPUT = "reports/trivy.html"
-          }
           steps {
-            sh 'mkdir -p reports && make scan'
+            // we always scan and create the full json report
+            sh 'TRIVY_FORMAT=json TRIVY_OUTPUT="reports/trivy.json" make scan'
+
+            // render custom full html report
+            sh 'trivy convert -f template -t @/home/jenkins/html.tpl -o reports/trivy.html reports/trivy.json'
+
             publishHTML target: [
               allowMissing: true,
               alwaysLinkToLastBuild: true,
@@ -48,26 +56,33 @@ def call(Map config=[:]) {
               reportName: 'TrivyScan',
               reportTitles: 'TrivyScan'
             ]
+            sh 'echo "Trivy report at: $BUILD_URL/TrivyScan"'
 
-            // Scan again and fail on CRITICAL vulns, if not overridden
+            // fail build if issues found above trivy threshold
             script {
-              if (config.trivyFail == 'NONE') {
+              if ( config.trivyFail ) {
-                echo 'trivyFail == NONE, review Trivy report manually. Proceeding ...'
+                sh "TRIVY_SEVERITY=${config.trivyFail} trivy convert --report summary --exit-code 1 reports/trivy.json"
-              } else {
-                sh "TRIVY_EXIT_CODE=1 TRIVY_SEVERITY=${config.trivyFail} make scan"
               }
             }
           }
         }
 
-        // Push to container registry, skip if PR
+        // Push to container registry if not PR
+        // incl. basic registry retention removing any untagged images
         stage('Push') {
           when { not { changeRequest() } }
           steps {
             sh 'make push'
+            sh 'make rm-remote-untagged'
           }
         }
 
+        // generic clean
+        stage('cleanup') {
+          steps {
+            sh 'make clean'
+          }
+        }
       }
     }
   }