.ci/ecr_public_lifecycle.py

@@ -1,63 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import boto3
-
-parser = argparse.ArgumentParser(
-    description='Implement basic public ECR lifecycle policy')
-parser.add_argument('--repo', dest='repositoryName', action='store', required=True,
-                    help='Name of the public ECR repository')
-parser.add_argument('--keep', dest='keep', action='store', default=10, type=int,
-                    help='number of tagged images to keep, default 10')
-parser.add_argument('--dev', dest='delete_dev', action='store_true',
-                    help='also delete in-development images only having tags like v0.1.1-commitNr-githash')
-
-args = parser.parse_args()
-
-client = boto3.client('ecr-public', region_name='us-east-1')
-
-images = client.describe_images(repositoryName=args.repositoryName)[
-    "imageDetails"]
-
-untagged = []
-kept = 0
-
-# actual Image
-# imageManifestMediaType: 'application/vnd.oci.image.manifest.v1+json'
-# image Index
-# imageManifestMediaType: 'application/vnd.oci.image.index.v1+json'
-
-# Sort by date uploaded
-for image in sorted(images, key=lambda d: d['imagePushedAt'], reverse=True):
-    # Remove all untagged
-    # if registry uses image index all actual images will be untagged anyways
-    if 'imageTags' not in image:
-        untagged.append({"imageDigest": image['imageDigest']})
-        # print("Delete untagged image {}".format(image["imageDigest"]))
-        continue
-
-    # check for dev tags
-    if args.delete_dev:
-        _delete = True
-        for tag in image["imageTags"]:
-            # Look for at least one tag NOT beign a SemVer dev tag
-            if "-" not in tag:
-                _delete = False
-        if _delete:
-            print("Deleting development image {}".format(image["imageTags"]))
-            untagged.append({"imageDigest": image['imageDigest']})
-            continue
-
-    if kept < args.keep:
-        kept = kept+1
-        print("Keeping tagged image {}".format(image["imageTags"]))
-        continue
-    else:
-        print("Deleting tagged image {}".format(image["imageTags"]))
-        untagged.append({"imageDigest": image['imageDigest']})
-
-deleted_images = client.batch_delete_image(
-    repositoryName=args.repositoryName, imageIds=untagged)
-
-if deleted_images["imageIds"]:
-    print("Deleted images: {}".format(deleted_images["imageIds"]))

.ci/podman.mk

@@ -1,84 +1,62 @@
 # Parse version from latest git semver tag
-GIT_TAG ?= $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)
+GTAG=$(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)
-GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
+TAG ?= $(shell echo $(GTAG) | awk -F '-' '{ print $$1 "-" $$2 }' | sed -e 's/-$$//')
 
-TAG ::= $(GIT_TAG)
+# EXTRA_TAGS supposed to be set at the caller, eg. $(shell echo $(TAG) | awk -F '.' '{ print $$1 "." $$2 }')
-# append branch name to tag if NOT main nor master
-ifeq (,$(filter main master, $(GIT_BRANCH)))
-	# If branch is substring of tag, omit branch name
-	ifeq ($(findstring $(GIT_BRANCH), $(GIT_TAG)),)
-		# only append branch name if not equal tag
-		ifneq ($(GIT_TAG), $(GIT_BRANCH))
-			# Sanitize GIT_BRANCH to allowed Docker tag character set
-			TAG = $(GIT_TAG)-$(shell echo $$GIT_BRANCH | sed -e 's/[^a-zA-Z0-9]/-/g')
-		endif
-	endif
-endif
-
-ARCH ::= amd64
-ALL_ARCHS ::= amd64 arm64
-_ARCH = $(or $(filter $(ARCH),$(ALL_ARCHS)),$(error $$ARCH [$(ARCH)] must be exactly one of "$(ALL_ARCHS)"))
 
 ifneq ($(TRIVY_REMOTE),)
-	TRIVY_OPTS ::= --server $(TRIVY_REMOTE)
+  TRIVY_OPTS := --server ${TRIVY_REMOTE}
 endif
 
-.SILENT: ; # no need for @
+.PHONY: build test scan push clean
-.ONESHELL: ; # recipes execute in same shell
-.NOTPARALLEL: ; # wait for this target to finish
-.EXPORT_ALL_VARIABLES: ; # send all vars to shell
-.PHONY: all # All targets are accessible for user
-.DEFAULT: help # Running Make will run the help target
 
-help: ## Show Help
+all: test
-	grep -E '^[a-zA-Z_-]+:.*?## .*$$' .ci/podman.mk | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-prepare:: ## custom step on the build agent before building
+build:
+	@docker image exists $(REGISTRY)/$(IMAGE):$(TAG) || \
+		docker build --rm -t $(REGISTRY)/$(IMAGE):$(TAG) --build-arg TAG=$(TAG) .
 
-fmt:: ## auto format source
+test: build rm-test-image
+	@test -f Dockerfile.test && \
+		{ docker build --rm -t $(REGISTRY)/$(IMAGE):$(TAG)-test --from=$(REGISTRY)/$(IMAGE):$(TAG) -f Dockerfile.test . && \
+			docker run --rm --env-host -t $(REGISTRY)/$(IMAGE):$(TAG)-test; } || \
+		echo "No Dockerfile.test found, skipping test"
 
-lint:: ## Lint source
+scan: build
+	@echo "Scanning $(REGISTRY)/$(IMAGE):$(TAG) using Trivy"
+	@trivy image $(TRIVY_OPTS) $(REGISTRY)/$(IMAGE):$(TAG)
 
-build: ## Build the app
+push: build
-	buildah build --rm --layers -t $(IMAGE):$(TAG)-$(_ARCH) --build-arg TAG=$(TAG) --build-arg ARCH=$(_ARCH) --platform linux/$(_ARCH) .
+	@aws ecr-public get-login-password --region $(REGION) | docker login --username AWS --password-stdin $(REGISTRY)
+	@for t in $(TAG) latest $(EXTRA_TAGS); do echo "tag and push: $$t"; docker tag $(IMAGE):$(TAG) $(REGISTRY)/$(IMAGE):$$t && docker push $(REGISTRY)/$(IMAGE):$$t; done
 
-test:: ## test built artificats
+clean: rm-test-image rm-image
 
-scan: ## Scan image using trivy
+# Delete all untagged images
-	echo "Scanning $(IMAGE):$(TAG)-$(_ARCH) using Trivy $(TRIVY_REMOTE)"
+.PHONY: rm-remote-untagged
-	trivy image $(TRIVY_OPTS) --quiet --no-progress localhost/$(IMAGE):$(TAG)-$(_ARCH)
+rm-remote-untagged:
-
+	@echo "Removing all untagged images from $(IMAGE) in $(REGION)"
-# first tag and push all actual images
+	@aws ecr-public batch-delete-image --repository-name $(IMAGE) --region $(REGION) --image-ids $$(for image in $$(aws ecr-public describe-images --repository-name $(IMAGE) --region $(REGION) --output json | jq -r '.imageDetails[] | select(.imageTags | not ).imageDigest'); do echo -n "imageDigest=$$image "; done)
-# create new manifest for each tag and add all available TAG-ARCH before pushing
-push: ecr-login ## push images to registry
-	for t in $(TAG) latest $(EXTRA_TAGS); do \
-		echo "Tagging image with $(REGISTRY)/$(IMAGE):$${t}-$(ARCH)"
-		buildah tag $(IMAGE):$(TAG)-$(_ARCH) $(REGISTRY)/$(IMAGE):$${t}-$(_ARCH); \
-		buildah manifest rm $(IMAGE):$$t || true; \
-		buildah manifest create $(IMAGE):$$t; \
-		for a in $(ALL_ARCHS); do \
-			buildah manifest add $(IMAGE):$$t $(REGISTRY)/$(IMAGE):$(TAG)-$$a; \
-		done; \
-		echo "Pushing manifest $(IMAGE):$$t"
-		buildah manifest push --all $(IMAGE):$$t docker://$(REGISTRY)/$(IMAGE):$$t; \
-	done
-
-ecr-login: ## log into AWS ECR public
-	aws ecr-public get-login-password --region $(REGION) | podman login --username AWS --password-stdin $(REGISTRY)
-
-rm-remote-untagged: ## delete all remote untagged and in-dev images, keep 10 tagged
-	echo "Removing all untagged and in-dev images from $(IMAGE) in $(REGION)"
-	.ci/ecr_public_lifecycle.py --repo $(IMAGE) --dev
-
-clean:: ## clean up source folder
 
+.PHONY: rm-image
 rm-image:
-	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH) > /dev/null
+	@test -z "$$(docker image ls -q $(IMAGE):$(TAG))" || docker image rm -f $(IMAGE):$(TAG) > /dev/null
-	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || echo "Error: Removing image failed"
+	@test -z "$$(docker image ls -q $(IMAGE):$(TAG))" || echo "Error: Removing image failed"
 
-## some useful tasks during development
+# Ensure we run the tests by removing any previous runs
-ci-pull-upstream: ## pull latest shared .ci subtree
+.PHONY: rm-test-image
-	git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash -m "Merge latest ci-tools-lib"
+rm-test-image:
+	@test -z "$$(docker image ls -q $(IMAGE):$(TAG)-test)" || docker image rm -f $(IMAGE):$(TAG)-test > /dev/null
+	@test -z "$$(docker image ls -q $(IMAGE):$(TAG)-test)" || echo "Error: Removing test image failed"
 
-create-repo: ## create new AWS ECR public repository
+# Convience task during dev of downstream projects
+.PHONY: ci-pull-upstream
+ci-pull-upstream:
+	git stash && git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash && git stash pop
+
+.PHONY: create-repo
+create-repo:
 	aws ecr-public create-repository --repository-name $(IMAGE) --region $(REGION)
+
+.DEFAULT:
+	@echo "$@ not implemented. NOOP"

.ci/vars/buildPodman.groovy

@@ -2,33 +2,24 @@
 
 def call(Map config=[:]) {
     pipeline {
-      options {
-        disableConcurrentBuilds()
-      }
       agent {
         node {
           label 'podman-aws-trivy'
         }
       }
+
       stages {
         stage('Prepare') {
+          // get tags
           steps {
-            sh 'mkdir -p reports'
+            sh 'git fetch -q --tags ${GIT_URL} +refs/heads/${BRANCH_NAME}:refs/remotes/origin/${BRANCH_NAME}'
-
-            // we set pull tags as project adv. options
-            // pull tags
-            //withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) {
-            //  sh 'git fetch -q --tags ${GIT_URL}'
-            //}
-            // Optional project specific preparations
-            sh 'make prepare'
           }
         }
 
         // Build using rootless podman
         stage('Build') {
           steps {
-            sh 'make build GIT_BRANCH=$GIT_BRANCH'
+            sh 'make build'
           }
         }
 
@@ -40,13 +31,13 @@ def call(Map config=[:]) {
 
         // Scan via trivy
         stage('Scan') {
+          environment {
+            TRIVY_FORMAT = "template"
+            TRIVY_OUTPUT = "reports/trivy.html"
+          }
           steps {
-            // we always scan and create the full json report
+            sh 'mkdir -p reports'
-            sh 'TRIVY_FORMAT=json TRIVY_OUTPUT="reports/trivy.json" make scan'
+            sh 'make scan'
-
-            // render custom full html report
-            sh 'trivy convert -f template -t @/home/jenkins/html.tpl -o reports/trivy.html reports/trivy.json'
-
             publishHTML target: [
               allowMissing: true,
               alwaysLinkToLastBuild: true,
@@ -56,33 +47,25 @@ def call(Map config=[:]) {
               reportName: 'TrivyScan',
               reportTitles: 'TrivyScan'
             ]
-            sh 'echo "Trivy report at: $BUILD_URL/TrivyScan"'
 
-            // fail build if issues found above trivy threshold
+            // Scan again and fail on CRITICAL vulns, if not overridden
             script {
-              if ( config.trivyFail ) {
+              if (config.trivyFail == 'NONE') {
-                sh "TRIVY_SEVERITY=${config.trivyFail} trivy convert --report summary --exit-code 1 reports/trivy.json"
+                echo 'trivyFail == NONE, review Trivy report manually. Proceeding ...'
+              } else {
+                sh "TRIVY_EXIT_CODE=1 TRIVY_SEVERITY=${config.trivyFail} make scan"
               }
             }
           }
         }
 
-        // Push to container registry if not PR
+        // Push to ECR
-        // incl. basic registry retention removing any untagged images
         stage('Push') {
-          when { not { changeRequest() } }
           steps {
             sh 'make push'
-            sh 'make rm-remote-untagged'
           }
         }
 
-        // generic clean
-        stage('cleanup') {
-          steps {
-            sh 'make clean'
-          }
-        }
       }
     }
   }

Dockerfile

@@ -1,4 +1,4 @@
-ARG ALPINE_VERSION=3.19
+ARG ALPINE_VERSION=3.16
 
 FROM alpine:${ALPINE_VERSION}
 
@@ -12,7 +12,7 @@ RUN cd /etc/apk/keys && \
       ca-certificates \
       docker-registry@kubezero
 
-USER docker-registry
+USER docker-registry 
 
 VOLUME ["/var/lib/registry"]
 EXPOSE 5000

Jenkinsfile

@@ -1,5 +0,0 @@
-library identifier: 'zdt-lib@master', retriever: modernSCM(
-  [$class: 'GitSCMSource',
-   remote: 'https://git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git'])
-
-buildPodman name: 'docker-repository'

renovate.json

@@ -1,10 +0,0 @@
-{
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": [
-    "config:recommended",
-    ":label(renovate)",
-    ":semanticCommits",
-    "group:allNonMajor"
-  ],
-  "prHourlyLimit": 0
-}