From d9320daa34dab4f9cde1ca25c66f90748dc19e3f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 25 Jun 2024 17:16:10 +0000 Subject: [PATCH] Squashed '.ci/' changes from 63421d1..2c44e4f 2c44e4f Disable concurrent builds 7144a42 Improve Trivy scanning logic c1a48a6 Remove auto stash push / pop as being too dangerous 318c19e Add merge comment for subtree 22ed100 Fix custom branch docker tags 227e39f Allow custom GIT_TAG 38a9cda Debug CI pipeline 3efcc81 Debug CI pipeline 5023473 Make branch detection work for tagged commits cdc32e0 Improve cleanup flow 8df60af Fix derp 748a4bd Migrate to :: to allow custom make steps, add generic stubs 955afa7 Apply pep8 5819ded Improve ECR public lifecycle handling via python script 5d4e4ad Make rm-remote-untagged less noisy f00e541 Add cleanup step to remove untagged images by default 0821e91 Ensure tag names are valid for remote branches like PRs 79eebe4 add ARCH support for tests aea1ccc Only add branch name to tags, if not part of actual tag a5875db Make EXTRA_TAGS work again git-subtree-dir: .ci git-subtree-split: 2c44e4fd8550d30fba503a2bcccec8e0bac1c151 --- ecr_public_lifecycle.py | 63 +++++++++++++++++++++++++++++++++++++++++ podman.mk | 53 +++++++++++++++++----------------- vars/buildPodman.groovy | 47 +++++++++++++++++++----------- 3 files changed, 120 insertions(+), 43 deletions(-) create mode 100755 ecr_public_lifecycle.py diff --git a/ecr_public_lifecycle.py b/ecr_public_lifecycle.py new file mode 100755 index 0000000..7397dc4 --- /dev/null +++ b/ecr_public_lifecycle.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +import argparse +import boto3 + +parser = argparse.ArgumentParser( + description='Implement basic public ECR lifecycle policy') +parser.add_argument('--repo', dest='repositoryName', action='store', required=True, + help='Name of the public ECR repository') +parser.add_argument('--keep', dest='keep', action='store', default=10, type=int, + help='number of tagged images to keep, default 10') +parser.add_argument('--dev', dest='delete_dev', action='store_true', + help='also delete in-development images only having tags like v0.1.1-commitNr-githash') + +args = parser.parse_args() + +client = boto3.client('ecr-public', region_name='us-east-1') + +images = client.describe_images(repositoryName=args.repositoryName)[ + "imageDetails"] + +untagged = [] +kept = 0 + +# actual Image +# imageManifestMediaType: 'application/vnd.oci.image.manifest.v1+json' +# image Index +# imageManifestMediaType: 'application/vnd.oci.image.index.v1+json' + +# Sort by date uploaded +for image in sorted(images, key=lambda d: d['imagePushedAt'], reverse=True): + # Remove all untagged + # if registry uses image index all actual images will be untagged anyways + if 'imageTags' not in image: + untagged.append({"imageDigest": image['imageDigest']}) + # print("Delete untagged image {}".format(image["imageDigest"])) + continue + + # check for dev tags + if args.delete_dev: + _delete = True + for tag in image["imageTags"]: + # Look for at least one tag NOT beign a SemVer dev tag + if "-" not in tag: + _delete = False + if _delete: + print("Deleting development image {}".format(image["imageTags"])) + untagged.append({"imageDigest": image['imageDigest']}) + continue + + if kept < args.keep: + kept = kept+1 + print("Keeping tagged image {}".format(image["imageTags"])) + continue + else: + print("Deleting tagged image {}".format(image["imageTags"])) + untagged.append({"imageDigest": image['imageDigest']}) + +deleted_images = client.batch_delete_image( + repositoryName=args.repositoryName, imageIds=untagged) + +if deleted_images["imageIds"]: + print("Deleted images: {}".format(deleted_images["imageIds"])) diff --git a/podman.mk b/podman.mk index 2ab4469..b51b762 100644 --- a/podman.mk +++ b/podman.mk @@ -1,24 +1,26 @@ # Parse version from latest git semver tag +GIT_TAG ?= $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null) GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) -GIT_TAG := $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null) +TAG ::= $(GIT_TAG) # append branch name to tag if NOT main nor master -TAG := $(GIT_TAG) ifeq (,$(filter main master, $(GIT_BRANCH))) - ifneq ($(GIT_TAG), $(GIT_BRANCH)) - TAG = $(GIT_TAG)-$(GIT_BRANCH) + # If branch is substring of tag, omit branch name + ifeq ($(findstring $(GIT_BRANCH), $(GIT_TAG)),) + # only append branch name if not equal tag + ifneq ($(GIT_TAG), $(GIT_BRANCH)) + # Sanitize GIT_BRANCH to allowed Docker tag character set + TAG = $(GIT_TAG)-$(shell echo $$GIT_BRANCH | sed -e 's/[^a-zA-Z0-9]/-/g') + endif endif endif -# optionally set by the caller -EXTRA_TAGS := - -ARCH := amd64 -ALL_ARCHS := amd64 arm64 +ARCH ::= amd64 +ALL_ARCHS ::= amd64 arm64 _ARCH = $(or $(filter $(ARCH),$(ALL_ARCHS)),$(error $$ARCH [$(ARCH)] must be exactly one of "$(ALL_ARCHS)")) ifneq ($(TRIVY_REMOTE),) - TRIVY_OPTS := --server $(TRIVY_REMOTE) + TRIVY_OPTS ::= --server $(TRIVY_REMOTE) endif .SILENT: ; # no need for @ @@ -31,18 +33,20 @@ endif help: ## Show Help grep -E '^[a-zA-Z_-]+:.*?## .*$$' .ci/podman.mk | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +prepare:: ## custom step on the build agent before building + +fmt:: ## auto format source + +lint:: ## Lint source + build: ## Build the app buildah build --rm --layers -t $(IMAGE):$(TAG)-$(_ARCH) --build-arg TAG=$(TAG) --build-arg ARCH=$(_ARCH) --platform linux/$(_ARCH) . -test: rm-test-image ## Execute Dockerfile.test - test -f Dockerfile.test && \ - { buildah build --rm --layers -t $(REGISTRY)/$(IMAGE):$(TAG)-test --from=$(REGISTRY)/$(IMAGE):$(TAG) -f Dockerfile.test --platform linux/$(_ARCH) . && \ - podman run --rm --env-host -t $(REGISTRY)/$(IMAGE):$(TAG)-$(_ARCH)-test; } || \ - echo "No Dockerfile.test found, skipping test" +test:: ## test built artificats scan: ## Scan image using trivy echo "Scanning $(IMAGE):$(TAG)-$(_ARCH) using Trivy $(TRIVY_REMOTE)" - trivy image $(TRIVY_OPTS) localhost/$(IMAGE):$(TAG)-$(_ARCH) + trivy image $(TRIVY_OPTS) --quiet --no-progress localhost/$(IMAGE):$(TAG)-$(_ARCH) # first tag and push all actual images # create new manifest for each tag and add all available TAG-ARCH before pushing @@ -62,24 +66,19 @@ push: ecr-login ## push images to registry ecr-login: ## log into AWS ECR public aws ecr-public get-login-password --region $(REGION) | podman login --username AWS --password-stdin $(REGISTRY) -clean: rm-test-image rm-image ## delete local built container and test images +rm-remote-untagged: ## delete all remote untagged and in-dev images, keep 10 tagged + echo "Removing all untagged and in-dev images from $(IMAGE) in $(REGION)" + .ci/ecr_public_lifecycle.py --repo $(IMAGE) --dev -rm-remote-untagged: ## delete all remote untagged images - echo "Removing all untagged images from $(IMAGE) in $(REGION)" - IMAGE_IDS=$$(for image in $$(aws ecr-public describe-images --repository-name $(IMAGE) --region $(REGION) --output json | jq -r '.imageDetails[] | select(.imageTags | not ).imageDigest'); do echo -n "imageDigest=$$image "; done) ; \ - [ -n "$$IMAGE_IDS" ] && aws ecr-public batch-delete-image --repository-name $(IMAGE) --region $(REGION) --image-ids $$IMAGE_IDS || echo "No image to remove" +clean:: ## clean up source folder rm-image: test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH) > /dev/null test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || echo "Error: Removing image failed" -# Ensure we run the tests by removing any previous runs -rm-test-image: - test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH)-test > /dev/null - test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || echo "Error: Removing test image failed" - +## some useful tasks during development ci-pull-upstream: ## pull latest shared .ci subtree - git stash && git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash && git stash pop + git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash -m "Merge latest ci-tools-lib" create-repo: ## create new AWS ECR public repository aws ecr-public create-repository --repository-name $(IMAGE) --region $(REGION) diff --git a/vars/buildPodman.groovy b/vars/buildPodman.groovy index e5ecff9..f1e8756 100644 --- a/vars/buildPodman.groovy +++ b/vars/buildPodman.groovy @@ -2,6 +2,9 @@ def call(Map config=[:]) { pipeline { + options { + disableConcurrentBuilds() + } agent { node { label 'podman-aws-trivy' @@ -10,18 +13,22 @@ def call(Map config=[:]) { stages { stage('Prepare') { steps { + sh 'mkdir -p reports' + + // we set pull tags as project adv. options // pull tags - withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) { - sh 'git fetch -q --tags ${GIT_URL}' - } - sh 'make prepare || true' + //withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) { + // sh 'git fetch -q --tags ${GIT_URL}' + //} + // Optional project specific preparations + sh 'make prepare' } } // Build using rootless podman stage('Build') { steps { - sh 'make build' + sh 'make build GIT_BRANCH=$GIT_BRANCH' } } @@ -33,12 +40,13 @@ def call(Map config=[:]) { // Scan via trivy stage('Scan') { - environment { - TRIVY_FORMAT = "template" - TRIVY_OUTPUT = "reports/trivy.html" - } steps { - sh 'mkdir -p reports && make scan' + // we always scan and create the full json report + sh 'TRIVY_FORMAT=json TRIVY_OUTPUT="reports/trivy.json" make scan' + + // render custom full html report + sh 'trivy convert -f template -t @/home/jenkins/html.tpl -o reports/trivy.html reports/trivy.json' + publishHTML target: [ allowMissing: true, alwaysLinkToLastBuild: true, @@ -48,26 +56,33 @@ def call(Map config=[:]) { reportName: 'TrivyScan', reportTitles: 'TrivyScan' ] + sh 'echo "Trivy report at: $BUILD_URL/TrivyScan"' - // Scan again and fail on CRITICAL vulns, if not overridden + // fail build if issues found above trivy threshold script { - if (config.trivyFail == 'NONE') { - echo 'trivyFail == NONE, review Trivy report manually. Proceeding ...' - } else { - sh "TRIVY_EXIT_CODE=1 TRIVY_SEVERITY=${config.trivyFail} make scan" + if ( config.trivyFail ) { + sh "TRIVY_SEVERITY=${config.trivyFail} trivy convert --report summary --exit-code 1 reports/trivy.json" } } } } - // Push to container registry, skip if PR + // Push to container registry if not PR + // incl. basic registry retention removing any untagged images stage('Push') { when { not { changeRequest() } } steps { sh 'make push' + sh 'make rm-remote-untagged' } } + // generic clean + stage('cleanup') { + steps { + sh 'make clean' + } + } } } }