From 80550a76c6d6d875d4a0915e9bc48f73b3565dcd Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 18 Aug 2023 15:19:17 +0000 Subject: [PATCH] Squashed '.ci/' changes from 79eebe4..318c19e 318c19e Add merge comment for subtree 22ed100 Fix custom branch docker tags 227e39f Allow custom GIT_TAG 38a9cda Debug CI pipeline 3efcc81 Debug CI pipeline 5023473 Make branch detection work for tagged commits cdc32e0 Improve cleanup flow 8df60af Fix derp 748a4bd Migrate to :: to allow custom make steps, add generic stubs 955afa7 Apply pep8 5819ded Improve ECR public lifecycle handling via python script 5d4e4ad Make rm-remote-untagged less noisy f00e541 Add cleanup step to remove untagged images by default 0821e91 Ensure tag names are valid for remote branches like PRs git-subtree-dir: .ci git-subtree-split: 318c19e5d7ac0fd5675a13652650faa8aced67f9 --- ecr_public_lifecycle.py | 63 +++++++++++++++++++++++++++++++++++++++++ podman.mk | 42 +++++++++++++-------------- vars/buildPodman.groovy | 22 ++++++++++---- 3 files changed, 99 insertions(+), 28 deletions(-) create mode 100755 ecr_public_lifecycle.py diff --git a/ecr_public_lifecycle.py b/ecr_public_lifecycle.py new file mode 100755 index 0000000..7397dc4 --- /dev/null +++ b/ecr_public_lifecycle.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +import argparse +import boto3 + +parser = argparse.ArgumentParser( + description='Implement basic public ECR lifecycle policy') +parser.add_argument('--repo', dest='repositoryName', action='store', required=True, + help='Name of the public ECR repository') +parser.add_argument('--keep', dest='keep', action='store', default=10, type=int, + help='number of tagged images to keep, default 10') +parser.add_argument('--dev', dest='delete_dev', action='store_true', + help='also delete in-development images only having tags like v0.1.1-commitNr-githash') + +args = parser.parse_args() + +client = boto3.client('ecr-public', region_name='us-east-1') + +images = client.describe_images(repositoryName=args.repositoryName)[ + "imageDetails"] + +untagged = [] +kept = 0 + +# actual Image +# imageManifestMediaType: 'application/vnd.oci.image.manifest.v1+json' +# image Index +# imageManifestMediaType: 'application/vnd.oci.image.index.v1+json' + +# Sort by date uploaded +for image in sorted(images, key=lambda d: d['imagePushedAt'], reverse=True): + # Remove all untagged + # if registry uses image index all actual images will be untagged anyways + if 'imageTags' not in image: + untagged.append({"imageDigest": image['imageDigest']}) + # print("Delete untagged image {}".format(image["imageDigest"])) + continue + + # check for dev tags + if args.delete_dev: + _delete = True + for tag in image["imageTags"]: + # Look for at least one tag NOT beign a SemVer dev tag + if "-" not in tag: + _delete = False + if _delete: + print("Deleting development image {}".format(image["imageTags"])) + untagged.append({"imageDigest": image['imageDigest']}) + continue + + if kept < args.keep: + kept = kept+1 + print("Keeping tagged image {}".format(image["imageTags"])) + continue + else: + print("Deleting tagged image {}".format(image["imageTags"])) + untagged.append({"imageDigest": image['imageDigest']}) + +deleted_images = client.batch_delete_image( + repositoryName=args.repositoryName, imageIds=untagged) + +if deleted_images["imageIds"]: + print("Deleted images: {}".format(deleted_images["imageIds"])) diff --git a/podman.mk b/podman.mk index 3554fd3..02e0475 100644 --- a/podman.mk +++ b/podman.mk @@ -1,25 +1,26 @@ # Parse version from latest git semver tag +GIT_TAG ?= $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null) GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) -GIT_TAG := $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null) -TAG := $(GIT_TAG) +TAG ::= $(GIT_TAG) # append branch name to tag if NOT main nor master ifeq (,$(filter main master, $(GIT_BRANCH))) # If branch is substring of tag, omit branch name ifeq ($(findstring $(GIT_BRANCH), $(GIT_TAG)),) # only append branch name if not equal tag ifneq ($(GIT_TAG), $(GIT_BRANCH)) - TAG = $(GIT_TAG)-$(GIT_BRANCH) + # Sanitize GIT_BRANCH to allowed Docker tag character set + TAG = $(GIT_TAG)-$(shell echo $$GIT_BRANCH | sed -e 's/[^a-zA-Z0-9]/-/g') endif endif endif -ARCH := amd64 -ALL_ARCHS := amd64 arm64 +ARCH ::= amd64 +ALL_ARCHS ::= amd64 arm64 _ARCH = $(or $(filter $(ARCH),$(ALL_ARCHS)),$(error $$ARCH [$(ARCH)] must be exactly one of "$(ALL_ARCHS)")) ifneq ($(TRIVY_REMOTE),) - TRIVY_OPTS := --server $(TRIVY_REMOTE) + TRIVY_OPTS ::= --server $(TRIVY_REMOTE) endif .SILENT: ; # no need for @ @@ -32,14 +33,16 @@ endif help: ## Show Help grep -E '^[a-zA-Z_-]+:.*?## .*$$' .ci/podman.mk | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +prepare:: ## custom step on the build agent before building + +fmt:: ## auto format source + +lint:: ## Lint source + build: ## Build the app buildah build --rm --layers -t $(IMAGE):$(TAG)-$(_ARCH) --build-arg TAG=$(TAG) --build-arg ARCH=$(_ARCH) --platform linux/$(_ARCH) . -test: rm-test-image ## Execute Dockerfile.test - test -f Dockerfile.test && \ - { buildah build --rm --layers -t $(REGISTRY)/$(IMAGE):$(TAG)-$(_ARCH)-test --from=$(REGISTRY)/$(IMAGE):$(TAG) -f Dockerfile.test --platform linux/$(_ARCH) . && \ - podman run --rm --env-host -t $(REGISTRY)/$(IMAGE):$(TAG)-$(_ARCH)-test; } || \ - echo "No Dockerfile.test found, skipping test" +test:: ## test built artificats scan: ## Scan image using trivy echo "Scanning $(IMAGE):$(TAG)-$(_ARCH) using Trivy $(TRIVY_REMOTE)" @@ -63,24 +66,19 @@ push: ecr-login ## push images to registry ecr-login: ## log into AWS ECR public aws ecr-public get-login-password --region $(REGION) | podman login --username AWS --password-stdin $(REGISTRY) -clean: rm-test-image rm-image ## delete local built container and test images +rm-remote-untagged: ## delete all remote untagged and in-dev images, keep 10 tagged + echo "Removing all untagged and in-dev images from $(IMAGE) in $(REGION)" + .ci/ecr_public_lifecycle.py --repo $(IMAGE) --dev -rm-remote-untagged: ## delete all remote untagged images - echo "Removing all untagged images from $(IMAGE) in $(REGION)" - IMAGE_IDS=$$(for image in $$(aws ecr-public describe-images --repository-name $(IMAGE) --region $(REGION) --output json | jq -r '.imageDetails[] | select(.imageTags | not ).imageDigest'); do echo -n "imageDigest=$$image "; done) ; \ - [ -n "$$IMAGE_IDS" ] && aws ecr-public batch-delete-image --repository-name $(IMAGE) --region $(REGION) --image-ids $$IMAGE_IDS || echo "No image to remove" +clean:: ## clean up source folder rm-image: test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH) > /dev/null test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || echo "Error: Removing image failed" -# Ensure we run the tests by removing any previous runs -rm-test-image: - test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH)-test > /dev/null - test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || echo "Error: Removing test image failed" - +## some useful tasks during development ci-pull-upstream: ## pull latest shared .ci subtree - git stash && git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash && git stash pop + git stash && git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash -m "Merge latest ci-tools-lib" && git stash pop create-repo: ## create new AWS ECR public repository aws ecr-public create-repository --repository-name $(IMAGE) --region $(REGION) diff --git a/vars/buildPodman.groovy b/vars/buildPodman.groovy index e5ecff9..e33c2cf 100644 --- a/vars/buildPodman.groovy +++ b/vars/buildPodman.groovy @@ -10,18 +10,20 @@ def call(Map config=[:]) { stages { stage('Prepare') { steps { + // we set pull tags as project adv. options // pull tags - withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) { - sh 'git fetch -q --tags ${GIT_URL}' - } - sh 'make prepare || true' + //withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) { + // sh 'git fetch -q --tags ${GIT_URL}' + //} + // Optional project specific preparations + sh 'make prepare' } } // Build using rootless podman stage('Build') { steps { - sh 'make build' + sh 'make build GIT_BRANCH=$GIT_BRANCH' } } @@ -60,14 +62,22 @@ def call(Map config=[:]) { } } - // Push to container registry, skip if PR + // Push to container registry if not PR + // incl. basic registry retention removing any untagged images stage('Push') { when { not { changeRequest() } } steps { sh 'make push' + sh 'make rm-remote-untagged' } } + // generic clean + stage('cleanup') { + steps { + sh 'make clean' + } + } } } }