From e355d2dff165d7a97951e65f02f68a07d61cdbf3 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 18 Aug 2023 11:53:14 +0000 Subject: [PATCH] Squashed '.ci/' changes from 47140c2..318c19e 318c19e Add merge comment for subtree 22ed100 Fix custom branch docker tags 227e39f Allow custom GIT_TAG 38a9cda Debug CI pipeline 3efcc81 Debug CI pipeline 5023473 Make branch detection work for tagged commits cdc32e0 Improve cleanup flow 8df60af Fix derp 748a4bd Migrate to :: to allow custom make steps, add generic stubs 955afa7 Apply pep8 5819ded Improve ECR public lifecycle handling via python script 5d4e4ad Make rm-remote-untagged less noisy f00e541 Add cleanup step to remove untagged images by default 0821e91 Ensure tag names are valid for remote branches like PRs 79eebe4 add ARCH support for tests aea1ccc Only add branch name to tags, if not part of actual tag a5875db Make EXTRA_TAGS work again 63421d1 fix: prevent branch_name equals tag git-subtree-dir: .ci git-subtree-split: 318c19e5d7ac0fd5675a13652650faa8aced67f9 --- ecr_public_lifecycle.py | 63 +++++++++++++++++++++++++++++++++++++++++ podman.mk | 59 +++++++++++++++++++------------------- vars/buildPodman.groovy | 22 ++++++++++---- 3 files changed, 109 insertions(+), 35 deletions(-) create mode 100755 ecr_public_lifecycle.py diff --git a/ecr_public_lifecycle.py b/ecr_public_lifecycle.py new file mode 100755 index 0000000..7397dc4 --- /dev/null +++ b/ecr_public_lifecycle.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +import argparse +import boto3 + +parser = argparse.ArgumentParser( + description='Implement basic public ECR lifecycle policy') +parser.add_argument('--repo', dest='repositoryName', action='store', required=True, + help='Name of the public ECR repository') +parser.add_argument('--keep', dest='keep', action='store', default=10, type=int, + help='number of tagged images to keep, default 10') +parser.add_argument('--dev', dest='delete_dev', action='store_true', + help='also delete in-development images only having tags like v0.1.1-commitNr-githash') + +args = parser.parse_args() + +client = boto3.client('ecr-public', region_name='us-east-1') + +images = client.describe_images(repositoryName=args.repositoryName)[ + "imageDetails"] + +untagged = [] +kept = 0 + +# actual Image +# imageManifestMediaType: 'application/vnd.oci.image.manifest.v1+json' +# image Index +# imageManifestMediaType: 'application/vnd.oci.image.index.v1+json' + +# Sort by date uploaded +for image in sorted(images, key=lambda d: d['imagePushedAt'], reverse=True): + # Remove all untagged + # if registry uses image index all actual images will be untagged anyways + if 'imageTags' not in image: + untagged.append({"imageDigest": image['imageDigest']}) + # print("Delete untagged image {}".format(image["imageDigest"])) + continue + + # check for dev tags + if args.delete_dev: + _delete = True + for tag in image["imageTags"]: + # Look for at least one tag NOT beign a SemVer dev tag + if "-" not in tag: + _delete = False + if _delete: + print("Deleting development image {}".format(image["imageTags"])) + untagged.append({"imageDigest": image['imageDigest']}) + continue + + if kept < args.keep: + kept = kept+1 + print("Keeping tagged image {}".format(image["imageTags"])) + continue + else: + print("Deleting tagged image {}".format(image["imageTags"])) + untagged.append({"imageDigest": image['imageDigest']}) + +deleted_images = client.batch_delete_image( + repositoryName=args.repositoryName, imageIds=untagged) + +if deleted_images["imageIds"]: + print("Deleted images: {}".format(deleted_images["imageIds"])) diff --git a/podman.mk b/podman.mk index 55cff91..02e0475 100644 --- a/podman.mk +++ b/podman.mk @@ -1,22 +1,26 @@ # Parse version from latest git semver tag -BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) -GIT_TAG := $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null) -TAG := $(GIT_TAG) +GIT_TAG ?= $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null) +GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) +TAG ::= $(GIT_TAG) # append branch name to tag if NOT main nor master -ifeq (,$(filter main master, $(BRANCH))) - TAG = $(GIT_TAG)-$(BRANCH) +ifeq (,$(filter main master, $(GIT_BRANCH))) + # If branch is substring of tag, omit branch name + ifeq ($(findstring $(GIT_BRANCH), $(GIT_TAG)),) + # only append branch name if not equal tag + ifneq ($(GIT_TAG), $(GIT_BRANCH)) + # Sanitize GIT_BRANCH to allowed Docker tag character set + TAG = $(GIT_TAG)-$(shell echo $$GIT_BRANCH | sed -e 's/[^a-zA-Z0-9]/-/g') + endif + endif endif -# optionally set by the caller -EXTRA_TAGS := - -ARCH := amd64 -ALL_ARCHS := amd64 arm64 +ARCH ::= amd64 +ALL_ARCHS ::= amd64 arm64 _ARCH = $(or $(filter $(ARCH),$(ALL_ARCHS)),$(error $$ARCH [$(ARCH)] must be exactly one of "$(ALL_ARCHS)")) ifneq ($(TRIVY_REMOTE),) - TRIVY_OPTS := --server $(TRIVY_REMOTE) + TRIVY_OPTS ::= --server $(TRIVY_REMOTE) endif .SILENT: ; # no need for @ @@ -29,14 +33,16 @@ endif help: ## Show Help grep -E '^[a-zA-Z_-]+:.*?## .*$$' .ci/podman.mk | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +prepare:: ## custom step on the build agent before building + +fmt:: ## auto format source + +lint:: ## Lint source + build: ## Build the app buildah build --rm --layers -t $(IMAGE):$(TAG)-$(_ARCH) --build-arg TAG=$(TAG) --build-arg ARCH=$(_ARCH) --platform linux/$(_ARCH) . -test: rm-test-image ## Execute Dockerfile.test - test -f Dockerfile.test && \ - { buildah build --rm --layers -t $(REGISTRY)/$(IMAGE):$(TAG)-test --from=$(REGISTRY)/$(IMAGE):$(TAG) -f Dockerfile.test --platform linux/$(_ARCH) . && \ - podman run --rm --env-host -t $(REGISTRY)/$(IMAGE):$(TAG)-$(_ARCH)-test; } || \ - echo "No Dockerfile.test found, skipping test" +test:: ## test built artificats scan: ## Scan image using trivy echo "Scanning $(IMAGE):$(TAG)-$(_ARCH) using Trivy $(TRIVY_REMOTE)" @@ -60,24 +66,19 @@ push: ecr-login ## push images to registry ecr-login: ## log into AWS ECR public aws ecr-public get-login-password --region $(REGION) | podman login --username AWS --password-stdin $(REGISTRY) -clean: rm-test-image rm-image ## delete local built container and test images +rm-remote-untagged: ## delete all remote untagged and in-dev images, keep 10 tagged + echo "Removing all untagged and in-dev images from $(IMAGE) in $(REGION)" + .ci/ecr_public_lifecycle.py --repo $(IMAGE) --dev -rm-remote-untagged: ## delete all remote untagged images - echo "Removing all untagged images from $(IMAGE) in $(REGION)" - IMAGE_IDS=$$(for image in $$(aws ecr-public describe-images --repository-name $(IMAGE) --region $(REGION) --output json | jq -r '.imageDetails[] | select(.imageTags | not ).imageDigest'); do echo -n "imageDigest=$$image "; done) ; \ - [ -n "$$IMAGE_IDS" ] && aws ecr-public batch-delete-image --repository-name $(IMAGE) --region $(REGION) --image-ids $$IMAGE_IDS || echo "No image to remove" +clean:: ## clean up source folder rm-image: - test -z "$$(docker image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH) > /dev/null - test -z "$$(docker image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || echo "Error: Removing image failed" - -# Ensure we run the tests by removing any previous runs -rm-test-image: - test -z "$$(docker image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH)-test > /dev/null - test -z "$$(docker image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || echo "Error: Removing test image failed" + test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH) > /dev/null + test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || echo "Error: Removing image failed" +## some useful tasks during development ci-pull-upstream: ## pull latest shared .ci subtree - git stash && git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash && git stash pop + git stash && git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash -m "Merge latest ci-tools-lib" && git stash pop create-repo: ## create new AWS ECR public repository aws ecr-public create-repository --repository-name $(IMAGE) --region $(REGION) diff --git a/vars/buildPodman.groovy b/vars/buildPodman.groovy index e5ecff9..e33c2cf 100644 --- a/vars/buildPodman.groovy +++ b/vars/buildPodman.groovy @@ -10,18 +10,20 @@ def call(Map config=[:]) { stages { stage('Prepare') { steps { + // we set pull tags as project adv. options // pull tags - withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) { - sh 'git fetch -q --tags ${GIT_URL}' - } - sh 'make prepare || true' + //withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) { + // sh 'git fetch -q --tags ${GIT_URL}' + //} + // Optional project specific preparations + sh 'make prepare' } } // Build using rootless podman stage('Build') { steps { - sh 'make build' + sh 'make build GIT_BRANCH=$GIT_BRANCH' } } @@ -60,14 +62,22 @@ def call(Map config=[:]) { } } - // Push to container registry, skip if PR + // Push to container registry if not PR + // incl. basic registry retention removing any untagged images stage('Push') { when { not { changeRequest() } } steps { sh 'make push' + sh 'make rm-remote-untagged' } } + // generic clean + stage('cleanup') { + steps { + sh 'make clean' + } + } } } }