From 174d1f9680e0d3da1c11e94bc1de92eae2e2203a Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 15 May 2023 12:21:08 +0000 Subject: [PATCH] Squashed '.ci/' changes from 98c8ec1..aea1ccc aea1ccc Only add branch name to tags, if not part of actual tag a5875db Make EXTRA_TAGS work again 63421d1 fix: prevent branch_name equals tag 47140c2 feat: append branch into tags if not main 4b62ada chore: improve messaging a49cc0c chore: improve messaging 194afb4 chore: get ci working again 8ec9769 chore: get ci working again fef4968 fix: do NOT push PRs to registry, other fixes 50a6d67 feat: ensure ARCH is only set to defined values 8fb40c7 fix: adjust trivy call to local podman 7378ea9 fix: fix trivy scan task to match new flow, add BRANCH env to Makefile 38cf7ab fix: more podman/buildah cleanups aece7fc fix: Improve multi-arch manifest handling 80dabc2 feat: remove implicit dependencies, add help target, cleanup 5554972 feat: multi-arch container images da15d68 feat: handle nothing to cleanup gracefully 01df38b feat: move ecr-login into its own task ea9c914 chore: test mermaid 19a782e chore: test mermaid a47929d feat: switch to latest trivy cli syntax cb5faca feat: add create-repo task to ease bootstrapping new project 49ea8c8 feat: Add support for custom EXTRA_TAGS dc2c208 fix: use absolute image URLs for some tasks bc72735 docs: add quickstart git-subtree-dir: .ci git-subtree-split: aea1cccfff35de2ef2eca138379a599c8bde39e0 --- README.md | 16 +++++++ podman.mk | 104 ++++++++++++++++++++++++++-------------- vars/buildPodman.groovy | 14 +++--- 3 files changed, 91 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index e08d660..689e535 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,22 @@ Various toolchain bits and pieces shared between projects +# Quickstart +Create top-level Makefile +``` +REGISTRY := +IMAGE := +REGION := + +include .ci/podman.mk +``` + +Add subtree to your project: +``` +git subtree add --prefix .ci https://git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash +``` + + ## Jenkins Shared groovy libraries diff --git a/podman.mk b/podman.mk index 9435847..b4b086e 100644 --- a/podman.mk +++ b/podman.mk @@ -1,56 +1,86 @@ # Parse version from latest git semver tag -GTAG=$(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null) -TAG ?= $(shell echo $(GTAG) | awk -F '-' '{ print $$1 "-" $$2 }' | sed -e 's/-$$//') +GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) +GIT_TAG := $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null) -ifeq ($(TRIVY_REMOTE),) - TRIVY_OPTS := image -else - TRIVY_OPTS := client --remote ${TRIVY_REMOTE} +TAG := $(GIT_TAG) +# append branch name to tag if NOT main nor master +ifeq (,$(filter main master, $(GIT_BRANCH))) + # If branch is substring of tag, omit branch name + ifeq ($(findstring $(GIT_BRANCH), $(GIT_TAG)),) + # only append branch name if not equal tag + ifneq ($(GIT_TAG), $(GIT_BRANCH)) + TAG = $(GIT_TAG)-$(GIT_BRANCH) + endif + endif endif -.PHONY: build test scan push clean +ARCH := amd64 +ALL_ARCHS := amd64 arm64 +_ARCH = $(or $(filter $(ARCH),$(ALL_ARCHS)),$(error $$ARCH [$(ARCH)] must be exactly one of "$(ALL_ARCHS)")) -all: test +ifneq ($(TRIVY_REMOTE),) + TRIVY_OPTS := --server $(TRIVY_REMOTE) +endif +.SILENT: ; # no need for @ +.ONESHELL: ; # recipes execute in same shell +.NOTPARALLEL: ; # wait for this target to finish +.EXPORT_ALL_VARIABLES: ; # send all vars to shell +.PHONY: all # All targets are accessible for user +.DEFAULT: help # Running Make will run the help target -build: - @docker image exists $(IMAGE):$(TAG) || \ - docker build --rm -t $(IMAGE):$(TAG) --build-arg TAG=$(TAG) . +help: ## Show Help + grep -E '^[a-zA-Z_-]+:.*?## .*$$' .ci/podman.mk | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -test: build rm-test-image - @test -f Dockerfile.test && \ - { docker build --rm -t $(IMAGE):$(TAG)-test --from=$(IMAGE):$(TAG) -f Dockerfile.test . && \ - docker run --rm --env-host -t $(IMAGE):$(TAG)-test; } || \ +build: ## Build the app + buildah build --rm --layers -t $(IMAGE):$(TAG)-$(_ARCH) --build-arg TAG=$(TAG) --build-arg ARCH=$(_ARCH) --platform linux/$(_ARCH) . + +test: rm-test-image ## Execute Dockerfile.test + test -f Dockerfile.test && \ + { buildah build --rm --layers -t $(REGISTRY)/$(IMAGE):$(TAG)-test --from=$(REGISTRY)/$(IMAGE):$(TAG) -f Dockerfile.test --platform linux/$(_ARCH) . && \ + podman run --rm --env-host -t $(REGISTRY)/$(IMAGE):$(TAG)-$(_ARCH)-test; } || \ echo "No Dockerfile.test found, skipping test" -scan: build - @echo "Scanning $(IMAGE):$(TAG) using Trivy" - @trivy $(TRIVY_OPTS) $(IMAGE):$(TAG) +scan: ## Scan image using trivy + echo "Scanning $(IMAGE):$(TAG)-$(_ARCH) using Trivy $(TRIVY_REMOTE)" + trivy image $(TRIVY_OPTS) localhost/$(IMAGE):$(TAG)-$(_ARCH) -push: build - @aws ecr-public get-login-password --region $(REGION) | docker login --username AWS --password-stdin $(REGISTRY) - @docker tag $(IMAGE):$(TAG) $(REGISTRY)/$(IMAGE):$(TAG) $(REGISTRY)/$(IMAGE):latest - docker push $(REGISTRY)/$(IMAGE):$(TAG) - docker push $(REGISTRY)/$(IMAGE):latest +# first tag and push all actual images +# create new manifest for each tag and add all available TAG-ARCH before pushing +push: ecr-login ## push images to registry + for t in $(TAG) latest $(EXTRA_TAGS); do \ + echo "Tagging image with $(REGISTRY)/$(IMAGE):$${t}-$(ARCH)" + buildah tag $(IMAGE):$(TAG)-$(_ARCH) $(REGISTRY)/$(IMAGE):$${t}-$(_ARCH); \ + buildah manifest rm $(IMAGE):$$t || true; \ + buildah manifest create $(IMAGE):$$t; \ + for a in $(ALL_ARCHS); do \ + buildah manifest add $(IMAGE):$$t $(REGISTRY)/$(IMAGE):$(TAG)-$$a; \ + done; \ + echo "Pushing manifest $(IMAGE):$$t" + buildah manifest push --all $(IMAGE):$$t docker://$(REGISTRY)/$(IMAGE):$$t; \ + done -clean: rm-test-image rm-image +ecr-login: ## log into AWS ECR public + aws ecr-public get-login-password --region $(REGION) | podman login --username AWS --password-stdin $(REGISTRY) -# Delete all untagged images -.PHONY: rm-remote-untagged -rm-remote-untagged: - @echo "Removing all untagged images from $(IMAGE) in $(REGION)" - @aws ecr-public batch-delete-image --repository-name $(IMAGE) --region $(REGION) --image-ids $$(for image in $$(aws ecr-public describe-images --repository-name $(IMAGE) --region $(REGION) --output json | jq -r '.imageDetails[] | select(.imageTags | not ).imageDigest'); do echo -n "imageDigest=$$image "; done) +clean: rm-test-image rm-image ## delete local built container and test images + +rm-remote-untagged: ## delete all remote untagged images + echo "Removing all untagged images from $(IMAGE) in $(REGION)" + IMAGE_IDS=$$(for image in $$(aws ecr-public describe-images --repository-name $(IMAGE) --region $(REGION) --output json | jq -r '.imageDetails[] | select(.imageTags | not ).imageDigest'); do echo -n "imageDigest=$$image "; done) ; \ + [ -n "$$IMAGE_IDS" ] && aws ecr-public batch-delete-image --repository-name $(IMAGE) --region $(REGION) --image-ids $$IMAGE_IDS || echo "No image to remove" -.PHONY: rm-image rm-image: - @test -z "$$(docker image ls -q $(IMAGE):$(TAG))" || docker image rm -f $(IMAGE):$(TAG) > /dev/null - @test -z "$$(docker image ls -q $(IMAGE):$(TAG))" || echo "Error: Removing image failed" + test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH) > /dev/null + test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || echo "Error: Removing image failed" # Ensure we run the tests by removing any previous runs -.PHONY: rm-test-image rm-test-image: - @test -z "$$(docker image ls -q $(IMAGE):$(TAG)-test)" || docker image rm -f $(IMAGE):$(TAG)-test > /dev/null - @test -z "$$(docker image ls -q $(IMAGE):$(TAG)-test)" || echo "Error: Removing test image failed" + test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH)-test > /dev/null + test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || echo "Error: Removing test image failed" -.DEFAULT: - @echo "$@ not implemented. NOOP" +ci-pull-upstream: ## pull latest shared .ci subtree + git stash && git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash && git stash pop + +create-repo: ## create new AWS ECR public repository + aws ecr-public create-repository --repository-name $(IMAGE) --region $(REGION) diff --git a/vars/buildPodman.groovy b/vars/buildPodman.groovy index 14a1b90..e5ecff9 100644 --- a/vars/buildPodman.groovy +++ b/vars/buildPodman.groovy @@ -7,12 +7,14 @@ def call(Map config=[:]) { label 'podman-aws-trivy' } } - stages { stage('Prepare') { - // get tags steps { - sh 'git fetch -q --tags ${GIT_URL} +refs/heads/${BRANCH_NAME}:refs/remotes/origin/${BRANCH_NAME}' + // pull tags + withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) { + sh 'git fetch -q --tags ${GIT_URL}' + } + sh 'make prepare || true' } } @@ -36,8 +38,7 @@ def call(Map config=[:]) { TRIVY_OUTPUT = "reports/trivy.html" } steps { - sh 'mkdir -p reports' - sh 'make scan' + sh 'mkdir -p reports && make scan' publishHTML target: [ allowMissing: true, alwaysLinkToLastBuild: true, @@ -59,8 +60,9 @@ def call(Map config=[:]) { } } - // Push to ECR + // Push to container registry, skip if PR stage('Push') { + when { not { changeRequest() } } steps { sh 'make push' }