Squashed '.ci/' changes from 63421d1..2c44e4f

2c44e4f Disable concurrent builds
7144a42 Improve Trivy scanning logic
c1a48a6 Remove auto stash push / pop as being too dangerous
318c19e Add merge comment for subtree
22ed100 Fix custom branch docker tags
227e39f Allow custom GIT_TAG
38a9cda Debug CI pipeline
3efcc81 Debug CI pipeline
5023473 Make branch detection work for tagged commits
cdc32e0 Improve cleanup flow
8df60af Fix derp
748a4bd Migrate to :: to allow custom make steps, add generic stubs
955afa7 Apply pep8
5819ded Improve ECR public lifecycle handling via python script
5d4e4ad Make rm-remote-untagged less noisy
f00e541 Add cleanup step to remove untagged images by default
0821e91 Ensure tag names are valid for remote branches like PRs
79eebe4 add ARCH support for tests
aea1ccc Only add branch name to tags, if not part of actual tag
a5875db Make EXTRA_TAGS work again

git-subtree-dir: .ci
git-subtree-split: 2c44e4fd8550d30fba503a2bcccec8e0bac1c151
This commit is contained in:
Stefan Reimer 2024-06-25 17:16:10 +00:00
parent 047e160d26
commit d9320daa34
3 changed files with 120 additions and 43 deletions

63
ecr_public_lifecycle.py Executable file
View File

@ -0,0 +1,63 @@
#!/usr/bin/env python3
import argparse
import boto3
parser = argparse.ArgumentParser(
description='Implement basic public ECR lifecycle policy')
parser.add_argument('--repo', dest='repositoryName', action='store', required=True,
help='Name of the public ECR repository')
parser.add_argument('--keep', dest='keep', action='store', default=10, type=int,
help='number of tagged images to keep, default 10')
parser.add_argument('--dev', dest='delete_dev', action='store_true',
help='also delete in-development images only having tags like v0.1.1-commitNr-githash')
args = parser.parse_args()
client = boto3.client('ecr-public', region_name='us-east-1')
images = client.describe_images(repositoryName=args.repositoryName)[
"imageDetails"]
untagged = []
kept = 0
# actual Image
# imageManifestMediaType: 'application/vnd.oci.image.manifest.v1+json'
# image Index
# imageManifestMediaType: 'application/vnd.oci.image.index.v1+json'
# Sort by date uploaded
for image in sorted(images, key=lambda d: d['imagePushedAt'], reverse=True):
# Remove all untagged
# if registry uses image index all actual images will be untagged anyways
if 'imageTags' not in image:
untagged.append({"imageDigest": image['imageDigest']})
# print("Delete untagged image {}".format(image["imageDigest"]))
continue
# check for dev tags
if args.delete_dev:
_delete = True
for tag in image["imageTags"]:
# Look for at least one tag NOT beign a SemVer dev tag
if "-" not in tag:
_delete = False
if _delete:
print("Deleting development image {}".format(image["imageTags"]))
untagged.append({"imageDigest": image['imageDigest']})
continue
if kept < args.keep:
kept = kept+1
print("Keeping tagged image {}".format(image["imageTags"]))
continue
else:
print("Deleting tagged image {}".format(image["imageTags"]))
untagged.append({"imageDigest": image['imageDigest']})
deleted_images = client.batch_delete_image(
repositoryName=args.repositoryName, imageIds=untagged)
if deleted_images["imageIds"]:
print("Deleted images: {}".format(deleted_images["imageIds"]))

View File

@ -1,24 +1,26 @@
# Parse version from latest git semver tag # Parse version from latest git semver tag
GIT_TAG ?= $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)
GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
GIT_TAG := $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)
TAG ::= $(GIT_TAG)
# append branch name to tag if NOT main nor master # append branch name to tag if NOT main nor master
TAG := $(GIT_TAG)
ifeq (,$(filter main master, $(GIT_BRANCH))) ifeq (,$(filter main master, $(GIT_BRANCH)))
ifneq ($(GIT_TAG), $(GIT_BRANCH)) # If branch is substring of tag, omit branch name
TAG = $(GIT_TAG)-$(GIT_BRANCH) ifeq ($(findstring $(GIT_BRANCH), $(GIT_TAG)),)
# only append branch name if not equal tag
ifneq ($(GIT_TAG), $(GIT_BRANCH))
# Sanitize GIT_BRANCH to allowed Docker tag character set
TAG = $(GIT_TAG)-$(shell echo $$GIT_BRANCH | sed -e 's/[^a-zA-Z0-9]/-/g')
endif
endif endif
endif endif
# optionally set by the caller ARCH ::= amd64
EXTRA_TAGS := ALL_ARCHS ::= amd64 arm64
ARCH := amd64
ALL_ARCHS := amd64 arm64
_ARCH = $(or $(filter $(ARCH),$(ALL_ARCHS)),$(error $$ARCH [$(ARCH)] must be exactly one of "$(ALL_ARCHS)")) _ARCH = $(or $(filter $(ARCH),$(ALL_ARCHS)),$(error $$ARCH [$(ARCH)] must be exactly one of "$(ALL_ARCHS)"))
ifneq ($(TRIVY_REMOTE),) ifneq ($(TRIVY_REMOTE),)
TRIVY_OPTS := --server $(TRIVY_REMOTE) TRIVY_OPTS ::= --server $(TRIVY_REMOTE)
endif endif
.SILENT: ; # no need for @ .SILENT: ; # no need for @
@ -31,18 +33,20 @@ endif
help: ## Show Help help: ## Show Help
grep -E '^[a-zA-Z_-]+:.*?## .*$$' .ci/podman.mk | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' grep -E '^[a-zA-Z_-]+:.*?## .*$$' .ci/podman.mk | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
prepare:: ## custom step on the build agent before building
fmt:: ## auto format source
lint:: ## Lint source
build: ## Build the app build: ## Build the app
buildah build --rm --layers -t $(IMAGE):$(TAG)-$(_ARCH) --build-arg TAG=$(TAG) --build-arg ARCH=$(_ARCH) --platform linux/$(_ARCH) . buildah build --rm --layers -t $(IMAGE):$(TAG)-$(_ARCH) --build-arg TAG=$(TAG) --build-arg ARCH=$(_ARCH) --platform linux/$(_ARCH) .
test: rm-test-image ## Execute Dockerfile.test test:: ## test built artificats
test -f Dockerfile.test && \
{ buildah build --rm --layers -t $(REGISTRY)/$(IMAGE):$(TAG)-test --from=$(REGISTRY)/$(IMAGE):$(TAG) -f Dockerfile.test --platform linux/$(_ARCH) . && \
podman run --rm --env-host -t $(REGISTRY)/$(IMAGE):$(TAG)-$(_ARCH)-test; } || \
echo "No Dockerfile.test found, skipping test"
scan: ## Scan image using trivy scan: ## Scan image using trivy
echo "Scanning $(IMAGE):$(TAG)-$(_ARCH) using Trivy $(TRIVY_REMOTE)" echo "Scanning $(IMAGE):$(TAG)-$(_ARCH) using Trivy $(TRIVY_REMOTE)"
trivy image $(TRIVY_OPTS) localhost/$(IMAGE):$(TAG)-$(_ARCH) trivy image $(TRIVY_OPTS) --quiet --no-progress localhost/$(IMAGE):$(TAG)-$(_ARCH)
# first tag and push all actual images # first tag and push all actual images
# create new manifest for each tag and add all available TAG-ARCH before pushing # create new manifest for each tag and add all available TAG-ARCH before pushing
@ -62,24 +66,19 @@ push: ecr-login ## push images to registry
ecr-login: ## log into AWS ECR public ecr-login: ## log into AWS ECR public
aws ecr-public get-login-password --region $(REGION) | podman login --username AWS --password-stdin $(REGISTRY) aws ecr-public get-login-password --region $(REGION) | podman login --username AWS --password-stdin $(REGISTRY)
clean: rm-test-image rm-image ## delete local built container and test images rm-remote-untagged: ## delete all remote untagged and in-dev images, keep 10 tagged
echo "Removing all untagged and in-dev images from $(IMAGE) in $(REGION)"
.ci/ecr_public_lifecycle.py --repo $(IMAGE) --dev
rm-remote-untagged: ## delete all remote untagged images clean:: ## clean up source folder
echo "Removing all untagged images from $(IMAGE) in $(REGION)"
IMAGE_IDS=$$(for image in $$(aws ecr-public describe-images --repository-name $(IMAGE) --region $(REGION) --output json | jq -r '.imageDetails[] | select(.imageTags | not ).imageDigest'); do echo -n "imageDigest=$$image "; done) ; \
[ -n "$$IMAGE_IDS" ] && aws ecr-public batch-delete-image --repository-name $(IMAGE) --region $(REGION) --image-ids $$IMAGE_IDS || echo "No image to remove"
rm-image: rm-image:
test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH) > /dev/null test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH) > /dev/null
test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || echo "Error: Removing image failed" test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || echo "Error: Removing image failed"
# Ensure we run the tests by removing any previous runs ## some useful tasks during development
rm-test-image:
test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH)-test > /dev/null
test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || echo "Error: Removing test image failed"
ci-pull-upstream: ## pull latest shared .ci subtree ci-pull-upstream: ## pull latest shared .ci subtree
git stash && git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash && git stash pop git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash -m "Merge latest ci-tools-lib"
create-repo: ## create new AWS ECR public repository create-repo: ## create new AWS ECR public repository
aws ecr-public create-repository --repository-name $(IMAGE) --region $(REGION) aws ecr-public create-repository --repository-name $(IMAGE) --region $(REGION)

View File

@ -2,6 +2,9 @@
def call(Map config=[:]) { def call(Map config=[:]) {
pipeline { pipeline {
options {
disableConcurrentBuilds()
}
agent { agent {
node { node {
label 'podman-aws-trivy' label 'podman-aws-trivy'
@ -10,18 +13,22 @@ def call(Map config=[:]) {
stages { stages {
stage('Prepare') { stage('Prepare') {
steps { steps {
sh 'mkdir -p reports'
// we set pull tags as project adv. options
// pull tags // pull tags
withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) { //withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) {
sh 'git fetch -q --tags ${GIT_URL}' // sh 'git fetch -q --tags ${GIT_URL}'
} //}
sh 'make prepare || true' // Optional project specific preparations
sh 'make prepare'
} }
} }
// Build using rootless podman // Build using rootless podman
stage('Build') { stage('Build') {
steps { steps {
sh 'make build' sh 'make build GIT_BRANCH=$GIT_BRANCH'
} }
} }
@ -33,12 +40,13 @@ def call(Map config=[:]) {
// Scan via trivy // Scan via trivy
stage('Scan') { stage('Scan') {
environment {
TRIVY_FORMAT = "template"
TRIVY_OUTPUT = "reports/trivy.html"
}
steps { steps {
sh 'mkdir -p reports && make scan' // we always scan and create the full json report
sh 'TRIVY_FORMAT=json TRIVY_OUTPUT="reports/trivy.json" make scan'
// render custom full html report
sh 'trivy convert -f template -t @/home/jenkins/html.tpl -o reports/trivy.html reports/trivy.json'
publishHTML target: [ publishHTML target: [
allowMissing: true, allowMissing: true,
alwaysLinkToLastBuild: true, alwaysLinkToLastBuild: true,
@ -48,26 +56,33 @@ def call(Map config=[:]) {
reportName: 'TrivyScan', reportName: 'TrivyScan',
reportTitles: 'TrivyScan' reportTitles: 'TrivyScan'
] ]
sh 'echo "Trivy report at: $BUILD_URL/TrivyScan"'
// Scan again and fail on CRITICAL vulns, if not overridden // fail build if issues found above trivy threshold
script { script {
if (config.trivyFail == 'NONE') { if ( config.trivyFail ) {
echo 'trivyFail == NONE, review Trivy report manually. Proceeding ...' sh "TRIVY_SEVERITY=${config.trivyFail} trivy convert --report summary --exit-code 1 reports/trivy.json"
} else {
sh "TRIVY_EXIT_CODE=1 TRIVY_SEVERITY=${config.trivyFail} make scan"
} }
} }
} }
} }
// Push to container registry, skip if PR // Push to container registry if not PR
// incl. basic registry retention removing any untagged images
stage('Push') { stage('Push') {
when { not { changeRequest() } } when { not { changeRequest() } }
steps { steps {
sh 'make push' sh 'make push'
sh 'make rm-remote-untagged'
} }
} }
// generic clean
stage('cleanup') {
steps {
sh 'make clean'
}
}
} }
} }
} }