feat: add support for OpenSearch

3feaf6f chore: migrate to main branch
a392836 feat: migrate all buildah cmds to podman only
d67a80e feat: make push and rm-image more resilient, prevent exit codes
8e202d4 fix: do not add non-existent images
6ef8d28 feat: ensure bash and safe exec
06fcff5 feat: improve image cleanup to incl. all tags and repositories
47b4da4 feat: add suport for trivyignore file

git-subtree-dir: .ci
git-subtree-split: 3feaf6fa96531b40c56945e3a7d6731f40bc52e2

.ci/podman.mk

@@ -1,3 +1,13 @@
+SHELL := bash
+.SHELLFLAGS := -eu -o pipefail -c
+.DELETE_ON_ERROR:
+.SILENT: ; # no need for @
+.ONESHELL: ; # recipes execute in same shell
+.NOTPARALLEL: ; # wait for this target to finish
+.EXPORT_ALL_VARIABLES: ; # send all vars to shell
+.PHONY: all # All targets are accessible for user
+.DEFAULT: help # Running Make will run the help target
+
 # Parse version from latest git semver tag
 GIT_TAG ?= $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)
 GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
@@ -23,13 +33,6 @@ ifneq ($(TRIVY_REMOTE),)
 	TRIVY_OPTS ::= --server $(TRIVY_REMOTE)
 endif
 
-.SILENT: ; # no need for @
-.ONESHELL: ; # recipes execute in same shell
-.NOTPARALLEL: ; # wait for this target to finish
-.EXPORT_ALL_VARIABLES: ; # send all vars to shell
-.PHONY: all # All targets are accessible for user
-.DEFAULT: help # Running Make will run the help target
-
 help: ## Show Help
 	grep -E '^[a-zA-Z_-]+:.*?## .*$$' .ci/podman.mk | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
@@ -40,27 +43,28 @@ fmt:: ## auto format source
 lint:: ## Lint source
 
 build: ## Build the app
-	buildah build --rm --layers -t $(IMAGE):$(TAG)-$(_ARCH) --build-arg TAG=$(TAG) --build-arg ARCH=$(_ARCH) --platform linux/$(_ARCH) .
+	podman build --rm --layers -t $(IMAGE):$(TAG)-$(_ARCH) --build-arg TAG=$(TAG) --build-arg ARCH=$(_ARCH) --platform linux/$(_ARCH) .
 
 test:: ## test built artificats
 
 scan: ## Scan image using trivy
 	echo "Scanning $(IMAGE):$(TAG)-$(_ARCH) using Trivy $(TRIVY_REMOTE)"
-	trivy image $(TRIVY_OPTS) --quiet --no-progress localhost/$(IMAGE):$(TAG)-$(_ARCH)
+	trivy image $(TRIVY_OPTS) --quiet --no-progress --ignorefile ./.trivyignore.yaml localhost/$(IMAGE):$(TAG)-$(_ARCH)
 
 # first tag and push all actual images
 # create new manifest for each tag and add all available TAG-ARCH before pushing
 push: ecr-login ## push images to registry
-	for t in $(TAG) latest $(EXTRA_TAGS); do \
+	for t in $(TAG) latest $(EXTRA_TAGS); do
 		echo "Tagging image with $(REGISTRY)/$(IMAGE):$${t}-$(ARCH)"
-		buildah tag $(IMAGE):$(TAG)-$(_ARCH) $(REGISTRY)/$(IMAGE):$${t}-$(_ARCH); \
-		buildah manifest rm $(IMAGE):$$t || true; \
-		buildah manifest create $(IMAGE):$$t; \
-		for a in $(ALL_ARCHS); do \
-			buildah manifest add $(IMAGE):$$t $(REGISTRY)/$(IMAGE):$(TAG)-$$a; \
-		done; \
+		podman tag $(IMAGE):$(TAG)-$(_ARCH) $(REGISTRY)/$(IMAGE):$${t}-$(_ARCH)
+		podman manifest rm $(IMAGE):$$t || true
+		podman manifest create $(IMAGE):$$t
+		for a in $(ALL_ARCHS); do
+			podman image exists $(REGISTRY)/$(IMAGE):$$t-$$a && \
+			podman manifest add $(IMAGE):$$t containers-storage:$(REGISTRY)/$(IMAGE):$$t-$$a
+		done
 		echo "Pushing manifest $(IMAGE):$$t"
-		buildah manifest push --all $(IMAGE):$$t docker://$(REGISTRY)/$(IMAGE):$$t; \
+		podman manifest push --all $(IMAGE):$$t docker://$(REGISTRY)/$(IMAGE):$$t
 	done
 
 ecr-login: ## log into AWS ECR public
@@ -73,12 +77,15 @@ rm-remote-untagged: ## delete all remote untagged and in-dev images, keep 10 tag
 clean:: ## clean up source folder
 
 rm-image:
-	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH) > /dev/null
-	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || echo "Error: Removing image failed"
+	for t in $(TAG) latest $(EXTRA_TAGS); do
+		for a in $(ALL_ARCHS); do
+			podman image exists $(IMAGE):$$t-$$a && podman image rm -f $(IMAGE):$$t-$$a || true
+		done
+	done
 
 ## some useful tasks during development
 ci-pull-upstream: ## pull latest shared .ci subtree
-	git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash -m "Merge latest ci-tools-lib"
+	git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git main --squash -m "Merge latest ci-tools-lib"
 
 create-repo: ## create new AWS ECR public repository
 	aws ecr-public create-repository --repository-name $(IMAGE) --region $(REGION)

Dockerfile

@@ -12,6 +12,7 @@ ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
 RUN apk update \
  && apk add --no-cache \
         ca-certificates \
+        libcurl \
         ruby ruby-irb ruby-etc ruby-webrick \
         tini ruby-bundler \
  && apk add --no-cache --virtual .build-deps \
@@ -31,7 +32,6 @@ RUN addgroup -S fluent && adduser -S -g fluent fluent \
     && mkdir -p /fluentd/etc /fluentd/plugins \
     && chown -R fluent /fluentd && chgrp -R fluent /fluentd
 
-
 COPY fluent.conf /fluentd/etc/
 COPY entrypoint.sh /bin/
 

Gemfile

@@ -7,6 +7,7 @@ gem "ffi"
 gem "fluentd", "1.16.5"
 gem "oj", "~> 3.16.0"
 gem "rexml", "~> 3.2.5"
+gem "faraday-typhoeus", "~> 1.1.0"
 gem "fluent-plugin-multi-format-parser", "~> 1.1.0"
 gem "fluent-plugin-concat", "~> 2.5.0"
 gem "fluent-plugin-grok-parser", "~> 2.6.2"

Jenkinsfile

@@ -1,4 +1,4 @@
-library identifier: 'zdt-lib@master', retriever: modernSCM(
+library identifier: 'zdt-lib@main', retriever: modernSCM(
   [$class: 'GitSCMSource',
    remote: 'https://git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git'])
 

README.md

@@ -9,5 +9,6 @@ Opinionated Fluentd container intended to work as central ingestion service fron
 - fluentd runs as non-root fluent user
 
 # Resources
+- https://github.com/fluent/fluent-plugin-opensearch
 - https://github.com/fluent/fluentd-docker-image/blob/master/v1.14/alpine/Dockerfile
 - https://github.com/fluent/fluentd-kubernetes-daemonset/tree/master/docker-image/v1.16/debian-elasticsearch7