From 472bfc98fa206e729d85a676a67978005a2c5f68 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 11 Dec 2023 19:02:42 +0000 Subject: [PATCH] first commit --- Dockerfile | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++ Gemfile | 21 ++++++++++++++++++++ Makefile | 23 +++++++++++++++++++++ README.md | 19 ++++++++++++++++++ entrypoint.sh | 33 +++++++++++++++++++++++++++++++ fluent.conf | 33 +++++++++++++++++++++++++++++++ 6 files changed, 184 insertions(+) create mode 100644 Dockerfile create mode 100644 Gemfile create mode 100644 Makefile create mode 100644 README.md create mode 100755 entrypoint.sh create mode 100644 fluent.conf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bac2767 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,55 @@ +#inspired from https://github.com/fluent/fluentd-docker-image/blob/master/v1.16/alpine/Dockerfile +FROM alpine:3.17 + +ARG version=1.16.0 + +LABEL maintainer "Stefan Reimer" +LABEL Description="Fluentd docker image intended as ingestion service" Vendor="Zero Down Time" Version="${version}" + +COPY Gemfile* /fluentd/ + +# skip runtime bundler installation +ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 + +RUN apk update \ + && apk add --no-cache \ + ca-certificates \ + ruby ruby-irb ruby-etc ruby-webrick \ + tini ruby-bundler \ + && apk add --no-cache --virtual .build-deps \ + build-base linux-headers \ + ruby-dev gnupg \ + && echo 'gem: --no-document' >> /etc/gemrc \ + && gem install oj -v 3.14.2 \ + && gem install json -v 2.6.3 \ + && gem install rexml -v 3.2.5 \ + && gem install async -v 1.30.3 \ + && gem install async-http -v 0.56.6 \ + && gem install fluentd -v ${version} \ + && gem install bigdecimal -v 1.4.4 \ + # Install additional gems from Gemfile + && bundle config silence_root_warning true \ + && bundle install --gemfile=/fluentd/Gemfile \ + && apk del .build-deps \ + && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test + +RUN addgroup -S fluent && adduser -S -g fluent fluent \ + # for log storage (maybe shared with host) + && mkdir -p /fluentd/log \ + # configuration/plugins path (default: copied from .) + && mkdir -p /fluentd/etc /fluentd/plugins \ + && chown -R fluent /fluentd && chgrp -R fluent /fluentd + + +COPY fluent.conf /fluentd/etc/ +COPY entrypoint.sh /bin/ + +ENV FLUENTD_CONF="fluent.conf" + +ENV LD_PRELOAD="" + +EXPOSE 24224 5140 + +USER fluent +ENTRYPOINT ["tini", "--", "/bin/entrypoint.sh"] +CMD ["fluentd"] diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..ae52c30 --- /dev/null +++ b/Gemfile @@ -0,0 +1,21 @@ +# AUTOMATICALLY GENERATED +# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb + +source "https://rubygems.org" + +gem "ffi" +gem "fluent-plugin-multi-format-parser" +gem "fluent-plugin-concat" +gem "fluent-plugin-grok-parser" +gem "fluent-plugin-prometheus" +gem 'fluent-plugin-json-in-json-2' +gem "fluent-plugin-record-modifier" +#gem "fluent-plugin-detect-exceptions" +gem "fluent-plugin-rewrite-tag-filter" +gem "fluent-plugin-parser-cri" +gem "fluent-plugin-dedot_filter" +gem "fluent-plugin-kubernetes_metadata_filter" +# Since 7.14 Elastic only allows their own ES services ;-( +gem "elasticsearch", '>= 7', '< 7.14' +gem "fluent-plugin-elasticsearch" +gem "fluent-plugin-s3" diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..37fdcf8 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +VERSION ?= 1.16.0 +REGISTRY := public.ecr.aws/zero-downtime +REPOSITORY := fluentd-concenter +TAG := $(REPOSITORY):v$(VERSION) + +.PHONY: build push clean scan + +all: build + +build: + buildah bud --build-arg version=$(VERSION) -t $(TAG) . + +push: + aws ecr-public get-login-password --region us-east-1 | buildah login --username AWS --password-stdin $(REGISTRY) + buildah tag $(TAG) $(REGISTRY)/$(TAG) + buildah push $(REGISTRY)/$(TAG) + +clean: + podman rmi -f $(TAG) + +scan: + podman system service& + sleep 5; trivy $(TAG) diff --git a/README.md b/README.md new file mode 100644 index 0000000..5a0b559 --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +# fluentd-concenter +Public Repo: https://gallery.ecr.aws/zero-downtime/fluentd-concenter + +## Abstract +Custom Fluentd container. + +Possible use as drop in replacement for `fluentd-kubernetes-daemonset` using the upstream fluentd Helm charts. + +## Changes from fluentd-kubernetes-daemonset +- Alpine 3.14 based +- passes Trivy scan as of Sep. 02 2021 +- fluentd runs as non-root fluent user +- remove fluent-plugin-detect-exceptions due to being unmaintained and pulling in insecure dependencies +- restrict ES Ruby SDK to <7.14 to allow connections to any OSS ES clusters +- latest plugins otherwise + +# Resources +- https://github.com/fluent/fluentd-docker-image/blob/master/v1.14/alpine/Dockerfile +- https://github.com/fluent/fluentd-kubernetes-daemonset/tree/master/docker-image/v1.14/debian-elasticsearch7 diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..6d3bfdb --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +#source vars if file exists +DEFAULT=/etc/default/fluentd + +if [ -r $DEFAULT ]; then + set -o allexport + . $DEFAULT + set +o allexport +fi + +# If the user has supplied only arguments append them to `fluentd` command +if [ "${1#-}" != "$1" ]; then + set -- fluentd "$@" +fi + +# If user does not supply config file or plugins, use the default +if [ "$1" = "fluentd" ]; then + if ! echo $@ | grep -e ' \-c' -e ' \-\-config' ; then + set -- "$@" --config /fluentd/etc/${FLUENTD_CONF} + fi + + if ! echo $@ | grep -e ' \-p' -e ' \-\-plugin' ; then + set -- "$@" --plugin /fluentd/plugins + fi + + SIMPLE_SNIFFER=$( gem contents fluent-plugin-elasticsearch | grep elasticsearch_simple_sniffer.rb ) + if [ -n "$SIMPLE_SNIFFER" -a -f "$SIMPLE_SNIFFER" ] ; then + set -- "$@" -r $SIMPLE_SNIFFER + fi +fi + +exec "$@" diff --git a/fluent.conf b/fluent.conf new file mode 100644 index 0000000..24a37b5 --- /dev/null +++ b/fluent.conf @@ -0,0 +1,33 @@ + + @type forward + @id input1 + @label @mainstream + port 24224 + + + + @type stdout + + +