first commit

Dockerfile

@@ -0,0 +1,55 @@
+#inspired from https://github.com/fluent/fluentd-docker-image/blob/master/v1.16/alpine/Dockerfile
+FROM alpine:3.17
+
+ARG version=1.16.0
+
+LABEL maintainer "Stefan Reimer<stefan@zero-downtime.net>"
+LABEL Description="Fluentd docker image intended as ingestion service" Vendor="Zero Down Time" Version="${version}"
+
+COPY Gemfile* /fluentd/
+
+# skip runtime bundler installation
+ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
+
+RUN apk update \
+ && apk add --no-cache \
+        ca-certificates \
+        ruby ruby-irb ruby-etc ruby-webrick \
+        tini ruby-bundler \
+ && apk add --no-cache --virtual .build-deps \
+        build-base linux-headers \
+        ruby-dev gnupg \
+ && echo 'gem: --no-document' >> /etc/gemrc \
+ && gem install oj -v 3.14.2 \
+ && gem install json -v 2.6.3 \
+ && gem install rexml -v 3.2.5 \
+ && gem install async -v 1.30.3 \
+ && gem install async-http -v 0.56.6 \
+ && gem install fluentd -v ${version} \
+ && gem install bigdecimal -v 1.4.4 \
+ # Install additional gems from Gemfile
+ && bundle config silence_root_warning true \
+ && bundle install --gemfile=/fluentd/Gemfile \
+ && apk del .build-deps \
+ && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem /usr/lib/ruby/gems/3.*/gems/fluentd-*/test
+
+RUN addgroup -S fluent && adduser -S -g fluent fluent \
+    # for log storage (maybe shared with host)
+    && mkdir -p /fluentd/log \
+    # configuration/plugins path (default: copied from .)
+    && mkdir -p /fluentd/etc /fluentd/plugins \
+    && chown -R fluent /fluentd && chgrp -R fluent /fluentd
+
+
+COPY fluent.conf /fluentd/etc/
+COPY entrypoint.sh /bin/
+
+ENV FLUENTD_CONF="fluent.conf"
+
+ENV LD_PRELOAD=""
+
+EXPOSE 24224 5140
+
+USER fluent
+ENTRYPOINT ["tini",  "--", "/bin/entrypoint.sh"]
+CMD ["fluentd"]

Gemfile

@@ -0,0 +1,21 @@
+# AUTOMATICALLY GENERATED
+# DO NOT EDIT THIS FILE DIRECTLY, USE /templates/Gemfile.erb
+
+source "https://rubygems.org"
+
+gem "ffi"
+gem "fluent-plugin-multi-format-parser"
+gem "fluent-plugin-concat"
+gem "fluent-plugin-grok-parser"
+gem "fluent-plugin-prometheus"
+gem 'fluent-plugin-json-in-json-2'
+gem "fluent-plugin-record-modifier"
+#gem "fluent-plugin-detect-exceptions"
+gem "fluent-plugin-rewrite-tag-filter"
+gem "fluent-plugin-parser-cri"
+gem "fluent-plugin-dedot_filter"
+gem "fluent-plugin-kubernetes_metadata_filter"
+# Since 7.14 Elastic only allows their own ES services ;-(
+gem "elasticsearch", '>= 7', '< 7.14'
+gem "fluent-plugin-elasticsearch"
+gem "fluent-plugin-s3"

Makefile

@@ -0,0 +1,23 @@
+VERSION ?= 1.16.0
+REGISTRY := public.ecr.aws/zero-downtime
+REPOSITORY := fluentd-concenter
+TAG := $(REPOSITORY):v$(VERSION)
+
+.PHONY: build push clean scan
+
+all: build
+
+build:
+	buildah bud --build-arg version=$(VERSION) -t $(TAG) .
+
+push:
+	aws ecr-public get-login-password --region us-east-1 | buildah login --username AWS --password-stdin $(REGISTRY)
+	buildah tag $(TAG) $(REGISTRY)/$(TAG)
+	buildah push $(REGISTRY)/$(TAG)
+
+clean:
+	podman rmi -f $(TAG)
+
+scan:
+	podman system service&
+	sleep 5; trivy $(TAG)

README.md

@@ -0,0 +1,19 @@
+# fluentd-concenter
+Public Repo: https://gallery.ecr.aws/zero-downtime/fluentd-concenter
+
+## Abstract
+Custom Fluentd container.  
+
+Possible use as drop in replacement for `fluentd-kubernetes-daemonset` using the upstream fluentd Helm charts.  
+
+## Changes from fluentd-kubernetes-daemonset
+- Alpine 3.14 based
+- passes Trivy scan as of Sep. 02 2021
+- fluentd runs as non-root fluent user
+- remove fluent-plugin-detect-exceptions due to being unmaintained and pulling in insecure dependencies
+- restrict ES Ruby SDK to <7.14 to allow connections to any OSS ES clusters
+- latest plugins otherwise
+
+# Resources
+- https://github.com/fluent/fluentd-docker-image/blob/master/v1.14/alpine/Dockerfile
+- https://github.com/fluent/fluentd-kubernetes-daemonset/tree/master/docker-image/v1.14/debian-elasticsearch7

entrypoint.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+#source vars if file exists
+DEFAULT=/etc/default/fluentd
+
+if [ -r $DEFAULT ]; then
+    set -o allexport
+    . $DEFAULT
+    set +o allexport
+fi
+
+# If the user has supplied only arguments append them to `fluentd` command
+if [ "${1#-}" != "$1" ]; then
+    set -- fluentd "$@"
+fi
+
+# If user does not supply config file or plugins, use the default
+if [ "$1" = "fluentd" ]; then
+    if ! echo $@ | grep -e ' \-c' -e ' \-\-config' ; then
+       set -- "$@" --config /fluentd/etc/${FLUENTD_CONF}
+    fi
+
+    if ! echo $@ | grep -e ' \-p' -e ' \-\-plugin' ; then
+       set -- "$@" --plugin /fluentd/plugins
+    fi
+
+    SIMPLE_SNIFFER=$( gem contents fluent-plugin-elasticsearch | grep elasticsearch_simple_sniffer.rb )
+    if [ -n "$SIMPLE_SNIFFER" -a -f "$SIMPLE_SNIFFER" ] ; then
+        set -- "$@" -r $SIMPLE_SNIFFER
+    fi
+fi
+
+exec "$@"

fluent.conf

@@ -0,0 +1,33 @@
+<source>
+  @type  forward
+  @id    input1
+  @label @mainstream
+  port  24224
+</source>
+
+<filter **>
+  @type stdout
+</filter>
+
+<label @mainstream>
+  <match docker.**>
+    @type file
+    @id   output_docker1
+    path         /fluentd/log/docker.*.log
+    symlink_path /fluentd/log/docker.log
+    append       true
+    time_slice_format %Y%m%d
+    time_slice_wait   1m
+    time_format       %Y%m%dT%H%M%S%z
+  </match>
+  <match **>
+    @type file
+    @id   output1
+    path         /fluentd/log/data.*.log
+    symlink_path /fluentd/log/data.log
+    append       true
+    time_slice_format %Y%m%d
+    time_slice_wait   10m
+    time_format       %Y%m%dT%H%M%S%z
+  </match>
+</label>