chore: fluent-bit,fluentd chart updates

charts/kubezero-logging/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-logging
 description: KubeZero Umbrella Chart for complete EFK stack
 type: application
-version: 0.7.6
+version: 0.7.7
 appVersion: 1.6.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@@ -23,9 +23,9 @@ dependencies:
     # repository: https://helm.elastic.co
     condition: eck-operator.enabled
   - name: fluentd
-    version: 0.2.6
+    version: 0.2.10
     condition: fluentd.enabled
   - name: fluent-bit
-    version: 0.15.15
+    version: 0.16.3
     condition: fluent-bit.enabled
 kubeVersion: ">= 1.18.0"

charts/kubezero-logging/charts/fluent-bit/Chart.yaml

@@ -1,8 +1,9 @@
 annotations:
   artifacthub.io/changes: |
-    - Upgrade fluent-bit image to v1.7.9
+    - kind: changed
+      description: Update image version to v1.8.3.
 apiVersion: v1
-appVersion: 1.7.9
+appVersion: 1.8.3
 description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD
   family operating systems.
 home: https://fluentbit.io/
@@ -21,4 +22,4 @@ maintainers:
 name: fluent-bit
 sources:
 - https://github.com/fluent/fluent-bit/
-version: 0.15.15
+version: 0.16.3

charts/kubezero-logging/charts/fluent-bit/ci/ci-values.yaml

@@ -0,0 +1 @@
+logLevel: debug

charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl

@@ -9,6 +9,8 @@ priorityClassName: {{ .Values.priorityClassName }}
 serviceAccountName: {{ include "fluent-bit.serviceAccountName" . }}
 securityContext:
   {{- toYaml .Values.podSecurityContext | nindent 2 }}
+hostNetwork: {{ .Values.hostNetwork }}
+dnsPolicy: {{ .Values.dnsPolicy }}
 {{- with .Values.dnsConfig }}
 dnsConfig:
   {{- toYaml . | nindent 2 }}
@@ -21,8 +23,6 @@ hostAliases:
 initContainers:
   {{- toYaml .Values.initContainers | nindent 2 }}
 {{- end }}
-hostNetwork: true
-dnsPolicy: ClusterFirstWithHostNet
 containers:
   - name: {{ .Chart.Name }}
     securityContext:
@@ -56,24 +56,10 @@ containers:
         protocol: {{ .protocol }}
       {{- end }}
     {{- end }}
-    {{- if .Values.livenessProbe }}
     livenessProbe:
       {{- toYaml .Values.livenessProbe | nindent 6 }}
-    {{- else }}
-    livenessProbe:
-      httpGet:
-        path: /
-        port: http
-    {{- end }}
-    {{- if .Values.readinessProbe }}
     readinessProbe:
       {{- toYaml .Values.readinessProbe | nindent 6 }}
-    {{- else }}
-    readinessProbe:
-      httpGet:
-        path: /
-        port: http
-    {{- end }}
     resources:
       {{- toYaml .Values.resources | nindent 6 }}
     volumeMounts:

charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml

@@ -9,10 +9,12 @@ rules:
   - apiGroups:
       - ""
     resources:
-      - pods
       - namespaces
+      - pods
+      {{- if .Values.rbac.nodeAccess }}
       - nodes
       - nodes/proxy
+      {{- end }}
     verbs:
       - get
       - list

charts/kubezero-logging/charts/fluent-bit/templates/daemonset.yaml

@@ -5,6 +5,13 @@ metadata:
   name: {{ include "fluent-bit.fullname" . }}
   labels:
     {{- include "fluent-bit.labels" . | nindent 4 }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
   selector:
     matchLabels:

charts/kubezero-logging/charts/fluent-bit/templates/deployment.yaml

@@ -5,6 +5,13 @@ metadata:
   name: {{ include "fluent-bit.fullname" . }}
   labels:
     {{- include "fluent-bit.labels" . | nindent 4 }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
   replicas: {{ .Values.replicaCount }}
   {{- with .Values.updateStrategy }}

charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml

@@ -17,7 +17,7 @@ spec:
     - ALL
   volumes:
     - '*'
-  hostNetwork: false
+  hostNetwork: {{ .Values.hostNetwork }}
   hostIPC: false
   hostPID: false
   runAsUser:

charts/kubezero-logging/charts/fluent-bit/values.yaml

@@ -8,8 +8,9 @@ replicaCount: 1
 
 image:
   repository: fluent/fluent-bit
+  # Overrides the image tag whose default is {{ .Chart.AppVersion }}
+  tag: ""
   pullPolicy: Always
-  # tag:
 
 testFramework:
   enabled: false
@@ -29,99 +30,98 @@ serviceAccount:
 
 rbac:
   create: true
+  nodeAccess: false
 
 podSecurityPolicy:
   create: false
   annotations: {}
 
-podSecurityContext:
+podSecurityContext: {}
-  {}
+#   fsGroup: 2000
-  # fsGroup: 2000
+
+hostNetwork: false
+dnsPolicy: ClusterFirst
+
 dnsConfig: {}
-  # nameservers:
+#   nameservers:
-  #   - 1.2.3.4
+#     - 1.2.3.4
-  # searches:
+#   searches:
-  #   - ns1.svc.cluster-domain.example
+#     - ns1.svc.cluster-domain.example
-  #   - my.dns.search.suffix
+#     - my.dns.search.suffix
-  # options:
+#   options:
-  #   - name: ndots
+#     - name: ndots
-#     value: "2"
+#       value: "2"
-#   - name: edns0
+#     - name: edns0
 
 hostAliases: []
-  # - ip: "1.2.3.4"
+#   - ip: "1.2.3.4"
-  #   hostnames:
+#     hostnames:
-  #   - "foo.local"
+#     - "foo.local"
-  #   - "bar.local"
+#     - "bar.local"
 
-securityContext:
+securityContext: {}
-  {}
+#   capabilities:
-  # capabilities:
+#     drop:
-  #   drop:
+#     - ALL
-  #   - ALL
+#   readOnlyRootFilesystem: true
-  # readOnlyRootFilesystem: true
+#   runAsNonRoot: true
-  # runAsNonRoot: true
+#   runAsUser: 1000
-  # runAsUser: 1000
 
 service:
   type: ClusterIP
   port: 2020
-  labels:
+  labels: {}
-    {}
+  annotations: {}
-  annotations:
+#   prometheus.io/path: "/api/v1/metrics/prometheus"
-    {}
+#   prometheus.io/port: "2020"
-    # prometheus.io/path: "/api/v1/metrics/prometheus"
+#   prometheus.io/scrape: "true"
-    # prometheus.io/port: "2020"
-    # prometheus.io/scrape: "true"
 
 serviceMonitor:
   enabled: false
-  # namespace: monitoring
+#   namespace: monitoring
-  # interval: 10s
+#   interval: 10s
-  # scrapeTimeout: 10s
+#   scrapeTimeout: 10s
-  # selector:
+#   selector:
-  #  prometheus: my-prometheus
+#    prometheus: my-prometheus
 
 prometheusRule:
   enabled: false
-  # namespace: ""
+#   namespace: ""
-  # additionnalLabels: {}
+#   additionnalLabels: {}
-  # rules:
+#   rules:
-  # - alert: NoOutputBytesProcessed
+#   - alert: NoOutputBytesProcessed
-  #   expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0
+#     expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0
-  #   annotations:
+#     annotations:
-  #     message: |
+#       message: |
-  #       Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any
+#         Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any
-  #       bytes for at least 15 minutes.
+#         bytes for at least 15 minutes.
-  #     summary: No Output Bytes Processed
+#       summary: No Output Bytes Processed
-  #   for: 15m
+#     for: 15m
-  #   labels:
+#     labels:
-  #     severity: critical
+#       severity: critical
 
 dashboards:
   enabled: false
   labelKey: grafana_dashboard
   annotations: {}
 
-
+livenessProbe:
-livenessProbe: {}
+  httpGet:
-  # httpGet:
+    path: /
-  #   path: /
+    port: http
-  #   port: http
 
 readinessProbe:
-  # httpGet:
+  httpGet:
-  #   path: /
+    path: /api/v1/health
-  #   port: http
+    port: http
 
-resources:
+resources: {}
-  {}
+#   limits:
-  # limits:
+#     cpu: 100m
-  #   cpu: 100m
+#     memory: 128Mi
-  #   memory: 128Mi
+#   requests:
-  # requests:
+#     cpu: 100m
-  #   cpu: 100m
+#     memory: 128Mi
-  #   memory: 128Mi
 
 nodeSelector: {}
 
@@ -129,6 +129,10 @@ tolerations: []
 
 affinity: {}
 
+labels: {}
+
+annotations: {}
+
 podAnnotations: {}
 
 podLabels: {}
@@ -140,9 +144,9 @@ env: []
 envFrom: []
 
 extraContainers: []
-  # - name: do-something
+#   - name: do-something
-  #   image: busybox
+#     image: busybox
-  #   command: ['do', 'something']
+#     command: ['do', 'something']
 
 extraPorts: []
 #   - port: 5170
@@ -155,17 +159,17 @@ extraVolumes: []
 extraVolumeMounts: []
 
 updateStrategy: {}
-  # type: RollingUpdate
+#   type: RollingUpdate
-  # rollingUpdate:
+#   rollingUpdate:
-  #   maxUnavailable: 1
+#     maxUnavailable: 1
 
 # Make use of a pre-defined configmap instead of the one templated here
 existingConfigMap: ""
 
 networkPolicy:
   enabled: false
-  # ingress:
+#   ingress:
-  #   from: []
+#     from: []
 
 luaScripts: {}
 
@@ -173,21 +177,22 @@ luaScripts: {}
 config:
   service: |
     [SERVICE]
-        Flush 1
         Daemon Off
-        Log_Level info
+        Flush 1
+        Log_Level {{ .Values.logLevel }}
         Parsers_File parsers.conf
         Parsers_File custom_parsers.conf
         HTTP_Server On
         HTTP_Listen 0.0.0.0
         HTTP_Port {{ .Values.service.port }}
+        Health_Check On
 
   ## https://docs.fluentbit.io/manual/pipeline/inputs
   inputs: |
     [INPUT]
         Name tail
         Path /var/log/containers/*.log
-        Parser docker
+        multiline.parser docker, cri
         Tag kube.*
         Mem_Buf_Limit 5MB
         Skip_Long_Lines On
@@ -270,6 +275,8 @@ args: []
 command: []
 
 initContainers: []
-  # - name: do-something
+#   - name: do-something
-  #   image: busybox
+#     image: busybox
-  #   command: ['do', 'something']
+#     command: ['do', 'something']
+
+logLevel: info

charts/kubezero-logging/charts/fluentd/Chart.yaml

@@ -6,8 +6,10 @@ icon: https://www.fluentd.org/assets/img/miscellany/fluentd-logo_2x.png
 maintainers:
 - email: eduardo@treasure-data.com
   name: edsiper
+- email: diogo.filipe.tomas.guerra@cern.ch
+  name: dioguerra
 name: fluentd
 sources:
 - https://github.com/fluent/fluentd/
 - https://github.com/fluent/fluentd-kubernetes-daemonset
-version: 0.2.6
+version: 0.2.10

charts/kubezero-logging/charts/fluentd/templates/daemonset.yaml

@@ -5,16 +5,28 @@ metadata:
   name: {{ include "fluentd.fullname" . }}
   labels:
     {{- include "fluentd.labels" . | nindent 4 }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
   selector:
     matchLabels:
       {{- include "fluentd.selectorLabels" . | nindent 6 }}
+  {{- with .Values.updateStrategy }}
+  updateStrategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   template:
     metadata:
-      {{- with .Values.podAnnotations }}
       annotations:
-        {{- toYaml . | nindent 8 }}
+        checksum/config: {{ include (print $.Template.BasePath "/fluentd-configurations-cm.yaml") . | sha256sum }}
-      {{- end }}
+        {{- with .Values.podAnnotations }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
       labels:
         {{- include "fluentd.selectorLabels" . | nindent 8 }}
         {{- with .Values.podLabels }}

charts/kubezero-logging/charts/fluentd/templates/deployment.yaml

@@ -5,8 +5,19 @@ metadata:
   name: {{ include "fluentd.fullname" . }}
   labels:
     {{- include "fluentd.labels" . | nindent 4 }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
   replicas: {{ .Values.replicaCount }}
+  {{- with .Values.updateStrategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "fluentd.selectorLabels" . | nindent 6 }}
@@ -14,9 +25,9 @@ spec:
     metadata:
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/fluentd-configurations-cm.yaml") . | sha256sum }}
-      {{- with .Values.podAnnotations }}
+        {{- with .Values.podAnnotations }}
-        {{- toYaml . | nindent 8 }}
+          {{- toYaml . | nindent 8 }}
-      {{- end }}
+        {{- end }}
       labels:
         {{- include "fluentd.selectorLabels" . | nindent 8 }}
         {{- with .Values.podLabels }}

charts/kubezero-logging/charts/fluentd/values.yaml

@@ -124,6 +124,14 @@ tolerations: []
 ##
 affinity: {}
 
+## Annotations to be added to fluentd DaemonSet/Deployment
+##
+annotations: {}
+
+## Labels to be added to fluentd DaemonSet/Deployment
+##
+labels: {}
+
 ## Annotations to be added to fluentd pods
 ##
 podAnnotations: {}
@@ -132,6 +140,13 @@ podAnnotations: {}
 ##
 podLabels: {}
 
+## Deployment strategy / DaemonSet updateStrategy
+##
+updateStrategy: {}
+#   type: RollingUpdate
+#   rollingUpdate:
+#     maxUnavailable: 1
+
 ## Additional environment variables to set for fluentd pods
 env:
 - name: "FLUENTD_CONF"

charts/kubezero-logging/fluent-bit.patch

@@ -1,27 +1,3 @@
-diff -tubr charts/fluent-bit/templates/_pod.tpl charts/fluent-bit.zdt/templates/_pod.tpl
---- charts/fluent-bit/templates/_pod.tpl	2021-06-22 12:25:10.000000000 +0200
-+++ charts/fluent-bit.zdt/templates/_pod.tpl	2021-07-20 12:44:55.349873171 +0200
-@@ -21,6 +21,8 @@
- initContainers:
-   {{- toYaml .Values.initContainers | nindent 2 }}
- {{- end }}
-+hostNetwork: true
-+dnsPolicy: ClusterFirstWithHostNet
- containers:
-   - name: {{ .Chart.Name }}
-     securityContext:
-diff -tubr charts/fluent-bit/templates/clusterrole.yaml charts/fluent-bit.zdt/templates/clusterrole.yaml
---- charts/fluent-bit/templates/clusterrole.yaml	2021-06-22 12:25:10.000000000 +0200
-+++ charts/fluent-bit.zdt/templates/clusterrole.yaml	2021-07-20 12:44:29.936278716 +0200
-@@ -11,6 +11,8 @@
-     resources:
-       - pods
-       - namespaces
-+      - nodes
-+      - nodes/proxy
-     verbs:
-       - get
-       - list
 diff -tubr charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent-bit.zdt/templates/tests/test-connection.yaml
 --- charts/fluent-bit/templates/tests/test-connection.yaml	2021-06-22 12:25:10.000000000 +0200
 +++ charts/fluent-bit.zdt/templates/tests/test-connection.yaml	2021-05-17 12:09:02.724057438 +0200

charts/kubezero-logging/fluentd.patch

@@ -1,17 +1,3 @@
-diff -tubrN charts/fluentd/templates/deployment.yaml charts/fluentd.zdt/templates/deployment.yaml
---- charts/fluentd/templates/deployment.yaml	2021-02-12 18:13:04.000000000 +0100
-+++ charts/fluentd.zdt/templates/deployment.yaml	2021-03-10 09:54:49.863586462 +0100
-@@ -12,8 +12,9 @@
-       {{- include "fluentd.selectorLabels" . | nindent 6 }}
-   template:
-     metadata:
--      {{- with .Values.podAnnotations }}
-       annotations:
-+        checksum/config: {{ include (print $.Template.BasePath "/fluentd-configurations-cm.yaml") . | sha256sum }}
-+      {{- with .Values.podAnnotations }}
-         {{- toYaml . | nindent 8 }}
-       {{- end }}
-       labels:
 diff -tubrN charts/fluentd/templates/files.conf/systemd.yaml charts/fluentd.zdt/templates/files.conf/systemd.yaml
 --- charts/fluentd/templates/files.conf/systemd.yaml	2021-02-12 18:13:04.000000000 +0100
 +++ charts/fluentd.zdt/templates/files.conf/systemd.yaml	1970-01-01 01:00:00.000000000 +0100

charts/kubezero-logging/update.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
 ECK_VERSION=1.6.0
-FLUENT_BIT_VERSION=0.15.15
+FLUENT_BIT_VERSION=0.16.3
-FLUENTD_VERSION=0.2.6
+FLUENTD_VERSION=0.2.10
 
 # fix ECK crds handling to adhere to proper helm v3 support which also fixes ArgoCD applying updates on upgrades
 helm repo list | grep elastic -qc || { helm repo add elastic https://helm.elastic.co; helm repo update; }

charts/kubezero-logging/values.yaml

@@ -225,15 +225,20 @@ fluentd:
 fluent-bit:
   enabled: false
 
-  image:
+  #image:
     #repository: public.ecr.aws/zero-downtime/fluent-bit
-    tag: 1.8.3
+    #tag: 1.8.3
 
   serviceMonitor:
     enabled: false
     selector:
       release: metrics
 
+  #rbac:
+  #  nodeAccess: true
+  #hostNetwork: true
+  #dnsPolicy: ClusterFirstWithHostNet
+
   tolerations:
   - key: node-role.kubernetes.io/master
     effect: NoSchedule
@@ -274,6 +279,7 @@ fluent-bit:
           HTTP_Server On
           HTTP_Listen 0.0.0.0
           HTTP_Port {{ .Values.service.port }}
+          Health_Check On
     inputs: |
       [INPUT]
           Name tail