chore: fluent-bit version bump to fix excessive logging

charts/kubezero-logging/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-logging
 description: KubeZero Umbrella Chart for complete EFK stack
 type: application
-version: 0.6.3
+version: 0.6.4
 appVersion: 1.4.1
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@@ -26,6 +26,6 @@ dependencies:
     version: 0.2.2
     condition: fluentd.enabled
   - name: fluent-bit
-    version: 0.12.3
+    version: 0.15.4
     condition: fluent-bit.enabled
 kubeVersion: ">= 1.18.0"

charts/kubezero-logging/charts/fluent-bit/Chart.yaml

@@ -1,8 +1,8 @@
 annotations:
   artifacthub.io/changes: |
-    - add custom annotations on dashboards configmap
+    - Upgrade fluent-bit image to v1.7.3
 apiVersion: v2
-appVersion: 1.7.2
+appVersion: 1.7.3
 description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
 home: https://fluentbit.io/
 icon: https://fluentbit.io/assets/img/logo1-default.png
@@ -20,4 +20,4 @@ maintainers:
 name: fluent-bit
 sources:
 - https://github.com/fluent/fluent-bit/
-version: 0.12.3
+version: 0.15.4

charts/kubezero-logging/charts/fluent-bit/dashboards/fluent-bit.json

@@ -122,7 +122,7 @@
       "pluginVersion": "7.2.1",
       "targets": [
         {
-          "expr": "sum(kube_pod_info{pod=~\".*fluent-bit.*\"})",
+          "expr": "sum(kube_pod_info{pod=~\".*{{ include "fluent-bit.fullname" . }}.*\"})",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -221,7 +221,7 @@
           "hide": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{ pod }}/{{name}}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}/{{"{{"}}name{{"}}"}}",
           "refId": "A"
         }
       ],
@@ -325,7 +325,7 @@
           "hide": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{ pod }}/{{name}}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}/{{"{{"}}name{{"}}"}}",
           "refId": "A"
         }
       ],
@@ -429,7 +429,7 @@
           "hide": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{ pod }}/{{name}}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}/{{"{{"}}name{{"}}"}}",
           "refId": "A"
         }
       ],
@@ -535,7 +535,7 @@
           "hide": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{ pod }}/{{name}}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}/{{"{{"}}name{{"}}"}}",
           "refId": "A"
         }
       ],
@@ -641,7 +641,7 @@
           "hide": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{pod}} Retries to {{name}}",
+          "legendFormat": "{{"{{"}}pod{{"}}"}} Retries to {{"{{"}}name{{"}}"}}",
           "refId": "A"
         },
         {
@@ -649,7 +649,7 @@
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{pod}} Failed Retries to {{ name }}",
+          "legendFormat": "{{"{{"}}pod{{"}}"}} Failed Retries to {{"{{"}} name {{"}}"}}",
           "refId": "B"
         }
       ],
@@ -756,7 +756,7 @@
           "hide": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{ pod }}/{{ name }}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}/{{"{{"}} name {{"}}"}}",
           "refId": "A"
         }
       ],
@@ -851,7 +851,7 @@
         {
           "expr": "sum(rate(fluentbit_filter_drop_records_total{pod=~\"$pod\"}[5m])) by (pod, instance, name)",
           "interval": "",
-          "legendFormat": "{{ pod }} / {{ name }}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}} / {{"{{"}} name {{"}}"}}",
           "refId": "A"
         }
       ],
@@ -946,7 +946,7 @@
         {
           "expr": "sum(rate(fluentbit_filter_add_records_total{pod=~\"$pod\"}[5m])) by (pod, instance, name)",
           "interval": "",
-          "legendFormat": "{{ pod }} / {{ name }}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}} / {{"{{"}} name {{"}}"}}",
           "refId": "A"
         }
       ],
@@ -1059,15 +1059,15 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "container_memory_working_set_bytes{pod=~\".*fluent-bit.*\",pod=~\"$pod\", image!=\"\", container!=\"POD\"}\n",
+          "expr": "container_memory_working_set_bytes{pod=~\".*{{ include "fluent-bit.fullname" . }}.*\",pod=~\"$pod\", image!=\"\", container!=\"POD\"}\n",
           "interval": "",
-          "legendFormat": "{{ pod }}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}",
           "refId": "A"
         },
         {
-          "expr": "avg(kube_pod_container_resource_requests_memory_bytes{pod=~\".*fluent-bit.*\",pod=~\"$pod\"}) by (pod)",
+          "expr": "avg(kube_pod_container_resource_requests_memory_bytes{pod=~\".*{{ include "fluent-bit.fullname" . }}.*\",pod=~\"$pod\"}) by (pod)",
           "interval": "",
-          "legendFormat": "{{ pod }} request",
+          "legendFormat": "{{"{{"}} pod {{"}}"}} request",
           "refId": "B"
         }
       ],
@@ -1075,7 +1075,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Memroy Usage",
+      "title": "Memory Usage",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -1166,15 +1166,15 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "rate(container_cpu_usage_seconds_total{pod=~\".*fluent-bit.*\",pod=~\"$pod\",image!=\"\",container!=\"POD\"}[5m])",
+          "expr": "rate(container_cpu_usage_seconds_total{pod=~\".*{{ include "fluent-bit.fullname" . }}.*\",pod=~\"$pod\",image!=\"\",container!=\"POD\"}[5m])",
           "interval": "",
-          "legendFormat": "{{ pod }}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}",
           "refId": "A"
         },
         {
           "expr": "avg(kube_pod_container_resource_requests_cpu_cores{pod=~\"$pod\"}) by (pod)",
           "interval": "",
-          "legendFormat": "{{ pod }} request",
+          "legendFormat": "{{"{{"}} pod {{"}}"}} request",
           "refId": "B"
         }
       ],
@@ -1249,14 +1249,14 @@
         "allValue": null,
         "current": {},
         "datasource": "$DS_PROMETHEUS",
-        "definition": "label_values(kube_pod_info{pod=~\".*fluent-bit.*\"}, pod)",
+        "definition": "label_values(kube_pod_info{pod=~\".*{{ include "fluent-bit.fullname" . }}.*\"}, pod)",
         "hide": 0,
         "includeAll": true,
         "label": "pod",
         "multi": false,
         "name": "pod",
         "options": [],
-        "query": "label_values(kube_pod_info{pod=~\".*fluent-bit.*\"}, pod)",
+        "query": "label_values(kube_pod_info{pod=~\".*{{ include "fluent-bit.fullname" . }}.*\"}, pod)",
         "refresh": 2,
         "regex": "",
         "skipUrlSync": false,
@@ -1299,7 +1299,7 @@
     ]
   },
   "timezone": "",
-  "title": "Fluent Bit",
+  "title": "{{ include "fluent-bit.fullname" . }}",
-  "uid": "fluentbit",
+  "uid": "{{ include "fluent-bit.fullname" . }}",
   "version": 2
 }

charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl

@@ -28,6 +28,14 @@ containers:
   {{- if .Values.envFrom }}
     envFrom:
     {{- toYaml .Values.envFrom | nindent 4 }}
+  {{- end }}
+  {{- if .Values.args }}
+    args:
+    {{- toYaml .Values.args | nindent 6 }}
+  {{- end}}
+  {{- if .Values.command }}
+    command:
+    {{- toYaml .Values.command | nindent 6 }}
   {{- end }}
     ports:
       - name: http
@@ -43,35 +51,39 @@ containers:
     {{- if .Values.livenessProbe }}
     livenessProbe:
       {{- toYaml .Values.livenessProbe | nindent 6 }}
+    {{- else }}
+    livenessProbe:
+      httpGet:
+        path: /
+        port: http
     {{- end }}
     {{- if .Values.readinessProbe }}
     readinessProbe:
       {{- toYaml .Values.readinessProbe | nindent 6 }}
+    {{- else }}
+    readinessProbe:
+      httpGet:
+        path: /
+        port: http
     {{- end }}
     resources:
       {{- toYaml .Values.resources | nindent 6 }}
     volumeMounts:
-      - name: config
+      {{- toYaml .Values.volumeMounts | nindent 6 }}
-        mountPath: /fluent-bit/etc/fluent-bit.conf
-        subPath: fluent-bit.conf
-      - name: config
-        mountPath: /fluent-bit/etc/custom_parsers.conf
-        subPath: custom_parsers.conf
     {{- range $key, $value := .Values.luaScripts }}
       - name: luascripts
         mountPath: /fluent-bit/scripts/{{ $key }}
         subPath: {{ $key }}
     {{- end }}
     {{- if eq .Values.kind "DaemonSet" }}
-      - name: varlog
+      {{- toYaml .Values.daemonSetVolumeMounts | nindent 6 }}
-        mountPath: /var/log
-      - name: etcmachineid
-        mountPath: /etc/machine-id
-        readOnly: true
     {{- end }}
     {{- if .Values.extraVolumeMounts }}
       {{- toYaml .Values.extraVolumeMounts | nindent 6 }}
     {{- end }}
+  {{- if .Values.extraContainers }}
+    {{- toYaml .Values.extraContainers | nindent 2 }}
+  {{- end }}
 volumes:
   - name: config
     configMap:
@@ -82,13 +94,7 @@ volumes:
       name: {{ include "fluent-bit.fullname" . }}-luascripts
 {{- end }}
 {{- if eq .Values.kind "DaemonSet" }}
-  - name: varlog
+  {{- toYaml .Values.daemonSetVolumes | nindent 2 }}
-    hostPath:
-      path: /var/log
-  - name: etcmachineid
-    hostPath:
-      path: /etc/machine-id
-      type: File
 {{- end }}
 {{- if .Values.extraVolumes }}
   {{- toYaml .Values.extraVolumes | nindent 2 }}

charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml

@@ -11,8 +11,8 @@ rules:
     resources:
       - pods
       - namespaces
-      - nodes
+      #- nodes
-      - nodes/proxy
+      #- nodes/proxy
     verbs:
       - get
       - list

charts/kubezero-logging/charts/fluent-bit/templates/configmap-dashboards.yaml

@@ -14,7 +14,7 @@ metadata:
     {{ $.Values.dashboards.labelKey }}: "1"
 data:
   {{ base $path }}: |
-    {{- $.Files.Get $path | nindent 4 }}
+    {{- tpl ($.Files.Get $path) $ | nindent 4 }}
 ---
 {{- end }}
 {{- end -}}

charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml

@@ -3,6 +3,10 @@ apiVersion: policy/v1beta1
 kind: PodSecurityPolicy
 metadata:
   name: {{ include "fluent-bit.fullname" . }}
+{{- if .Values.podSecurityPolicy.annotations }}
+  annotations:
+    {{- toYaml .Values.podSecurityPolicy.annotations | nindent 4 }}
+{{- end }}
 spec:
   privileged: false
   # Required to prevent escalations to root.

charts/kubezero-logging/charts/fluent-bit/values.yaml

@@ -32,6 +32,7 @@ rbac:
 
 podSecurityPolicy:
   create: false
+  annotations: {}
 
 podSecurityContext:
   {}
@@ -96,15 +97,15 @@ dashboards:
   annotations: {}
 
 
-livenessProbe:
+livenessProbe: {}
-  httpGet:
+  # httpGet:
-    path: /
+  #   path: /
-    port: http
+  #   port: http
 
 readinessProbe:
-  httpGet:
+  # httpGet:
-    path: /
+  #   path: /
-    port: http
+  #   port: http
 
 resources:
   {}
@@ -131,6 +132,11 @@ env: []
 
 envFrom: []
 
+extraContainers: []
+  # - name: do-something
+  #   image: busybox
+  #   command: ['do', 'something']
+
 extraPorts: []
 #   - port: 5170
 #     containerPort: 5170
@@ -220,3 +226,38 @@ config:
         Time_Keep Off
         Time_Key time
         Time_Format %Y-%m-%dT%H:%M:%S.%L
+
+# The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname"
+volumeMounts:
+  - name: config
+    mountPath: /fluent-bit/etc/fluent-bit.conf
+    subPath: fluent-bit.conf
+  - name: config
+    mountPath: /fluent-bit/etc/custom_parsers.conf
+    subPath: custom_parsers.conf
+
+daemonSetVolumes:
+  - name: varlog
+    hostPath:
+      path: /var/log
+  - name: varlibdockercontainers
+    hostPath:
+      path: /var/lib/docker/containers
+  - name: etcmachineid
+    hostPath:
+      path: /etc/machine-id
+      type: File
+
+daemonSetVolumeMounts:
+  - name: varlog
+    mountPath: /var/log
+  - name: varlibdockercontainers
+    mountPath: /var/lib/docker/containers
+    readOnly: true
+  - name: etcmachineid
+    mountPath: /etc/machine-id
+    readOnly: true
+
+args: []
+
+command: []

charts/kubezero-logging/fluent-bit.patch

@@ -1,20 +1,18 @@
-diff -tubNr charts/fluent-bit/Chart.yaml charts/fluent-bit.zdt/Chart.yaml
+diff -tubrN charts/fluent-bit/Chart.yaml charts/fluent-bit.zdt/Chart.yaml
---- charts/fluent-bit/Chart.yaml	2021-02-23 14:58:39.000000000 +0100
+--- charts/fluent-bit/Chart.yaml	2021-04-06 13:55:48.000000000 +0200
-+++ charts/fluent-bit.zdt/Chart.yaml	2021-03-07 12:32:34.360084252 +0100
++++ charts/fluent-bit.zdt/Chart.yaml	2021-04-07 11:49:25.955685229 +0200
-@@ -1,8 +1,8 @@
+@@ -1,7 +1,7 @@
  annotations:
    artifacthub.io/changes: |
-     - add custom annotations on dashboards configmap
+     - Upgrade fluent-bit image to v1.7.3
 -apiVersion: v1
--appVersion: 1.7.1
 +apiVersion: v2
-+appVersion: 1.7.2
+ appVersion: 1.7.3
  description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
  home: https://fluentbit.io/
- icon: https://fluentbit.io/assets/img/logo1-default.png
+diff -tubrN charts/fluent-bit/templates/_pod.tpl charts/fluent-bit.zdt/templates/_pod.tpl
-diff -tubNr charts/fluent-bit/templates/_pod.tpl charts/fluent-bit.zdt/templates/_pod.tpl
+--- charts/fluent-bit/templates/_pod.tpl	2021-04-06 13:55:48.000000000 +0200
---- charts/fluent-bit/templates/_pod.tpl	2021-02-23 14:58:39.000000000 +0100
++++ charts/fluent-bit.zdt/templates/_pod.tpl	2021-04-07 11:53:45.609030378 +0200
-+++ charts/fluent-bit.zdt/templates/_pod.tpl	2021-03-07 12:45:11.766785251 +0100
 @@ -13,6 +13,8 @@
  dnsConfig:
    {{- toYaml . | nindent 2 }}
@@ -24,41 +22,21 @@ diff -tubNr charts/fluent-bit/templates/_pod.tpl charts/fluent-bit.zdt/templates
  containers:
    - name: {{ .Chart.Name }}
      securityContext:
-@@ -63,9 +65,6 @@
+diff -tubrN charts/fluent-bit/templates/clusterrole.yaml charts/fluent-bit.zdt/templates/clusterrole.yaml
-     {{- if eq .Values.kind "DaemonSet" }}
+--- charts/fluent-bit/templates/clusterrole.yaml	2021-04-06 13:55:48.000000000 +0200
-       - name: varlog
++++ charts/fluent-bit.zdt/templates/clusterrole.yaml	2021-04-07 11:53:26.049029491 +0200
-         mountPath: /var/log
--      - name: varlibdockercontainers
--        mountPath: /var/lib/docker/containers
--        readOnly: true
-       - name: etcmachineid
-         mountPath: /etc/machine-id
-         readOnly: true
-@@ -86,9 +85,6 @@
-   - name: varlog
-     hostPath:
-       path: /var/log
--  - name: varlibdockercontainers
--    hostPath:
--      path: /var/lib/docker/containers
-   - name: etcmachineid
-     hostPath:
-       path: /etc/machine-id
-diff -tubNr charts/fluent-bit/templates/clusterrole.yaml charts/fluent-bit.zdt/templates/clusterrole.yaml
---- charts/fluent-bit/templates/clusterrole.yaml	2021-02-23 14:58:39.000000000 +0100
-+++ charts/fluent-bit.zdt/templates/clusterrole.yaml	2021-03-07 12:32:34.363417586 +0100
 @@ -11,6 +11,8 @@
      resources:
        - pods
        - namespaces
-+      - nodes
++      #- nodes
-+      - nodes/proxy
++      #- nodes/proxy
      verbs:
        - get
        - list
-diff -tubNr charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent-bit.zdt/templates/tests/test-connection.yaml
+diff -tubrN charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent-bit.zdt/templates/tests/test-connection.yaml
---- charts/fluent-bit/templates/tests/test-connection.yaml	2021-02-23 14:58:39.000000000 +0100
+--- charts/fluent-bit/templates/tests/test-connection.yaml	2021-04-06 13:55:48.000000000 +0200
-+++ charts/fluent-bit.zdt/templates/tests/test-connection.yaml	2021-03-07 12:32:34.363417586 +0100
++++ charts/fluent-bit.zdt/templates/tests/test-connection.yaml	2021-04-07 11:52:34.199027132 +0200
 @@ -1,3 +1,4 @@
 +{{- if .Values.testFramework.enabled }}
  apiVersion: v1
@@ -69,9 +47,9 @@ diff -tubNr charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent
    {{- end }}
    restartPolicy: Never
 +{{- end }}
-diff -tubNr charts/fluent-bit/values.yaml charts/fluent-bit.zdt/values.yaml
+diff -tubrN charts/fluent-bit/values.yaml charts/fluent-bit.zdt/values.yaml
---- charts/fluent-bit/values.yaml	2021-02-23 14:58:39.000000000 +0100
+--- charts/fluent-bit/values.yaml	2021-04-06 13:55:48.000000000 +0200
-+++ charts/fluent-bit.zdt/values.yaml	2021-03-07 12:32:34.363417586 +0100
++++ charts/fluent-bit.zdt/values.yaml	2021-04-07 11:52:54.742361399 +0200
 @@ -12,6 +12,7 @@
    # tag:
  

charts/kubezero-logging/update.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-FLUENT_BIT_VERSION=0.12.3
+FLUENT_BIT_VERSION=0.15.4
 FLUENTD_VERSION=0.2.2
 
 # Fluent Bit

charts/kubezero-logging/values.yaml

@@ -350,3 +350,19 @@ fluent-bit:
           end
           return 1, timestamp, new_record
       end
+
+  daemonSetVolumes:
+  - name: varlog
+    hostPath:
+      path: /var/log
+  - name: etcmachineid
+    hostPath:
+      path: /etc/machine-id
+      type: File
+
+  daemonSetVolumeMounts:
+  - name: varlog
+    mountPath: /var/log
+  - name: etcmachineid
+    mountPath: /etc/machine-id
+    readOnly: true