chore: fluent-bit,fluentd chart updates

2021-08-27 12:14:46 +02:00 · 2021-08-27 12:14:46 +02:00 · 32e12eaeb2
commit 32e12eaeb2
parent ee379aa802
17 changed files with 172 additions and 153 deletions
--- a/charts/kubezero-logging/Chart.yaml
+++ b/charts/kubezero-logging/Chart.yaml
@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-logging
 description: KubeZero Umbrella Chart for complete EFK stack
 type: application
-version: 0.7.6
+version: 0.7.7
 appVersion: 1.6.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@ -23,9 +23,9 @@ dependencies:
    # repository: https://helm.elastic.co
    condition: eck-operator.enabled
  - name: fluentd
-    version: 0.2.6
+    version: 0.2.10
    condition: fluentd.enabled
  - name: fluent-bit
-    version: 0.15.15
+    version: 0.16.3
    condition: fluent-bit.enabled
 kubeVersion: ">= 1.18.0"
--- a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml
@ -1,8 +1,9 @@
 annotations:
  artifacthub.io/changes: |
-    - Upgrade fluent-bit image to v1.7.9
+    - kind: changed
+      description: Update image version to v1.8.3.
 apiVersion: v1
-appVersion: 1.7.9
+appVersion: 1.8.3
 description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD
  family operating systems.
 home: https://fluentbit.io/
@ -21,4 +22,4 @@ maintainers:
 name: fluent-bit
 sources:
 - https://github.com/fluent/fluent-bit/
-version: 0.15.15
+version: 0.16.3
--- a/charts/kubezero-logging/charts/fluent-bit/ci/ci-values.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/ci/ci-values.yaml
@ -0,0 +1 @@
+logLevel: debug
--- a/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl
@ -9,6 +9,8 @@ priorityClassName: {{ .Values.priorityClassName }}
 serviceAccountName: {{ include "fluent-bit.serviceAccountName" . }}
 securityContext:
  {{- toYaml .Values.podSecurityContext | nindent 2 }}
+hostNetwork: {{ .Values.hostNetwork }}
+dnsPolicy: {{ .Values.dnsPolicy }}
 {{- with .Values.dnsConfig }}
 dnsConfig:
  {{- toYaml . | nindent 2 }}
@ -21,8 +23,6 @@ hostAliases:
 initContainers:
  {{- toYaml .Values.initContainers | nindent 2 }}
 {{- end }}
-hostNetwork: true
-dnsPolicy: ClusterFirstWithHostNet
 containers:
  - name: {{ .Chart.Name }}
    securityContext:
@ -56,24 +56,10 @@ containers:
        protocol: {{ .protocol }}
      {{- end }}
    {{- end }}
-    {{- if .Values.livenessProbe }}
    livenessProbe:
      {{- toYaml .Values.livenessProbe | nindent 6 }}
-    {{- else }}
-    livenessProbe:
-      httpGet:
-        path: /
-        port: http
-    {{- end }}
-    {{- if .Values.readinessProbe }}
    readinessProbe:
      {{- toYaml .Values.readinessProbe | nindent 6 }}
-    {{- else }}
-    readinessProbe:
-      httpGet:
-        path: /
-        port: http
-    {{- end }}
    resources:
      {{- toYaml .Values.resources | nindent 6 }}
    volumeMounts:
--- a/charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml
@ -9,10 +9,12 @@ rules:
  - apiGroups:
      - ""
    resources:
-      - pods
      - namespaces
+      - pods
+      {{- if .Values.rbac.nodeAccess }}
      - nodes
      - nodes/proxy
+      {{- end }}
    verbs:
      - get
      - list
--- a/charts/kubezero-logging/charts/fluent-bit/templates/daemonset.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/daemonset.yaml
@ -5,6 +5,13 @@ metadata:
  name: {{ include "fluent-bit.fullname" . }}
  labels:
    {{- include "fluent-bit.labels" . | nindent 4 }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
  selector:
    matchLabels:
--- a/charts/kubezero-logging/charts/fluent-bit/templates/deployment.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/deployment.yaml
@ -5,6 +5,13 @@ metadata:
  name: {{ include "fluent-bit.fullname" . }}
  labels:
    {{- include "fluent-bit.labels" . | nindent 4 }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
  replicas: {{ .Values.replicaCount }}
  {{- with .Values.updateStrategy }}
--- a/charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml
@ -17,7 +17,7 @@ spec:
    - ALL
  volumes:
    - '*'
-  hostNetwork: false
+  hostNetwork: {{ .Values.hostNetwork }}
  hostIPC: false
  hostPID: false
  runAsUser:
--- a/charts/kubezero-logging/charts/fluent-bit/values.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/values.yaml
@ -8,8 +8,9 @@ replicaCount: 1

 image:
  repository: fluent/fluent-bit
+  # Overrides the image tag whose default is {{ .Chart.AppVersion }}
+  tag: ""
  pullPolicy: Always
-  # tag:

 testFramework:
  enabled: false
@ -29,99 +30,98 @@ serviceAccount:

 rbac:
  create: true
+  nodeAccess: false

 podSecurityPolicy:
  create: false
  annotations: {}

-podSecurityContext:
-  {}
-  # fsGroup: 2000
+podSecurityContext: {}
+#   fsGroup: 2000
+
+hostNetwork: false
+dnsPolicy: ClusterFirst
+
 dnsConfig: {}
-  # nameservers:
-  #   - 1.2.3.4
-  # searches:
-  #   - ns1.svc.cluster-domain.example
-  #   - my.dns.search.suffix
-  # options:
-  #   - name: ndots
-#     value: "2"
-#   - name: edns0
+#   nameservers:
+#     - 1.2.3.4
+#   searches:
+#     - ns1.svc.cluster-domain.example
+#     - my.dns.search.suffix
+#   options:
+#     - name: ndots
+#       value: "2"
+#     - name: edns0

 hostAliases: []
-  # - ip: "1.2.3.4"
-  #   hostnames:
-  #   - "foo.local"
-  #   - "bar.local"
+#   - ip: "1.2.3.4"
+#     hostnames:
+#     - "foo.local"
+#     - "bar.local"

-securityContext:
-  {}
-  # capabilities:
-  #   drop:
-  #   - ALL
-  # readOnlyRootFilesystem: true
-  # runAsNonRoot: true
-  # runAsUser: 1000
+securityContext: {}
+#   capabilities:
+#     drop:
+#     - ALL
+#   readOnlyRootFilesystem: true
+#   runAsNonRoot: true
+#   runAsUser: 1000

 service:
  type: ClusterIP
  port: 2020
-  labels:
-    {}
-  annotations:
-    {}
-    # prometheus.io/path: "/api/v1/metrics/prometheus"
-    # prometheus.io/port: "2020"
-    # prometheus.io/scrape: "true"
+  labels: {}
+  annotations: {}
+#   prometheus.io/path: "/api/v1/metrics/prometheus"
+#   prometheus.io/port: "2020"
+#   prometheus.io/scrape: "true"

 serviceMonitor:
  enabled: false
-  # namespace: monitoring
-  # interval: 10s
-  # scrapeTimeout: 10s
-  # selector:
-  #  prometheus: my-prometheus
+#   namespace: monitoring
+#   interval: 10s
+#   scrapeTimeout: 10s
+#   selector:
+#    prometheus: my-prometheus

 prometheusRule:
  enabled: false
-  # namespace: ""
-  # additionnalLabels: {}
-  # rules:
-  # - alert: NoOutputBytesProcessed
-  #   expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0
-  #   annotations:
-  #     message: |
-  #       Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any
-  #       bytes for at least 15 minutes.
-  #     summary: No Output Bytes Processed
-  #   for: 15m
-  #   labels:
-  #     severity: critical
+#   namespace: ""
+#   additionnalLabels: {}
+#   rules:
+#   - alert: NoOutputBytesProcessed
+#     expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0
+#     annotations:
+#       message: |
+#         Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any
+#         bytes for at least 15 minutes.
+#       summary: No Output Bytes Processed
+#     for: 15m
+#     labels:
+#       severity: critical

 dashboards:
  enabled: false
  labelKey: grafana_dashboard
  annotations: {}

-
-livenessProbe: {}
-  # httpGet:
-  #   path: /
-  #   port: http
+livenessProbe:
+  httpGet:
+    path: /
+    port: http

 readinessProbe:
-  # httpGet:
-  #   path: /
-  #   port: http
+  httpGet:
+    path: /api/v1/health
+    port: http

-resources:
-  {}
-  # limits:
-  #   cpu: 100m
-  #   memory: 128Mi
-  # requests:
-  #   cpu: 100m
-  #   memory: 128Mi
+resources: {}
+#   limits:
+#     cpu: 100m
+#     memory: 128Mi
+#   requests:
+#     cpu: 100m
+#     memory: 128Mi

 nodeSelector: {}

@ -129,6 +129,10 @@ tolerations: []

 affinity: {}

+labels: {}
+
+annotations: {}
+
 podAnnotations: {}

 podLabels: {}
@ -140,9 +144,9 @@ env: []
 envFrom: []

 extraContainers: []
-  # - name: do-something
-  #   image: busybox
-  #   command: ['do', 'something']
+#   - name: do-something
+#     image: busybox
+#     command: ['do', 'something']

 extraPorts: []
 #   - port: 5170
@ -155,17 +159,17 @@ extraVolumes: []
 extraVolumeMounts: []

 updateStrategy: {}
-  # type: RollingUpdate
-  # rollingUpdate:
-  #   maxUnavailable: 1
+#   type: RollingUpdate
+#   rollingUpdate:
+#     maxUnavailable: 1

 # Make use of a pre-defined configmap instead of the one templated here
 existingConfigMap: ""

 networkPolicy:
  enabled: false
-  # ingress:
-  #   from: []
+#   ingress:
+#     from: []

 luaScripts: {}

@ -173,21 +177,22 @@ luaScripts: {}
 config:
  service: |
    [SERVICE]
-        Flush 1
        Daemon Off
-        Log_Level info
+        Flush 1
+        Log_Level {{ .Values.logLevel }}
        Parsers_File parsers.conf
        Parsers_File custom_parsers.conf
        HTTP_Server On
        HTTP_Listen 0.0.0.0
        HTTP_Port {{ .Values.service.port }}
+        Health_Check On

  ## https://docs.fluentbit.io/manual/pipeline/inputs
  inputs: |
    [INPUT]
        Name tail
        Path /var/log/containers/*.log
-        Parser docker
+        multiline.parser docker, cri
        Tag kube.*
        Mem_Buf_Limit 5MB
        Skip_Long_Lines On
@ -270,6 +275,8 @@ args: []
 command: []

 initContainers: []
-  # - name: do-something
-  #   image: busybox
-  #   command: ['do', 'something']
+#   - name: do-something
+#     image: busybox
+#     command: ['do', 'something']
+
+logLevel: info
--- a/charts/kubezero-logging/charts/fluentd/Chart.yaml
+++ b/charts/kubezero-logging/charts/fluentd/Chart.yaml
@ -6,8 +6,10 @@ icon: https://www.fluentd.org/assets/img/miscellany/fluentd-logo_2x.png
 maintainers:
 - email: eduardo@treasure-data.com
  name: edsiper
+- email: diogo.filipe.tomas.guerra@cern.ch
+  name: dioguerra
 name: fluentd
 sources:
 - https://github.com/fluent/fluentd/
 - https://github.com/fluent/fluentd-kubernetes-daemonset
-version: 0.2.6
+version: 0.2.10
--- a/charts/kubezero-logging/charts/fluentd/templates/daemonset.yaml
+++ b/charts/kubezero-logging/charts/fluentd/templates/daemonset.yaml
@ -5,16 +5,28 @@ metadata:
  name: {{ include "fluentd.fullname" . }}
  labels:
    {{- include "fluentd.labels" . | nindent 4 }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
  selector:
    matchLabels:
      {{- include "fluentd.selectorLabels" . | nindent 6 }}
+  {{- with .Values.updateStrategy }}
+  updateStrategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
  template:
    metadata:
-      {{- with .Values.podAnnotations }}
      annotations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
+        checksum/config: {{ include (print $.Template.BasePath "/fluentd-configurations-cm.yaml") . | sha256sum }}
+        {{- with .Values.podAnnotations }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
      labels:
        {{- include "fluentd.selectorLabels" . | nindent 8 }}
        {{- with .Values.podLabels }}
--- a/charts/kubezero-logging/charts/fluentd/templates/deployment.yaml
+++ b/charts/kubezero-logging/charts/fluentd/templates/deployment.yaml
@ -5,8 +5,19 @@ metadata:
  name: {{ include "fluentd.fullname" . }}
  labels:
    {{- include "fluentd.labels" . | nindent 4 }}
+    {{- with .Values.labels }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
  replicas: {{ .Values.replicaCount }}
+  {{- with .Values.updateStrategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
  selector:
    matchLabels:
      {{- include "fluentd.selectorLabels" . | nindent 6 }}
@ -14,9 +25,9 @@ spec:
    metadata:
      annotations:
        checksum/config: {{ include (print $.Template.BasePath "/fluentd-configurations-cm.yaml") . | sha256sum }}
-      {{- with .Values.podAnnotations }}
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
+        {{- with .Values.podAnnotations }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
      labels:
        {{- include "fluentd.selectorLabels" . | nindent 8 }}
        {{- with .Values.podLabels }}
--- a/charts/kubezero-logging/charts/fluentd/values.yaml
+++ b/charts/kubezero-logging/charts/fluentd/values.yaml
@ -124,6 +124,14 @@ tolerations: []
 ##
 affinity: {}

+## Annotations to be added to fluentd DaemonSet/Deployment
+##
+annotations: {}
+
+## Labels to be added to fluentd DaemonSet/Deployment
+##
+labels: {}
+
 ## Annotations to be added to fluentd pods
 ##
 podAnnotations: {}
@ -132,6 +140,13 @@ podAnnotations: {}
 ##
 podLabels: {}

+## Deployment strategy / DaemonSet updateStrategy
+##
+updateStrategy: {}
+#   type: RollingUpdate
+#   rollingUpdate:
+#     maxUnavailable: 1
+
 ## Additional environment variables to set for fluentd pods
 env:
 - name: "FLUENTD_CONF"
--- a/charts/kubezero-logging/fluent-bit.patch
+++ b/charts/kubezero-logging/fluent-bit.patch
@ -1,27 +1,3 @@
-diff -tubr charts/fluent-bit/templates/_pod.tpl charts/fluent-bit.zdt/templates/_pod.tpl
--- charts/fluent-bit/templates/_pod.tpl	2021-06-22 12:25:10.000000000 +0200
-+++ charts/fluent-bit.zdt/templates/_pod.tpl	2021-07-20 12:44:55.349873171 +0200
-@@ -21,6 +21,8 @@
- initContainers:
-   {{- toYaml .Values.initContainers | nindent 2 }}
- {{- end }}
-+hostNetwork: true
-+dnsPolicy: ClusterFirstWithHostNet
- containers:
-   - name: {{ .Chart.Name }}
-     securityContext:
-diff -tubr charts/fluent-bit/templates/clusterrole.yaml charts/fluent-bit.zdt/templates/clusterrole.yaml
--- charts/fluent-bit/templates/clusterrole.yaml	2021-06-22 12:25:10.000000000 +0200
-+++ charts/fluent-bit.zdt/templates/clusterrole.yaml	2021-07-20 12:44:29.936278716 +0200
-@@ -11,6 +11,8 @@
-     resources:
-       - pods
-       - namespaces
-+      - nodes
-+      - nodes/proxy
-     verbs:
-       - get
-       - list
 diff -tubr charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent-bit.zdt/templates/tests/test-connection.yaml
 --- charts/fluent-bit/templates/tests/test-connection.yaml	2021-06-22 12:25:10.000000000 +0200
 +++ charts/fluent-bit.zdt/templates/tests/test-connection.yaml	2021-05-17 12:09:02.724057438 +0200
--- a/charts/kubezero-logging/fluentd.patch
+++ b/charts/kubezero-logging/fluentd.patch
@ -1,17 +1,3 @@
-diff -tubrN charts/fluentd/templates/deployment.yaml charts/fluentd.zdt/templates/deployment.yaml
--- charts/fluentd/templates/deployment.yaml	2021-02-12 18:13:04.000000000 +0100
-+++ charts/fluentd.zdt/templates/deployment.yaml	2021-03-10 09:54:49.863586462 +0100
-@@ -12,8 +12,9 @@
-       {{- include "fluentd.selectorLabels" . | nindent 6 }}
-   template:
-     metadata:
-      {{- with .Values.podAnnotations }}
-       annotations:
-+        checksum/config: {{ include (print $.Template.BasePath "/fluentd-configurations-cm.yaml") . | sha256sum }}
-+      {{- with .Values.podAnnotations }}
-         {{- toYaml . | nindent 8 }}
-       {{- end }}
-       labels:
 diff -tubrN charts/fluentd/templates/files.conf/systemd.yaml charts/fluentd.zdt/templates/files.conf/systemd.yaml
 --- charts/fluentd/templates/files.conf/systemd.yaml	2021-02-12 18:13:04.000000000 +0100
 +++ charts/fluentd.zdt/templates/files.conf/systemd.yaml	1970-01-01 01:00:00.000000000 +0100
--- a/charts/kubezero-logging/update.sh
+++ b/charts/kubezero-logging/update.sh
@ -1,8 +1,8 @@
 #!/bin/bash

 ECK_VERSION=1.6.0
-FLUENT_BIT_VERSION=0.15.15
-FLUENTD_VERSION=0.2.6
+FLUENT_BIT_VERSION=0.16.3
+FLUENTD_VERSION=0.2.10

 # fix ECK crds handling to adhere to proper helm v3 support which also fixes ArgoCD applying updates on upgrades
 helm repo list | grep elastic -qc || { helm repo add elastic https://helm.elastic.co; helm repo update; }
--- a/charts/kubezero-logging/values.yaml
+++ b/charts/kubezero-logging/values.yaml
@ -225,15 +225,20 @@ fluentd:
 fluent-bit:
  enabled: false

-  image:
+  #image:
    #repository: public.ecr.aws/zero-downtime/fluent-bit
-    tag: 1.8.3
+    #tag: 1.8.3

  serviceMonitor:
    enabled: false
    selector:
      release: metrics

+  #rbac:
+  #  nodeAccess: true
+  #hostNetwork: true
+  #dnsPolicy: ClusterFirstWithHostNet
+
  tolerations:
  - key: node-role.kubernetes.io/master
    effect: NoSchedule
@ -274,6 +279,7 @@ fluent-bit:
          HTTP_Server On
          HTTP_Listen 0.0.0.0
          HTTP_Port {{ .Values.service.port }}
+          Health_Check On
    inputs: |
      [INPUT]
          Name tail