Add fluent-bit support to kuberzero-logging, istio fixes

charts/fluent-bit

@@ -0,0 +1 @@
+../../helm-charts/charts/fluent-bit

charts/kubezero-istio/templates/istio-private-ingress.yaml

@@ -45,11 +45,11 @@ spec:
         {{- end }}
         resources:
           limits:
-            cpu: 2000m
+            # cpu: 2000m
             memory: 1024Mi
           requests:
             cpu: 100m
-            memory: 128Mi
+            memory: 64Mi
         strategy:
           rollingUpdate:
             maxSurge: 100%
@@ -115,11 +115,11 @@ spec:
           image: node-agent-k8s
           resources:
             limits:
-              #cpu: 2000m
+              cpu: 2000m
               memory: 1024Mi
             requests:
               cpu: 100m
-              memory: 64Mi
+              memory: 128Mi
         secretVolumes:
         - mountPath: /etc/istio/ingressgateway-certs
           name: ingressgateway-certs

charts/kubezero-lib/templates/_helpers.tpl

@@ -2,7 +2,7 @@
 Common set of labels
 */ -}}
 {{- define "kubezero-lib.labels" -}}
-helm.sh/chart: {{ include "kubezero-lib.chart . }}
+helm.sh/chart: {{ include "kubezero-lib.chart" . }}
 app.kubernetes.io/name: {{ include "kubezero-lib.name" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 app.kubernetes.io/part-of: kubezero

charts/kubezero-logging/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-logging
 description: KubeZero Umbrella Chart for complete EFK stack
 type: application
-version: 0.2.0
+version: 0.3.1
 appVersion: 1.2.1
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@@ -22,4 +22,9 @@ dependencies:
     version: 2.5.1
     repository: https://kubernetes-charts.storage.googleapis.com/
     condition: fluentd.enabled
+  - name: fluent-bit
+    version: 0.6.3
+    repository: https://zero-down-time.github.io/kubezero/
+    # repository: https://fluent.github.io/helm-charts
+    condition: fluent-bit.enabled
 kubeVersion: ">= 1.16.0"

charts/kubezero-logging/values-fluentbit.yaml

@@ -0,0 +1,8 @@
+fluent-bit:
+  enabled: true
+
+  metrics:
+    enabled: true
+
+  url: fluentd.example.com
+

charts/kubezero-logging/values.yaml

@@ -125,7 +125,7 @@ fluentd:
         id_key id
         remove_keys id
 
-        # This pipeline incl. eg. GeoIP
+        # KubeZero pipeline incl. GeoIP etc.
         pipeline fluentd
 
         host "#{ENV['OUTPUT_HOST']}"
@@ -193,3 +193,109 @@ fluentd:
 #        </grok>
 #      </parse>
 #    </filter>
+
+
+fluent-bit:
+  enabled: true
+  test:
+    enabled: false
+
+  config:
+    outputs: |
+      [OUTPUT]
+          Match *
+          Name forward
+          Host fluentd
+          Port 24224
+          tls on
+          tls.verify off
+          Shared_Key cloudbender
+
+    inputs: |
+      [INPUT]
+          Name tail
+          Path /var/log/containers/*.log
+          Parser cri
+          Tag kube.*
+          Mem_Buf_Limit 5MB
+          Skip_Long_Lines On
+          Refresh_Interval 10
+          DB /var/log/flb_kube.db
+          DB.Sync Normal
+
+    filters: |
+      [FILTER]
+          Name kubernetes
+          Match kube.*
+          Merge_Log On
+          Keep_Log Off
+          K8S-Logging.Parser On
+          K8S-Logging.Exclude On
+
+      [FILTER]
+          Name    lua
+          Match   kube.*
+          script  /fluent-bit/etc/functions.lua
+          call    dedot
+
+    customParsers: |
+      [PARSER]
+          # http://rubular.com/r/tjUt3Awgg4
+          Name cri
+          Format regex
+          Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
+          Time_Key    time
+          Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+          # Decode_Field_As json log
+
+    service: |
+      [SERVICE]
+          Flush 5
+          Daemon Off
+          Log_Level warn
+          Parsers_File parsers.conf
+          Parsers_File custom_parsers.conf
+          HTTP_Server On
+          HTTP_Listen 0.0.0.0
+          HTTP_Port 2020
+
+    lua: |
+      function dedot(tag, timestamp, record)
+          if record["kubernetes"] == nil then
+              return 0, 0, 0
+          end
+          dedot_keys(record["kubernetes"]["annotations"])
+          dedot_keys(record["kubernetes"]["labels"])
+          return 1, timestamp, record
+      end
+
+      function dedot_keys(map)
+          if map == nil then
+              return
+          end
+          local new_map = {}
+          local changed_keys = {}
+          for k, v in pairs(map) do
+              local dedotted = string.gsub(k, "%.", "_")
+              if dedotted ~= k then
+                  new_map[dedotted] = v
+                  changed_keys[k] = true
+              end
+          end
+          for k in pairs(changed_keys) do
+              map[k] = nil
+          end
+          for k, v in pairs(new_map) do
+              map[k] = v
+          end
+      end
+
+  serviceMonitor:
+    enabled: true
+    namespace: monitoring
+    selector:
+      release: metrics
+
+  tolerations:
+  - key: node-role.kubernetes.io/master
+    effect: NoSchedule

deploy/templates/values.yaml

@@ -129,7 +129,9 @@ kubezero:
   logging:
     enabled: {{ .Values.logging.enabled }}
     values:
+      {{- if .Values.logging.elastic_password }}
       elastic_password: {{ .Values.logging.elastic_password }}
+      {{- end }}
 
       {{- if .Values.logging.version }}
       version: {{ .Values.logging.version }}
@@ -176,6 +178,16 @@ kubezero:
           {{- end }}
         {{- end }}
 
+      fluent-bit:
+        enabled: {{ index .Values.logging "fluent-bit" "enabled" }}
+        metrics:
+          enabled: {{ .Values.metrics.enabled }}
+        {{- if index .Values.logging "fluent-bit" "config" }}
+        config:
+          {{- with index .Values.logging "fluent-bit" "config" }}
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+        {{- end }}
         
 argo-cd:
   controller:

deploy/values.yaml

@@ -37,6 +37,8 @@ logging:
   enabled: false
   fluentd:
    enabled: false
+  fluent-bit:
+   enabled: false
 
 argo-cd:
   server: {}

scripts/publish.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 set -eux
 
+CHARTS=${1:-'.*'}
 # all credits go to the argoproj Helm guys https://github.com/argoproj/argo-helm
 
 SRCROOT="$(cd "$(dirname "$0")/.." && pwd)"
@@ -16,10 +17,10 @@ helm repo add jetstack https://charts.jetstack.io
 helm repo add uswitch https://uswitch.github.io/kiam-helm-charts/charts/
 helm repo update
 
-for dir in $(find $SRCROOT/charts -mindepth 1 -maxdepth 1 -type d);
+for dir in $(find -L $SRCROOT/charts -mindepth 1 -maxdepth 1 -type d);
 do
     name=$(basename $dir)
-
+    [[ $name =~ $CHARTS ]] || continue
     if [ $(helm dep list $dir 2>/dev/null| wc -l) -gt 1 ]
     then
         echo "Processing chart dependencies"