feat: logging version bump

charts/kubezero-logging/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-logging
 description: KubeZero Umbrella Chart for complete EFK stack
 type: application
-version: 0.7.12
+version: 0.7.13
 appVersion: 1.6.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@@ -26,6 +26,6 @@ dependencies:
     version: 0.2.10
     condition: fluentd.enabled
   - name: fluent-bit
-    version: 0.16.3
+    version: 0.17.0
     condition: fluent-bit.enabled
 kubeVersion: ">= 1.18.0"

charts/kubezero-logging/README.md

@@ -1,6 +1,6 @@
 # kubezero-logging
 
-![Version: 0.7.11](https://img.shields.io/badge/Version-0.7.11-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6.0](https://img.shields.io/badge/AppVersion-1.6.0-informational?style=flat-square)
+![Version: 0.7.13](https://img.shields.io/badge/Version-0.7.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6.0](https://img.shields.io/badge/AppVersion-1.6.0-informational?style=flat-square)
 
 KubeZero Umbrella Chart for complete EFK stack
 
@@ -19,7 +19,7 @@ Kubernetes: `>= 1.18.0`
 | Repository | Name | Version |
 |------------|------|---------|
 |  | eck-operator | 1.6.0 |
-|  | fluent-bit | 0.16.3 |
+|  | fluent-bit | 0.17.0 |
 |  | fluentd | 0.2.10 |
 | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 |
 
@@ -90,7 +90,7 @@ Kubernetes: `>= 1.18.0`
 | fluent-bit.daemonSetVolumes[1].hostPath.type | string | `"File"` |  |
 | fluent-bit.daemonSetVolumes[1].name | string | `"etcmachineid"` |  |
 | fluent-bit.enabled | bool | `false` |  |
-| fluent-bit.image.tag | string | `"1.8.5"` |  |
+| fluent-bit.image.tag | string | `"1.8.7"` |  |
 | fluent-bit.luaScripts."kubezero.lua" | string | `"function nest_k8s_ns(tag, timestamp, record)\n    if not record['kubernetes']['namespace_name'] then\n        return 0, 0, 0\n    end\n    new_record = {}\n    for key, val in pairs(record) do\n        if key == 'kube' then\n            new_record[key] = {}\n            new_record[key][record['kubernetes']['namespace_name']] = record[key]\n        else\n            new_record[key] = record[key]\n        end\n    end\n    return 1, timestamp, new_record\nend\n"` |  |
 | fluent-bit.resources.limits.memory | string | `"64Mi"` |  |
 | fluent-bit.resources.requests.cpu | string | `"20m"` |  |
@@ -109,10 +109,10 @@ Kubernetes: `>= 1.18.0`
 | fluentd.env[1].name | string | `"OUTPUT_PASSWORD"` |  |
 | fluentd.env[1].valueFrom.secretKeyRef.key | string | `"elastic"` |  |
 | fluentd.env[1].valueFrom.secretKeyRef.name | string | `"logging-es-elastic-user"` |  |
-| fluentd.fileConfigs."00_system.conf" | string | `"<system>\n  root_dir /var/log/fluentd\n  log_level info\n  ignore_repeated_log_interval 60s\n  ignore_same_log_interval 60s\n  workers 1\n</system>"` |  |
+| fluentd.fileConfigs."00_system.conf" | string | `"<system>\n  root_dir /fluentd/log\n  log_level info\n  ignore_repeated_log_interval 60s\n  ignore_same_log_interval 60s\n  workers 1\n</system>"` |  |
 | fluentd.fileConfigs."01_sources.conf" | string | `"<source>\n  @type http\n  @label @KUBERNETES\n  port 9880\n  bind 0.0.0.0\n  keepalive_timeout 30\n</source>\n\n<source>\n  @type forward\n  @label @KUBERNETES\n  port 24224\n  bind 0.0.0.0\n  # skip_invalid_event true\n  send_keepalive_packet true\n  <security>\n    self_hostname \"#{ENV['HOSTNAME']}\"\n    shared_key {{ .Values.shared_key }}\n  </security>\n</source>"` |  |
 | fluentd.fileConfigs."02_filters.conf" | string | `"<label @KUBERNETES>\n  # prevent log feedback loops eg. ES has issues etc.\n  # discard logs from our own pods\n  <match kube.logging.fluentd>\n    @type relabel\n    @label @FLUENT_LOG\n  </match>\n\n  # Exclude current fluent-bit multiline noise\n  <filter kube.logging.fluent-bit>\n    @type grep\n    <exclude>\n      key log\n      pattern /could not append content to multiline context/\n    </exclude>\n  </filter>\n\n  # Generate Hash ID to break endless loop for already ingested events during retries\n  <filter **>\n    @type elasticsearch_genid\n    use_entire_record true\n  </filter>\n\n  # Route through DISPATCH for Prometheus metrics\n  <match **>\n    @type relabel\n    @label @DISPATCH\n  </match>\n</label>"` |  |
-| fluentd.fileConfigs."04_outputs.conf" | string | `"<label @OUTPUT>\n  <match **>\n    @id out_es\n    @type elasticsearch\n    # @log_level debug\n    include_tag_key true\n\n    id_key _hash\n    remove_keys _hash\n    write_operation create\n\n    # KubeZero pipeline incl. GeoIP etc.\n    pipeline fluentd\n\n    hosts \"{{ .Values.output.host }}\"\n    port 9200\n    scheme http\n    user elastic\n    password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n    log_es_400_reason\n    logstash_format true\n    reconnect_on_error true\n    reload_on_failure true\n    request_timeout 60s\n    slow_flush_log_threshold 55.0\n\n    with_transporter_log true\n\n    verify_es_version_at_startup false\n    default_elasticsearch_version 7\n    suppress_type_name true\n\n    # Retry failed bulk requests\n    # https://github.com/uken/fluent-plugin-elasticsearch#unrecoverable-error-types\n    unrecoverable_error_types [\"out_of_memory_error\"]\n    bulk_message_request_threshold 2097152\n\n    <buffer>\n      @type file\n\n      flush_mode interval\n      flush_thread_count 2\n      flush_interval 30s\n\n      chunk_limit_size 4MB\n      total_limit_size 2GB\n\n      flush_at_shutdown true\n      retry_type exponential_backoff\n      retry_timeout 6h\n      overflow_action drop_oldest_chunk\n      disable_chunk_backup true\n    </buffer>\n  </match>\n</label>"` |  |
+| fluentd.fileConfigs."04_outputs.conf" | string | `"<label @OUTPUT>\n  <match **>\n    @id out_es\n    @type elasticsearch\n    # @log_level debug\n    include_tag_key true\n\n    id_key _hash\n    remove_keys _hash\n    write_operation create\n\n    # KubeZero pipeline incl. GeoIP etc.\n    pipeline fluentd\n\n    hosts \"{{ .Values.output.host }}\"\n    port 9200\n    scheme http\n    user elastic\n    password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n    log_es_400_reason\n    logstash_format true\n    reconnect_on_error true\n    reload_on_failure true\n    request_timeout 300s\n    slow_flush_log_threshold 55.0\n\n    #with_transporter_log true\n\n    verify_es_version_at_startup false\n    default_elasticsearch_version 7\n    suppress_type_name true\n\n    # Retry failed bulk requests\n    # https://github.com/uken/fluent-plugin-elasticsearch#unrecoverable-error-types\n    unrecoverable_error_types [\"out_of_memory_error\"]\n    bulk_message_request_threshold 1048576\n\n    <buffer>\n      @type file\n\n      flush_mode interval\n      flush_thread_count 2\n      flush_interval 10s\n\n      chunk_limit_size 2MB\n      total_limit_size 1GB\n\n      flush_at_shutdown true\n      retry_type exponential_backoff\n      retry_timeout 6h\n      overflow_action drop_oldest_chunk\n      disable_chunk_backup true\n    </buffer>\n  </match>\n</label>"` |  |
 | fluentd.image.repository | string | `"public.ecr.aws/zero-downtime/fluentd-concenter"` |  |
 | fluentd.image.tag | string | `"v1.14.1"` |  |
 | fluentd.istio.enabled | bool | `false` |  |

charts/kubezero-logging/charts/fluent-bit/Chart.yaml

@@ -1,9 +1,9 @@
 annotations:
   artifacthub.io/changes: |
     - kind: changed
-      description: Update image version to v1.8.3.
+      description: "Update image version to v1.8.6."
 apiVersion: v1
-appVersion: 1.8.3
+appVersion: 1.8.6
 description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD
   family operating systems.
 home: https://fluentbit.io/
@@ -22,4 +22,4 @@ maintainers:
 name: fluent-bit
 sources:
 - https://github.com/fluent/fluent-bit/
-version: 0.16.3
+version: 0.17.0

charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl

@@ -64,6 +64,11 @@ containers:
       {{- toYaml .Values.resources | nindent 6 }}
     volumeMounts:
       {{- toYaml .Values.volumeMounts | nindent 6 }}
+    {{- range $key, $val := .Values.config.extraFiles }}
+      - name: config
+        mountPath: /fluent-bit/etc/{{ $key }}
+        subPath: {{ $key }}
+    {{- end }}
     {{- range $key, $value := .Values.luaScripts }}
       - name: luascripts
         mountPath: /fluent-bit/scripts/{{ $key }}

charts/kubezero-logging/charts/fluent-bit/templates/configmap.yaml

@@ -13,4 +13,8 @@ data:
     {{- (tpl .Values.config.inputs $)  | nindent 4 }}
     {{- (tpl .Values.config.filters $)  | nindent 4 }}
     {{- (tpl .Values.config.outputs $)  | nindent 4 }}
+  {{- range $key, $val := .Values.config.extraFiles }}
+  {{ $key }}: |
+    {{- $val | nindent 4 }}
+  {{- end }}
 {{- end -}}

charts/kubezero-logging/charts/fluent-bit/templates/service.yaml

@@ -18,12 +18,18 @@ spec:
       targetPort: http
       protocol: TCP
       name: http
+      {{- if and (eq .Values.service.type "NodePort") (.Values.service.nodePort) }}
+      nodePort: {{ .Values.service.nodePort }}
+      {{- end }}
   {{- if .Values.extraPorts }}
     {{- range .Values.extraPorts }}
     - name: {{ .name }}
       targetPort: {{ .name }}
       protocol: {{ .protocol }}
       port: {{ .port }}
+      {{- if and (eq $.Values.service.type "NodePort") (.nodePort) }}
+      nodePort: {{ .nodePort }}
+      {{- end }}
     {{- end }}
   {{- end }}
   selector:

charts/kubezero-logging/charts/fluent-bit/values.yaml

@@ -71,6 +71,7 @@ service:
   type: ClusterIP
   port: 2020
   labels: {}
+  # nodePort: 30020
   annotations: {}
 #   prometheus.io/path: "/api/v1/metrics/prometheus"
 #   prometheus.io/port: "2020"
@@ -153,6 +154,7 @@ extraPorts: []
 #     containerPort: 5170
 #     protocol: TCP
 #     name: tcp
+#     nodePort: 30517
 
 extraVolumes: []
 
@@ -239,6 +241,15 @@ config:
         Time_Key time
         Time_Format %Y-%m-%dT%H:%M:%S.%L
 
+  # This allows adding more files with arbitary filenames to /fluent-bit/etc by providing key/value pairs.
+  # The key becomes the filename, the value becomes the file content.
+  extraFiles: {}
+#     example.conf: |
+#       [OUTPUT]
+#           Name example
+#           Match foo.*
+#           Host bar
+
 # The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname"
 volumeMounts:
   - name: config

charts/kubezero-logging/update.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
-ECK_VERSION=1.6.0
+ECK_VERSION=$(yq r Chart.yaml dependencies.name==eck-operator.version)
-FLUENT_BIT_VERSION=0.16.3
+FLUENT_BIT_VERSION=$(yq r Chart.yaml dependencies.name==fluent-bit.version)
-FLUENTD_VERSION=0.2.10
+FLUENTD_VERSION=$(yq r Chart.yaml dependencies.name==fluentd.version)
 
 # fix ECK crds handling to adhere to proper helm v3 support which also fixes ArgoCD applying updates on upgrades
 helm repo list | grep elastic -qc || { helm repo add elastic https://helm.elastic.co; helm repo update; }

charts/kubezero-logging/values.yaml

@@ -244,7 +244,7 @@ fluent-bit:
 
   image:
     #repository: public.ecr.aws/zero-downtime/fluent-bit
-    tag: 1.8.5
+    tag: 1.8.7
 
   serviceMonitor:
     enabled: false