fix: more fine tuning for fluent pipeline

charts/kubezero-logging/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-logging
 description: KubeZero Umbrella Chart for complete EFK stack
 type: application
-version: 0.7.0
+version: 0.7.1
 appVersion: 1.6.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@@ -26,6 +26,6 @@ dependencies:
     version: 0.2.6
     condition: fluentd.enabled
   - name: fluent-bit
-    version: 0.15.14
+    version: 0.15.15
     condition: fluent-bit.enabled
 kubeVersion: ">= 1.18.0"

charts/kubezero-logging/charts/fluent-bit/Chart.yaml

@@ -1,8 +1,8 @@
 annotations:
   artifacthub.io/changes: |
-    - Upgrade fluent-bit image to v1.7.8
+    - Upgrade fluent-bit image to v1.7.9
 apiVersion: v1
-appVersion: 1.7.8
+appVersion: 1.7.9
 description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD
   family operating systems.
 home: https://fluentbit.io/
@@ -21,4 +21,4 @@ maintainers:
 name: fluent-bit
 sources:
 - https://github.com/fluent/fluent-bit/
-version: 0.15.14
+version: 0.15.15

charts/kubezero-logging/update.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 ECK_VERSION=1.6.0
-FLUENT_BIT_VERSION=0.15.14
+FLUENT_BIT_VERSION=0.15.15
 FLUENTD_VERSION=0.2.6
 
 # fix ECK crds handling to adhere to proper helm v3 support which also fixes ArgoCD applyong updates on upgrades

charts/kubezero-logging/values.yaml

@@ -120,7 +120,9 @@ fluentd:
     00_system.conf: |-
       <system>
         root_dir /var/log/fluentd
-        # log_level debug
+        log_level info
+        ignore_repeated_log_interval 60s
+        ignore_same_log_interval 60s
         workers 2
       </system>
     01_sources.conf: |-
@@ -165,7 +167,7 @@ fluentd:
         <match **>
           @id out_es
           @type elasticsearch
-          @log_level info
+          # @log_level debug
           include_tag_key true
           id_key id
           remove_keys id
@@ -185,7 +187,7 @@ fluentd:
           reload_on_failure true
           request_timeout 60s
           suppress_type_name true
-          slow_flush_log_threshold 50.0
+          slow_flush_log_threshold 55.0
 
           # Retry failed bulk requests
           # https://github.com/uken/fluent-plugin-elasticsearch#unrecoverable-error-types
@@ -199,12 +201,12 @@ fluentd:
             flush_thread_count 1
             flush_interval 30s
 
-            chunk_limit_size 4MB
+            chunk_limit_size 8MB
             total_limit_size 2GB
 
             flush_at_shutdown true
             retry_type exponential_backoff
-            retry_timeout 2h
+            retry_timeout 6h
             overflow_action drop_oldest_chunk
             disable_chunk_backup true
           </buffer>