From 32d0095d3a283d6603251d31273c894c3bcef5d2 Mon Sep 17 00:00:00 2001
From: Stefan Reimer <stefan@zero-downtime.net>
Date: Wed, 14 Jul 2021 17:38:32 +0200
Subject: [PATCH] fix: more fine tuning for fluent pipeline

---
 charts/kubezero-logging/Chart.yaml                   |  4 ++--
 charts/kubezero-logging/charts/fluent-bit/Chart.yaml |  6 +++---
 charts/kubezero-logging/update.sh                    |  2 +-
 charts/kubezero-logging/values.yaml                  | 12 +++++++-----
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml
index ced132b0..e8460994 100644
--- a/charts/kubezero-logging/Chart.yaml
+++ b/charts/kubezero-logging/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-logging
 description: KubeZero Umbrella Chart for complete EFK stack
 type: application
-version: 0.7.0
+version: 0.7.1
 appVersion: 1.6.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@@ -26,6 +26,6 @@ dependencies:
     version: 0.2.6
     condition: fluentd.enabled
   - name: fluent-bit
-    version: 0.15.14
+    version: 0.15.15
     condition: fluent-bit.enabled
 kubeVersion: ">= 1.18.0"
diff --git a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml
index e3e351d9..140229e5 100644
--- a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml
@@ -1,8 +1,8 @@
 annotations:
   artifacthub.io/changes: |
-    - Upgrade fluent-bit image to v1.7.8
+    - Upgrade fluent-bit image to v1.7.9
 apiVersion: v1
-appVersion: 1.7.8
+appVersion: 1.7.9
 description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD
   family operating systems.
 home: https://fluentbit.io/
@@ -21,4 +21,4 @@ maintainers:
 name: fluent-bit
 sources:
 - https://github.com/fluent/fluent-bit/
-version: 0.15.14
+version: 0.15.15
diff --git a/charts/kubezero-logging/update.sh b/charts/kubezero-logging/update.sh
index 9c0dd233..19473dc0 100755
--- a/charts/kubezero-logging/update.sh
+++ b/charts/kubezero-logging/update.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 ECK_VERSION=1.6.0
-FLUENT_BIT_VERSION=0.15.14
+FLUENT_BIT_VERSION=0.15.15
 FLUENTD_VERSION=0.2.6
 
 # fix ECK crds handling to adhere to proper helm v3 support which also fixes ArgoCD applyong updates on upgrades
diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml
index df80173f..72a09992 100644
--- a/charts/kubezero-logging/values.yaml
+++ b/charts/kubezero-logging/values.yaml
@@ -120,7 +120,9 @@ fluentd:
     00_system.conf: |-
       <system>
         root_dir /var/log/fluentd
-        # log_level debug
+        log_level info
+        ignore_repeated_log_interval 60s
+        ignore_same_log_interval 60s
         workers 2
       </system>
     01_sources.conf: |-
@@ -165,7 +167,7 @@ fluentd:
         <match **>
           @id out_es
           @type elasticsearch
-          @log_level info
+          # @log_level debug
           include_tag_key true
           id_key id
           remove_keys id
@@ -185,7 +187,7 @@ fluentd:
           reload_on_failure true
           request_timeout 60s
           suppress_type_name true
-          slow_flush_log_threshold 50.0
+          slow_flush_log_threshold 55.0
 
           # Retry failed bulk requests
           # https://github.com/uken/fluent-plugin-elasticsearch#unrecoverable-error-types
@@ -199,12 +201,12 @@ fluentd:
             flush_thread_count 1
             flush_interval 30s
 
-            chunk_limit_size 4MB
+            chunk_limit_size 8MB
             total_limit_size 2GB
 
             flush_at_shutdown true
             retry_type exponential_backoff
-            retry_timeout 2h
+            retry_timeout 6h
             overflow_action drop_oldest_chunk
             disable_chunk_backup true
           </buffer>