From 5d57cf548065f3a56be9d9c9364ef31c8b79e224 Mon Sep 17 00:00:00 2001
From: Stefan Reimer <stefan@zero-downtime.net>
Date: Fri, 5 Mar 2021 16:53:02 +0100
Subject: [PATCH] Version upgrade ES/Kibana and Fluentbit, various tunings

---
 charts/kubezero-logging/Chart.yaml            |    8 +-
 .../charts/fluent-bit/Chart.yaml              |   35 +-
 .../fluent-bit/conf/custom_parsers.conf       |    6 -
 .../charts/fluent-bit/conf/fluent-bit.conf    |   76 -
 .../charts/fluent-bit/conf/functions.lua      |   28 -
 .../fluent-bit/dashboards/fluent-bit.json     | 1305 +++++++++++++++++
 .../charts/fluent-bit/templates/_pod.tpl      |   29 +-
 .../templates/configmap-dashboards.yaml       |   20 +
 .../templates/configmap-luascripts.yaml       |   12 +
 .../fluent-bit/templates/configmap.yaml       |    9 +-
 .../fluent-bit/templates/daemonset.yaml       |    1 +
 .../fluent-bit/templates/deployment.yaml      |    1 +
 .../fluent-bit/templates/networkpolicy.yaml   |   22 +
 .../fluent-bit/templates/prometheusrule.yaml  |   20 +
 .../templates/tests/test-connection.yaml      |    9 +-
 .../charts/fluent-bit/values.yaml             |  126 +-
 charts/kubezero-logging/fluent-bit.patch      |   37 +
 .../templates/eck/elasticsearch.yaml          |   26 +-
 .../templates/eck/kibana.yaml                 |    2 +-
 charts/kubezero-logging/update.sh             |    8 +
 charts/kubezero-logging/values.yaml           |  131 +-
 21 files changed, 1734 insertions(+), 177 deletions(-)
 delete mode 100644 charts/kubezero-logging/charts/fluent-bit/conf/custom_parsers.conf
 delete mode 100644 charts/kubezero-logging/charts/fluent-bit/conf/fluent-bit.conf
 delete mode 100644 charts/kubezero-logging/charts/fluent-bit/conf/functions.lua
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/dashboards/fluent-bit.json
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/configmap-dashboards.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/configmap-luascripts.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/networkpolicy.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/prometheusrule.yaml
 create mode 100644 charts/kubezero-logging/fluent-bit.patch
 create mode 100755 charts/kubezero-logging/update.sh

diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml
index a78f85d..f030c77 100644
--- a/charts/kubezero-logging/Chart.yaml
+++ b/charts/kubezero-logging/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: kubezero-logging
 description: KubeZero Umbrella Chart for complete EFK stack
 type: application
-version: 0.5.3
-appVersion: 1.3.1
+version: 0.6.0
+appVersion: 1.4.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -19,13 +19,13 @@ dependencies:
     version: ">= 0.1.3"
     repository: https://zero-down-time.github.io/kubezero/
   - name: eck-operator
-    version: 1.3.1
+    version: 1.4.0
     repository: https://helm.elastic.co
     condition: eck-operator.enabled
   - name: fluentd
     version: 2.5.3
     condition: fluentd.enabled
   - name: fluent-bit
-    version: 0.7.14
+    version: 0.12.3
     condition: fluent-bit.enabled
 kubeVersion: ">= 1.16.0"
diff --git a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml
index f29b870..f83e759 100644
--- a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml
@@ -1,20 +1,23 @@
+annotations:
+  artifacthub.io/changes: |
+    - add custom annotations on dashboards configmap
 apiVersion: v2
-name: fluent-bit
+appVersion: 1.7.1
 description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
-keywords:
-  - logging
-  - fluent-bit
-  - fluentd
-version: 0.7.14
-appVersion: 1.6.8
-icon: https://fluentbit.io/assets/img/logo1-default.png
 home: https://fluentbit.io/
-sources:
-  - https://github.com/fluent/fluent-bit/
+icon: https://fluentbit.io/assets/img/logo1-default.png
+keywords:
+- logging
+- fluent-bit
+- fluentd
 maintainers:
-  - name: edsiper
-    email: eduardo@treasure-data.com
-  - name: naseemkullah
-    email: naseem@transit.app
-  - name: Towmeykaw
-    email: towmeykaw@gmail.com
+- email: eduardo@treasure-data.com
+  name: edsiper
+- email: naseem@transit.app
+  name: naseemkullah
+- email: towmeykaw@gmail.com
+  name: Towmeykaw
+name: fluent-bit
+sources:
+- https://github.com/fluent/fluent-bit/
+version: 0.12.3
diff --git a/charts/kubezero-logging/charts/fluent-bit/conf/custom_parsers.conf b/charts/kubezero-logging/charts/fluent-bit/conf/custom_parsers.conf
deleted file mode 100644
index a2055db..0000000
--- a/charts/kubezero-logging/charts/fluent-bit/conf/custom_parsers.conf
+++ /dev/null
@@ -1,6 +0,0 @@
-[PARSER]
-    Name cri-log
-    Format regex
-    Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
-    Time_Key    time
-    Time_Format %Y-%m-%dT%H:%M:%S.%L%z
diff --git a/charts/kubezero-logging/charts/fluent-bit/conf/fluent-bit.conf b/charts/kubezero-logging/charts/fluent-bit/conf/fluent-bit.conf
deleted file mode 100644
index 23d86ee..0000000
--- a/charts/kubezero-logging/charts/fluent-bit/conf/fluent-bit.conf
+++ /dev/null
@@ -1,76 +0,0 @@
-[SERVICE]
-    Flush {{ .Values.config.flushInterval }}
-    Daemon Off
-    Log_Level {{ .Values.config.logLevel }}
-    Parsers_File parsers.conf
-    Parsers_File custom_parsers.conf
-    HTTP_Server On
-    HTTP_Listen 0.0.0.0
-    HTTP_Port 2020
-
-[INPUT]
-    Name tail
-    Path /var/log/containers/*.log
-    Parser cri-log
-    Tag cri.*
-    Skip_Long_Lines On
-    DB /var/log/flb_kube.db
-    DB.Sync Normal
-    {{- with .Values.config.input }}
-    Mem_Buf_Limit {{ default "16MB" .memBufLimit }}
-    Refresh_Interval {{ default 10 .refreshInterval }}
-    {{- end }}
-
-[FILTER]
-    Name    lua
-    Match   cri.*
-    script  /fluent-bit/etc/functions.lua
-    call    reassemble_cri_logs
-
-[FILTER]
-    Name kubernetes
-    Match cri.*
-    Merge_Log On
-    Merge_Log_Key kube
-    Kube_Tag_Prefix cri.var.log.containers.
-    Keep_Log Off
-    K8S-Logging.Parser Off
-    K8S-Logging.Exclude Off
-
-{{- if index .Values "config" "extraRecords" }}
-
-[FILTER]
-    Name record_modifier
-    Match cri.*
-    {{- range $k,$v := index .Values "config" "extraRecords" }}
-    Record {{ $k }} {{ $v }}
-    {{- end }}
-{{- end }}
-
-[FILTER]
-    Name rewrite_tag
-    Match cri.*
-    Emitter_Name kube_tag_rewriter
-    Rule logtag F kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false
-
-[FILTER]
-    Name    lua
-    Match   kube.*
-    script  /fluent-bit/etc/functions.lua
-    call    nest_k8s_ns
-
-{{- if .Values.config.outputs }}
-{{ .Values.config.outputs }}
-{{- else }}
-{{- with .Values.config.output }}
-[OUTPUT]
-    Match *
-    Name forward
-    Host {{ .host }}
-    Port 24224
-    Shared_Key {{ .sharedKey }}
-    tls {{ ternary "on" "off" .tls }}
-    Send_options true
-    Require_ack_response true
-{{- end }}
-{{- end }}
diff --git a/charts/kubezero-logging/charts/fluent-bit/conf/functions.lua b/charts/kubezero-logging/charts/fluent-bit/conf/functions.lua
deleted file mode 100644
index c7171fe..0000000
--- a/charts/kubezero-logging/charts/fluent-bit/conf/functions.lua
+++ /dev/null
@@ -1,28 +0,0 @@
-local reassemble_state = {}
-
-function reassemble_cri_logs(tag, timestamp, record)
-   local reassemble_key = tag
-   if record.logtag == 'P' then
-      reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or "" .. record.log
-      return -1, 0, 0
-   end
-   record.log = reassemble_state[reassemble_key] or "" .. (record.log or "")
-   reassemble_state[reassemble_key] = nil
-   return 1, timestamp, record
-end
-
-function nest_k8s_ns(tag, timestamp, record)
-    if not record['kubernetes']['namespace_name'] then
-        return 0, 0, 0
-    end
-    new_record = {}
-    for key, val in pairs(record) do
-        if key == 'kube' then
-            new_record[key] = {}
-            new_record[key][record['kubernetes']['namespace_name']] = record[key]
-        else
-            new_record[key] = record[key]
-        end
-    end
-    return 1, timestamp, new_record
-end
diff --git a/charts/kubezero-logging/charts/fluent-bit/dashboards/fluent-bit.json b/charts/kubezero-logging/charts/fluent-bit/dashboards/fluent-bit.json
new file mode 100644
index 0000000..0952678
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/dashboards/fluent-bit.json
@@ -0,0 +1,1305 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "7.2.1"
+    },
+    {
+      "type": "panel",
+      "id": "graph",
+      "name": "Graph",
+      "version": ""
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "stat",
+      "name": "Stat",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "${DS_PROMETHEUS}",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Inspired by https://grafana.com/grafana/dashboards/7752",
+  "editable": true,
+  "gnetId": 7752,
+  "graphTooltip": 1,
+  "id": null,
+  "iteration": 1612355253484,
+  "links": [],
+  "panels": [
+    {
+      "collapsed": false,
+      "datasource": null,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 45,
+      "panels": [],
+      "title": "Cluster Status",
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "datasource": "$DS_PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "mappings": [],
+          "nullValueMode": "connected",
+          "thresholds": {
+            "mode": "percentage",
+            "steps": [
+              {
+                "color": "#d44a3a",
+                "value": null
+              },
+              {
+                "color": "#299c46",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 24,
+        "x": 0,
+        "y": 1
+      },
+      "id": 6,
+      "interval": null,
+      "links": [],
+      "maxDataPoints": 100,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "vertical",
+        "reduceOptions": {
+          "calcs": [
+            "last"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "7.2.1",
+      "targets": [
+        {
+          "expr": "sum(kube_pod_info{pod=~\".*fluent-bit.*\"})",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "Active Fluent-bit",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})",
+          "interval": "",
+          "legendFormat": "Ready Nodes",
+          "refId": "B"
+        },
+        {
+          "expr": "sum(kube_node_status_condition{condition!=\"Ready\",status=\"true\"})",
+          "interval": "",
+          "legendFormat": "Non-Ready Nodes",
+          "refId": "C"
+        }
+      ],
+      "title": "",
+      "transparent": true,
+      "type": "stat"
+    },
+    {
+      "collapsed": false,
+      "datasource": null,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "id": 43,
+      "panels": [],
+      "title": "FluentBit metrics",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$DS_PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 5,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 0,
+        "y": 5
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": true,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": false,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(fluentbit_input_bytes_total{pod=~\"$pod\"}[5m])) by (pod, instance, name)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{ pod }}/{{name}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Input Bytes Processing Rate",
+      "tooltip": {
+        "shared": true,
+        "sort": 2,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$DS_PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 5,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 5
+      },
+      "hiddenSeries": false,
+      "id": 9,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": true,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": false,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(fluentbit_output_proc_bytes_total{pod=~\"$pod\"}[5m])) by (pod, instance, name)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{ pod }}/{{name}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Output Bytes Processing Rate",
+      "tooltip": {
+        "shared": true,
+        "sort": 2,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "Bps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$DS_PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 5,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 0,
+        "y": 11
+      },
+      "hiddenSeries": false,
+      "id": 40,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": true,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": false,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(fluentbit_input_records_total{pod=~\"$pod\"}[5m])) by (pod, instance, name)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{ pod }}/{{name}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Input Records Processing Rate",
+      "tooltip": {
+        "shared": true,
+        "sort": 2,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "rps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$DS_PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 5,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 11
+      },
+      "hiddenSeries": false,
+      "id": 41,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": true,
+        "hideEmpty": false,
+        "hideZero": false,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": false,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(fluentbit_output_proc_records_total{pod=~\"$pod\"}[5m])) by (pod, instance, name)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{ pod }}/{{name}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Output Record Processing Rate",
+      "tooltip": {
+        "shared": true,
+        "sort": 2,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "rps",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$DS_PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 0,
+        "y": 17
+      },
+      "hiddenSeries": false,
+      "id": 11,
+      "legend": {
+        "alignAsTable": false,
+        "avg": true,
+        "current": true,
+        "hideEmpty": true,
+        "hideZero": true,
+        "max": true,
+        "min": false,
+        "rightSide": false,
+        "show": false,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(fluentbit_output_retries_total{pod=~\"$pod\"}[1m])) by (pod, instance, name)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{pod}} Retries to {{name}}",
+          "refId": "A"
+        },
+        {
+          "expr": "sum(rate(fluentbit_output_retries_failed_total{pod=~\"$pod\"}[1m])) by (pod, instance, name)",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{pod}} Failed Retries to {{ name }}",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Output Retry/Failed Rates",
+      "tooltip": {
+        "shared": true,
+        "sort": 2,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": null,
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": false
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$DS_PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {},
+          "links": []
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 17
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": true,
+        "hideEmpty": true,
+        "hideZero": true,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "sort": "current",
+        "sortDesc": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 5,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(fluentbit_output_errors_total{pod=~\"$pod\"}[1m])) by (pod, instance, name)",
+          "format": "time_series",
+          "hide": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{ pod }}/{{ name }}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Output Error Rate",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "errors/sec",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": "0",
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$DS_PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 23
+      },
+      "hiddenSeries": false,
+      "id": 47,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(fluentbit_filter_drop_records_total{pod=~\"$pod\"}[5m])) by (pod, instance, name)",
+          "interval": "",
+          "legendFormat": "{{ pod }} / {{ name }}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Filter Drop",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$DS_PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 23
+      },
+      "hiddenSeries": false,
+      "id": 48,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": true,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "sum(rate(fluentbit_filter_add_records_total{pod=~\"$pod\"}[5m])) by (pod, instance, name)",
+          "interval": "",
+          "legendFormat": "{{ pod }} / {{ name }}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Filter Add",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "collapsed": false,
+      "datasource": null,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 31
+      },
+      "id": 53,
+      "panels": [],
+      "title": "Kubernetes metrics",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$DS_PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 32
+      },
+      "hiddenSeries": false,
+      "id": 51,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [
+        {
+          "alias": "/.* request/",
+          "color": "#F2CC0C",
+          "fill": 0
+        }
+      ],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "container_memory_working_set_bytes{pod=~\".*fluent-bit.*\",pod=~\"$pod\", image!=\"\", container!=\"POD\"}\n",
+          "interval": "",
+          "legendFormat": "{{ pod }}",
+          "refId": "A"
+        },
+        {
+          "expr": "avg(kube_pod_container_resource_requests_memory_bytes{pod=~\".*fluent-bit.*\",pod=~\"$pod\"}) by (pod)",
+          "interval": "",
+          "legendFormat": "{{ pod }} request",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Memroy Usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "decbytes",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "$DS_PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "hiddenSeries": false,
+      "id": 50,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": false,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "7.2.1",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [
+        {
+          "alias": "/.* request/",
+          "color": "#F2CC0C",
+          "fill": 0
+        }
+      ],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "rate(container_cpu_usage_seconds_total{pod=~\".*fluent-bit.*\",pod=~\"$pod\",image!=\"\",container!=\"POD\"}[5m])",
+          "interval": "",
+          "legendFormat": "{{ pod }}",
+          "refId": "A"
+        },
+        {
+          "expr": "avg(kube_pod_container_resource_requests_cpu_cores{pod=~\"$pod\"}) by (pod)",
+          "interval": "",
+          "legendFormat": "{{ pod }} request",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "CPU Usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 26,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": "prometheus",
+          "value": "prometheus"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "DS_PROMETHEUS",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "$DS_PROMETHEUS",
+        "definition": "label_values(kube_pod_info{pod=~\".*fluent-bit.*\"}, pod)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "pod",
+        "multi": false,
+        "name": "pod",
+        "options": [],
+        "query": "label_values(kube_pod_info{pod=~\".*fluent-bit.*\"}, pod)",
+        "refresh": 2,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "Fluent Bit",
+  "uid": "fluentbit",
+  "version": 2
+}
\ No newline at end of file
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl b/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl
index 962c9ed..7acb111 100644
--- a/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl
@@ -9,6 +9,10 @@ priorityClassName: {{ .Values.priorityClassName }}
 serviceAccountName: {{ include "fluent-bit.serviceAccountName" . }}
 securityContext:
   {{- toYaml .Values.podSecurityContext | nindent 2 }}
+{{- with .Values.dnsConfig }}
+dnsConfig:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
 containers:
   - name: {{ .Chart.Name }}
     securityContext:
@@ -34,14 +38,14 @@ containers:
         protocol: {{ .protocol }}
       {{- end }}
     {{- end }}
+    {{- if .Values.livenessProbe }}
     livenessProbe:
-      httpGet:
-        path: /
-        port: http
+      {{- toYaml .Values.livenessProbe | nindent 6 }}
+    {{- end }}
+    {{- if .Values.readinessProbe }}
     readinessProbe:
-      httpGet:
-        path: /
-        port: http
+      {{- toYaml .Values.readinessProbe | nindent 6 }}
+    {{- end }}
     resources:
       {{- toYaml .Values.resources | nindent 6 }}
     volumeMounts:
@@ -51,9 +55,11 @@ containers:
       - name: config
         mountPath: /fluent-bit/etc/custom_parsers.conf
         subPath: custom_parsers.conf
-      - name: config
-        mountPath: /fluent-bit/etc/functions.lua
-        subPath: functions.lua
+    {{- range $key, $value := .Values.luaScripts }}
+      - name: luascripts
+        mountPath: /fluent-bit/scripts/{{ $key }}
+        subPath: {{ $key }}
+    {{- end }}
     {{- if eq .Values.kind "DaemonSet" }}
       - name: varlog
         mountPath: /var/log
@@ -71,6 +77,11 @@ volumes:
   - name: config
     configMap:
       name: {{ if .Values.existingConfigMap }}{{ .Values.existingConfigMap }}{{- else }}{{ include "fluent-bit.fullname" . }}{{- end }}
+{{- if gt (len .Values.luaScripts) 0 }}
+  - name: luascripts
+    configMap:
+      name: {{ include "fluent-bit.fullname" . }}-luascripts
+{{- end }}
 {{- if eq .Values.kind "DaemonSet" }}
   - name: varlog
     hostPath:
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/configmap-dashboards.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/configmap-dashboards.yaml
new file mode 100644
index 0000000..1af4343
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/configmap-dashboards.yaml
@@ -0,0 +1,20 @@
+
+{{- if .Values.dashboards.enabled -}}
+{{- range $path, $_ :=  .Files.Glob "dashboards/*.json" }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dashboard-{{ trimSuffix ".json" (base $path) }}
+  {{- with $.Values.dashboards.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 -}}
+  {{- end }}
+  labels:
+    {{- include "fluent-bit.labels" $ | nindent 4 }}
+    {{ $.Values.dashboards.labelKey }}: "1"
+data:
+  {{ base $path }}: |
+    {{- $.Files.Get $path | nindent 4 }}
+---
+{{- end }}
+{{- end -}}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/configmap-luascripts.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/configmap-luascripts.yaml
new file mode 100644
index 0000000..0350b4f
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/configmap-luascripts.yaml
@@ -0,0 +1,12 @@
+{{- if gt (len .Values.luaScripts) 0 -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "fluent-bit.fullname" . }}-luascripts
+  labels:
+    {{- include "fluent-bit.labels" . | nindent 4 }}
+data:
+  {{ range $key, $value := .Values.luaScripts }}
+  {{ $key }}: {{ $value | quote }}
+  {{ end }}
+{{- end -}}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/configmap.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/configmap.yaml
index b54e91b..59de91d 100644
--- a/charts/kubezero-logging/charts/fluent-bit/templates/configmap.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/configmap.yaml
@@ -7,9 +7,10 @@ metadata:
     {{- include "fluent-bit.labels" . | nindent 4 }}
 data:
   custom_parsers.conf: |
-    {{ tpl (.Files.Get "conf/custom_parsers.conf") . | nindent 4 }}
+    {{- (tpl .Values.config.customParsers $) | nindent 4 }}
   fluent-bit.conf: |
-    {{ tpl (.Files.Get "conf/fluent-bit.conf") . | nindent 4 }}
-  functions.lua: |
-    {{ tpl (.Files.Get "conf/functions.lua") . | nindent 4 }}
+    {{- (tpl .Values.config.service $)  | nindent 4 }}
+    {{- (tpl .Values.config.inputs $)  | nindent 4 }}
+    {{- (tpl .Values.config.filters $)  | nindent 4 }}
+    {{- (tpl .Values.config.outputs $)  | nindent 4 }}
 {{- end -}}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/daemonset.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/daemonset.yaml
index 9227c73..9003911 100644
--- a/charts/kubezero-logging/charts/fluent-bit/templates/daemonset.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/daemonset.yaml
@@ -17,6 +17,7 @@ spec:
     metadata:
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        checksum/luascripts: {{ include (print $.Template.BasePath "/configmap-luascripts.yaml") . | sha256sum }}
       {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/deployment.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/deployment.yaml
index 3af288f..c9bbe9c 100644
--- a/charts/kubezero-logging/charts/fluent-bit/templates/deployment.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/deployment.yaml
@@ -18,6 +18,7 @@ spec:
     metadata:
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        checksum/luascripts: {{ include (print $.Template.BasePath "/configmap-luascripts.yaml") . | sha256sum }}
       {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/networkpolicy.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/networkpolicy.yaml
new file mode 100644
index 0000000..92c0bf3
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/networkpolicy.yaml
@@ -0,0 +1,22 @@
+{{- if .Values.networkPolicy.enabled }}
+apiVersion: "networking.k8s.io/v1"
+kind: "NetworkPolicy"
+metadata:
+  name: {{ include "fluent-bit.fullname" . | quote }}
+  labels:
+    {{- include "fluent-bit.labels" . | nindent 4 }}
+spec:
+  policyTypes:
+    - "Ingress"
+  podSelector:
+    matchLabels:
+      {{- include "fluent-bit.selectorLabels" . | nindent 6 }}
+  ingress:
+    {{- with .Values.networkPolicy.ingress }}
+    - from:
+        {{- with .from }}{{- . | toYaml | nindent 8 }}{{- else }} []{{- end }}
+      ports:
+        - protocol: "TCP"
+          port: {{ $.Values.service.port }}
+    {{- end }}
+{{- end }}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/prometheusrule.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/prometheusrule.yaml
new file mode 100644
index 0000000..d8dd951
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/prometheusrule.yaml
@@ -0,0 +1,20 @@
+{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.prometheusRule.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ include "fluent-bit.fullname" . }}
+  {{- with .Values.prometheusRule.namespace }}
+  namespace: {{ . }}
+  {{- end }}
+  labels:
+    {{- include "fluent-bit.labels" . | nindent 4 }}
+  {{- if .Values.prometheusRule.additionalLabels }}
+    {{- toYaml .Values.prometheusRule.additionalLabels | nindent 4 }}
+  {{- end }}
+spec:
+{{- if .Values.prometheusRule.rules }}
+  groups:
+  - name: {{ template "fluent-bit.name" . }}
+    rules: {{- toYaml .Values.prometheusRule.rules | nindent 4 }}
+{{- end }}
+{{- end }}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/tests/test-connection.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/tests/test-connection.yaml
index db53e09..659f970 100644
--- a/charts/kubezero-logging/charts/fluent-bit/templates/tests/test-connection.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/tests/test-connection.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.test.enabled }}
+{{- if .Values.testFramework.enabled }}
 apiVersion: v1
 kind: Pod
 metadata:
@@ -10,8 +10,13 @@ metadata:
 spec:
   containers:
     - name: wget
-      image: busybox
+      image: "{{ .Values.testFramework.image.repository }}:{{ .Values.testFramework.image.tag }}"
+      imagePullPolicy: {{ .Values.testFramework.image.pullPolicy }}
       command: ['wget']
       args: ['{{ include "fluent-bit.fullname" . }}:{{ .Values.service.port }}']
+  {{- with .Values.imagePullSecrets }}
+  imagePullSecrets:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   restartPolicy: Never
 {{- end }}
diff --git a/charts/kubezero-logging/charts/fluent-bit/values.yaml b/charts/kubezero-logging/charts/fluent-bit/values.yaml
index f9bcd9c..0a01f64 100644
--- a/charts/kubezero-logging/charts/fluent-bit/values.yaml
+++ b/charts/kubezero-logging/charts/fluent-bit/values.yaml
@@ -6,14 +6,18 @@ kind: DaemonSet
 # replicaCount -- Only applicable if kind=Deployment
 replicaCount: 1
 
-test:
-  enabled: true
-
 image:
   repository: fluent/fluent-bit
   pullPolicy: Always
   # tag:
 
+testFramework:
+  enabled: false
+  image:
+    repository: busybox
+    pullPolicy: Always
+    tag: latest
+
 imagePullSecrets: []
 nameOverride: ""
 fullnameOverride: ""
@@ -32,7 +36,16 @@ podSecurityPolicy:
 podSecurityContext:
   {}
   # fsGroup: 2000
-
+dnsConfig: {}
+  # nameservers:
+  #   - 1.2.3.4
+  # searches:
+  #   - ns1.svc.cluster-domain.example
+  #   - my.dns.search.suffix
+  # options:
+  #   - name: ndots
+#     value: "2"
+#   - name: edns0
 securityContext:
   {}
   # capabilities:
@@ -61,6 +74,38 @@ serviceMonitor:
   # selector:
   #  prometheus: my-prometheus
 
+prometheusRule:
+  enabled: false
+  # namespace: ""
+  # additionnalLabels: {}
+  # rules:
+  # - alert: NoOutputBytesProcessed
+  #   expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0
+  #   annotations:
+  #     message: |
+  #       Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any
+  #       bytes for at least 15 minutes.
+  #     summary: No Output Bytes Processed
+  #   for: 15m
+  #   labels:
+  #     severity: critical
+
+dashboards:
+  enabled: false
+  labelKey: grafana_dashboard
+  annotations: {}
+
+
+livenessProbe:
+  httpGet:
+    path: /
+    port: http
+
+readinessProbe:
+  httpGet:
+    path: /
+    port: http
+
 resources:
   {}
   # limits:
@@ -104,5 +149,74 @@ updateStrategy: {}
 # Make use of a pre-defined configmap instead of the one templated here
 existingConfigMap: ""
 
-## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit
-config: {}
+networkPolicy:
+  enabled: false
+  # ingress:
+  #   from: []
+
+luaScripts: {}
+
+## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file
+config:
+  service: |
+    [SERVICE]
+        Flush 1
+        Daemon Off
+        Log_Level info
+        Parsers_File parsers.conf
+        Parsers_File custom_parsers.conf
+        HTTP_Server On
+        HTTP_Listen 0.0.0.0
+        HTTP_Port {{ .Values.service.port }}
+
+  ## https://docs.fluentbit.io/manual/pipeline/inputs
+  inputs: |
+    [INPUT]
+        Name tail
+        Path /var/log/containers/*.log
+        Parser docker
+        Tag kube.*
+        Mem_Buf_Limit 5MB
+        Skip_Long_Lines On
+
+    [INPUT]
+        Name systemd
+        Tag host.*
+        Systemd_Filter _SYSTEMD_UNIT=kubelet.service
+        Read_From_Tail On
+
+  ## https://docs.fluentbit.io/manual/pipeline/filters
+  filters: |
+    [FILTER]
+        Name kubernetes
+        Match kube.*
+        Merge_Log On
+        Keep_Log Off
+        K8S-Logging.Parser On
+        K8S-Logging.Exclude On
+
+  ## https://docs.fluentbit.io/manual/pipeline/outputs
+  outputs: |
+    [OUTPUT]
+        Name es
+        Match kube.*
+        Host elasticsearch-master
+        Logstash_Format On
+        Retry_Limit False
+
+    [OUTPUT]
+        Name es
+        Match host.*
+        Host elasticsearch-master
+        Logstash_Format On
+        Logstash_Prefix node
+        Retry_Limit False
+
+  ## https://docs.fluentbit.io/manual/pipeline/parsers
+  customParsers: |
+    [PARSER]
+        Name docker_no_time
+        Format json
+        Time_Keep Off
+        Time_Key time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L
diff --git a/charts/kubezero-logging/fluent-bit.patch b/charts/kubezero-logging/fluent-bit.patch
new file mode 100644
index 0000000..a008045
--- /dev/null
+++ b/charts/kubezero-logging/fluent-bit.patch
@@ -0,0 +1,37 @@
+Binary files charts/fluent-bit/.values.yaml.swp and charts/fluent-bit.zdt/.values.yaml.swp differ
+diff -rtubN charts/fluent-bit/Chart.yaml charts/fluent-bit.zdt/Chart.yaml
+--- charts/fluent-bit/Chart.yaml	2021-02-23 14:58:39.000000000 +0100
++++ charts/fluent-bit.zdt/Chart.yaml	2021-03-05 14:48:07.082648075 +0100
+@@ -1,7 +1,7 @@
+ annotations:
+   artifacthub.io/changes: |
+     - add custom annotations on dashboards configmap
+-apiVersion: v1
++apiVersion: v2
+ appVersion: 1.7.1
+ description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
+ home: https://fluentbit.io/
+diff -rtubN charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent-bit.zdt/templates/tests/test-connection.yaml
+--- charts/fluent-bit/templates/tests/test-connection.yaml	2021-02-23 14:58:39.000000000 +0100
++++ charts/fluent-bit.zdt/templates/tests/test-connection.yaml	2021-03-05 14:51:34.749324131 +0100
+@@ -1,3 +1,4 @@
++{{- if .Values.testFramework.enabled }}
+ apiVersion: v1
+ kind: Pod
+ metadata:
+@@ -18,3 +19,4 @@
+     {{- toYaml . | nindent 4 }}
+   {{- end }}
+   restartPolicy: Never
++{{- end }}
+diff -rtubN charts/fluent-bit/values.yaml charts/fluent-bit.zdt/values.yaml
+--- charts/fluent-bit/values.yaml	2021-02-23 14:58:39.000000000 +0100
++++ charts/fluent-bit.zdt/values.yaml	2021-03-05 14:50:40.652655018 +0100
+@@ -12,6 +12,7 @@
+   # tag:
+ 
+ testFramework:
++  enabled: false
+   image:
+     repository: busybox
+     pullPolicy: Always
diff --git a/charts/kubezero-logging/templates/eck/elasticsearch.yaml b/charts/kubezero-logging/templates/eck/elasticsearch.yaml
index bc83c1d..553de90 100644
--- a/charts/kubezero-logging/templates/eck/elasticsearch.yaml
+++ b/charts/kubezero-logging/templates/eck/elasticsearch.yaml
@@ -12,10 +12,7 @@ spec:
   {{- range .Values.es.nodeSets }}
   - name: {{ .name }}
     config:
-      node.master: true
-      node.data: true
-      node.ingest: true
-      node.ml: false
+      node.roles: [ master, data, ingest ]
       {{- if $.Values.es.prometheus }}
       prometheus.indices: false
       {{- end }}
@@ -24,7 +21,10 @@ spec:
       cluster.routing.allocation.awareness.attributes: zone
       {{- end }}
       transport.compress: true
-      node.processors: {{ default 1 .processors }}
+      {{- if .processors }}
+      node.processors: {{ .processors }}
+      {{- end }}
+      indices.memory.index_buffer_size: "30%"
     podTemplate:
       {{- if $.Values.es.s3Snapshot.iamrole }}
       metadata:
@@ -51,18 +51,14 @@ spec:
           securityContext:
             capabilities:
               add: ["SYS_CHROOT"]
-          resources:
-            requests:
-              cpu: {{ default "200m" .cpu_request }}
-              memory: {{ default ( mul 2 ( default "2" .jvm_heap ) ) .memory_request }}Gi
-            limits:
-              {{- if .cpu_limit }}
-              cpu: {{ .cpu_limit }}
-              {{- end }}
-              memory: {{ default ( mul 2 ( default "2" .jvm_heap ) ) .memory_limit }}Gi
+          {{- with .resources }}
+          resources: {{ toYaml . | nindent 12 }}
+          {{- end }}
+          {{- if .jvm_heap }}
           env:
           - name: ES_JAVA_OPTS
-            value: -Xms{{ default "2" .jvm_heap }}g -Xmx{{ default "2" .jvm_heap }}g
+            value: -Xms{{ .jvm_heap }}g -Xmx{{ .jvm_heap }}g
+          {{- end }}
         affinity:
           podAntiAffinity:
             requiredDuringSchedulingIgnoredDuringExecution:
diff --git a/charts/kubezero-logging/templates/eck/kibana.yaml b/charts/kubezero-logging/templates/eck/kibana.yaml
index 1a24230..a532295 100644
--- a/charts/kubezero-logging/templates/eck/kibana.yaml
+++ b/charts/kubezero-logging/templates/eck/kibana.yaml
@@ -26,7 +26,7 @@ spec:
         resources:
           requests:
             memory: 1Gi
-            cpu: 100m
+            cpu: 200m
           limits:
             memory: 2Gi
   http:
diff --git a/charts/kubezero-logging/update.sh b/charts/kubezero-logging/update.sh
new file mode 100755
index 0000000..4117137
--- /dev/null
+++ b/charts/kubezero-logging/update.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+FLUENT_BIT_VERSION=0.12.3
+
+rm -rf charts/fluent-bit
+curl -L -s -o - https://github.com/fluent/helm-charts/releases/download/fluent-bit-${FLUENT_BIT_VERSION}/fluent-bit-${FLUENT_BIT_VERSION}.tgz | tar xfz - -C charts
+
+patch -i fluent-bit.patch -p0 --no-backup-if-mismatch
diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml
index b43662c..5fbee4d 100644
--- a/charts/kubezero-logging/values.yaml
+++ b/charts/kubezero-logging/values.yaml
@@ -11,7 +11,7 @@ eck-operator:
     node-role.kubernetes.io/master: ""
 
 # Version for ElasticSearch and Kibana have to match so we define it at top-level
-version: 7.10.1
+version: 7.11.1
 
 elastic_password: "" # super_secret_elastic_password
 
@@ -192,8 +192,15 @@ fluentd:
 
 fluent-bit:
   enabled: false
-  test:
-    enabled: false
+
+  serviceMonitor:
+    enabled: true
+    selector:
+      release: metrics
+
+  tolerations:
+  - key: node-role.kubernetes.io/master
+    effect: NoSchedule
 
   config:
     output:
@@ -211,11 +218,115 @@ fluent-bit:
     #extraRecords:
     #  source.clustername: MyKubeCluster
 
-  serviceMonitor:
-    enabled: true
-    selector:
-      release: metrics
+    service: |
+      [SERVICE]
+          Flush {{ .Values.config.flushInterval }}
+          Daemon Off
+          Log_Level {{ .Values.config.logLevel }}
+          Parsers_File parsers.conf
+          Parsers_File custom_parsers.conf
+          HTTP_Server On
+          HTTP_Listen 0.0.0.0
+          HTTP_Port {{ .Values.service.port }}
+    inputs: |
+      [INPUT]
+          Name tail
+          Path /var/log/containers/*.log
+          Parser cri-log
+          Tag cri.*
+          Skip_Long_Lines On
+          DB /var/log/flb_kube.db
+          DB.Sync Normal
+          {{- with .Values.config.input }}
+          Mem_Buf_Limit {{ default "16MB" .memBufLimit }}
+          Refresh_Interval {{ default 10 .refreshInterval }}
+          {{- end }}
 
-  tolerations:
-  - key: node-role.kubernetes.io/master
-    effect: NoSchedule
+    filters: |
+      [FILTER]
+          Name    lua
+          Match   cri.*
+          script  /fluent-bit/scripts/kubezero.lua
+          call    reassemble_cri_logs
+
+      [FILTER]
+          Name kubernetes
+          Match cri.*
+          Merge_Log On
+          Merge_Log_Key kube
+          Kube_Tag_Prefix cri.var.log.containers.
+          Keep_Log Off
+          K8S-Logging.Parser Off
+          K8S-Logging.Exclude Off
+
+      {{- if index .Values "config" "extraRecords" }}
+
+      [FILTER]
+          Name record_modifier
+          Match cri.*
+          {{- range $k,$v := index .Values "config" "extraRecords" }}
+          Record {{ $k }} {{ $v }}
+          {{- end }}
+      {{- end }}
+
+      [FILTER]
+          Name rewrite_tag
+          Match cri.*
+          Emitter_Name kube_tag_rewriter
+          Rule logtag F kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false
+
+      [FILTER]
+          Name    lua
+          Match   kube.*
+          script  /fluent-bit/scripts/kubezero.lua
+          call    nest_k8s_ns
+
+    outputs: |
+      [OUTPUT]
+          Match *
+          Name forward
+          Host {{ .Values.config.output.host }}
+          Port 24224
+          Shared_Key {{ .Values.config.output.sharedKey }}
+          tls {{ ternary "on" "off" .Values.config.output.tls }}
+          Send_options true
+          Require_ack_response true
+
+    customParsers: |
+      [PARSER]
+          Name cri-log
+          Format regex
+          Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
+          Time_Key    time
+          Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+
+  luaScripts:
+    kubezero.lua: |
+      local reassemble_state = {}
+
+      function reassemble_cri_logs(tag, timestamp, record)
+         local reassemble_key = tag
+         if record.logtag == 'P' then
+            reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or "" .. record.log
+            return -1, 0, 0
+         end
+         record.log = reassemble_state[reassemble_key] or "" .. (record.log or "")
+         reassemble_state[reassemble_key] = nil
+         return 1, timestamp, record
+      end
+
+      function nest_k8s_ns(tag, timestamp, record)
+          if not record['kubernetes']['namespace_name'] then
+              return 0, 0, 0
+          end
+          new_record = {}
+          for key, val in pairs(record) do
+              if key == 'kube' then
+                  new_record[key] = {}
+                  new_record[key][record['kubernetes']['namespace_name']] = record[key]
+              else
+                  new_record[key] = record[key]
+              end
+          end
+          return 1, timestamp, new_record
+      end