From 77a7ba2ed6970c7db483f691da64175dd9c059cd Mon Sep 17 00:00:00 2001
From: Stefan Reimer <stefan@zero-downtime.net>
Date: Tue, 8 Dec 2020 07:05:25 -0800
Subject: [PATCH] Integrare fluent-bit into logging to allow better config

---
 charts/kubezero-logging/Chart.yaml            |   8 +-
 charts/kubezero-logging/README.md             |  20 +-
 .../charts/fluent-bit/.helmignore             |  23 +++
 .../charts/fluent-bit/Chart.yaml              |  20 ++
 .../charts/fluent-bit/README.md               |  23 +++
 .../fluent-bit/conf/custom_parsers.conf       |   6 +
 .../charts/fluent-bit/conf/fluent-bit.conf    |  72 ++++++++
 .../charts/fluent-bit/conf/functions.lua      |  28 +++
 .../charts/fluent-bit/templates/NOTES.txt     |   5 +
 .../charts/fluent-bit/templates/_helpers.tpl  |  63 +++++++
 .../charts/fluent-bit/templates/_pod.tpl      | 103 +++++++++++
 .../fluent-bit/templates/clusterrole.yaml     |  28 +++
 .../templates/clusterrolebinding.yaml         |  16 ++
 .../fluent-bit/templates/configmap.yaml       |  15 ++
 .../fluent-bit/templates/daemonset.yaml       |  30 +++
 .../fluent-bit/templates/deployment.yaml      |  31 ++++
 .../charts/fluent-bit/templates/psp.yaml      |  38 ++++
 .../charts/fluent-bit/templates/service.yaml  |  30 +++
 .../fluent-bit/templates/serviceaccount.yaml  |  12 ++
 .../fluent-bit/templates/servicemonitor.yaml  |  30 +++
 .../templates/tests/test-connection.yaml      |  17 ++
 .../charts/fluent-bit/values.yaml             | 171 ++++++++++++++++++
 charts/kubezero-logging/values.yaml           | 108 ++---------
 23 files changed, 788 insertions(+), 109 deletions(-)
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/.helmignore
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/Chart.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/README.md
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/conf/custom_parsers.conf
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/conf/fluent-bit.conf
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/conf/functions.lua
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/NOTES.txt
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/_helpers.tpl
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/clusterrolebinding.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/configmap.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/daemonset.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/deployment.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/service.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/serviceaccount.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/servicemonitor.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/tests/test-connection.yaml
 create mode 100644 charts/kubezero-logging/charts/fluent-bit/values.yaml

diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml
index c6fc86c0..1c0c77e1 100644
--- a/charts/kubezero-logging/Chart.yaml
+++ b/charts/kubezero-logging/Chart.yaml
@@ -26,9 +26,9 @@ dependencies:
     version: 2.5.1
     repository: https://kubernetes-charts.storage.googleapis.com/
     condition: fluentd.enabled
-  - name: fluent-bit
-    version: 0.7.10
-    repository: https://zero-down-time.github.io/kubezero/
+#  - name: fluent-bit
+#    version: 0.7.10
+#    repository: https://zero-down-time.github.io/kubezero/
     # repository: https://fluent.github.io/helm-charts
-    condition: fluent-bit.enabled
+#    condition: fluent-bit.enabled
 kubeVersion: ">= 1.16.0"
diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md
index 10ab547e..9178c0a1 100644
--- a/charts/kubezero-logging/README.md
+++ b/charts/kubezero-logging/README.md
@@ -20,7 +20,6 @@ Kubernetes: `>= 1.16.0`
 |------------|------|---------|
 | https://helm.elastic.co | eck-operator | 1.3.0 |
 | https://kubernetes-charts.storage.googleapis.com/ | fluentd | 2.5.1 |
-| https://zero-down-time.github.io/kubezero/ | fluent-bit | 0.7.10 |
 | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 |
 
 ## Changes from upstream
@@ -57,7 +56,8 @@ Kubernetes: `>= 1.16.0`
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
-| eck-operator.enabled | bool | `true` |  |
+| eck-operator.enabled | bool | `false` |  |
+| eck-operator.installCRDs | bool | `false` |  |
 | eck-operator.nodeSelector."node-role.kubernetes.io/master" | string | `""` |  |
 | eck-operator.tolerations[0].effect | string | `"NoSchedule"` |  |
 | eck-operator.tolerations[0].key | string | `"node-role.kubernetes.io/master"` |  |
@@ -66,12 +66,14 @@ Kubernetes: `>= 1.16.0`
 | es.prometheus | bool | `false` |  |
 | es.s3Snapshot.enabled | bool | `false` |  |
 | es.s3Snapshot.iamrole | string | `""` |  |
-| fluent-bit.config.customParsers | string | `"[PARSER]\n    Name cri-log\n    Format regex\n    Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$\n    Time_Key    time\n    Time_Format %Y-%m-%dT%H:%M:%S.%L%z\n"` |  |
-| fluent-bit.config.filters | string | `"[FILTER]\n    Name    lua\n    Match   cri.*\n    script  /fluent-bit/etc/functions.lua\n    call    reassemble_cri_logs\n\n[FILTER]\n    Name kubernetes\n    Match cri.*\n    Merge_Log On\n    Merge_Log_Key kube\n    Kube_Tag_Prefix cri.var.log.containers.\n    Keep_Log Off\n    K8S-Logging.Parser Off\n    K8S-Logging.Exclude Off\n\n[FILTER]\n    Name rewrite_tag\n    Match cri.*\n    Emitter_Name kube_tag_rewriter\n    Rule logtag F kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false\n\n[FILTER]\n    Name    lua\n    Match   kube.*\n    script  /fluent-bit/etc/functions.lua\n    call    nest_k8s_ns\n"` |  |
-| fluent-bit.config.inputs | string | `"[INPUT]\n    Name tail\n    Path /var/log/containers/*.log\n    Parser cri-log\n    Tag cri.*\n    Mem_Buf_Limit 16MB\n    Skip_Long_Lines On\n    Refresh_Interval 10\n    DB /var/log/flb_kube.db\n    DB.Sync Normal\n"` |  |
-| fluent-bit.config.lua | string | `"local reassemble_state = {}\n\nfunction reassemble_cri_logs(tag, timestamp, record)\n   local reassemble_key = tag\n   if record.logtag == 'P' then\n      reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or \"\" .. record.log\n      return -1, 0, 0\n   end\n   record.log = reassemble_state[reassemble_key] or \"\" .. (record.log or \"\")\n   reassemble_state[reassemble_key] = nil\n   return 1, timestamp, record\nend\n\nfunction nest_k8s_ns(tag, timestamp, record)\n    if not record['kubernetes']['namespace_name'] then\n        return 0, 0, 0\n    end\n    new_record = {}\n    for key, val in pairs(record) do\n        if key == 'kube' then\n            new_record[key] = {}\n            new_record[key][record['kubernetes']['namespace_name']] = record[key]\n        else\n            new_record[key] = record[key]\n        end\n    end\n    return 1, timestamp, new_record\nend\n"` |  |
-| fluent-bit.config.outputs | string | `"[OUTPUT]\n    Match *\n    Name forward\n    Host logging-fluentd\n    Port 24224\n    Shared_Key cloudbender\n    Send_options true\n    Require_ack_response true\n"` |  |
-| fluent-bit.config.service | string | `"[SERVICE]\n    Flush 1\n    Daemon Off\n    Log_Level warn\n    Parsers_File parsers.conf\n    Parsers_File custom_parsers.conf\n    HTTP_Server On\n    HTTP_Listen 0.0.0.0\n    HTTP_Port 2020\n"` |  |
+| fluent-bit.config.flushInterval | int | `1` |  |
+| fluent-bit.config.input.memBufLimit | string | `"16MB"` |  |
+| fluent-bit.config.input.refreshInterval | int | `10` |  |
+| fluent-bit.config.logLevel | string | `"warn"` |  |
+| fluent-bit.config.output.host | string | `"logging-fluentd"` |  |
+| fluent-bit.config.output.sharedKey | string | `"cloudbender"` |  |
+| fluent-bit.config.output.tls | bool | `false` |  |
+| fluent-bit.config.outputs | object | `{}` |  |
 | fluent-bit.enabled | bool | `false` |  |
 | fluent-bit.serviceMonitor.enabled | bool | `true` |  |
 | fluent-bit.serviceMonitor.selector.release | string | `"metrics"` |  |
@@ -81,7 +83,7 @@ Kubernetes: `>= 1.16.0`
 | fluentd.configMaps."filter.conf" | string | `"<filter disabled.kube.**>\n  @type parser\n  key_name message\n  remove_key_name_field true\n  reserve_data true\n  reserve_time true\n  # inject_key_prefix message_json.\n  emit_invalid_record_to_error false\n  <parse>\n    @type json\n  </parse>\n</filter>\n"` |  |
 | fluentd.configMaps."forward-input.conf" | string | `"<source>\n  @type forward\n  port 24224\n  bind 0.0.0.0\n  skip_invalid_event true\n  send_keepalive_packet true\n  <security>\n    self_hostname \"#{ENV['HOSTNAME']}\"\n    shared_key \"#{ENV['FLUENTD_SHARED_KEY']}\"\n  </security>\n</source>\n"` |  |
 | fluentd.configMaps."general.conf" | string | `"<label @FLUENT_LOG>\n  <match **>\n    @type null\n  </match>\n</label>\n<source>\n  @type http\n  port 9880\n  bind 0.0.0.0\n  keepalive_timeout 30\n</source>\n<source>\n  @type monitor_agent\n  bind 0.0.0.0\n  port 24220\n  tag fluentd.monitor.metrics\n</source>\n"` |  |
-| fluentd.configMaps."output.conf" | string | `"<match **>\n  @id elasticsearch\n  @type elasticsearch\n  @log_level info\n  include_tag_key true\n  id_key id\n  remove_keys id\n\n  # KubeZero pipeline incl. GeoIP etc.\n  # pipeline fluentd\n\n  host \"#{ENV['OUTPUT_HOST']}\"\n  port \"#{ENV['OUTPUT_PORT']}\"\n  scheme \"#{ENV['OUTPUT_SCHEME']}\"\n  ssl_version \"#{ENV['OUTPUT_SSL_VERSION']}\"\n  ssl_verify \"#{ENV['OUTPUT_SSL_VERIFY']}\"\n  user \"#{ENV['OUTPUT_USER']}\"\n  password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n  log_es_400_reason\n  logstash_format true\n  reconnect_on_error true\n  # reload_on_failure true\n  request_timeout 15s\n  suppress_type_name true\n\n  <buffer tag>\n    @type file_single\n    path /var/log/fluentd-buffers/kubernetes.system.buffer\n    flush_mode interval\n    flush_thread_count 2\n    flush_interval 30s\n    flush_at_shutdown true\n    retry_type exponential_backoff\n    retry_timeout 60m\n    overflow_action drop_oldest_chunk\n  </buffer>\n</match>\n"` |  |
+| fluentd.configMaps."output.conf" | string | `"<match **>\n  @id elasticsearch\n  @type elasticsearch\n  @log_level info\n  include_tag_key true\n  id_key id\n  remove_keys id\n\n  # KubeZero pipeline incl. GeoIP etc.\n  pipeline fluentd\n\n  host \"#{ENV['OUTPUT_HOST']}\"\n  port \"#{ENV['OUTPUT_PORT']}\"\n  scheme \"#{ENV['OUTPUT_SCHEME']}\"\n  ssl_version \"#{ENV['OUTPUT_SSL_VERSION']}\"\n  ssl_verify \"#{ENV['OUTPUT_SSL_VERIFY']}\"\n  user \"#{ENV['OUTPUT_USER']}\"\n  password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n  log_es_400_reason\n  logstash_format true\n  reconnect_on_error true\n  # reload_on_failure true\n  request_timeout 15s\n  suppress_type_name true\n\n  <buffer tag>\n    @type file_single\n    path /var/log/fluentd-buffers/kubernetes.system.buffer\n    flush_mode interval\n    flush_thread_count 2\n    flush_interval 30s\n    flush_at_shutdown true\n    retry_type exponential_backoff\n    retry_timeout 60m\n    overflow_action drop_oldest_chunk\n  </buffer>\n</match>\n"` |  |
 | fluentd.enabled | bool | `false` |  |
 | fluentd.env.OUTPUT_SSL_VERIFY | string | `"false"` |  |
 | fluentd.env.OUTPUT_USER | string | `"elastic"` |  |
diff --git a/charts/kubezero-logging/charts/fluent-bit/.helmignore b/charts/kubezero-logging/charts/fluent-bit/.helmignore
new file mode 100644
index 00000000..0e8a0eb3
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml
new file mode 100644
index 00000000..58a0c169
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml
@@ -0,0 +1,20 @@
+apiVersion: v2
+name: fluent-bit
+description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
+keywords:
+  - logging
+  - fluent-bit
+  - fluentd
+version: 0.7.13
+appVersion: 1.6.8
+icon: https://fluentbit.io/assets/img/logo1-default.png
+home: https://fluentbit.io/
+sources:
+  - https://github.com/fluent/fluent-bit/
+maintainers:
+  - name: edsiper
+    email: eduardo@treasure-data.com
+  - name: naseemkullah
+    email: naseem@transit.app
+  - name: Towmeykaw
+    email: towmeykaw@gmail.com
diff --git a/charts/kubezero-logging/charts/fluent-bit/README.md b/charts/kubezero-logging/charts/fluent-bit/README.md
new file mode 100644
index 00000000..c4b61729
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/README.md
@@ -0,0 +1,23 @@
+# Fluent Bit Helm Chart
+
+[Fluent Bit](https://fluentbit.io) is a fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
+
+## Installation
+
+To add the `fluent` helm repo, run:
+
+```sh
+helm repo add fluent https://fluent.github.io/helm-charts
+```
+
+To install a release named `fluent-bit`, run:
+
+```sh
+helm install fluent-bit fluent/fluent-bit
+```
+
+## Chart Values
+
+```sh
+helm show values fluent/fluent-bit
+```
diff --git a/charts/kubezero-logging/charts/fluent-bit/conf/custom_parsers.conf b/charts/kubezero-logging/charts/fluent-bit/conf/custom_parsers.conf
new file mode 100644
index 00000000..a2055dba
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/conf/custom_parsers.conf
@@ -0,0 +1,6 @@
+[PARSER]
+    Name cri-log
+    Format regex
+    Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
+    Time_Key    time
+    Time_Format %Y-%m-%dT%H:%M:%S.%L%z
diff --git a/charts/kubezero-logging/charts/fluent-bit/conf/fluent-bit.conf b/charts/kubezero-logging/charts/fluent-bit/conf/fluent-bit.conf
new file mode 100644
index 00000000..9ce6b677
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/conf/fluent-bit.conf
@@ -0,0 +1,72 @@
+[SERVICE]
+    Flush {{ .Values.config.flushInterval }}
+    Daemon Off
+    Log_Level {{ .Values.config.logLevel }}
+    Parsers_File parsers.conf
+    Parsers_File custom_parsers.conf
+    HTTP_Server On
+    HTTP_Listen 0.0.0.0
+    HTTP_Port 2020
+
+[INPUT]
+    Name tail
+    Path /var/log/containers/*.log
+    Parser cri-log
+    Tag cri.*
+    Mem_Buf_Limit {{ .Values.config.input.memBufLimit }}
+    Skip_Long_Lines On
+    Refresh_Interval {{ .Values.config.input.refreshInterval }}
+    DB /var/log/flb_kube.db
+    DB.Sync Normal
+
+[FILTER]
+    Name    lua
+    Match   cri.*
+    script  /fluent-bit/etc/functions.lua
+    call    reassemble_cri_logs
+
+[FILTER]
+    Name kubernetes
+    Match cri.*
+    Merge_Log On
+    Merge_Log_Key kube
+    Kube_Tag_Prefix cri.var.log.containers.
+    Keep_Log Off
+    K8S-Logging.Parser Off
+    K8S-Logging.Exclude Off
+
+{{- if index .Values "config" "extraRecords" }}
+
+[FILTER]
+    Name record_modifier
+    Match cri.*
+    {{- range $k,$v := index .Values "config" "extraRecords" }}
+    Record {{ $k }} {{ $v }}
+    {{- end }}
+{{- end }}
+
+[FILTER]
+    Name rewrite_tag
+    Match cri.*
+    Emitter_Name kube_tag_rewriter
+    Rule logtag F kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false
+
+[FILTER]
+    Name    lua
+    Match   kube.*
+    script  /fluent-bit/etc/functions.lua
+    call    nest_k8s_ns
+
+{{- if .Values.config.outputs }}
+{{ tpl .Values.config.outputs . }}
+{{- else }}
+[OUTPUT]
+    Match *
+    Name forward
+    Host {{ .Values.config.output.host }}
+    Port 24224
+    Shared_Key {{ .Values.config.output.sharedKey }}
+    tls {{ ternary "on" "off" .Values.config.output.tls }}
+    Send_options true
+    Require_ack_response true
+{{- end }}
diff --git a/charts/kubezero-logging/charts/fluent-bit/conf/functions.lua b/charts/kubezero-logging/charts/fluent-bit/conf/functions.lua
new file mode 100644
index 00000000..c7171fe3
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/conf/functions.lua
@@ -0,0 +1,28 @@
+local reassemble_state = {}
+
+function reassemble_cri_logs(tag, timestamp, record)
+   local reassemble_key = tag
+   if record.logtag == 'P' then
+      reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or "" .. record.log
+      return -1, 0, 0
+   end
+   record.log = reassemble_state[reassemble_key] or "" .. (record.log or "")
+   reassemble_state[reassemble_key] = nil
+   return 1, timestamp, record
+end
+
+function nest_k8s_ns(tag, timestamp, record)
+    if not record['kubernetes']['namespace_name'] then
+        return 0, 0, 0
+    end
+    new_record = {}
+    for key, val in pairs(record) do
+        if key == 'kube' then
+            new_record[key] = {}
+            new_record[key][record['kubernetes']['namespace_name']] = record[key]
+        else
+            new_record[key] = record[key]
+        end
+    end
+    return 1, timestamp, new_record
+end
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/NOTES.txt b/charts/kubezero-logging/charts/fluent-bit/templates/NOTES.txt
new file mode 100644
index 00000000..a06cff8c
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/NOTES.txt
@@ -0,0 +1,5 @@
+Get Fluent Bit build information by running these commands:
+
+export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "fluent-bit.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+echo "curl http://127.0.0.1:2020 for Fluent Bit build information"
+kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 2020:2020
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/_helpers.tpl b/charts/kubezero-logging/charts/fluent-bit/templates/_helpers.tpl
new file mode 100644
index 00000000..ebc39e71
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/_helpers.tpl
@@ -0,0 +1,63 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "fluent-bit.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "fluent-bit.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "fluent-bit.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "fluent-bit.labels" -}}
+helm.sh/chart: {{ include "fluent-bit.chart" . }}
+{{ include "fluent-bit.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "fluent-bit.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "fluent-bit.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "fluent-bit.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "fluent-bit.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl b/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl
new file mode 100644
index 00000000..a55df22b
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl
@@ -0,0 +1,103 @@
+{{- define "fluent-bit.pod" -}}
+{{- with .Values.imagePullSecrets }}
+imagePullSecrets:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- if .Values.priorityClassName }}
+priorityClassName: {{ .Values.priorityClassName }}
+{{- end }}
+serviceAccountName: {{ include "fluent-bit.serviceAccountName" . }}
+securityContext:
+  {{- toYaml .Values.podSecurityContext | nindent 2 }}
+containers:
+  - name: {{ .Chart.Name }}
+    securityContext:
+      {{- toYaml .Values.securityContext | nindent 6 }}
+    image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}"
+    imagePullPolicy: {{ .Values.image.pullPolicy }}
+  {{- if .Values.env }}
+    env:
+    {{- toYaml .Values.env | nindent 4 }}
+  {{- end }}
+  {{- if .Values.envFrom }}
+    envFrom:
+    {{- toYaml .Values.envFrom | nindent 4 }}
+  {{- end }}
+    ports:
+      - name: http
+        containerPort: 2020
+        protocol: TCP
+    {{- if .Values.extraPorts }}
+      {{- range .Values.extraPorts }}
+      - name: {{ .name }}
+        containerPort: {{ .containerPort }}
+        protocol: {{ .protocol }}
+      {{- end }}
+    {{- end }}
+    livenessProbe:
+      httpGet:
+        path: /
+        port: http
+    readinessProbe:
+      httpGet:
+        path: /
+        port: http
+    resources:
+      {{- toYaml .Values.resources | nindent 6 }}
+    volumeMounts:
+      - name: config
+        mountPath: /fluent-bit/etc/fluent-bit.conf
+        subPath: fluent-bit.conf
+      - name: config
+        mountPath: /fluent-bit/etc/custom_parsers.conf
+        subPath: custom_parsers.conf
+    {{- if .Values.config.lua }}
+      - name: config
+        mountPath: /fluent-bit/etc/functions.lua
+        subPath: functions.lua
+    {{- end }}
+    {{- if eq .Values.kind "DaemonSet" }}
+      - name: varlog
+        mountPath: /var/log
+      - name: varlibdockercontainers
+        mountPath: /var/lib/docker/containers
+        readOnly: true
+      - name: etcmachineid
+        mountPath: /etc/machine-id
+        readOnly: true
+    {{- end }}
+    {{- if .Values.extraVolumeMounts }}
+      {{- toYaml .Values.extraVolumeMounts | nindent 6 }}
+    {{- end }}
+volumes:
+  - name: config
+    configMap:
+      name: {{ if .Values.existingConfigMap }}{{ .Values.existingConfigMap }}{{- else }}{{ include "fluent-bit.fullname" . }}{{- end }}
+{{- if eq .Values.kind "DaemonSet" }}
+  - name: varlog
+    hostPath:
+      path: /var/log
+  - name: varlibdockercontainers
+    hostPath:
+      path: /var/lib/docker/containers
+  - name: etcmachineid
+    hostPath:
+      path: /etc/machine-id
+      type: File
+{{- end }}
+{{- if .Values.extraVolumes }}
+  {{- toYaml .Values.extraVolumes | nindent 2 }}
+{{- end }}
+{{- with .Values.nodeSelector }}
+nodeSelector:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- with .Values.affinity }}
+affinity:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- with .Values.tolerations }}
+tolerations:
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- end -}}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml
new file mode 100644
index 00000000..2ef2248a
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml
@@ -0,0 +1,28 @@
+{{- if .Values.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "fluent-bit.fullname" . }}
+  labels:
+    {{- include "fluent-bit.labels" . | nindent 4 }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+  {{- if .Values.podSecurityPolicy.create }}
+  - apiGroups:
+      - policy
+    resources:
+      - podsecuritypolicies
+    resourceNames:
+      - {{ include "fluent-bit.fullname" . }}
+    verbs:
+      - use
+  {{- end }}
+{{- end -}}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/clusterrolebinding.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/clusterrolebinding.yaml
new file mode 100644
index 00000000..e4b55b20
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/clusterrolebinding.yaml
@@ -0,0 +1,16 @@
+{{- if .Values.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "fluent-bit.fullname" . }}
+  labels:
+    {{- include "fluent-bit.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "fluent-bit.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "fluent-bit.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end -}}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/configmap.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/configmap.yaml
new file mode 100644
index 00000000..b54e91b3
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/configmap.yaml
@@ -0,0 +1,15 @@
+{{- if (empty .Values.existingConfigMap) -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "fluent-bit.fullname" . }}
+  labels:
+    {{- include "fluent-bit.labels" . | nindent 4 }}
+data:
+  custom_parsers.conf: |
+    {{ tpl (.Files.Get "conf/custom_parsers.conf") . | nindent 4 }}
+  fluent-bit.conf: |
+    {{ tpl (.Files.Get "conf/fluent-bit.conf") . | nindent 4 }}
+  functions.lua: |
+    {{ tpl (.Files.Get "conf/functions.lua") . | nindent 4 }}
+{{- end -}}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/daemonset.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/daemonset.yaml
new file mode 100644
index 00000000..9227c73e
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/daemonset.yaml
@@ -0,0 +1,30 @@
+{{- if eq .Values.kind "DaemonSet" }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ include "fluent-bit.fullname" . }}
+  labels:
+    {{- include "fluent-bit.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "fluent-bit.selectorLabels" . | nindent 6 }}
+  {{- with .Values.updateStrategy }}
+  updateStrategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  template:
+    metadata:
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+      {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "fluent-bit.selectorLabels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- include "fluent-bit.pod" . | nindent 6 }}
+{{- end }}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/deployment.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/deployment.yaml
new file mode 100644
index 00000000..3af288ff
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/deployment.yaml
@@ -0,0 +1,31 @@
+{{- if eq .Values.kind "Deployment" }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "fluent-bit.fullname" . }}
+  labels:
+    {{- include "fluent-bit.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  {{- with .Values.updateStrategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "fluent-bit.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+      {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "fluent-bit.selectorLabels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- include "fluent-bit.pod" . | nindent 6 }}
+{{- end }}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml
new file mode 100644
index 00000000..ed57cc18
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml
@@ -0,0 +1,38 @@
+{{- if .Values.podSecurityPolicy.create }}
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: {{ include "fluent-bit.fullname" . }}
+spec:
+  privileged: false
+  # Required to prevent escalations to root.
+  allowPrivilegeEscalation: false
+  # This is redundant with non-root + disallow privilege escalation,
+  # but we can provide it for defense in depth.
+  requiredDropCapabilities:
+    - ALL
+  volumes:
+    - '*'
+  hostNetwork: false
+  hostIPC: false
+  hostPID: false
+  runAsUser:
+    # TODO: Require the container to run without root privileges.
+    rule: 'RunAsAny'
+  seLinux:
+    # This policy assumes the nodes are using AppArmor rather than SELinux.
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'MustRunAs'
+    ranges:
+      # Forbid adding the root group.
+      - min: 1
+        max: 65535
+  fsGroup:
+    rule: 'MustRunAs'
+    ranges:
+      # Forbid adding the root group.
+      - min: 1
+        max: 65535
+  readOnlyRootFilesystem: false
+{{- end }}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/service.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/service.yaml
new file mode 100644
index 00000000..408358bc
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/service.yaml
@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "fluent-bit.fullname" . }}
+  labels:
+    {{- include "fluent-bit.labels" . | nindent 4 }}
+    {{- with .Values.service.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  {{- with .Values.service.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  {{- if .Values.extraPorts }}
+    {{- range .Values.extraPorts }}
+    - name: {{ .name }}
+      targetPort: {{ .name }}
+      protocol: {{ .protocol }}
+      port: {{ .port }}
+    {{- end }}
+  {{- end }}
+  selector:
+    {{- include "fluent-bit.selectorLabels" . | nindent 4 }}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/serviceaccount.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/serviceaccount.yaml
new file mode 100644
index 00000000..1be6e749
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/serviceaccount.yaml
@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "fluent-bit.serviceAccountName" . }}
+  labels:
+    {{- include "fluent-bit.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end -}}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/servicemonitor.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/servicemonitor.yaml
new file mode 100644
index 00000000..e564faa8
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/servicemonitor.yaml
@@ -0,0 +1,30 @@
+{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "fluent-bit.fullname" . }}
+  {{- with .Values.serviceMonitor.namespace }}
+  namespace: {{ . }}
+  {{- end }}
+  labels:
+    {{- include "fluent-bit.labels" . | nindent 4 }}
+    {{- with .Values.serviceMonitor.selector }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+    - port: http
+      path: /api/v1/metrics/prometheus
+      {{- with .Values.serviceMonitor.interval }}
+      interval: {{ . }}
+      {{- end }}
+      {{- with .Values.serviceMonitor.scrapeTimeout }}
+      scrapeTimeout: {{ . }}
+      {{- end }}
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      {{- include "fluent-bit.selectorLabels" . | nindent 6 }}
+{{- end }}
diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/tests/test-connection.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/tests/test-connection.yaml
new file mode 100644
index 00000000..db53e098
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/templates/tests/test-connection.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.test.enabled }}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "fluent-bit.fullname" . }}-test-connection"
+  labels:
+    {{- include "fluent-bit.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test-success
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args: ['{{ include "fluent-bit.fullname" . }}:{{ .Values.service.port }}']
+  restartPolicy: Never
+{{- end }}
diff --git a/charts/kubezero-logging/charts/fluent-bit/values.yaml b/charts/kubezero-logging/charts/fluent-bit/values.yaml
new file mode 100644
index 00000000..3fc4b76d
--- /dev/null
+++ b/charts/kubezero-logging/charts/fluent-bit/values.yaml
@@ -0,0 +1,171 @@
+# Default values for fluent-bit.
+
+# kind -- DaemonSet or Deployment
+kind: DaemonSet
+
+# replicaCount -- Only applicable if kind=Deployment
+replicaCount: 1
+
+test:
+  enabled: true
+
+image:
+  repository: fluent/fluent-bit
+  pullPolicy: Always
+  # tag:
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  create: true
+  annotations: {}
+  name:
+
+rbac:
+  create: true
+
+podSecurityPolicy:
+  create: false
+
+podSecurityContext:
+  {}
+  # fsGroup: 2000
+
+securityContext:
+  {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+service:
+  type: ClusterIP
+  port: 2020
+  labels:
+    {}
+  annotations:
+    {}
+    # prometheus.io/path: "/api/v1/metrics/prometheus"
+    # prometheus.io/port: "2020"
+    # prometheus.io/scrape: "true"
+
+serviceMonitor:
+  enabled: false
+  # namespace: monitoring
+  # interval: 10s
+  # scrapeTimeout: 10s
+  # selector:
+  #  prometheus: my-prometheus
+
+resources:
+  {}
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+podAnnotations: {}
+
+podLabels: {}
+
+priorityClassName: ""
+
+env: []
+
+envFrom: []
+
+extraPorts: []
+#   - port: 5170
+#     containerPort: 5170
+#     protocol: TCP
+#     name: tcp
+
+extraVolumes: []
+
+extraVolumeMounts: []
+
+updateStrategy: {}
+  # type: RollingUpdate
+  # rollingUpdate:
+  #   maxUnavailable: 1
+
+# Make use of a pre-defined configmap instead of the one templated here
+existingConfigMap: ""
+
+## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit
+config:
+  ## https://docs.fluentbit.io/manual/service
+  service: |
+    [SERVICE]
+        Flush 1
+        Daemon Off
+        Log_Level info
+        Parsers_File parsers.conf
+        Parsers_File custom_parsers.conf
+        HTTP_Server On
+        HTTP_Listen 0.0.0.0
+        HTTP_Port 2020
+
+  ## https://docs.fluentbit.io/manual/pipeline/inputs
+  inputs: |
+    [INPUT]
+        Name tail
+        Path /var/log/containers/*.log
+        Parser docker
+        Tag kube.*
+        Mem_Buf_Limit 5MB
+        Skip_Long_Lines On
+
+    [INPUT]
+        Name systemd
+        Tag host.*
+        Systemd_Filter _SYSTEMD_UNIT=kubelet.service
+        Read_From_Tail On
+
+  ## https://docs.fluentbit.io/manual/pipeline/filters
+  filters: |
+    [FILTER]
+        Name kubernetes
+        Match kube.*
+        Merge_Log On
+        Keep_Log Off
+        K8S-Logging.Parser On
+        K8S-Logging.Exclude On
+
+  ## https://docs.fluentbit.io/manual/pipeline/outputs
+  outputs: |
+    [OUTPUT]
+        Name es
+        Match kube.*
+        Host elasticsearch-master
+        Logstash_Format On
+        Retry_Limit False
+
+    [OUTPUT]
+        Name es
+        Match host.*
+        Host elasticsearch-master
+        Logstash_Format On
+        Logstash_Prefix node
+        Retry_Limit False
+
+  ## https://docs.fluentbit.io/manual/pipeline/parsers
+  customParsers: |
+    [PARSER]
+        Name docker_no_time
+        Format json
+        Time_Keep Off
+        Time_Key time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L
diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml
index e2fa0e08..884a1cf8 100644
--- a/charts/kubezero-logging/values.yaml
+++ b/charts/kubezero-logging/values.yaml
@@ -193,105 +193,21 @@ fluent-bit:
     enabled: false
 
   config:
-    outputs: |
-      [OUTPUT]
-          Match *
-          Name forward
-          Host logging-fluentd
-          Port 24224
-          Shared_Key cloudbender
-          Send_options true
-          Require_ack_response true
+    outputs: {}
+    output:
+      host: logging-fluentd
+      sharedKey: cloudbender
+      tls: false
 
-    customParsers: |
-      [PARSER]
-          Name cri-log
-          Format regex
-          Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
-          Time_Key    time
-          Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+    input:
+      memBufLimit: 16MB
+      refreshInterval: 10
 
-    inputs: |
-      [INPUT]
-          Name tail
-          Path /var/log/containers/*.log
-          Parser cri-log
-          Tag cri.*
-          Mem_Buf_Limit 16MB
-          Skip_Long_Lines On
-          Refresh_Interval 10
-          DB /var/log/flb_kube.db
-          DB.Sync Normal
+    logLevel: warn
+    flushInterval: 1
 
-    filters: |
-      [FILTER]
-          Name    lua
-          Match   cri.*
-          script  /fluent-bit/etc/functions.lua
-          call    reassemble_cri_logs
-
-      [FILTER]
-          Name kubernetes
-          Match cri.*
-          Merge_Log On
-          Merge_Log_Key kube
-          Kube_Tag_Prefix cri.var.log.containers.
-          Keep_Log Off
-          K8S-Logging.Parser Off
-          K8S-Logging.Exclude Off
-
-      [FILTER]
-          Name rewrite_tag
-          Match cri.*
-          Emitter_Name kube_tag_rewriter
-          Rule logtag F kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false
-
-      [FILTER]
-          Name    lua
-          Match   kube.*
-          script  /fluent-bit/etc/functions.lua
-          call    nest_k8s_ns
-
-    service: |
-      [SERVICE]
-          Flush 1
-          Daemon Off
-          Log_Level warn
-          Parsers_File parsers.conf
-          Parsers_File custom_parsers.conf
-          HTTP_Server On
-          HTTP_Listen 0.0.0.0
-          HTTP_Port 2020
-
-    lua: |
-      local reassemble_state = {}
-
-      function reassemble_cri_logs(tag, timestamp, record)
-         local reassemble_key = tag
-         if record.logtag == 'P' then
-            reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or "" .. record.log
-            return -1, 0, 0
-         end
-         record.log = reassemble_state[reassemble_key] or "" .. (record.log or "")
-         reassemble_state[reassemble_key] = nil
-         return 1, timestamp, record
-      end
-
-      function nest_k8s_ns(tag, timestamp, record)
-          if not record['kubernetes']['namespace_name'] then
-              return 0, 0, 0
-          end
-          new_record = {}
-          for key, val in pairs(record) do
-              if key == 'kube' then
-                  new_record[key] = {}
-                  new_record[key][record['kubernetes']['namespace_name']] = record[key]
-              else
-                  new_record[key] = record[key]
-              end
-          end
-          return 1, timestamp, new_record
-      end
+    #extraRecords:
+    #  source.clustername: MyKubeCluster
 
   serviceMonitor:
     enabled: true