From 6b1b02a74348f33317ca33ba0c82c9f5a3a48391 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 8 Sep 2020 12:40:28 +0100 Subject: [PATCH 1/8] Fluent-bit version bump and support for api audit logs --- charts/kubezero-logging/Chart.yaml | 4 ++-- charts/kubezero-logging/README.md | 14 ++++++++++---- charts/kubezero-logging/README.md.gotmpl | 8 +++++++- charts/kubezero-logging/delete_zdt-logging.sh | 5 ----- charts/kubezero-logging/values.yaml | 10 ++++++++++ charts/kubezero-metrics/delete_kube-prometheus.sh | 9 --------- 6 files changed, 29 insertions(+), 21 deletions(-) delete mode 100755 charts/kubezero-logging/delete_zdt-logging.sh delete mode 100755 charts/kubezero-metrics/delete_kube-prometheus.sh diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index 71cdf36..9d82968 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application -version: 0.3.1 +version: 0.3.2 appVersion: 1.2.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -23,7 +23,7 @@ dependencies: repository: https://kubernetes-charts.storage.googleapis.com/ condition: fluentd.enabled - name: fluent-bit - version: 0.6.3 + version: 0.6.4 repository: https://zero-down-time.github.io/kubezero/ # repository: https://fluent.github.io/helm-charts condition: fluent-bit.enabled diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md index 5d33343..f0017dd 100644 --- a/charts/kubezero-logging/README.md +++ b/charts/kubezero-logging/README.md @@ -2,7 +2,7 @@ kubezero-logging ================ KubeZero Umbrella Chart for complete EFK stack -Current chart version is `0.3.1` +Current chart version is `0.3.2` Source code can be found [here](https://kubezero.com) @@ -11,7 +11,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| | https://kubernetes-charts.storage.googleapis.com/ | fluentd | 2.5.1 | -| https://zero-down-time.github.io/kubezero/ | fluent-bit | 0.6.3 | +| https://zero-down-time.github.io/kubezero/ | fluent-bit | 0.6.4 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Changes from upstream @@ -32,10 +32,16 @@ Source code can be found [here](https://kubezero.com) - increased timeout to ES to 3 minutes +### FluentD + + +### Fluent-bit +- support for dedot Lua filter to replace "." with "_" for all annotations and labels +- support for api audit log ## Manual tasks ATM -- Install index template +- install index template - setup Kibana - create `logstash-*` Index Pattern @@ -50,7 +56,7 @@ Source code can be found [here](https://kubezero.com) | es.s3Snapshot.enabled | bool | `false` | | | es.s3Snapshot.iamrole | string | `""` | | | fluent-bit.config.filters | string | `"[FILTER]\n Name kubernetes\n Match kube.*\n Merge_Log On\n Keep_Log Off\n K8S-Logging.Parser On\n K8S-Logging.Exclude On\n\n[FILTER]\n Name lua\n Match kube.*\n script /fluent-bit/etc/functions.lua\n call dedot\n"` | | -| fluent-bit.config.inputs | string | `"[INPUT]\n Name tail\n Path /var/log/containers/*.log\n Parser cri\n Tag kube.*\n Mem_Buf_Limit 5MB\n Skip_Long_Lines On\n Refresh_Interval 10\n DB /var/log/flb_kube.db\n DB.Sync Normal\n"` | | +| fluent-bit.config.inputs | string | `"[INPUT]\n Name tail\n Path /var/log/containers/*.log\n Parser cri\n Tag kube.*\n Mem_Buf_Limit 5MB\n Skip_Long_Lines On\n Refresh_Interval 10\n DB /var/log/flb_kube.db\n DB.Sync Normal\n[INPUT]\n Name tail\n Path /var/log/kubernetes/audit.log\n Parser json\n Tag kube.api.audit\n Mem_Buf_Limit 5MB\n Skip_Long_Lines On\n Refresh_Interval 60\n DB /var/log/flb_kube_audit.db\n DB.Sync Normal\n"` | | | fluent-bit.config.lua | string | `"function dedot(tag, timestamp, record)\n if record[\"kubernetes\"] == nil then\n return 0, 0, 0\n end\n dedot_keys(record[\"kubernetes\"][\"annotations\"])\n dedot_keys(record[\"kubernetes\"][\"labels\"])\n return 1, timestamp, record\nend\n\nfunction dedot_keys(map)\n if map == nil then\n return\n end\n local new_map = {}\n local changed_keys = {}\n for k, v in pairs(map) do\n local dedotted = string.gsub(k, \"%.\", \"_\")\n if dedotted ~= k then\n new_map[dedotted] = v\n changed_keys[k] = true\n end\n end\n for k in pairs(changed_keys) do\n map[k] = nil\n end\n for k, v in pairs(new_map) do\n map[k] = v\n end\nend\n"` | | | fluent-bit.config.outputs | string | `"[OUTPUT]\n Match *\n Name forward\n Host logging-fluentd\n Port 24224\n tls on\n tls.verify off\n Shared_Key cloudbender\n"` | | | fluent-bit.config.service | string | `"[SERVICE]\n Flush 5\n Daemon Off\n Log_Level warn\n Parsers_File parsers.conf\n Parsers_File custom_parsers.conf\n HTTP_Server On\n HTTP_Listen 0.0.0.0\n HTTP_Port 2020\n"` | | diff --git a/charts/kubezero-logging/README.md.gotmpl b/charts/kubezero-logging/README.md.gotmpl index ae55a2b..b29d9e9 100644 --- a/charts/kubezero-logging/README.md.gotmpl +++ b/charts/kubezero-logging/README.md.gotmpl @@ -25,10 +25,16 @@ - increased timeout to ES to 3 minutes +### FluentD + + +### Fluent-bit +- support for dedot Lua filter to replace "." with "_" for all annotations and labels +- support for api audit log ## Manual tasks ATM -- Install index template +- install index template - setup Kibana - create `logstash-*` Index Pattern diff --git a/charts/kubezero-logging/delete_zdt-logging.sh b/charts/kubezero-logging/delete_zdt-logging.sh deleted file mode 100755 index 0931e5d..0000000 --- a/charts/kubezero-logging/delete_zdt-logging.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -# We only need to delete the service monitor and virtual service, others will be taken over by the new chart and we dont loose data -kubectl delete -n logging VirtualService kibana-logging -kubectl delete -n logging ServiceMonitor es-logging diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 2592c21..595d94d 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -222,6 +222,16 @@ fluent-bit: Refresh_Interval 10 DB /var/log/flb_kube.db DB.Sync Normal + [INPUT] + Name tail + Path /var/log/kubernetes/audit.log + Parser json + Tag kube.api.audit + Mem_Buf_Limit 5MB + Skip_Long_Lines On + Refresh_Interval 60 + DB /var/log/flb_kube_audit.db + DB.Sync Normal filters: | [FILTER] diff --git a/charts/kubezero-metrics/delete_kube-prometheus.sh b/charts/kubezero-metrics/delete_kube-prometheus.sh deleted file mode 100755 index e7de5b4..0000000 --- a/charts/kubezero-metrics/delete_kube-prometheus.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -[ -f istio-authorization-policy.yaml ] && kubectl delete -f istio-authorization-policy.yaml -[ -f istio-service.yaml ] && kubectl delete -f istio-service.yaml - -kubectl delete -f manifests -kubectl delete -f manifests/setup - -kubectl delete namespace monitoring -- 2.40.1 From e09935a819f32373d6a27e153fdca830df6ff954 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 8 Sep 2020 13:12:21 +0100 Subject: [PATCH 2/8] Add Lua functions to reassemble partial cri-o logs --- charts/kubezero-logging/values.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 595d94d..1cb9b32 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -234,6 +234,12 @@ fluent-bit: DB.Sync Normal filters: | + [FILTER] + Name lua + Match kube.* + script /fluent-bit/etc/functions.lua + call reassemble_cri_logs + [FILTER] Name kubernetes Match kube.* @@ -290,6 +296,25 @@ fluent-bit: end end + local reassemble_state = {} + + function reassemble_cri_logs(tag, timestamp, record) + -- IMPORTANT: reassemble_key must be unique for each parser stream + -- otherwise entries from different sources will get mixed up. + -- Either make sure that your parser tags satisfy this or construct + -- reassemble_key some other way + local reassemble_key = tag + -- if partial line, accumulate + if record.logtag == 'P' then + reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or "" .. record.message + return -1, 0, 0 + end + -- otherwise it's a full line, concatenate with accumulated partial lines if any + record.message = reassemble_state[reassemble_key] or "" .. record.message + reassemble_state[reassemble_key] = nil + return 1, timestamp, record + end + serviceMonitor: enabled: true namespace: monitoring -- 2.40.1 From 9e0e819fd6c25cdcabc5f17c133431630ad5b07c Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 8 Sep 2020 13:40:09 +0100 Subject: [PATCH 3/8] Handle empty message events --- charts/kubezero-logging/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 1cb9b32..f54e407 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -310,7 +310,7 @@ fluent-bit: return -1, 0, 0 end -- otherwise it's a full line, concatenate with accumulated partial lines if any - record.message = reassemble_state[reassemble_key] or "" .. record.message + record.message = reassemble_state[reassemble_key] or "" .. (record.message or "") reassemble_state[reassemble_key] = nil return 1, timestamp, record end -- 2.40.1 From b7feeae83c46d52f263d5662168c43b812ad3980 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 8 Sep 2020 13:44:31 +0100 Subject: [PATCH 4/8] Remove CRD property to fix OutofSync Argo --- charts/kubezero-logging/templates/eck/eck-operator.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/kubezero-logging/templates/eck/eck-operator.yaml b/charts/kubezero-logging/templates/eck/eck-operator.yaml index 055d1c3..366a36e 100644 --- a/charts/kubezero-logging/templates/eck/eck-operator.yaml +++ b/charts/kubezero-logging/templates/eck/eck-operator.yaml @@ -542,7 +542,6 @@ spec: description: DaemonSet specifies the Beat should be deployed as a DaemonSet, and allows providing its spec. Cannot be used along with `deployment`. If both are absent a default for the Type is used. - properties: {} type: object deployment: description: Deployment specifies the Beat should be deployed as a Deployment, -- 2.40.1 From 3b438711dc35ea9281d8b310fe1cf0c50088d729 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 8 Sep 2020 15:05:31 +0100 Subject: [PATCH 5/8] Update fluentd to latest quay.io image, add json parser for message --- charts/kubezero-logging/README.md.gotmpl | 1 + charts/kubezero-logging/values.yaml | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-logging/README.md.gotmpl b/charts/kubezero-logging/README.md.gotmpl index b29d9e9..e1d1e66 100644 --- a/charts/kubezero-logging/README.md.gotmpl +++ b/charts/kubezero-logging/README.md.gotmpl @@ -45,3 +45,4 @@ - https://www.elastic.co/downloads/elastic-cloud-kubernetes - https://github.com/elastic/cloud-on-k8s +- https://grafana.com/grafana/dashboards/7752 diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index f54e407..3fba750 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -31,7 +31,7 @@ fluentd: enabled: false image: repository: quay.io/fluentd_elasticsearch/fluentd - tag: v2.9.0 + tag: v3.0.4 istio: enabled: false @@ -158,7 +158,15 @@ fluentd: -# filter.conf: | + filter.conf: | + + key_name message + remove_key_name_field true + + @type json + + + # # @type parser # key_name message -- 2.40.1 From a1af1a2753d34ec293bf5cd8571296671b981a28 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 8 Sep 2020 15:07:17 +0100 Subject: [PATCH 6/8] Fix fluentd typo --- charts/kubezero-logging/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 3fba750..d95b0d7 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -160,6 +160,7 @@ fluentd: filter.conf: | + @type parser key_name message remove_key_name_field true -- 2.40.1 From da503ab38cb157eab628cab871856068844f65dd Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 8 Sep 2020 15:34:16 +0100 Subject: [PATCH 7/8] Fix fluentd parsing of json --- charts/kubezero-logging/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index d95b0d7..c967317 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -159,10 +159,11 @@ fluentd: filter.conf: | - + @type parser key_name message remove_key_name_field true + emit_invalid_record_to_error false @type json -- 2.40.1 From 9e043a62414436a23f131e61cdb76c692dc57d70 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 8 Sep 2020 15:41:20 +0100 Subject: [PATCH 8/8] Dont remove other fields for valid json --- charts/kubezero-logging/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index c967317..076c78d 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -163,6 +163,7 @@ fluentd: @type parser key_name message remove_key_name_field true + reserve_data true emit_invalid_record_to_error false @type json -- 2.40.1