Fluent-bit version bump and support for api audit logs

charts/kubezero-logging/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-logging
 description: KubeZero Umbrella Chart for complete EFK stack
 type: application
-version: 0.3.1
+version: 0.3.2
 appVersion: 1.2.1
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@@ -23,7 +23,7 @@ dependencies:
     repository: https://kubernetes-charts.storage.googleapis.com/
     condition: fluentd.enabled
   - name: fluent-bit
-    version: 0.6.3
+    version: 0.6.4
     repository: https://zero-down-time.github.io/kubezero/
     # repository: https://fluent.github.io/helm-charts
     condition: fluent-bit.enabled

charts/kubezero-logging/README.md

@@ -2,7 +2,7 @@ kubezero-logging
 ================
 KubeZero Umbrella Chart for complete EFK stack
 
-Current chart version is `0.3.1`
+Current chart version is `0.3.2`
 
 Source code can be found [here](https://kubezero.com)
 
@@ -11,7 +11,7 @@ Source code can be found [here](https://kubezero.com)
 | Repository | Name | Version |
 |------------|------|---------|
 | https://kubernetes-charts.storage.googleapis.com/ | fluentd | 2.5.1 |
-| https://zero-down-time.github.io/kubezero/ | fluent-bit | 0.6.3 |
+| https://zero-down-time.github.io/kubezero/ | fluent-bit | 0.6.4 |
 | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 |
 
 ## Changes from upstream
@@ -32,10 +32,16 @@ Source code can be found [here](https://kubezero.com)
 
 - increased timeout to ES to 3 minutes
  
+### FluentD
+
+
+### Fluent-bit
+- support for dedot Lua filter to replace "." with "_" for all annotations and labels
+- support for api audit log
 
 ## Manual tasks ATM
 
--  Install index template
+- install index template
 - setup Kibana
 - create `logstash-*` Index Pattern
 
@@ -50,7 +56,7 @@ Source code can be found [here](https://kubezero.com)
 | es.s3Snapshot.enabled | bool | `false` |  |
 | es.s3Snapshot.iamrole | string | `""` |  |
 | fluent-bit.config.filters | string | `"[FILTER]\n    Name kubernetes\n    Match kube.*\n    Merge_Log On\n    Keep_Log Off\n    K8S-Logging.Parser On\n    K8S-Logging.Exclude On\n\n[FILTER]\n    Name    lua\n    Match   kube.*\n    script  /fluent-bit/etc/functions.lua\n    call    dedot\n"` |  |
-| fluent-bit.config.inputs | string | `"[INPUT]\n    Name tail\n    Path /var/log/containers/*.log\n    Parser cri\n    Tag kube.*\n    Mem_Buf_Limit 5MB\n    Skip_Long_Lines On\n    Refresh_Interval 10\n    DB /var/log/flb_kube.db\n    DB.Sync Normal\n"` |  |
+| fluent-bit.config.inputs | string | `"[INPUT]\n    Name tail\n    Path /var/log/containers/*.log\n    Parser cri\n    Tag kube.*\n    Mem_Buf_Limit 5MB\n    Skip_Long_Lines On\n    Refresh_Interval 10\n    DB /var/log/flb_kube.db\n    DB.Sync Normal\n[INPUT]\n    Name tail\n    Path /var/log/kubernetes/audit.log\n    Parser json\n    Tag kube.api.audit\n    Mem_Buf_Limit 5MB\n    Skip_Long_Lines On\n    Refresh_Interval 60\n    DB /var/log/flb_kube_audit.db\n    DB.Sync Normal\n"` |  |
 | fluent-bit.config.lua | string | `"function dedot(tag, timestamp, record)\n    if record[\"kubernetes\"] == nil then\n        return 0, 0, 0\n    end\n    dedot_keys(record[\"kubernetes\"][\"annotations\"])\n    dedot_keys(record[\"kubernetes\"][\"labels\"])\n    return 1, timestamp, record\nend\n\nfunction dedot_keys(map)\n    if map == nil then\n        return\n    end\n    local new_map = {}\n    local changed_keys = {}\n    for k, v in pairs(map) do\n        local dedotted = string.gsub(k, \"%.\", \"_\")\n        if dedotted ~= k then\n            new_map[dedotted] = v\n            changed_keys[k] = true\n        end\n    end\n    for k in pairs(changed_keys) do\n        map[k] = nil\n    end\n    for k, v in pairs(new_map) do\n        map[k] = v\n    end\nend\n"` |  |
 | fluent-bit.config.outputs | string | `"[OUTPUT]\n    Match *\n    Name forward\n    Host logging-fluentd\n    Port 24224\n    tls on\n    tls.verify off\n    Shared_Key cloudbender\n"` |  |
 | fluent-bit.config.service | string | `"[SERVICE]\n    Flush 5\n    Daemon Off\n    Log_Level warn\n    Parsers_File parsers.conf\n    Parsers_File custom_parsers.conf\n    HTTP_Server On\n    HTTP_Listen 0.0.0.0\n    HTTP_Port 2020\n"` |  |

charts/kubezero-logging/README.md.gotmpl

@@ -25,10 +25,16 @@
 
 - increased timeout to ES to 3 minutes
  
+### FluentD
+
+
+### Fluent-bit
+- support for dedot Lua filter to replace "." with "_" for all annotations and labels
+- support for api audit log
 
 ## Manual tasks ATM
 
--  Install index template
+- install index template
 - setup Kibana
 - create `logstash-*` Index Pattern
 

charts/kubezero-logging/delete_zdt-logging.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-# We only need to delete the service monitor and virtual service, others will be taken over by the new chart and we dont loose data
-kubectl delete -n logging VirtualService kibana-logging
-kubectl delete -n logging ServiceMonitor es-logging

charts/kubezero-logging/values.yaml

@@ -222,6 +222,16 @@ fluent-bit:
           Refresh_Interval 10
           DB /var/log/flb_kube.db
           DB.Sync Normal
+      [INPUT]
+          Name tail
+          Path /var/log/kubernetes/audit.log
+          Parser json
+          Tag kube.api.audit
+          Mem_Buf_Limit 5MB
+          Skip_Long_Lines On
+          Refresh_Interval 60
+          DB /var/log/flb_kube_audit.db
+          DB.Sync Normal
 
     filters: |
       [FILTER]

charts/kubezero-metrics/delete_kube-prometheus.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-[ -f istio-authorization-policy.yaml ] && kubectl delete -f istio-authorization-policy.yaml
-[ -f istio-service.yaml ] && kubectl delete -f istio-service.yaml
-
-kubectl delete -f manifests
-kubectl delete -f manifests/setup
-
-kubectl delete namespace monitoring