Logging Improvements #18
@ -2,7 +2,7 @@ apiVersion: v2
|
|||||||
name: kubezero-logging
|
name: kubezero-logging
|
||||||
description: KubeZero Umbrella Chart for complete EFK stack
|
description: KubeZero Umbrella Chart for complete EFK stack
|
||||||
type: application
|
type: application
|
||||||
version: 0.3.1
|
version: 0.3.2
|
||||||
appVersion: 1.2.1
|
appVersion: 1.2.1
|
||||||
home: https://kubezero.com
|
home: https://kubezero.com
|
||||||
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
|
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
|
||||||
@ -23,7 +23,7 @@ dependencies:
|
|||||||
repository: https://kubernetes-charts.storage.googleapis.com/
|
repository: https://kubernetes-charts.storage.googleapis.com/
|
||||||
condition: fluentd.enabled
|
condition: fluentd.enabled
|
||||||
- name: fluent-bit
|
- name: fluent-bit
|
||||||
version: 0.6.3
|
version: 0.6.4
|
||||||
repository: https://zero-down-time.github.io/kubezero/
|
repository: https://zero-down-time.github.io/kubezero/
|
||||||
# repository: https://fluent.github.io/helm-charts
|
# repository: https://fluent.github.io/helm-charts
|
||||||
condition: fluent-bit.enabled
|
condition: fluent-bit.enabled
|
||||||
|
@ -2,7 +2,7 @@ kubezero-logging
|
|||||||
================
|
================
|
||||||
KubeZero Umbrella Chart for complete EFK stack
|
KubeZero Umbrella Chart for complete EFK stack
|
||||||
|
|
||||||
Current chart version is `0.3.1`
|
Current chart version is `0.3.2`
|
||||||
|
|
||||||
Source code can be found [here](https://kubezero.com)
|
Source code can be found [here](https://kubezero.com)
|
||||||
|
|
||||||
@ -11,7 +11,7 @@ Source code can be found [here](https://kubezero.com)
|
|||||||
| Repository | Name | Version |
|
| Repository | Name | Version |
|
||||||
|------------|------|---------|
|
|------------|------|---------|
|
||||||
| https://kubernetes-charts.storage.googleapis.com/ | fluentd | 2.5.1 |
|
| https://kubernetes-charts.storage.googleapis.com/ | fluentd | 2.5.1 |
|
||||||
| https://zero-down-time.github.io/kubezero/ | fluent-bit | 0.6.3 |
|
| https://zero-down-time.github.io/kubezero/ | fluent-bit | 0.6.4 |
|
||||||
| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 |
|
| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 |
|
||||||
|
|
||||||
## Changes from upstream
|
## Changes from upstream
|
||||||
@ -32,10 +32,16 @@ Source code can be found [here](https://kubezero.com)
|
|||||||
|
|
||||||
- increased timeout to ES to 3 minutes
|
- increased timeout to ES to 3 minutes
|
||||||
|
|
||||||
|
### FluentD
|
||||||
|
|
||||||
|
|
||||||
|
### Fluent-bit
|
||||||
|
- support for dedot Lua filter to replace "." with "_" for all annotations and labels
|
||||||
|
- support for api audit log
|
||||||
|
|
||||||
## Manual tasks ATM
|
## Manual tasks ATM
|
||||||
|
|
||||||
- Install index template
|
- install index template
|
||||||
- setup Kibana
|
- setup Kibana
|
||||||
- create `logstash-*` Index Pattern
|
- create `logstash-*` Index Pattern
|
||||||
|
|
||||||
@ -50,7 +56,7 @@ Source code can be found [here](https://kubezero.com)
|
|||||||
| es.s3Snapshot.enabled | bool | `false` | |
|
| es.s3Snapshot.enabled | bool | `false` | |
|
||||||
| es.s3Snapshot.iamrole | string | `""` | |
|
| es.s3Snapshot.iamrole | string | `""` | |
|
||||||
| fluent-bit.config.filters | string | `"[FILTER]\n Name kubernetes\n Match kube.*\n Merge_Log On\n Keep_Log Off\n K8S-Logging.Parser On\n K8S-Logging.Exclude On\n\n[FILTER]\n Name lua\n Match kube.*\n script /fluent-bit/etc/functions.lua\n call dedot\n"` | |
|
| fluent-bit.config.filters | string | `"[FILTER]\n Name kubernetes\n Match kube.*\n Merge_Log On\n Keep_Log Off\n K8S-Logging.Parser On\n K8S-Logging.Exclude On\n\n[FILTER]\n Name lua\n Match kube.*\n script /fluent-bit/etc/functions.lua\n call dedot\n"` | |
|
||||||
| fluent-bit.config.inputs | string | `"[INPUT]\n Name tail\n Path /var/log/containers/*.log\n Parser cri\n Tag kube.*\n Mem_Buf_Limit 5MB\n Skip_Long_Lines On\n Refresh_Interval 10\n DB /var/log/flb_kube.db\n DB.Sync Normal\n"` | |
|
| fluent-bit.config.inputs | string | `"[INPUT]\n Name tail\n Path /var/log/containers/*.log\n Parser cri\n Tag kube.*\n Mem_Buf_Limit 5MB\n Skip_Long_Lines On\n Refresh_Interval 10\n DB /var/log/flb_kube.db\n DB.Sync Normal\n[INPUT]\n Name tail\n Path /var/log/kubernetes/audit.log\n Parser json\n Tag kube.api.audit\n Mem_Buf_Limit 5MB\n Skip_Long_Lines On\n Refresh_Interval 60\n DB /var/log/flb_kube_audit.db\n DB.Sync Normal\n"` | |
|
||||||
| fluent-bit.config.lua | string | `"function dedot(tag, timestamp, record)\n if record[\"kubernetes\"] == nil then\n return 0, 0, 0\n end\n dedot_keys(record[\"kubernetes\"][\"annotations\"])\n dedot_keys(record[\"kubernetes\"][\"labels\"])\n return 1, timestamp, record\nend\n\nfunction dedot_keys(map)\n if map == nil then\n return\n end\n local new_map = {}\n local changed_keys = {}\n for k, v in pairs(map) do\n local dedotted = string.gsub(k, \"%.\", \"_\")\n if dedotted ~= k then\n new_map[dedotted] = v\n changed_keys[k] = true\n end\n end\n for k in pairs(changed_keys) do\n map[k] = nil\n end\n for k, v in pairs(new_map) do\n map[k] = v\n end\nend\n"` | |
|
| fluent-bit.config.lua | string | `"function dedot(tag, timestamp, record)\n if record[\"kubernetes\"] == nil then\n return 0, 0, 0\n end\n dedot_keys(record[\"kubernetes\"][\"annotations\"])\n dedot_keys(record[\"kubernetes\"][\"labels\"])\n return 1, timestamp, record\nend\n\nfunction dedot_keys(map)\n if map == nil then\n return\n end\n local new_map = {}\n local changed_keys = {}\n for k, v in pairs(map) do\n local dedotted = string.gsub(k, \"%.\", \"_\")\n if dedotted ~= k then\n new_map[dedotted] = v\n changed_keys[k] = true\n end\n end\n for k in pairs(changed_keys) do\n map[k] = nil\n end\n for k, v in pairs(new_map) do\n map[k] = v\n end\nend\n"` | |
|
||||||
| fluent-bit.config.outputs | string | `"[OUTPUT]\n Match *\n Name forward\n Host logging-fluentd\n Port 24224\n tls on\n tls.verify off\n Shared_Key cloudbender\n"` | |
|
| fluent-bit.config.outputs | string | `"[OUTPUT]\n Match *\n Name forward\n Host logging-fluentd\n Port 24224\n tls on\n tls.verify off\n Shared_Key cloudbender\n"` | |
|
||||||
| fluent-bit.config.service | string | `"[SERVICE]\n Flush 5\n Daemon Off\n Log_Level warn\n Parsers_File parsers.conf\n Parsers_File custom_parsers.conf\n HTTP_Server On\n HTTP_Listen 0.0.0.0\n HTTP_Port 2020\n"` | |
|
| fluent-bit.config.service | string | `"[SERVICE]\n Flush 5\n Daemon Off\n Log_Level warn\n Parsers_File parsers.conf\n Parsers_File custom_parsers.conf\n HTTP_Server On\n HTTP_Listen 0.0.0.0\n HTTP_Port 2020\n"` | |
|
||||||
|
@ -25,10 +25,16 @@
|
|||||||
|
|
||||||
- increased timeout to ES to 3 minutes
|
- increased timeout to ES to 3 minutes
|
||||||
|
|
||||||
|
### FluentD
|
||||||
|
|
||||||
|
|
||||||
|
### Fluent-bit
|
||||||
|
- support for dedot Lua filter to replace "." with "_" for all annotations and labels
|
||||||
|
- support for api audit log
|
||||||
|
|
||||||
## Manual tasks ATM
|
## Manual tasks ATM
|
||||||
|
|
||||||
- Install index template
|
- install index template
|
||||||
- setup Kibana
|
- setup Kibana
|
||||||
- create `logstash-*` Index Pattern
|
- create `logstash-*` Index Pattern
|
||||||
|
|
||||||
@ -39,3 +45,4 @@
|
|||||||
|
|
||||||
- https://www.elastic.co/downloads/elastic-cloud-kubernetes
|
- https://www.elastic.co/downloads/elastic-cloud-kubernetes
|
||||||
- https://github.com/elastic/cloud-on-k8s
|
- https://github.com/elastic/cloud-on-k8s
|
||||||
|
- https://grafana.com/grafana/dashboards/7752
|
||||||
|
@ -1,5 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# We only need to delete the service monitor and virtual service, others will be taken over by the new chart and we dont loose data
|
|
||||||
kubectl delete -n logging VirtualService kibana-logging
|
|
||||||
kubectl delete -n logging ServiceMonitor es-logging
|
|
@ -542,7 +542,6 @@ spec:
|
|||||||
description: DaemonSet specifies the Beat should be deployed as a DaemonSet,
|
description: DaemonSet specifies the Beat should be deployed as a DaemonSet,
|
||||||
and allows providing its spec. Cannot be used along with `deployment`.
|
and allows providing its spec. Cannot be used along with `deployment`.
|
||||||
If both are absent a default for the Type is used.
|
If both are absent a default for the Type is used.
|
||||||
properties: {}
|
|
||||||
type: object
|
type: object
|
||||||
deployment:
|
deployment:
|
||||||
description: Deployment specifies the Beat should be deployed as a Deployment,
|
description: Deployment specifies the Beat should be deployed as a Deployment,
|
||||||
|
@ -31,7 +31,7 @@ fluentd:
|
|||||||
enabled: false
|
enabled: false
|
||||||
image:
|
image:
|
||||||
repository: quay.io/fluentd_elasticsearch/fluentd
|
repository: quay.io/fluentd_elasticsearch/fluentd
|
||||||
tag: v2.9.0
|
tag: v3.0.4
|
||||||
istio:
|
istio:
|
||||||
enabled: false
|
enabled: false
|
||||||
|
|
||||||
@ -158,7 +158,18 @@ fluentd:
|
|||||||
</buffer>
|
</buffer>
|
||||||
</match>
|
</match>
|
||||||
|
|
||||||
# filter.conf: |
|
filter.conf: |
|
||||||
|
<filter kube.**>
|
||||||
|
@type parser
|
||||||
|
key_name message
|
||||||
|
remove_key_name_field true
|
||||||
|
reserve_data true
|
||||||
|
emit_invalid_record_to_error false
|
||||||
|
<parse>
|
||||||
|
@type json
|
||||||
|
</parse>
|
||||||
|
</filter>
|
||||||
|
|
||||||
# <filter auth system.auth>
|
# <filter auth system.auth>
|
||||||
# @type parser
|
# @type parser
|
||||||
# key_name message
|
# key_name message
|
||||||
@ -222,8 +233,24 @@ fluent-bit:
|
|||||||
Refresh_Interval 10
|
Refresh_Interval 10
|
||||||
DB /var/log/flb_kube.db
|
DB /var/log/flb_kube.db
|
||||||
DB.Sync Normal
|
DB.Sync Normal
|
||||||
|
[INPUT]
|
||||||
|
Name tail
|
||||||
|
Path /var/log/kubernetes/audit.log
|
||||||
|
Parser json
|
||||||
|
Tag kube.api.audit
|
||||||
|
Mem_Buf_Limit 5MB
|
||||||
|
Skip_Long_Lines On
|
||||||
|
Refresh_Interval 60
|
||||||
|
DB /var/log/flb_kube_audit.db
|
||||||
|
DB.Sync Normal
|
||||||
|
|
||||||
filters: |
|
filters: |
|
||||||
|
[FILTER]
|
||||||
|
Name lua
|
||||||
|
Match kube.*
|
||||||
|
script /fluent-bit/etc/functions.lua
|
||||||
|
call reassemble_cri_logs
|
||||||
|
|
||||||
[FILTER]
|
[FILTER]
|
||||||
Name kubernetes
|
Name kubernetes
|
||||||
Match kube.*
|
Match kube.*
|
||||||
@ -280,6 +307,25 @@ fluent-bit:
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local reassemble_state = {}
|
||||||
|
|
||||||
|
function reassemble_cri_logs(tag, timestamp, record)
|
||||||
|
-- IMPORTANT: reassemble_key must be unique for each parser stream
|
||||||
|
-- otherwise entries from different sources will get mixed up.
|
||||||
|
-- Either make sure that your parser tags satisfy this or construct
|
||||||
|
-- reassemble_key some other way
|
||||||
|
local reassemble_key = tag
|
||||||
|
-- if partial line, accumulate
|
||||||
|
if record.logtag == 'P' then
|
||||||
|
reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or "" .. record.message
|
||||||
|
return -1, 0, 0
|
||||||
|
end
|
||||||
|
-- otherwise it's a full line, concatenate with accumulated partial lines if any
|
||||||
|
record.message = reassemble_state[reassemble_key] or "" .. (record.message or "")
|
||||||
|
reassemble_state[reassemble_key] = nil
|
||||||
|
return 1, timestamp, record
|
||||||
|
end
|
||||||
|
|
||||||
serviceMonitor:
|
serviceMonitor:
|
||||||
enabled: true
|
enabled: true
|
||||||
namespace: monitoring
|
namespace: monitoring
|
||||||
|
@ -1,9 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
[ -f istio-authorization-policy.yaml ] && kubectl delete -f istio-authorization-policy.yaml
|
|
||||||
[ -f istio-service.yaml ] && kubectl delete -f istio-service.yaml
|
|
||||||
|
|
||||||
kubectl delete -f manifests
|
|
||||||
kubectl delete -f manifests/setup
|
|
||||||
|
|
||||||
kubectl delete namespace monitoring
|
|
Loading…
Reference in New Issue
Block a user