ZeroDownTime/KubeZero
3
0
Fork 0
Code Issues 1 Pull Requests 21 Packages Projects Releases Wiki Activity

Version upgrade ES/Kibana and Fluentbit, various tunings

Browse Source
This commit is contained in:
Stefan Reimer 2021-03-05 16:53:02 +01:00
parent 532710b77b
commit 50ffcf28eb
21 changed files with 1734 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
charts/kubezero-logging/Chart.yaml
View File

@ -2,8 +2,8 @@ apiVersion: v2
name: kubezero-logging
description: KubeZero Umbrella Chart for complete EFK stack
type: application
version: 0.5.3
appVersion: 1.3.1
version: 0.6.0
appVersion: 1.4.0
home: https://kubezero.com
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
keywords:
@ -19,13 +19,13 @@ dependencies:
version: ">= 0.1.3"
repository: https://zero-down-time.github.io/kubezero/
- name: eck-operator
version: 1.3.1
version: 1.4.0
repository: https://helm.elastic.co
condition: eck-operator.enabled
- name: fluentd
version: 2.5.3
condition: fluentd.enabled
- name: fluent-bit
version: 0.7.14
version: 0.12.3
condition: fluent-bit.enabled
kubeVersion: ">= 1.16.0"

35
charts/kubezero-logging/charts/fluent-bit/Chart.yaml
View File

@ -1,20 +1,23 @@
annotations:
artifacthub.io/changes: |
- add custom annotations on dashboards configmap
apiVersion: v2
name: fluent-bit
appVersion: 1.7.1
description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
keywords:
- logging
- fluent-bit
- fluentd
version: 0.7.14
appVersion: 1.6.8
icon: https://fluentbit.io/assets/img/logo1-default.png
home: https://fluentbit.io/
sources:
- https://github.com/fluent/fluent-bit/
icon: https://fluentbit.io/assets/img/logo1-default.png
keywords:
- logging
- fluent-bit
- fluentd
maintainers:
- name: edsiper
email: eduardo@treasure-data.com
- name: naseemkullah
email: naseem@transit.app
- name: Towmeykaw
email: towmeykaw@gmail.com
- email: eduardo@treasure-data.com
name: edsiper
- email: naseem@transit.app
name: naseemkullah
- email: towmeykaw@gmail.com
name: Towmeykaw
name: fluent-bit
sources:
- https://github.com/fluent/fluent-bit/
version: 0.12.3

6
charts/kubezero-logging/charts/fluent-bit/conf/custom_parsers.conf
View File

@ -1,6 +0,0 @@
[PARSER]
Name cri-log
Format regex
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z

76
charts/kubezero-logging/charts/fluent-bit/conf/fluent-bit.conf
View File

@ -1,76 +0,0 @@
[SERVICE]
Flush {{ .Values.config.flushInterval }}
Daemon Off
Log_Level {{ .Values.config.logLevel }}
Parsers_File parsers.conf
Parsers_File custom_parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020
[INPUT]
Name tail
Path /var/log/containers/*.log
Parser cri-log
Tag cri.*
Skip_Long_Lines On
DB /var/log/flb_kube.db
DB.Sync Normal
{{- with .Values.config.input }}
Mem_Buf_Limit {{ default "16MB" .memBufLimit }}
Refresh_Interval {{ default 10 .refreshInterval }}
{{- end }}
[FILTER]
Name lua
Match cri.*
script /fluent-bit/etc/functions.lua
call reassemble_cri_logs
[FILTER]
Name kubernetes
Match cri.*
Merge_Log On
Merge_Log_Key kube
Kube_Tag_Prefix cri.var.log.containers.
Keep_Log Off
K8S-Logging.Parser Off
K8S-Logging.Exclude Off
{{- if index .Values "config" "extraRecords" }}
[FILTER]
Name record_modifier
Match cri.*
{{- range $k,$v := index .Values "config" "extraRecords" }}
Record {{ $k }} {{ $v }}
{{- end }}
{{- end }}
[FILTER]
Name rewrite_tag
Match cri.*
Emitter_Name kube_tag_rewriter
Rule logtag F kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false
[FILTER]
Name lua
Match kube.*
script /fluent-bit/etc/functions.lua
call nest_k8s_ns
{{- if .Values.config.outputs }}
{{ .Values.config.outputs }}
{{- else }}
{{- with .Values.config.output }}
[OUTPUT]
Match *
Name forward
Host {{ .host }}
Port 24224
Shared_Key {{ .sharedKey }}
tls {{ ternary "on" "off" .tls }}
Send_options true
Require_ack_response true
{{- end }}
{{- end }}

28
charts/kubezero-logging/charts/fluent-bit/conf/functions.lua
View File

@ -1,28 +0,0 @@
local reassemble_state = {}
function reassemble_cri_logs(tag, timestamp, record)
local reassemble_key = tag
if record.logtag == 'P' then
reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or "" .. record.log
return -1, 0, 0
end
record.log = reassemble_state[reassemble_key] or "" .. (record.log or "")
reassemble_state[reassemble_key] = nil
return 1, timestamp, record
end
function nest_k8s_ns(tag, timestamp, record)
if not record['kubernetes']['namespace_name'] then
return 0, 0, 0
end
new_record = {}
for key, val in pairs(record) do
if key == 'kube' then
new_record[key] = {}
new_record[key][record['kubernetes']['namespace_name']] = record[key]
else
new_record[key] = record[key]
end
end
return 1, timestamp, new_record
end

1305
charts/kubezero-logging/charts/fluent-bit/dashboards/fluent-bit.json Normal file
View File

File diff suppressed because it is too large Load Diff

29
charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl
View File

@ -9,6 +9,10 @@ priorityClassName: {{ .Values.priorityClassName }}
serviceAccountName: {{ include "fluent-bit.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 2 }}
{{- with .Values.dnsConfig }}
dnsConfig:
{{- toYaml . | nindent 2 }}
{{- end }}
containers:
- name: {{ .Chart.Name }}
securityContext:
@ -34,14 +38,14 @@ containers:
protocol: {{ .protocol }}
{{- end }}
{{- end }}
{{- if .Values.livenessProbe }}
livenessProbe:
httpGet:
path: /
port: http
{{- toYaml .Values.livenessProbe | nindent 6 }}
{{- end }}
{{- if .Values.readinessProbe }}
readinessProbe:
httpGet:
path: /
port: http
{{- toYaml .Values.readinessProbe | nindent 6 }}
{{- end }}
resources:
{{- toYaml .Values.resources | nindent 6 }}
volumeMounts:
@ -51,9 +55,11 @@ containers:
- name: config
mountPath: /fluent-bit/etc/custom_parsers.conf
subPath: custom_parsers.conf
- name: config
mountPath: /fluent-bit/etc/functions.lua
subPath: functions.lua
{{- range $key, $value := .Values.luaScripts }}
- name: luascripts
mountPath: /fluent-bit/scripts/{{ $key }}
subPath: {{ $key }}
{{- end }}
{{- if eq .Values.kind "DaemonSet" }}
- name: varlog
mountPath: /var/log
@ -71,6 +77,11 @@ volumes:
- name: config
configMap:
name: {{ if .Values.existingConfigMap }}{{ .Values.existingConfigMap }}{{- else }}{{ include "fluent-bit.fullname" . }}{{- end }}
{{- if gt (len .Values.luaScripts) 0 }}
- name: luascripts
configMap:
name: {{ include "fluent-bit.fullname" . }}-luascripts
{{- end }}
{{- if eq .Values.kind "DaemonSet" }}
- name: varlog
hostPath:

20
charts/kubezero-logging/charts/fluent-bit/templates/configmap-dashboards.yaml Normal file
View File

@ -0,0 +1,20 @@
{{- if .Values.dashboards.enabled -}}
{{- range $path, $_ := .Files.Glob "dashboards/*.json" }}
apiVersion: v1
kind: ConfigMap
metadata:
name: dashboard-{{ trimSuffix ".json" (base $path) }}
{{- with $.Values.dashboards.annotations }}
annotations:
{{- toYaml . | nindent 4 -}}
{{- end }}
labels:
{{- include "fluent-bit.labels" $ | nindent 4 }}
{{ $.Values.dashboards.labelKey }}: "1"
data:
{{ base $path }}: |
{{- $.Files.Get $path | nindent 4 }}
---
{{- end }}
{{- end -}}

12
charts/kubezero-logging/charts/fluent-bit/templates/configmap-luascripts.yaml Normal file
View File

@ -0,0 +1,12 @@
{{- if gt (len .Values.luaScripts) 0 -}}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "fluent-bit.fullname" . }}-luascripts
labels:
{{- include "fluent-bit.labels" . | nindent 4 }}
data:
{{ range $key, $value := .Values.luaScripts }}
{{ $key }}: {{ $value | quote }}
{{ end }}
{{- end -}}

9
charts/kubezero-logging/charts/fluent-bit/templates/configmap.yaml
View File

@ -7,9 +7,10 @@ metadata:
{{- include "fluent-bit.labels" . | nindent 4 }}
data:
custom_parsers.conf: |
{{ tpl (.Files.Get "conf/custom_parsers.conf") . | nindent 4 }}
{{- (tpl .Values.config.customParsers $) | nindent 4 }}
fluent-bit.conf: |
{{ tpl (.Files.Get "conf/fluent-bit.conf") . | nindent 4 }}
functions.lua: |
{{ tpl (.Files.Get "conf/functions.lua") . | nindent 4 }}
{{- (tpl .Values.config.service $) | nindent 4 }}
{{- (tpl .Values.config.inputs $) | nindent 4 }}
{{- (tpl .Values.config.filters $) | nindent 4 }}
{{- (tpl .Values.config.outputs $) | nindent 4 }}
{{- end -}}

1
charts/kubezero-logging/charts/fluent-bit/templates/daemonset.yaml
View File

@ -17,6 +17,7 @@ spec:
metadata:
annotations:
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/luascripts: {{ include (print $.Template.BasePath "/configmap-luascripts.yaml") . | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}

1
charts/kubezero-logging/charts/fluent-bit/templates/deployment.yaml
View File

@ -18,6 +18,7 @@ spec:
metadata:
annotations:
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
checksum/luascripts: {{ include (print $.Template.BasePath "/configmap-luascripts.yaml") . | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}

22
charts/kubezero-logging/charts/fluent-bit/templates/networkpolicy.yaml Normal file
View File

@ -0,0 +1,22 @@
{{- if .Values.networkPolicy.enabled }}
apiVersion: "networking.k8s.io/v1"
kind: "NetworkPolicy"
metadata:
name: {{ include "fluent-bit.fullname" . | quote }}
labels:
{{- include "fluent-bit.labels" . | nindent 4 }}
spec:
policyTypes:
- "Ingress"
podSelector:
matchLabels:
{{- include "fluent-bit.selectorLabels" . | nindent 6 }}
ingress:
{{- with .Values.networkPolicy.ingress }}
- from:
{{- with .from }}{{- . | toYaml | nindent 8 }}{{- else }} []{{- end }}
ports:
- protocol: "TCP"
port: {{ $.Values.service.port }}
{{- end }}
{{- end }}

20
charts/kubezero-logging/charts/fluent-bit/templates/prometheusrule.yaml Normal file
View File

@ -0,0 +1,20 @@
{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.prometheusRule.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: {{ include "fluent-bit.fullname" . }}
{{- with .Values.prometheusRule.namespace }}
namespace: {{ . }}
{{- end }}
labels:
{{- include "fluent-bit.labels" . | nindent 4 }}
{{- if .Values.prometheusRule.additionalLabels }}
{{- toYaml .Values.prometheusRule.additionalLabels | nindent 4 }}
{{- end }}
spec:
{{- if .Values.prometheusRule.rules }}
groups:
- name: {{ template "fluent-bit.name" . }}
rules: {{- toYaml .Values.prometheusRule.rules | nindent 4 }}
{{- end }}
{{- end }}

9
charts/kubezero-logging/charts/fluent-bit/templates/tests/test-connection.yaml
View File

@ -1,4 +1,4 @@
{{- if .Values.test.enabled }}
{{- if .Values.testFramework.enabled }}
apiVersion: v1
kind: Pod
metadata:
@ -10,8 +10,13 @@ metadata:
spec:
containers:
- name: wget
image: busybox
image: "{{ .Values.testFramework.image.repository }}:{{ .Values.testFramework.image.tag }}"
imagePullPolicy: {{ .Values.testFramework.image.pullPolicy }}
command: ['wget']
args: ['{{ include "fluent-bit.fullname" . }}:{{ .Values.service.port }}']
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 4 }}
{{- end }}
restartPolicy: Never
{{- end }}

126
charts/kubezero-logging/charts/fluent-bit/values.yaml
View File

@ -6,14 +6,18 @@ kind: DaemonSet
# replicaCount -- Only applicable if kind=Deployment
replicaCount: 1
test:
enabled: true
image:
repository: fluent/fluent-bit
pullPolicy: Always
# tag:
testFramework:
enabled: false
image:
repository: busybox
pullPolicy: Always
tag: latest
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
@ -32,7 +36,16 @@ podSecurityPolicy:
podSecurityContext:
{}
# fsGroup: 2000
dnsConfig: {}
# nameservers:
# - 1.2.3.4
# searches:
# - ns1.svc.cluster-domain.example
# - my.dns.search.suffix
# options:
# - name: ndots
# value: "2"
# - name: edns0
securityContext:
{}
# capabilities:
@ -61,6 +74,38 @@ serviceMonitor:
# selector:
# prometheus: my-prometheus
prometheusRule:
enabled: false
# namespace: ""
# additionnalLabels: {}
# rules:
# - alert: NoOutputBytesProcessed
# expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0
# annotations:
# message: |
# Fluent Bit instance {{ $labels.instance }}'s output plugin {{ $labels.name }} has not processed any
# bytes for at least 15 minutes.
# summary: No Output Bytes Processed
# for: 15m
# labels:
# severity: critical
dashboards:
enabled: false
labelKey: grafana_dashboard
annotations: {}
livenessProbe:
httpGet:
path: /
port: http
readinessProbe:
httpGet:
path: /
port: http
resources:
{}
# limits:
@ -104,5 +149,74 @@ updateStrategy: {}
# Make use of a pre-defined configmap instead of the one templated here
existingConfigMap: ""
## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit
config: {}
networkPolicy:
enabled: false
# ingress:
# from: []
luaScripts: {}
## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file
config:
service: |
[SERVICE]
Flush 1
Daemon Off
Log_Level info
Parsers_File parsers.conf
Parsers_File custom_parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port {{ .Values.service.port }}
## https://docs.fluentbit.io/manual/pipeline/inputs
inputs: |
[INPUT]
Name tail
Path /var/log/containers/*.log
Parser docker
Tag kube.*
Mem_Buf_Limit 5MB
Skip_Long_Lines On
[INPUT]
Name systemd
Tag host.*
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
Read_From_Tail On
## https://docs.fluentbit.io/manual/pipeline/filters
filters: |
[FILTER]
Name kubernetes
Match kube.*
Merge_Log On
Keep_Log Off
K8S-Logging.Parser On
K8S-Logging.Exclude On
## https://docs.fluentbit.io/manual/pipeline/outputs
outputs: |
[OUTPUT]
Name es
Match kube.*
Host elasticsearch-master
Logstash_Format On
Retry_Limit False
[OUTPUT]
Name es
Match host.*
Host elasticsearch-master
Logstash_Format On
Logstash_Prefix node
Retry_Limit False
## https://docs.fluentbit.io/manual/pipeline/parsers
customParsers: |
[PARSER]
Name docker_no_time
Format json
Time_Keep Off
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L

37
charts/kubezero-logging/fluent-bit.patch Normal file
View File

@ -0,0 +1,37 @@
Binary files charts/fluent-bit/.values.yaml.swp and charts/fluent-bit.zdt/.values.yaml.swp differ
diff -rtubN charts/fluent-bit/Chart.yaml charts/fluent-bit.zdt/Chart.yaml
--- charts/fluent-bit/Chart.yaml 2021-02-23 14:58:39.000000000 +0100
+++ charts/fluent-bit.zdt/Chart.yaml 2021-03-05 14:48:07.082648075 +0100
@@ -1,7 +1,7 @@
annotations:
artifacthub.io/changes: |
- add custom annotations on dashboards configmap
-apiVersion: v1
+apiVersion: v2
appVersion: 1.7.1
description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
home: https://fluentbit.io/
diff -rtubN charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent-bit.zdt/templates/tests/test-connection.yaml
--- charts/fluent-bit/templates/tests/test-connection.yaml 2021-02-23 14:58:39.000000000 +0100
+++ charts/fluent-bit.zdt/templates/tests/test-connection.yaml 2021-03-05 14:51:34.749324131 +0100
@@ -1,3 +1,4 @@
+{{- if .Values.testFramework.enabled }}
apiVersion: v1
kind: Pod
metadata:
@@ -18,3 +19,4 @@
{{- toYaml . | nindent 4 }}
{{- end }}
restartPolicy: Never
+{{- end }}
diff -rtubN charts/fluent-bit/values.yaml charts/fluent-bit.zdt/values.yaml
--- charts/fluent-bit/values.yaml 2021-02-23 14:58:39.000000000 +0100
+++ charts/fluent-bit.zdt/values.yaml 2021-03-05 14:50:40.652655018 +0100
@@ -12,6 +12,7 @@
# tag:
testFramework:
+ enabled: false
image:
repository: busybox
pullPolicy: Always

26
charts/kubezero-logging/templates/eck/elasticsearch.yaml
View File

@ -12,10 +12,7 @@ spec:
{{- range .Values.es.nodeSets }}
- name: {{ .name }}
config:
node.master: true
node.data: true
node.ingest: true
node.ml: false
node.roles: [ master, data, ingest ]
{{- if $.Values.es.prometheus }}
prometheus.indices: false
{{- end }}
@ -24,7 +21,10 @@ spec:
cluster.routing.allocation.awareness.attributes: zone
{{- end }}
transport.compress: true
node.processors: {{ default 1 .processors }}
{{- if .processors }}
node.processors: {{ .processors }}
{{- end }}
indices.memory.index_buffer_size: "30%"
podTemplate:
{{- if $.Values.es.s3Snapshot.iamrole }}
metadata:
@ -51,18 +51,14 @@ spec:
securityContext:
capabilities:
add: ["SYS_CHROOT"]
resources:
requests:
cpu: {{ default "200m" .cpu_request }}
memory: {{ default ( mul 2 ( default "2" .jvm_heap ) ) .memory_request }}Gi
limits:
{{- if .cpu_limit }}
cpu: {{ .cpu_limit }}
{{- end }}
memory: {{ default ( mul 2 ( default "2" .jvm_heap ) ) .memory_limit }}Gi
{{- with .resources }}
resources: {{ toYaml . | nindent 12 }}
{{- end }}
{{- if .jvm_heap }}
env:
- name: ES_JAVA_OPTS
value: -Xms{{ default "2" .jvm_heap }}g -Xmx{{ default "2" .jvm_heap }}g
value: -Xms{{ .jvm_heap }}g -Xmx{{ .jvm_heap }}g
{{- end }}
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:

2
charts/kubezero-logging/templates/eck/kibana.yaml
View File

@ -26,7 +26,7 @@ spec:
resources:
requests:
memory: 1Gi
cpu: 100m
cpu: 200m
limits:
memory: 2Gi
http:

8
charts/kubezero-logging/update.sh Executable file
View File

@ -0,0 +1,8 @@
#!/bin/bash
FLUENT_BIT_VERSION=0.12.3
rm -rf charts/fluent-bit
curl -L -s -o - https://github.com/fluent/helm-charts/releases/download/fluent-bit-${FLUENT_BIT_VERSION}/fluent-bit-${FLUENT_BIT_VERSION}.tgz | tar xfz - -C charts
patch -i fluent-bit.patch -p0 --no-backup-if-mismatch

131
charts/kubezero-logging/values.yaml
View File

@ -11,7 +11,7 @@ eck-operator:
node-role.kubernetes.io/master: ""
# Version for ElasticSearch and Kibana have to match so we define it at top-level
version: 7.10.1
version: 7.11.1
elastic_password: "" # super_secret_elastic_password
@ -192,8 +192,15 @@ fluentd:
fluent-bit:
enabled: false
test:
enabled: false
serviceMonitor:
enabled: true
selector:
release: metrics
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
config:
output:
@ -211,11 +218,115 @@ fluent-bit:
#extraRecords:
# source.clustername: MyKubeCluster
serviceMonitor:
enabled: true
selector:
release: metrics
service: |
[SERVICE]
Flush {{ .Values.config.flushInterval }}
Daemon Off
Log_Level {{ .Values.config.logLevel }}
Parsers_File parsers.conf
Parsers_File custom_parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port {{ .Values.service.port }}
inputs: |
[INPUT]
Name tail
Path /var/log/containers/*.log
Parser cri-log
Tag cri.*
Skip_Long_Lines On
DB /var/log/flb_kube.db
DB.Sync Normal
{{- with .Values.config.input }}
Mem_Buf_Limit {{ default "16MB" .memBufLimit }}
Refresh_Interval {{ default 10 .refreshInterval }}
{{- end }}
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
filters: |
[FILTER]
Name lua
Match cri.*
script /fluent-bit/scripts/kubezero.lua
call reassemble_cri_logs
[FILTER]
Name kubernetes
Match cri.*
Merge_Log On
Merge_Log_Key kube
Kube_Tag_Prefix cri.var.log.containers.
Keep_Log Off
K8S-Logging.Parser Off
K8S-Logging.Exclude Off
{{- if index .Values "config" "extraRecords" }}
[FILTER]
Name record_modifier
Match cri.*
{{- range $k,$v := index .Values "config" "extraRecords" }}
Record {{ $k }} {{ $v }}
{{- end }}
{{- end }}
[FILTER]
Name rewrite_tag
Match cri.*
Emitter_Name kube_tag_rewriter
Rule logtag F kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false
[FILTER]
Name lua
Match kube.*
script /fluent-bit/scripts/kubezero.lua
call nest_k8s_ns
outputs: |
[OUTPUT]
Match *
Name forward
Host {{ .Values.config.output.host }}
Port 24224
Shared_Key {{ .Values.config.output.sharedKey }}
tls {{ ternary "on" "off" .Values.config.output.tls }}
Send_options true
Require_ack_response true
customParsers: |
[PARSER]
Name cri-log
Format regex
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
luaScripts:
kubezero.lua: |
local reassemble_state = {}
function reassemble_cri_logs(tag, timestamp, record)
local reassemble_key = tag
if record.logtag == 'P' then
reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or "" .. record.log
return -1, 0, 0
end
record.log = reassemble_state[reassemble_key] or "" .. (record.log or "")
reassemble_state[reassemble_key] = nil
return 1, timestamp, record
end
function nest_k8s_ns(tag, timestamp, record)
if not record['kubernetes']['namespace_name'] then
return 0, 0, 0
end
new_record = {}
for key, val in pairs(record) do
if key == 'kube' then
new_record[key] = {}
new_record[key][record['kubernetes']['namespace_name']] = record[key]
else
new_record[key] = record[key]
end
end
return 1, timestamp, new_record
end