Integrare fluent-bit into logging to allow better config
This commit is contained in:
parent
f78c382be6
commit
77a7ba2ed6
@ -26,9 +26,9 @@ dependencies:
|
|||||||
version: 2.5.1
|
version: 2.5.1
|
||||||
repository: https://kubernetes-charts.storage.googleapis.com/
|
repository: https://kubernetes-charts.storage.googleapis.com/
|
||||||
condition: fluentd.enabled
|
condition: fluentd.enabled
|
||||||
- name: fluent-bit
|
# - name: fluent-bit
|
||||||
version: 0.7.10
|
# version: 0.7.10
|
||||||
repository: https://zero-down-time.github.io/kubezero/
|
# repository: https://zero-down-time.github.io/kubezero/
|
||||||
# repository: https://fluent.github.io/helm-charts
|
# repository: https://fluent.github.io/helm-charts
|
||||||
condition: fluent-bit.enabled
|
# condition: fluent-bit.enabled
|
||||||
kubeVersion: ">= 1.16.0"
|
kubeVersion: ">= 1.16.0"
|
||||||
|
@ -20,7 +20,6 @@ Kubernetes: `>= 1.16.0`
|
|||||||
|------------|------|---------|
|
|------------|------|---------|
|
||||||
| https://helm.elastic.co | eck-operator | 1.3.0 |
|
| https://helm.elastic.co | eck-operator | 1.3.0 |
|
||||||
| https://kubernetes-charts.storage.googleapis.com/ | fluentd | 2.5.1 |
|
| https://kubernetes-charts.storage.googleapis.com/ | fluentd | 2.5.1 |
|
||||||
| https://zero-down-time.github.io/kubezero/ | fluent-bit | 0.7.10 |
|
|
||||||
| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 |
|
| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 |
|
||||||
|
|
||||||
## Changes from upstream
|
## Changes from upstream
|
||||||
@ -57,7 +56,8 @@ Kubernetes: `>= 1.16.0`
|
|||||||
|
|
||||||
| Key | Type | Default | Description |
|
| Key | Type | Default | Description |
|
||||||
|-----|------|---------|-------------|
|
|-----|------|---------|-------------|
|
||||||
| eck-operator.enabled | bool | `true` | |
|
| eck-operator.enabled | bool | `false` | |
|
||||||
|
| eck-operator.installCRDs | bool | `false` | |
|
||||||
| eck-operator.nodeSelector."node-role.kubernetes.io/master" | string | `""` | |
|
| eck-operator.nodeSelector."node-role.kubernetes.io/master" | string | `""` | |
|
||||||
| eck-operator.tolerations[0].effect | string | `"NoSchedule"` | |
|
| eck-operator.tolerations[0].effect | string | `"NoSchedule"` | |
|
||||||
| eck-operator.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | |
|
| eck-operator.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | |
|
||||||
@ -66,12 +66,14 @@ Kubernetes: `>= 1.16.0`
|
|||||||
| es.prometheus | bool | `false` | |
|
| es.prometheus | bool | `false` | |
|
||||||
| es.s3Snapshot.enabled | bool | `false` | |
|
| es.s3Snapshot.enabled | bool | `false` | |
|
||||||
| es.s3Snapshot.iamrole | string | `""` | |
|
| es.s3Snapshot.iamrole | string | `""` | |
|
||||||
| fluent-bit.config.customParsers | string | `"[PARSER]\n Name cri-log\n Format regex\n Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$\n Time_Key time\n Time_Format %Y-%m-%dT%H:%M:%S.%L%z\n"` | |
|
| fluent-bit.config.flushInterval | int | `1` | |
|
||||||
| fluent-bit.config.filters | string | `"[FILTER]\n Name lua\n Match cri.*\n script /fluent-bit/etc/functions.lua\n call reassemble_cri_logs\n\n[FILTER]\n Name kubernetes\n Match cri.*\n Merge_Log On\n Merge_Log_Key kube\n Kube_Tag_Prefix cri.var.log.containers.\n Keep_Log Off\n K8S-Logging.Parser Off\n K8S-Logging.Exclude Off\n\n[FILTER]\n Name rewrite_tag\n Match cri.*\n Emitter_Name kube_tag_rewriter\n Rule logtag F kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false\n\n[FILTER]\n Name lua\n Match kube.*\n script /fluent-bit/etc/functions.lua\n call nest_k8s_ns\n"` | |
|
| fluent-bit.config.input.memBufLimit | string | `"16MB"` | |
|
||||||
| fluent-bit.config.inputs | string | `"[INPUT]\n Name tail\n Path /var/log/containers/*.log\n Parser cri-log\n Tag cri.*\n Mem_Buf_Limit 16MB\n Skip_Long_Lines On\n Refresh_Interval 10\n DB /var/log/flb_kube.db\n DB.Sync Normal\n"` | |
|
| fluent-bit.config.input.refreshInterval | int | `10` | |
|
||||||
| fluent-bit.config.lua | string | `"local reassemble_state = {}\n\nfunction reassemble_cri_logs(tag, timestamp, record)\n local reassemble_key = tag\n if record.logtag == 'P' then\n reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or \"\" .. record.log\n return -1, 0, 0\n end\n record.log = reassemble_state[reassemble_key] or \"\" .. (record.log or \"\")\n reassemble_state[reassemble_key] = nil\n return 1, timestamp, record\nend\n\nfunction nest_k8s_ns(tag, timestamp, record)\n if not record['kubernetes']['namespace_name'] then\n return 0, 0, 0\n end\n new_record = {}\n for key, val in pairs(record) do\n if key == 'kube' then\n new_record[key] = {}\n new_record[key][record['kubernetes']['namespace_name']] = record[key]\n else\n new_record[key] = record[key]\n end\n end\n return 1, timestamp, new_record\nend\n"` | |
|
| fluent-bit.config.logLevel | string | `"warn"` | |
|
||||||
| fluent-bit.config.outputs | string | `"[OUTPUT]\n Match *\n Name forward\n Host logging-fluentd\n Port 24224\n Shared_Key cloudbender\n Send_options true\n Require_ack_response true\n"` | |
|
| fluent-bit.config.output.host | string | `"logging-fluentd"` | |
|
||||||
| fluent-bit.config.service | string | `"[SERVICE]\n Flush 1\n Daemon Off\n Log_Level warn\n Parsers_File parsers.conf\n Parsers_File custom_parsers.conf\n HTTP_Server On\n HTTP_Listen 0.0.0.0\n HTTP_Port 2020\n"` | |
|
| fluent-bit.config.output.sharedKey | string | `"cloudbender"` | |
|
||||||
|
| fluent-bit.config.output.tls | bool | `false` | |
|
||||||
|
| fluent-bit.config.outputs | object | `{}` | |
|
||||||
| fluent-bit.enabled | bool | `false` | |
|
| fluent-bit.enabled | bool | `false` | |
|
||||||
| fluent-bit.serviceMonitor.enabled | bool | `true` | |
|
| fluent-bit.serviceMonitor.enabled | bool | `true` | |
|
||||||
| fluent-bit.serviceMonitor.selector.release | string | `"metrics"` | |
|
| fluent-bit.serviceMonitor.selector.release | string | `"metrics"` | |
|
||||||
@ -81,7 +83,7 @@ Kubernetes: `>= 1.16.0`
|
|||||||
| fluentd.configMaps."filter.conf" | string | `"<filter disabled.kube.**>\n @type parser\n key_name message\n remove_key_name_field true\n reserve_data true\n reserve_time true\n # inject_key_prefix message_json.\n emit_invalid_record_to_error false\n <parse>\n @type json\n </parse>\n</filter>\n"` | |
|
| fluentd.configMaps."filter.conf" | string | `"<filter disabled.kube.**>\n @type parser\n key_name message\n remove_key_name_field true\n reserve_data true\n reserve_time true\n # inject_key_prefix message_json.\n emit_invalid_record_to_error false\n <parse>\n @type json\n </parse>\n</filter>\n"` | |
|
||||||
| fluentd.configMaps."forward-input.conf" | string | `"<source>\n @type forward\n port 24224\n bind 0.0.0.0\n skip_invalid_event true\n send_keepalive_packet true\n <security>\n self_hostname \"#{ENV['HOSTNAME']}\"\n shared_key \"#{ENV['FLUENTD_SHARED_KEY']}\"\n </security>\n</source>\n"` | |
|
| fluentd.configMaps."forward-input.conf" | string | `"<source>\n @type forward\n port 24224\n bind 0.0.0.0\n skip_invalid_event true\n send_keepalive_packet true\n <security>\n self_hostname \"#{ENV['HOSTNAME']}\"\n shared_key \"#{ENV['FLUENTD_SHARED_KEY']}\"\n </security>\n</source>\n"` | |
|
||||||
| fluentd.configMaps."general.conf" | string | `"<label @FLUENT_LOG>\n <match **>\n @type null\n </match>\n</label>\n<source>\n @type http\n port 9880\n bind 0.0.0.0\n keepalive_timeout 30\n</source>\n<source>\n @type monitor_agent\n bind 0.0.0.0\n port 24220\n tag fluentd.monitor.metrics\n</source>\n"` | |
|
| fluentd.configMaps."general.conf" | string | `"<label @FLUENT_LOG>\n <match **>\n @type null\n </match>\n</label>\n<source>\n @type http\n port 9880\n bind 0.0.0.0\n keepalive_timeout 30\n</source>\n<source>\n @type monitor_agent\n bind 0.0.0.0\n port 24220\n tag fluentd.monitor.metrics\n</source>\n"` | |
|
||||||
| fluentd.configMaps."output.conf" | string | `"<match **>\n @id elasticsearch\n @type elasticsearch\n @log_level info\n include_tag_key true\n id_key id\n remove_keys id\n\n # KubeZero pipeline incl. GeoIP etc.\n # pipeline fluentd\n\n host \"#{ENV['OUTPUT_HOST']}\"\n port \"#{ENV['OUTPUT_PORT']}\"\n scheme \"#{ENV['OUTPUT_SCHEME']}\"\n ssl_version \"#{ENV['OUTPUT_SSL_VERSION']}\"\n ssl_verify \"#{ENV['OUTPUT_SSL_VERIFY']}\"\n user \"#{ENV['OUTPUT_USER']}\"\n password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n log_es_400_reason\n logstash_format true\n reconnect_on_error true\n # reload_on_failure true\n request_timeout 15s\n suppress_type_name true\n\n <buffer tag>\n @type file_single\n path /var/log/fluentd-buffers/kubernetes.system.buffer\n flush_mode interval\n flush_thread_count 2\n flush_interval 30s\n flush_at_shutdown true\n retry_type exponential_backoff\n retry_timeout 60m\n overflow_action drop_oldest_chunk\n </buffer>\n</match>\n"` | |
|
| fluentd.configMaps."output.conf" | string | `"<match **>\n @id elasticsearch\n @type elasticsearch\n @log_level info\n include_tag_key true\n id_key id\n remove_keys id\n\n # KubeZero pipeline incl. GeoIP etc.\n pipeline fluentd\n\n host \"#{ENV['OUTPUT_HOST']}\"\n port \"#{ENV['OUTPUT_PORT']}\"\n scheme \"#{ENV['OUTPUT_SCHEME']}\"\n ssl_version \"#{ENV['OUTPUT_SSL_VERSION']}\"\n ssl_verify \"#{ENV['OUTPUT_SSL_VERIFY']}\"\n user \"#{ENV['OUTPUT_USER']}\"\n password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n log_es_400_reason\n logstash_format true\n reconnect_on_error true\n # reload_on_failure true\n request_timeout 15s\n suppress_type_name true\n\n <buffer tag>\n @type file_single\n path /var/log/fluentd-buffers/kubernetes.system.buffer\n flush_mode interval\n flush_thread_count 2\n flush_interval 30s\n flush_at_shutdown true\n retry_type exponential_backoff\n retry_timeout 60m\n overflow_action drop_oldest_chunk\n </buffer>\n</match>\n"` | |
|
||||||
| fluentd.enabled | bool | `false` | |
|
| fluentd.enabled | bool | `false` | |
|
||||||
| fluentd.env.OUTPUT_SSL_VERIFY | string | `"false"` | |
|
| fluentd.env.OUTPUT_SSL_VERIFY | string | `"false"` | |
|
||||||
| fluentd.env.OUTPUT_USER | string | `"elastic"` | |
|
| fluentd.env.OUTPUT_USER | string | `"elastic"` | |
|
||||||
|
23
charts/kubezero-logging/charts/fluent-bit/.helmignore
Normal file
23
charts/kubezero-logging/charts/fluent-bit/.helmignore
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
# Patterns to ignore when building packages.
|
||||||
|
# This supports shell glob matching, relative path matching, and
|
||||||
|
# negation (prefixed with !). Only one pattern per line.
|
||||||
|
.DS_Store
|
||||||
|
# Common VCS dirs
|
||||||
|
.git/
|
||||||
|
.gitignore
|
||||||
|
.bzr/
|
||||||
|
.bzrignore
|
||||||
|
.hg/
|
||||||
|
.hgignore
|
||||||
|
.svn/
|
||||||
|
# Common backup files
|
||||||
|
*.swp
|
||||||
|
*.bak
|
||||||
|
*.tmp
|
||||||
|
*.orig
|
||||||
|
*~
|
||||||
|
# Various IDEs
|
||||||
|
.project
|
||||||
|
.idea/
|
||||||
|
*.tmproj
|
||||||
|
.vscode/
|
20
charts/kubezero-logging/charts/fluent-bit/Chart.yaml
Normal file
20
charts/kubezero-logging/charts/fluent-bit/Chart.yaml
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
apiVersion: v2
|
||||||
|
name: fluent-bit
|
||||||
|
description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
|
||||||
|
keywords:
|
||||||
|
- logging
|
||||||
|
- fluent-bit
|
||||||
|
- fluentd
|
||||||
|
version: 0.7.13
|
||||||
|
appVersion: 1.6.8
|
||||||
|
icon: https://fluentbit.io/assets/img/logo1-default.png
|
||||||
|
home: https://fluentbit.io/
|
||||||
|
sources:
|
||||||
|
- https://github.com/fluent/fluent-bit/
|
||||||
|
maintainers:
|
||||||
|
- name: edsiper
|
||||||
|
email: eduardo@treasure-data.com
|
||||||
|
- name: naseemkullah
|
||||||
|
email: naseem@transit.app
|
||||||
|
- name: Towmeykaw
|
||||||
|
email: towmeykaw@gmail.com
|
23
charts/kubezero-logging/charts/fluent-bit/README.md
Normal file
23
charts/kubezero-logging/charts/fluent-bit/README.md
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
# Fluent Bit Helm Chart
|
||||||
|
|
||||||
|
[Fluent Bit](https://fluentbit.io) is a fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
To add the `fluent` helm repo, run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
helm repo add fluent https://fluent.github.io/helm-charts
|
||||||
|
```
|
||||||
|
|
||||||
|
To install a release named `fluent-bit`, run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
helm install fluent-bit fluent/fluent-bit
|
||||||
|
```
|
||||||
|
|
||||||
|
## Chart Values
|
||||||
|
|
||||||
|
```sh
|
||||||
|
helm show values fluent/fluent-bit
|
||||||
|
```
|
@ -0,0 +1,6 @@
|
|||||||
|
[PARSER]
|
||||||
|
Name cri-log
|
||||||
|
Format regex
|
||||||
|
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
|
||||||
|
Time_Key time
|
||||||
|
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
|
@ -0,0 +1,72 @@
|
|||||||
|
[SERVICE]
|
||||||
|
Flush {{ .Values.config.flushInterval }}
|
||||||
|
Daemon Off
|
||||||
|
Log_Level {{ .Values.config.logLevel }}
|
||||||
|
Parsers_File parsers.conf
|
||||||
|
Parsers_File custom_parsers.conf
|
||||||
|
HTTP_Server On
|
||||||
|
HTTP_Listen 0.0.0.0
|
||||||
|
HTTP_Port 2020
|
||||||
|
|
||||||
|
[INPUT]
|
||||||
|
Name tail
|
||||||
|
Path /var/log/containers/*.log
|
||||||
|
Parser cri-log
|
||||||
|
Tag cri.*
|
||||||
|
Mem_Buf_Limit {{ .Values.config.input.memBufLimit }}
|
||||||
|
Skip_Long_Lines On
|
||||||
|
Refresh_Interval {{ .Values.config.input.refreshInterval }}
|
||||||
|
DB /var/log/flb_kube.db
|
||||||
|
DB.Sync Normal
|
||||||
|
|
||||||
|
[FILTER]
|
||||||
|
Name lua
|
||||||
|
Match cri.*
|
||||||
|
script /fluent-bit/etc/functions.lua
|
||||||
|
call reassemble_cri_logs
|
||||||
|
|
||||||
|
[FILTER]
|
||||||
|
Name kubernetes
|
||||||
|
Match cri.*
|
||||||
|
Merge_Log On
|
||||||
|
Merge_Log_Key kube
|
||||||
|
Kube_Tag_Prefix cri.var.log.containers.
|
||||||
|
Keep_Log Off
|
||||||
|
K8S-Logging.Parser Off
|
||||||
|
K8S-Logging.Exclude Off
|
||||||
|
|
||||||
|
{{- if index .Values "config" "extraRecords" }}
|
||||||
|
|
||||||
|
[FILTER]
|
||||||
|
Name record_modifier
|
||||||
|
Match cri.*
|
||||||
|
{{- range $k,$v := index .Values "config" "extraRecords" }}
|
||||||
|
Record {{ $k }} {{ $v }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
|
||||||
|
[FILTER]
|
||||||
|
Name rewrite_tag
|
||||||
|
Match cri.*
|
||||||
|
Emitter_Name kube_tag_rewriter
|
||||||
|
Rule logtag F kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false
|
||||||
|
|
||||||
|
[FILTER]
|
||||||
|
Name lua
|
||||||
|
Match kube.*
|
||||||
|
script /fluent-bit/etc/functions.lua
|
||||||
|
call nest_k8s_ns
|
||||||
|
|
||||||
|
{{- if .Values.config.outputs }}
|
||||||
|
{{ tpl .Values.config.outputs . }}
|
||||||
|
{{- else }}
|
||||||
|
[OUTPUT]
|
||||||
|
Match *
|
||||||
|
Name forward
|
||||||
|
Host {{ .Values.config.output.host }}
|
||||||
|
Port 24224
|
||||||
|
Shared_Key {{ .Values.config.output.sharedKey }}
|
||||||
|
tls {{ ternary "on" "off" .Values.config.output.tls }}
|
||||||
|
Send_options true
|
||||||
|
Require_ack_response true
|
||||||
|
{{- end }}
|
28
charts/kubezero-logging/charts/fluent-bit/conf/functions.lua
Normal file
28
charts/kubezero-logging/charts/fluent-bit/conf/functions.lua
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
local reassemble_state = {}
|
||||||
|
|
||||||
|
function reassemble_cri_logs(tag, timestamp, record)
|
||||||
|
local reassemble_key = tag
|
||||||
|
if record.logtag == 'P' then
|
||||||
|
reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or "" .. record.log
|
||||||
|
return -1, 0, 0
|
||||||
|
end
|
||||||
|
record.log = reassemble_state[reassemble_key] or "" .. (record.log or "")
|
||||||
|
reassemble_state[reassemble_key] = nil
|
||||||
|
return 1, timestamp, record
|
||||||
|
end
|
||||||
|
|
||||||
|
function nest_k8s_ns(tag, timestamp, record)
|
||||||
|
if not record['kubernetes']['namespace_name'] then
|
||||||
|
return 0, 0, 0
|
||||||
|
end
|
||||||
|
new_record = {}
|
||||||
|
for key, val in pairs(record) do
|
||||||
|
if key == 'kube' then
|
||||||
|
new_record[key] = {}
|
||||||
|
new_record[key][record['kubernetes']['namespace_name']] = record[key]
|
||||||
|
else
|
||||||
|
new_record[key] = record[key]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
return 1, timestamp, new_record
|
||||||
|
end
|
@ -0,0 +1,5 @@
|
|||||||
|
Get Fluent Bit build information by running these commands:
|
||||||
|
|
||||||
|
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "fluent-bit.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
|
||||||
|
echo "curl http://127.0.0.1:2020 for Fluent Bit build information"
|
||||||
|
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 2020:2020
|
@ -0,0 +1,63 @@
|
|||||||
|
{{/* vim: set filetype=mustache: */}}
|
||||||
|
{{/*
|
||||||
|
Expand the name of the chart.
|
||||||
|
*/}}
|
||||||
|
{{- define "fluent-bit.name" -}}
|
||||||
|
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Create a default fully qualified app name.
|
||||||
|
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||||
|
If release name contains chart name it will be used as a full name.
|
||||||
|
*/}}
|
||||||
|
{{- define "fluent-bit.fullname" -}}
|
||||||
|
{{- if .Values.fullnameOverride -}}
|
||||||
|
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||||
|
{{- if contains $name .Release.Name -}}
|
||||||
|
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- else -}}
|
||||||
|
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Create chart name and version as used by the chart label.
|
||||||
|
*/}}
|
||||||
|
{{- define "fluent-bit.chart" -}}
|
||||||
|
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Common labels
|
||||||
|
*/}}
|
||||||
|
{{- define "fluent-bit.labels" -}}
|
||||||
|
helm.sh/chart: {{ include "fluent-bit.chart" . }}
|
||||||
|
{{ include "fluent-bit.selectorLabels" . }}
|
||||||
|
{{- if .Chart.AppVersion }}
|
||||||
|
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||||
|
{{- end }}
|
||||||
|
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Selector labels
|
||||||
|
*/}}
|
||||||
|
{{- define "fluent-bit.selectorLabels" -}}
|
||||||
|
app.kubernetes.io/name: {{ include "fluent-bit.name" . }}
|
||||||
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||||
|
{{- end -}}
|
||||||
|
|
||||||
|
{{/*
|
||||||
|
Create the name of the service account to use
|
||||||
|
*/}}
|
||||||
|
{{- define "fluent-bit.serviceAccountName" -}}
|
||||||
|
{{- if .Values.serviceAccount.create -}}
|
||||||
|
{{ default (include "fluent-bit.fullname" .) .Values.serviceAccount.name }}
|
||||||
|
{{- else -}}
|
||||||
|
{{ default "default" .Values.serviceAccount.name }}
|
||||||
|
{{- end -}}
|
||||||
|
{{- end -}}
|
103
charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl
Normal file
103
charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl
Normal file
@ -0,0 +1,103 @@
|
|||||||
|
{{- define "fluent-bit.pod" -}}
|
||||||
|
{{- with .Values.imagePullSecrets }}
|
||||||
|
imagePullSecrets:
|
||||||
|
{{- toYaml . | nindent 2 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.priorityClassName }}
|
||||||
|
priorityClassName: {{ .Values.priorityClassName }}
|
||||||
|
{{- end }}
|
||||||
|
serviceAccountName: {{ include "fluent-bit.serviceAccountName" . }}
|
||||||
|
securityContext:
|
||||||
|
{{- toYaml .Values.podSecurityContext | nindent 2 }}
|
||||||
|
containers:
|
||||||
|
- name: {{ .Chart.Name }}
|
||||||
|
securityContext:
|
||||||
|
{{- toYaml .Values.securityContext | nindent 6 }}
|
||||||
|
image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}"
|
||||||
|
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||||
|
{{- if .Values.env }}
|
||||||
|
env:
|
||||||
|
{{- toYaml .Values.env | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.envFrom }}
|
||||||
|
envFrom:
|
||||||
|
{{- toYaml .Values.envFrom | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
ports:
|
||||||
|
- name: http
|
||||||
|
containerPort: 2020
|
||||||
|
protocol: TCP
|
||||||
|
{{- if .Values.extraPorts }}
|
||||||
|
{{- range .Values.extraPorts }}
|
||||||
|
- name: {{ .name }}
|
||||||
|
containerPort: {{ .containerPort }}
|
||||||
|
protocol: {{ .protocol }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /
|
||||||
|
port: http
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /
|
||||||
|
port: http
|
||||||
|
resources:
|
||||||
|
{{- toYaml .Values.resources | nindent 6 }}
|
||||||
|
volumeMounts:
|
||||||
|
- name: config
|
||||||
|
mountPath: /fluent-bit/etc/fluent-bit.conf
|
||||||
|
subPath: fluent-bit.conf
|
||||||
|
- name: config
|
||||||
|
mountPath: /fluent-bit/etc/custom_parsers.conf
|
||||||
|
subPath: custom_parsers.conf
|
||||||
|
{{- if .Values.config.lua }}
|
||||||
|
- name: config
|
||||||
|
mountPath: /fluent-bit/etc/functions.lua
|
||||||
|
subPath: functions.lua
|
||||||
|
{{- end }}
|
||||||
|
{{- if eq .Values.kind "DaemonSet" }}
|
||||||
|
- name: varlog
|
||||||
|
mountPath: /var/log
|
||||||
|
- name: varlibdockercontainers
|
||||||
|
mountPath: /var/lib/docker/containers
|
||||||
|
readOnly: true
|
||||||
|
- name: etcmachineid
|
||||||
|
mountPath: /etc/machine-id
|
||||||
|
readOnly: true
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.extraVolumeMounts }}
|
||||||
|
{{- toYaml .Values.extraVolumeMounts | nindent 6 }}
|
||||||
|
{{- end }}
|
||||||
|
volumes:
|
||||||
|
- name: config
|
||||||
|
configMap:
|
||||||
|
name: {{ if .Values.existingConfigMap }}{{ .Values.existingConfigMap }}{{- else }}{{ include "fluent-bit.fullname" . }}{{- end }}
|
||||||
|
{{- if eq .Values.kind "DaemonSet" }}
|
||||||
|
- name: varlog
|
||||||
|
hostPath:
|
||||||
|
path: /var/log
|
||||||
|
- name: varlibdockercontainers
|
||||||
|
hostPath:
|
||||||
|
path: /var/lib/docker/containers
|
||||||
|
- name: etcmachineid
|
||||||
|
hostPath:
|
||||||
|
path: /etc/machine-id
|
||||||
|
type: File
|
||||||
|
{{- end }}
|
||||||
|
{{- if .Values.extraVolumes }}
|
||||||
|
{{- toYaml .Values.extraVolumes | nindent 2 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.nodeSelector }}
|
||||||
|
nodeSelector:
|
||||||
|
{{- toYaml . | nindent 2 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.affinity }}
|
||||||
|
affinity:
|
||||||
|
{{- toYaml . | nindent 2 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.tolerations }}
|
||||||
|
tolerations:
|
||||||
|
{{- toYaml . | nindent 2 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end -}}
|
@ -0,0 +1,28 @@
|
|||||||
|
{{- if .Values.rbac.create -}}
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRole
|
||||||
|
metadata:
|
||||||
|
name: {{ include "fluent-bit.fullname" . }}
|
||||||
|
labels:
|
||||||
|
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||||
|
rules:
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- pods
|
||||||
|
- namespaces
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- list
|
||||||
|
- watch
|
||||||
|
{{- if .Values.podSecurityPolicy.create }}
|
||||||
|
- apiGroups:
|
||||||
|
- policy
|
||||||
|
resources:
|
||||||
|
- podsecuritypolicies
|
||||||
|
resourceNames:
|
||||||
|
- {{ include "fluent-bit.fullname" . }}
|
||||||
|
verbs:
|
||||||
|
- use
|
||||||
|
{{- end }}
|
||||||
|
{{- end -}}
|
@ -0,0 +1,16 @@
|
|||||||
|
{{- if .Values.rbac.create -}}
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
metadata:
|
||||||
|
name: {{ include "fluent-bit.fullname" . }}
|
||||||
|
labels:
|
||||||
|
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: {{ include "fluent-bit.fullname" . }}
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: {{ include "fluent-bit.serviceAccountName" . }}
|
||||||
|
namespace: {{ .Release.Namespace }}
|
||||||
|
{{- end -}}
|
@ -0,0 +1,15 @@
|
|||||||
|
{{- if (empty .Values.existingConfigMap) -}}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: {{ include "fluent-bit.fullname" . }}
|
||||||
|
labels:
|
||||||
|
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||||
|
data:
|
||||||
|
custom_parsers.conf: |
|
||||||
|
{{ tpl (.Files.Get "conf/custom_parsers.conf") . | nindent 4 }}
|
||||||
|
fluent-bit.conf: |
|
||||||
|
{{ tpl (.Files.Get "conf/fluent-bit.conf") . | nindent 4 }}
|
||||||
|
functions.lua: |
|
||||||
|
{{ tpl (.Files.Get "conf/functions.lua") . | nindent 4 }}
|
||||||
|
{{- end -}}
|
@ -0,0 +1,30 @@
|
|||||||
|
{{- if eq .Values.kind "DaemonSet" }}
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: DaemonSet
|
||||||
|
metadata:
|
||||||
|
name: {{ include "fluent-bit.fullname" . }}
|
||||||
|
labels:
|
||||||
|
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
{{- include "fluent-bit.selectorLabels" . | nindent 6 }}
|
||||||
|
{{- with .Values.updateStrategy }}
|
||||||
|
updateStrategy:
|
||||||
|
{{- toYaml . | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||||
|
{{- with .Values.podAnnotations }}
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
labels:
|
||||||
|
{{- include "fluent-bit.selectorLabels" . | nindent 8 }}
|
||||||
|
{{- with .Values.podLabels }}
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
{{- include "fluent-bit.pod" . | nindent 6 }}
|
||||||
|
{{- end }}
|
@ -0,0 +1,31 @@
|
|||||||
|
{{- if eq .Values.kind "Deployment" }}
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: {{ include "fluent-bit.fullname" . }}
|
||||||
|
labels:
|
||||||
|
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||||
|
spec:
|
||||||
|
replicas: {{ .Values.replicaCount }}
|
||||||
|
{{- with .Values.updateStrategy }}
|
||||||
|
strategy:
|
||||||
|
{{- toYaml . | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
{{- include "fluent-bit.selectorLabels" . | nindent 6 }}
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||||
|
{{- with .Values.podAnnotations }}
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
labels:
|
||||||
|
{{- include "fluent-bit.selectorLabels" . | nindent 8 }}
|
||||||
|
{{- with .Values.podLabels }}
|
||||||
|
{{- toYaml . | nindent 8 }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
{{- include "fluent-bit.pod" . | nindent 6 }}
|
||||||
|
{{- end }}
|
38
charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml
Normal file
38
charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
{{- if .Values.podSecurityPolicy.create }}
|
||||||
|
apiVersion: policy/v1beta1
|
||||||
|
kind: PodSecurityPolicy
|
||||||
|
metadata:
|
||||||
|
name: {{ include "fluent-bit.fullname" . }}
|
||||||
|
spec:
|
||||||
|
privileged: false
|
||||||
|
# Required to prevent escalations to root.
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
# This is redundant with non-root + disallow privilege escalation,
|
||||||
|
# but we can provide it for defense in depth.
|
||||||
|
requiredDropCapabilities:
|
||||||
|
- ALL
|
||||||
|
volumes:
|
||||||
|
- '*'
|
||||||
|
hostNetwork: false
|
||||||
|
hostIPC: false
|
||||||
|
hostPID: false
|
||||||
|
runAsUser:
|
||||||
|
# TODO: Require the container to run without root privileges.
|
||||||
|
rule: 'RunAsAny'
|
||||||
|
seLinux:
|
||||||
|
# This policy assumes the nodes are using AppArmor rather than SELinux.
|
||||||
|
rule: 'RunAsAny'
|
||||||
|
supplementalGroups:
|
||||||
|
rule: 'MustRunAs'
|
||||||
|
ranges:
|
||||||
|
# Forbid adding the root group.
|
||||||
|
- min: 1
|
||||||
|
max: 65535
|
||||||
|
fsGroup:
|
||||||
|
rule: 'MustRunAs'
|
||||||
|
ranges:
|
||||||
|
# Forbid adding the root group.
|
||||||
|
- min: 1
|
||||||
|
max: 65535
|
||||||
|
readOnlyRootFilesystem: false
|
||||||
|
{{- end }}
|
@ -0,0 +1,30 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: {{ include "fluent-bit.fullname" . }}
|
||||||
|
labels:
|
||||||
|
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||||
|
{{- with .Values.service.labels }}
|
||||||
|
{{- toYaml . | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.service.annotations }}
|
||||||
|
annotations:
|
||||||
|
{{- toYaml . | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
type: {{ .Values.service.type }}
|
||||||
|
ports:
|
||||||
|
- port: {{ .Values.service.port }}
|
||||||
|
targetPort: http
|
||||||
|
protocol: TCP
|
||||||
|
name: http
|
||||||
|
{{- if .Values.extraPorts }}
|
||||||
|
{{- range .Values.extraPorts }}
|
||||||
|
- name: {{ .name }}
|
||||||
|
targetPort: {{ .name }}
|
||||||
|
protocol: {{ .protocol }}
|
||||||
|
port: {{ .port }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
selector:
|
||||||
|
{{- include "fluent-bit.selectorLabels" . | nindent 4 }}
|
@ -0,0 +1,12 @@
|
|||||||
|
{{- if .Values.serviceAccount.create -}}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: {{ include "fluent-bit.serviceAccountName" . }}
|
||||||
|
labels:
|
||||||
|
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||||
|
{{- with .Values.serviceAccount.annotations }}
|
||||||
|
annotations:
|
||||||
|
{{- toYaml . | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end -}}
|
@ -0,0 +1,30 @@
|
|||||||
|
{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.serviceMonitor.enabled }}
|
||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: ServiceMonitor
|
||||||
|
metadata:
|
||||||
|
name: {{ template "fluent-bit.fullname" . }}
|
||||||
|
{{- with .Values.serviceMonitor.namespace }}
|
||||||
|
namespace: {{ . }}
|
||||||
|
{{- end }}
|
||||||
|
labels:
|
||||||
|
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||||
|
{{- with .Values.serviceMonitor.selector }}
|
||||||
|
{{- toYaml . | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
spec:
|
||||||
|
endpoints:
|
||||||
|
- port: http
|
||||||
|
path: /api/v1/metrics/prometheus
|
||||||
|
{{- with .Values.serviceMonitor.interval }}
|
||||||
|
interval: {{ . }}
|
||||||
|
{{- end }}
|
||||||
|
{{- with .Values.serviceMonitor.scrapeTimeout }}
|
||||||
|
scrapeTimeout: {{ . }}
|
||||||
|
{{- end }}
|
||||||
|
namespaceSelector:
|
||||||
|
matchNames:
|
||||||
|
- {{ .Release.Namespace }}
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
{{- include "fluent-bit.selectorLabels" . | nindent 6 }}
|
||||||
|
{{- end }}
|
@ -0,0 +1,17 @@
|
|||||||
|
{{- if .Values.test.enabled }}
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: "{{ include "fluent-bit.fullname" . }}-test-connection"
|
||||||
|
labels:
|
||||||
|
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||||
|
annotations:
|
||||||
|
"helm.sh/hook": test-success
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: wget
|
||||||
|
image: busybox
|
||||||
|
command: ['wget']
|
||||||
|
args: ['{{ include "fluent-bit.fullname" . }}:{{ .Values.service.port }}']
|
||||||
|
restartPolicy: Never
|
||||||
|
{{- end }}
|
171
charts/kubezero-logging/charts/fluent-bit/values.yaml
Normal file
171
charts/kubezero-logging/charts/fluent-bit/values.yaml
Normal file
@ -0,0 +1,171 @@
|
|||||||
|
# Default values for fluent-bit.
|
||||||
|
|
||||||
|
# kind -- DaemonSet or Deployment
|
||||||
|
kind: DaemonSet
|
||||||
|
|
||||||
|
# replicaCount -- Only applicable if kind=Deployment
|
||||||
|
replicaCount: 1
|
||||||
|
|
||||||
|
test:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
image:
|
||||||
|
repository: fluent/fluent-bit
|
||||||
|
pullPolicy: Always
|
||||||
|
# tag:
|
||||||
|
|
||||||
|
imagePullSecrets: []
|
||||||
|
nameOverride: ""
|
||||||
|
fullnameOverride: ""
|
||||||
|
|
||||||
|
serviceAccount:
|
||||||
|
create: true
|
||||||
|
annotations: {}
|
||||||
|
name:
|
||||||
|
|
||||||
|
rbac:
|
||||||
|
create: true
|
||||||
|
|
||||||
|
podSecurityPolicy:
|
||||||
|
create: false
|
||||||
|
|
||||||
|
podSecurityContext:
|
||||||
|
{}
|
||||||
|
# fsGroup: 2000
|
||||||
|
|
||||||
|
securityContext:
|
||||||
|
{}
|
||||||
|
# capabilities:
|
||||||
|
# drop:
|
||||||
|
# - ALL
|
||||||
|
# readOnlyRootFilesystem: true
|
||||||
|
# runAsNonRoot: true
|
||||||
|
# runAsUser: 1000
|
||||||
|
|
||||||
|
service:
|
||||||
|
type: ClusterIP
|
||||||
|
port: 2020
|
||||||
|
labels:
|
||||||
|
{}
|
||||||
|
annotations:
|
||||||
|
{}
|
||||||
|
# prometheus.io/path: "/api/v1/metrics/prometheus"
|
||||||
|
# prometheus.io/port: "2020"
|
||||||
|
# prometheus.io/scrape: "true"
|
||||||
|
|
||||||
|
serviceMonitor:
|
||||||
|
enabled: false
|
||||||
|
# namespace: monitoring
|
||||||
|
# interval: 10s
|
||||||
|
# scrapeTimeout: 10s
|
||||||
|
# selector:
|
||||||
|
# prometheus: my-prometheus
|
||||||
|
|
||||||
|
resources:
|
||||||
|
{}
|
||||||
|
# limits:
|
||||||
|
# cpu: 100m
|
||||||
|
# memory: 128Mi
|
||||||
|
# requests:
|
||||||
|
# cpu: 100m
|
||||||
|
# memory: 128Mi
|
||||||
|
|
||||||
|
nodeSelector: {}
|
||||||
|
|
||||||
|
tolerations: []
|
||||||
|
|
||||||
|
affinity: {}
|
||||||
|
|
||||||
|
podAnnotations: {}
|
||||||
|
|
||||||
|
podLabels: {}
|
||||||
|
|
||||||
|
priorityClassName: ""
|
||||||
|
|
||||||
|
env: []
|
||||||
|
|
||||||
|
envFrom: []
|
||||||
|
|
||||||
|
extraPorts: []
|
||||||
|
# - port: 5170
|
||||||
|
# containerPort: 5170
|
||||||
|
# protocol: TCP
|
||||||
|
# name: tcp
|
||||||
|
|
||||||
|
extraVolumes: []
|
||||||
|
|
||||||
|
extraVolumeMounts: []
|
||||||
|
|
||||||
|
updateStrategy: {}
|
||||||
|
# type: RollingUpdate
|
||||||
|
# rollingUpdate:
|
||||||
|
# maxUnavailable: 1
|
||||||
|
|
||||||
|
# Make use of a pre-defined configmap instead of the one templated here
|
||||||
|
existingConfigMap: ""
|
||||||
|
|
||||||
|
## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit
|
||||||
|
config:
|
||||||
|
## https://docs.fluentbit.io/manual/service
|
||||||
|
service: |
|
||||||
|
[SERVICE]
|
||||||
|
Flush 1
|
||||||
|
Daemon Off
|
||||||
|
Log_Level info
|
||||||
|
Parsers_File parsers.conf
|
||||||
|
Parsers_File custom_parsers.conf
|
||||||
|
HTTP_Server On
|
||||||
|
HTTP_Listen 0.0.0.0
|
||||||
|
HTTP_Port 2020
|
||||||
|
|
||||||
|
## https://docs.fluentbit.io/manual/pipeline/inputs
|
||||||
|
inputs: |
|
||||||
|
[INPUT]
|
||||||
|
Name tail
|
||||||
|
Path /var/log/containers/*.log
|
||||||
|
Parser docker
|
||||||
|
Tag kube.*
|
||||||
|
Mem_Buf_Limit 5MB
|
||||||
|
Skip_Long_Lines On
|
||||||
|
|
||||||
|
[INPUT]
|
||||||
|
Name systemd
|
||||||
|
Tag host.*
|
||||||
|
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
|
||||||
|
Read_From_Tail On
|
||||||
|
|
||||||
|
## https://docs.fluentbit.io/manual/pipeline/filters
|
||||||
|
filters: |
|
||||||
|
[FILTER]
|
||||||
|
Name kubernetes
|
||||||
|
Match kube.*
|
||||||
|
Merge_Log On
|
||||||
|
Keep_Log Off
|
||||||
|
K8S-Logging.Parser On
|
||||||
|
K8S-Logging.Exclude On
|
||||||
|
|
||||||
|
## https://docs.fluentbit.io/manual/pipeline/outputs
|
||||||
|
outputs: |
|
||||||
|
[OUTPUT]
|
||||||
|
Name es
|
||||||
|
Match kube.*
|
||||||
|
Host elasticsearch-master
|
||||||
|
Logstash_Format On
|
||||||
|
Retry_Limit False
|
||||||
|
|
||||||
|
[OUTPUT]
|
||||||
|
Name es
|
||||||
|
Match host.*
|
||||||
|
Host elasticsearch-master
|
||||||
|
Logstash_Format On
|
||||||
|
Logstash_Prefix node
|
||||||
|
Retry_Limit False
|
||||||
|
|
||||||
|
## https://docs.fluentbit.io/manual/pipeline/parsers
|
||||||
|
customParsers: |
|
||||||
|
[PARSER]
|
||||||
|
Name docker_no_time
|
||||||
|
Format json
|
||||||
|
Time_Keep Off
|
||||||
|
Time_Key time
|
||||||
|
Time_Format %Y-%m-%dT%H:%M:%S.%L
|
@ -193,105 +193,21 @@ fluent-bit:
|
|||||||
enabled: false
|
enabled: false
|
||||||
|
|
||||||
config:
|
config:
|
||||||
outputs: |
|
outputs: {}
|
||||||
[OUTPUT]
|
output:
|
||||||
Match *
|
host: logging-fluentd
|
||||||
Name forward
|
sharedKey: cloudbender
|
||||||
Host logging-fluentd
|
tls: false
|
||||||
Port 24224
|
|
||||||
Shared_Key cloudbender
|
|
||||||
Send_options true
|
|
||||||
Require_ack_response true
|
|
||||||
|
|
||||||
customParsers: |
|
input:
|
||||||
[PARSER]
|
memBufLimit: 16MB
|
||||||
Name cri-log
|
refreshInterval: 10
|
||||||
Format regex
|
|
||||||
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
|
|
||||||
Time_Key time
|
|
||||||
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
|
|
||||||
|
|
||||||
inputs: |
|
logLevel: warn
|
||||||
[INPUT]
|
flushInterval: 1
|
||||||
Name tail
|
|
||||||
Path /var/log/containers/*.log
|
|
||||||
Parser cri-log
|
|
||||||
Tag cri.*
|
|
||||||
Mem_Buf_Limit 16MB
|
|
||||||
Skip_Long_Lines On
|
|
||||||
Refresh_Interval 10
|
|
||||||
DB /var/log/flb_kube.db
|
|
||||||
DB.Sync Normal
|
|
||||||
|
|
||||||
filters: |
|
#extraRecords:
|
||||||
[FILTER]
|
# source.clustername: MyKubeCluster
|
||||||
Name lua
|
|
||||||
Match cri.*
|
|
||||||
script /fluent-bit/etc/functions.lua
|
|
||||||
call reassemble_cri_logs
|
|
||||||
|
|
||||||
[FILTER]
|
|
||||||
Name kubernetes
|
|
||||||
Match cri.*
|
|
||||||
Merge_Log On
|
|
||||||
Merge_Log_Key kube
|
|
||||||
Kube_Tag_Prefix cri.var.log.containers.
|
|
||||||
Keep_Log Off
|
|
||||||
K8S-Logging.Parser Off
|
|
||||||
K8S-Logging.Exclude Off
|
|
||||||
|
|
||||||
[FILTER]
|
|
||||||
Name rewrite_tag
|
|
||||||
Match cri.*
|
|
||||||
Emitter_Name kube_tag_rewriter
|
|
||||||
Rule logtag F kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false
|
|
||||||
|
|
||||||
[FILTER]
|
|
||||||
Name lua
|
|
||||||
Match kube.*
|
|
||||||
script /fluent-bit/etc/functions.lua
|
|
||||||
call nest_k8s_ns
|
|
||||||
|
|
||||||
service: |
|
|
||||||
[SERVICE]
|
|
||||||
Flush 1
|
|
||||||
Daemon Off
|
|
||||||
Log_Level warn
|
|
||||||
Parsers_File parsers.conf
|
|
||||||
Parsers_File custom_parsers.conf
|
|
||||||
HTTP_Server On
|
|
||||||
HTTP_Listen 0.0.0.0
|
|
||||||
HTTP_Port 2020
|
|
||||||
|
|
||||||
lua: |
|
|
||||||
local reassemble_state = {}
|
|
||||||
|
|
||||||
function reassemble_cri_logs(tag, timestamp, record)
|
|
||||||
local reassemble_key = tag
|
|
||||||
if record.logtag == 'P' then
|
|
||||||
reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or "" .. record.log
|
|
||||||
return -1, 0, 0
|
|
||||||
end
|
|
||||||
record.log = reassemble_state[reassemble_key] or "" .. (record.log or "")
|
|
||||||
reassemble_state[reassemble_key] = nil
|
|
||||||
return 1, timestamp, record
|
|
||||||
end
|
|
||||||
|
|
||||||
function nest_k8s_ns(tag, timestamp, record)
|
|
||||||
if not record['kubernetes']['namespace_name'] then
|
|
||||||
return 0, 0, 0
|
|
||||||
end
|
|
||||||
new_record = {}
|
|
||||||
for key, val in pairs(record) do
|
|
||||||
if key == 'kube' then
|
|
||||||
new_record[key] = {}
|
|
||||||
new_record[key][record['kubernetes']['namespace_name']] = record[key]
|
|
||||||
else
|
|
||||||
new_record[key] = record[key]
|
|
||||||
end
|
|
||||||
end
|
|
||||||
return 1, timestamp, new_record
|
|
||||||
end
|
|
||||||
|
|
||||||
serviceMonitor:
|
serviceMonitor:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
Loading…
Reference in New Issue
Block a user