Integrare fluent-bit into logging to allow better config
This commit is contained in:
parent
f78c382be6
commit
77a7ba2ed6
@ -26,9 +26,9 @@ dependencies:
|
||||
version: 2.5.1
|
||||
repository: https://kubernetes-charts.storage.googleapis.com/
|
||||
condition: fluentd.enabled
|
||||
- name: fluent-bit
|
||||
version: 0.7.10
|
||||
repository: https://zero-down-time.github.io/kubezero/
|
||||
# - name: fluent-bit
|
||||
# version: 0.7.10
|
||||
# repository: https://zero-down-time.github.io/kubezero/
|
||||
# repository: https://fluent.github.io/helm-charts
|
||||
condition: fluent-bit.enabled
|
||||
# condition: fluent-bit.enabled
|
||||
kubeVersion: ">= 1.16.0"
|
||||
|
@ -20,7 +20,6 @@ Kubernetes: `>= 1.16.0`
|
||||
|------------|------|---------|
|
||||
| https://helm.elastic.co | eck-operator | 1.3.0 |
|
||||
| https://kubernetes-charts.storage.googleapis.com/ | fluentd | 2.5.1 |
|
||||
| https://zero-down-time.github.io/kubezero/ | fluent-bit | 0.7.10 |
|
||||
| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 |
|
||||
|
||||
## Changes from upstream
|
||||
@ -57,7 +56,8 @@ Kubernetes: `>= 1.16.0`
|
||||
|
||||
| Key | Type | Default | Description |
|
||||
|-----|------|---------|-------------|
|
||||
| eck-operator.enabled | bool | `true` | |
|
||||
| eck-operator.enabled | bool | `false` | |
|
||||
| eck-operator.installCRDs | bool | `false` | |
|
||||
| eck-operator.nodeSelector."node-role.kubernetes.io/master" | string | `""` | |
|
||||
| eck-operator.tolerations[0].effect | string | `"NoSchedule"` | |
|
||||
| eck-operator.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | |
|
||||
@ -66,12 +66,14 @@ Kubernetes: `>= 1.16.0`
|
||||
| es.prometheus | bool | `false` | |
|
||||
| es.s3Snapshot.enabled | bool | `false` | |
|
||||
| es.s3Snapshot.iamrole | string | `""` | |
|
||||
| fluent-bit.config.customParsers | string | `"[PARSER]\n Name cri-log\n Format regex\n Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$\n Time_Key time\n Time_Format %Y-%m-%dT%H:%M:%S.%L%z\n"` | |
|
||||
| fluent-bit.config.filters | string | `"[FILTER]\n Name lua\n Match cri.*\n script /fluent-bit/etc/functions.lua\n call reassemble_cri_logs\n\n[FILTER]\n Name kubernetes\n Match cri.*\n Merge_Log On\n Merge_Log_Key kube\n Kube_Tag_Prefix cri.var.log.containers.\n Keep_Log Off\n K8S-Logging.Parser Off\n K8S-Logging.Exclude Off\n\n[FILTER]\n Name rewrite_tag\n Match cri.*\n Emitter_Name kube_tag_rewriter\n Rule logtag F kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false\n\n[FILTER]\n Name lua\n Match kube.*\n script /fluent-bit/etc/functions.lua\n call nest_k8s_ns\n"` | |
|
||||
| fluent-bit.config.inputs | string | `"[INPUT]\n Name tail\n Path /var/log/containers/*.log\n Parser cri-log\n Tag cri.*\n Mem_Buf_Limit 16MB\n Skip_Long_Lines On\n Refresh_Interval 10\n DB /var/log/flb_kube.db\n DB.Sync Normal\n"` | |
|
||||
| fluent-bit.config.lua | string | `"local reassemble_state = {}\n\nfunction reassemble_cri_logs(tag, timestamp, record)\n local reassemble_key = tag\n if record.logtag == 'P' then\n reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or \"\" .. record.log\n return -1, 0, 0\n end\n record.log = reassemble_state[reassemble_key] or \"\" .. (record.log or \"\")\n reassemble_state[reassemble_key] = nil\n return 1, timestamp, record\nend\n\nfunction nest_k8s_ns(tag, timestamp, record)\n if not record['kubernetes']['namespace_name'] then\n return 0, 0, 0\n end\n new_record = {}\n for key, val in pairs(record) do\n if key == 'kube' then\n new_record[key] = {}\n new_record[key][record['kubernetes']['namespace_name']] = record[key]\n else\n new_record[key] = record[key]\n end\n end\n return 1, timestamp, new_record\nend\n"` | |
|
||||
| fluent-bit.config.outputs | string | `"[OUTPUT]\n Match *\n Name forward\n Host logging-fluentd\n Port 24224\n Shared_Key cloudbender\n Send_options true\n Require_ack_response true\n"` | |
|
||||
| fluent-bit.config.service | string | `"[SERVICE]\n Flush 1\n Daemon Off\n Log_Level warn\n Parsers_File parsers.conf\n Parsers_File custom_parsers.conf\n HTTP_Server On\n HTTP_Listen 0.0.0.0\n HTTP_Port 2020\n"` | |
|
||||
| fluent-bit.config.flushInterval | int | `1` | |
|
||||
| fluent-bit.config.input.memBufLimit | string | `"16MB"` | |
|
||||
| fluent-bit.config.input.refreshInterval | int | `10` | |
|
||||
| fluent-bit.config.logLevel | string | `"warn"` | |
|
||||
| fluent-bit.config.output.host | string | `"logging-fluentd"` | |
|
||||
| fluent-bit.config.output.sharedKey | string | `"cloudbender"` | |
|
||||
| fluent-bit.config.output.tls | bool | `false` | |
|
||||
| fluent-bit.config.outputs | object | `{}` | |
|
||||
| fluent-bit.enabled | bool | `false` | |
|
||||
| fluent-bit.serviceMonitor.enabled | bool | `true` | |
|
||||
| fluent-bit.serviceMonitor.selector.release | string | `"metrics"` | |
|
||||
@ -81,7 +83,7 @@ Kubernetes: `>= 1.16.0`
|
||||
| fluentd.configMaps."filter.conf" | string | `"<filter disabled.kube.**>\n @type parser\n key_name message\n remove_key_name_field true\n reserve_data true\n reserve_time true\n # inject_key_prefix message_json.\n emit_invalid_record_to_error false\n <parse>\n @type json\n </parse>\n</filter>\n"` | |
|
||||
| fluentd.configMaps."forward-input.conf" | string | `"<source>\n @type forward\n port 24224\n bind 0.0.0.0\n skip_invalid_event true\n send_keepalive_packet true\n <security>\n self_hostname \"#{ENV['HOSTNAME']}\"\n shared_key \"#{ENV['FLUENTD_SHARED_KEY']}\"\n </security>\n</source>\n"` | |
|
||||
| fluentd.configMaps."general.conf" | string | `"<label @FLUENT_LOG>\n <match **>\n @type null\n </match>\n</label>\n<source>\n @type http\n port 9880\n bind 0.0.0.0\n keepalive_timeout 30\n</source>\n<source>\n @type monitor_agent\n bind 0.0.0.0\n port 24220\n tag fluentd.monitor.metrics\n</source>\n"` | |
|
||||
| fluentd.configMaps."output.conf" | string | `"<match **>\n @id elasticsearch\n @type elasticsearch\n @log_level info\n include_tag_key true\n id_key id\n remove_keys id\n\n # KubeZero pipeline incl. GeoIP etc.\n # pipeline fluentd\n\n host \"#{ENV['OUTPUT_HOST']}\"\n port \"#{ENV['OUTPUT_PORT']}\"\n scheme \"#{ENV['OUTPUT_SCHEME']}\"\n ssl_version \"#{ENV['OUTPUT_SSL_VERSION']}\"\n ssl_verify \"#{ENV['OUTPUT_SSL_VERIFY']}\"\n user \"#{ENV['OUTPUT_USER']}\"\n password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n log_es_400_reason\n logstash_format true\n reconnect_on_error true\n # reload_on_failure true\n request_timeout 15s\n suppress_type_name true\n\n <buffer tag>\n @type file_single\n path /var/log/fluentd-buffers/kubernetes.system.buffer\n flush_mode interval\n flush_thread_count 2\n flush_interval 30s\n flush_at_shutdown true\n retry_type exponential_backoff\n retry_timeout 60m\n overflow_action drop_oldest_chunk\n </buffer>\n</match>\n"` | |
|
||||
| fluentd.configMaps."output.conf" | string | `"<match **>\n @id elasticsearch\n @type elasticsearch\n @log_level info\n include_tag_key true\n id_key id\n remove_keys id\n\n # KubeZero pipeline incl. GeoIP etc.\n pipeline fluentd\n\n host \"#{ENV['OUTPUT_HOST']}\"\n port \"#{ENV['OUTPUT_PORT']}\"\n scheme \"#{ENV['OUTPUT_SCHEME']}\"\n ssl_version \"#{ENV['OUTPUT_SSL_VERSION']}\"\n ssl_verify \"#{ENV['OUTPUT_SSL_VERIFY']}\"\n user \"#{ENV['OUTPUT_USER']}\"\n password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n log_es_400_reason\n logstash_format true\n reconnect_on_error true\n # reload_on_failure true\n request_timeout 15s\n suppress_type_name true\n\n <buffer tag>\n @type file_single\n path /var/log/fluentd-buffers/kubernetes.system.buffer\n flush_mode interval\n flush_thread_count 2\n flush_interval 30s\n flush_at_shutdown true\n retry_type exponential_backoff\n retry_timeout 60m\n overflow_action drop_oldest_chunk\n </buffer>\n</match>\n"` | |
|
||||
| fluentd.enabled | bool | `false` | |
|
||||
| fluentd.env.OUTPUT_SSL_VERIFY | string | `"false"` | |
|
||||
| fluentd.env.OUTPUT_USER | string | `"elastic"` | |
|
||||
|
23
charts/kubezero-logging/charts/fluent-bit/.helmignore
Normal file
23
charts/kubezero-logging/charts/fluent-bit/.helmignore
Normal file
@ -0,0 +1,23 @@
|
||||
# Patterns to ignore when building packages.
|
||||
# This supports shell glob matching, relative path matching, and
|
||||
# negation (prefixed with !). Only one pattern per line.
|
||||
.DS_Store
|
||||
# Common VCS dirs
|
||||
.git/
|
||||
.gitignore
|
||||
.bzr/
|
||||
.bzrignore
|
||||
.hg/
|
||||
.hgignore
|
||||
.svn/
|
||||
# Common backup files
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*.orig
|
||||
*~
|
||||
# Various IDEs
|
||||
.project
|
||||
.idea/
|
||||
*.tmproj
|
||||
.vscode/
|
20
charts/kubezero-logging/charts/fluent-bit/Chart.yaml
Normal file
20
charts/kubezero-logging/charts/fluent-bit/Chart.yaml
Normal file
@ -0,0 +1,20 @@
|
||||
apiVersion: v2
|
||||
name: fluent-bit
|
||||
description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
|
||||
keywords:
|
||||
- logging
|
||||
- fluent-bit
|
||||
- fluentd
|
||||
version: 0.7.13
|
||||
appVersion: 1.6.8
|
||||
icon: https://fluentbit.io/assets/img/logo1-default.png
|
||||
home: https://fluentbit.io/
|
||||
sources:
|
||||
- https://github.com/fluent/fluent-bit/
|
||||
maintainers:
|
||||
- name: edsiper
|
||||
email: eduardo@treasure-data.com
|
||||
- name: naseemkullah
|
||||
email: naseem@transit.app
|
||||
- name: Towmeykaw
|
||||
email: towmeykaw@gmail.com
|
23
charts/kubezero-logging/charts/fluent-bit/README.md
Normal file
23
charts/kubezero-logging/charts/fluent-bit/README.md
Normal file
@ -0,0 +1,23 @@
|
||||
# Fluent Bit Helm Chart
|
||||
|
||||
[Fluent Bit](https://fluentbit.io) is a fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
|
||||
|
||||
## Installation
|
||||
|
||||
To add the `fluent` helm repo, run:
|
||||
|
||||
```sh
|
||||
helm repo add fluent https://fluent.github.io/helm-charts
|
||||
```
|
||||
|
||||
To install a release named `fluent-bit`, run:
|
||||
|
||||
```sh
|
||||
helm install fluent-bit fluent/fluent-bit
|
||||
```
|
||||
|
||||
## Chart Values
|
||||
|
||||
```sh
|
||||
helm show values fluent/fluent-bit
|
||||
```
|
@ -0,0 +1,6 @@
|
||||
[PARSER]
|
||||
Name cri-log
|
||||
Format regex
|
||||
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
|
||||
Time_Key time
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
|
@ -0,0 +1,72 @@
|
||||
[SERVICE]
|
||||
Flush {{ .Values.config.flushInterval }}
|
||||
Daemon Off
|
||||
Log_Level {{ .Values.config.logLevel }}
|
||||
Parsers_File parsers.conf
|
||||
Parsers_File custom_parsers.conf
|
||||
HTTP_Server On
|
||||
HTTP_Listen 0.0.0.0
|
||||
HTTP_Port 2020
|
||||
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/containers/*.log
|
||||
Parser cri-log
|
||||
Tag cri.*
|
||||
Mem_Buf_Limit {{ .Values.config.input.memBufLimit }}
|
||||
Skip_Long_Lines On
|
||||
Refresh_Interval {{ .Values.config.input.refreshInterval }}
|
||||
DB /var/log/flb_kube.db
|
||||
DB.Sync Normal
|
||||
|
||||
[FILTER]
|
||||
Name lua
|
||||
Match cri.*
|
||||
script /fluent-bit/etc/functions.lua
|
||||
call reassemble_cri_logs
|
||||
|
||||
[FILTER]
|
||||
Name kubernetes
|
||||
Match cri.*
|
||||
Merge_Log On
|
||||
Merge_Log_Key kube
|
||||
Kube_Tag_Prefix cri.var.log.containers.
|
||||
Keep_Log Off
|
||||
K8S-Logging.Parser Off
|
||||
K8S-Logging.Exclude Off
|
||||
|
||||
{{- if index .Values "config" "extraRecords" }}
|
||||
|
||||
[FILTER]
|
||||
Name record_modifier
|
||||
Match cri.*
|
||||
{{- range $k,$v := index .Values "config" "extraRecords" }}
|
||||
Record {{ $k }} {{ $v }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
[FILTER]
|
||||
Name rewrite_tag
|
||||
Match cri.*
|
||||
Emitter_Name kube_tag_rewriter
|
||||
Rule logtag F kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false
|
||||
|
||||
[FILTER]
|
||||
Name lua
|
||||
Match kube.*
|
||||
script /fluent-bit/etc/functions.lua
|
||||
call nest_k8s_ns
|
||||
|
||||
{{- if .Values.config.outputs }}
|
||||
{{ tpl .Values.config.outputs . }}
|
||||
{{- else }}
|
||||
[OUTPUT]
|
||||
Match *
|
||||
Name forward
|
||||
Host {{ .Values.config.output.host }}
|
||||
Port 24224
|
||||
Shared_Key {{ .Values.config.output.sharedKey }}
|
||||
tls {{ ternary "on" "off" .Values.config.output.tls }}
|
||||
Send_options true
|
||||
Require_ack_response true
|
||||
{{- end }}
|
28
charts/kubezero-logging/charts/fluent-bit/conf/functions.lua
Normal file
28
charts/kubezero-logging/charts/fluent-bit/conf/functions.lua
Normal file
@ -0,0 +1,28 @@
|
||||
local reassemble_state = {}
|
||||
|
||||
function reassemble_cri_logs(tag, timestamp, record)
|
||||
local reassemble_key = tag
|
||||
if record.logtag == 'P' then
|
||||
reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or "" .. record.log
|
||||
return -1, 0, 0
|
||||
end
|
||||
record.log = reassemble_state[reassemble_key] or "" .. (record.log or "")
|
||||
reassemble_state[reassemble_key] = nil
|
||||
return 1, timestamp, record
|
||||
end
|
||||
|
||||
function nest_k8s_ns(tag, timestamp, record)
|
||||
if not record['kubernetes']['namespace_name'] then
|
||||
return 0, 0, 0
|
||||
end
|
||||
new_record = {}
|
||||
for key, val in pairs(record) do
|
||||
if key == 'kube' then
|
||||
new_record[key] = {}
|
||||
new_record[key][record['kubernetes']['namespace_name']] = record[key]
|
||||
else
|
||||
new_record[key] = record[key]
|
||||
end
|
||||
end
|
||||
return 1, timestamp, new_record
|
||||
end
|
@ -0,0 +1,5 @@
|
||||
Get Fluent Bit build information by running these commands:
|
||||
|
||||
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "fluent-bit.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
|
||||
echo "curl http://127.0.0.1:2020 for Fluent Bit build information"
|
||||
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 2020:2020
|
@ -0,0 +1,63 @@
|
||||
{{/* vim: set filetype=mustache: */}}
|
||||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "fluent-bit.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "fluent-bit.fullname" -}}
|
||||
{{- if .Values.fullnameOverride -}}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||
{{- if contains $name .Release.Name -}}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
{{- define "fluent-bit.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "fluent-bit.labels" -}}
|
||||
helm.sh/chart: {{ include "fluent-bit.chart" . }}
|
||||
{{ include "fluent-bit.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Selector labels
|
||||
*/}}
|
||||
{{- define "fluent-bit.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "fluent-bit.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create the name of the service account to use
|
||||
*/}}
|
||||
{{- define "fluent-bit.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccount.create -}}
|
||||
{{ default (include "fluent-bit.fullname" .) .Values.serviceAccount.name }}
|
||||
{{- else -}}
|
||||
{{ default "default" .Values.serviceAccount.name }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
103
charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl
Normal file
103
charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl
Normal file
@ -0,0 +1,103 @@
|
||||
{{- define "fluent-bit.pod" -}}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- if .Values.priorityClassName }}
|
||||
priorityClassName: {{ .Values.priorityClassName }}
|
||||
{{- end }}
|
||||
serviceAccountName: {{ include "fluent-bit.serviceAccountName" . }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.podSecurityContext | nindent 2 }}
|
||||
containers:
|
||||
- name: {{ .Chart.Name }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.securityContext | nindent 6 }}
|
||||
image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
{{- if .Values.env }}
|
||||
env:
|
||||
{{- toYaml .Values.env | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.envFrom }}
|
||||
envFrom:
|
||||
{{- toYaml .Values.envFrom | nindent 4 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 2020
|
||||
protocol: TCP
|
||||
{{- if .Values.extraPorts }}
|
||||
{{- range .Values.extraPorts }}
|
||||
- name: {{ .name }}
|
||||
containerPort: {{ .containerPort }}
|
||||
protocol: {{ .protocol }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
resources:
|
||||
{{- toYaml .Values.resources | nindent 6 }}
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /fluent-bit/etc/fluent-bit.conf
|
||||
subPath: fluent-bit.conf
|
||||
- name: config
|
||||
mountPath: /fluent-bit/etc/custom_parsers.conf
|
||||
subPath: custom_parsers.conf
|
||||
{{- if .Values.config.lua }}
|
||||
- name: config
|
||||
mountPath: /fluent-bit/etc/functions.lua
|
||||
subPath: functions.lua
|
||||
{{- end }}
|
||||
{{- if eq .Values.kind "DaemonSet" }}
|
||||
- name: varlog
|
||||
mountPath: /var/log
|
||||
- name: varlibdockercontainers
|
||||
mountPath: /var/lib/docker/containers
|
||||
readOnly: true
|
||||
- name: etcmachineid
|
||||
mountPath: /etc/machine-id
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.extraVolumeMounts }}
|
||||
{{- toYaml .Values.extraVolumeMounts | nindent 6 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: {{ if .Values.existingConfigMap }}{{ .Values.existingConfigMap }}{{- else }}{{ include "fluent-bit.fullname" . }}{{- end }}
|
||||
{{- if eq .Values.kind "DaemonSet" }}
|
||||
- name: varlog
|
||||
hostPath:
|
||||
path: /var/log
|
||||
- name: varlibdockercontainers
|
||||
hostPath:
|
||||
path: /var/lib/docker/containers
|
||||
- name: etcmachineid
|
||||
hostPath:
|
||||
path: /etc/machine-id
|
||||
type: File
|
||||
{{- end }}
|
||||
{{- if .Values.extraVolumes }}
|
||||
{{- toYaml .Values.extraVolumes | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- with .Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- with .Values.affinity }}
|
||||
affinity:
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- with .Values.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
@ -0,0 +1,28 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "fluent-bit.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- namespaces
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
{{- if .Values.podSecurityPolicy.create }}
|
||||
- apiGroups:
|
||||
- policy
|
||||
resources:
|
||||
- podsecuritypolicies
|
||||
resourceNames:
|
||||
- {{ include "fluent-bit.fullname" . }}
|
||||
verbs:
|
||||
- use
|
||||
{{- end }}
|
||||
{{- end -}}
|
@ -0,0 +1,16 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "fluent-bit.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "fluent-bit.fullname" . }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "fluent-bit.serviceAccountName" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end -}}
|
@ -0,0 +1,15 @@
|
||||
{{- if (empty .Values.existingConfigMap) -}}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "fluent-bit.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||
data:
|
||||
custom_parsers.conf: |
|
||||
{{ tpl (.Files.Get "conf/custom_parsers.conf") . | nindent 4 }}
|
||||
fluent-bit.conf: |
|
||||
{{ tpl (.Files.Get "conf/fluent-bit.conf") . | nindent 4 }}
|
||||
functions.lua: |
|
||||
{{ tpl (.Files.Get "conf/functions.lua") . | nindent 4 }}
|
||||
{{- end -}}
|
@ -0,0 +1,30 @@
|
||||
{{- if eq .Values.kind "DaemonSet" }}
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: {{ include "fluent-bit.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "fluent-bit.selectorLabels" . | nindent 6 }}
|
||||
{{- with .Values.updateStrategy }}
|
||||
updateStrategy:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "fluent-bit.selectorLabels" . | nindent 8 }}
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- include "fluent-bit.pod" . | nindent 6 }}
|
||||
{{- end }}
|
@ -0,0 +1,31 @@
|
||||
{{- if eq .Values.kind "Deployment" }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "fluent-bit.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||
spec:
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
{{- with .Values.updateStrategy }}
|
||||
strategy:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "fluent-bit.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "fluent-bit.selectorLabels" . | nindent 8 }}
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- include "fluent-bit.pod" . | nindent 6 }}
|
||||
{{- end }}
|
38
charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml
Normal file
38
charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml
Normal file
@ -0,0 +1,38 @@
|
||||
{{- if .Values.podSecurityPolicy.create }}
|
||||
apiVersion: policy/v1beta1
|
||||
kind: PodSecurityPolicy
|
||||
metadata:
|
||||
name: {{ include "fluent-bit.fullname" . }}
|
||||
spec:
|
||||
privileged: false
|
||||
# Required to prevent escalations to root.
|
||||
allowPrivilegeEscalation: false
|
||||
# This is redundant with non-root + disallow privilege escalation,
|
||||
# but we can provide it for defense in depth.
|
||||
requiredDropCapabilities:
|
||||
- ALL
|
||||
volumes:
|
||||
- '*'
|
||||
hostNetwork: false
|
||||
hostIPC: false
|
||||
hostPID: false
|
||||
runAsUser:
|
||||
# TODO: Require the container to run without root privileges.
|
||||
rule: 'RunAsAny'
|
||||
seLinux:
|
||||
# This policy assumes the nodes are using AppArmor rather than SELinux.
|
||||
rule: 'RunAsAny'
|
||||
supplementalGroups:
|
||||
rule: 'MustRunAs'
|
||||
ranges:
|
||||
# Forbid adding the root group.
|
||||
- min: 1
|
||||
max: 65535
|
||||
fsGroup:
|
||||
rule: 'MustRunAs'
|
||||
ranges:
|
||||
# Forbid adding the root group.
|
||||
- min: 1
|
||||
max: 65535
|
||||
readOnlyRootFilesystem: false
|
||||
{{- end }}
|
@ -0,0 +1,30 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "fluent-bit.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||
{{- with .Values.service.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.service.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.service.type }}
|
||||
ports:
|
||||
- port: {{ .Values.service.port }}
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
name: http
|
||||
{{- if .Values.extraPorts }}
|
||||
{{- range .Values.extraPorts }}
|
||||
- name: {{ .name }}
|
||||
targetPort: {{ .name }}
|
||||
protocol: {{ .protocol }}
|
||||
port: {{ .port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
selector:
|
||||
{{- include "fluent-bit.selectorLabels" . | nindent 4 }}
|
@ -0,0 +1,12 @@
|
||||
{{- if .Values.serviceAccount.create -}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "fluent-bit.serviceAccountName" . }}
|
||||
labels:
|
||||
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||
{{- with .Values.serviceAccount.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
@ -0,0 +1,30 @@
|
||||
{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.serviceMonitor.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: {{ template "fluent-bit.fullname" . }}
|
||||
{{- with .Values.serviceMonitor.namespace }}
|
||||
namespace: {{ . }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||
{{- with .Values.serviceMonitor.selector }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /api/v1/metrics/prometheus
|
||||
{{- with .Values.serviceMonitor.interval }}
|
||||
interval: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.serviceMonitor.scrapeTimeout }}
|
||||
scrapeTimeout: {{ . }}
|
||||
{{- end }}
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- {{ .Release.Namespace }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "fluent-bit.selectorLabels" . | nindent 6 }}
|
||||
{{- end }}
|
@ -0,0 +1,17 @@
|
||||
{{- if .Values.test.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: "{{ include "fluent-bit.fullname" . }}-test-connection"
|
||||
labels:
|
||||
{{- include "fluent-bit.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/hook": test-success
|
||||
spec:
|
||||
containers:
|
||||
- name: wget
|
||||
image: busybox
|
||||
command: ['wget']
|
||||
args: ['{{ include "fluent-bit.fullname" . }}:{{ .Values.service.port }}']
|
||||
restartPolicy: Never
|
||||
{{- end }}
|
171
charts/kubezero-logging/charts/fluent-bit/values.yaml
Normal file
171
charts/kubezero-logging/charts/fluent-bit/values.yaml
Normal file
@ -0,0 +1,171 @@
|
||||
# Default values for fluent-bit.
|
||||
|
||||
# kind -- DaemonSet or Deployment
|
||||
kind: DaemonSet
|
||||
|
||||
# replicaCount -- Only applicable if kind=Deployment
|
||||
replicaCount: 1
|
||||
|
||||
test:
|
||||
enabled: true
|
||||
|
||||
image:
|
||||
repository: fluent/fluent-bit
|
||||
pullPolicy: Always
|
||||
# tag:
|
||||
|
||||
imagePullSecrets: []
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
serviceAccount:
|
||||
create: true
|
||||
annotations: {}
|
||||
name:
|
||||
|
||||
rbac:
|
||||
create: true
|
||||
|
||||
podSecurityPolicy:
|
||||
create: false
|
||||
|
||||
podSecurityContext:
|
||||
{}
|
||||
# fsGroup: 2000
|
||||
|
||||
securityContext:
|
||||
{}
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# readOnlyRootFilesystem: true
|
||||
# runAsNonRoot: true
|
||||
# runAsUser: 1000
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 2020
|
||||
labels:
|
||||
{}
|
||||
annotations:
|
||||
{}
|
||||
# prometheus.io/path: "/api/v1/metrics/prometheus"
|
||||
# prometheus.io/port: "2020"
|
||||
# prometheus.io/scrape: "true"
|
||||
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
# namespace: monitoring
|
||||
# interval: 10s
|
||||
# scrapeTimeout: 10s
|
||||
# selector:
|
||||
# prometheus: my-prometheus
|
||||
|
||||
resources:
|
||||
{}
|
||||
# limits:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
|
||||
nodeSelector: {}
|
||||
|
||||
tolerations: []
|
||||
|
||||
affinity: {}
|
||||
|
||||
podAnnotations: {}
|
||||
|
||||
podLabels: {}
|
||||
|
||||
priorityClassName: ""
|
||||
|
||||
env: []
|
||||
|
||||
envFrom: []
|
||||
|
||||
extraPorts: []
|
||||
# - port: 5170
|
||||
# containerPort: 5170
|
||||
# protocol: TCP
|
||||
# name: tcp
|
||||
|
||||
extraVolumes: []
|
||||
|
||||
extraVolumeMounts: []
|
||||
|
||||
updateStrategy: {}
|
||||
# type: RollingUpdate
|
||||
# rollingUpdate:
|
||||
# maxUnavailable: 1
|
||||
|
||||
# Make use of a pre-defined configmap instead of the one templated here
|
||||
existingConfigMap: ""
|
||||
|
||||
## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit
|
||||
config:
|
||||
## https://docs.fluentbit.io/manual/service
|
||||
service: |
|
||||
[SERVICE]
|
||||
Flush 1
|
||||
Daemon Off
|
||||
Log_Level info
|
||||
Parsers_File parsers.conf
|
||||
Parsers_File custom_parsers.conf
|
||||
HTTP_Server On
|
||||
HTTP_Listen 0.0.0.0
|
||||
HTTP_Port 2020
|
||||
|
||||
## https://docs.fluentbit.io/manual/pipeline/inputs
|
||||
inputs: |
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/containers/*.log
|
||||
Parser docker
|
||||
Tag kube.*
|
||||
Mem_Buf_Limit 5MB
|
||||
Skip_Long_Lines On
|
||||
|
||||
[INPUT]
|
||||
Name systemd
|
||||
Tag host.*
|
||||
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
|
||||
Read_From_Tail On
|
||||
|
||||
## https://docs.fluentbit.io/manual/pipeline/filters
|
||||
filters: |
|
||||
[FILTER]
|
||||
Name kubernetes
|
||||
Match kube.*
|
||||
Merge_Log On
|
||||
Keep_Log Off
|
||||
K8S-Logging.Parser On
|
||||
K8S-Logging.Exclude On
|
||||
|
||||
## https://docs.fluentbit.io/manual/pipeline/outputs
|
||||
outputs: |
|
||||
[OUTPUT]
|
||||
Name es
|
||||
Match kube.*
|
||||
Host elasticsearch-master
|
||||
Logstash_Format On
|
||||
Retry_Limit False
|
||||
|
||||
[OUTPUT]
|
||||
Name es
|
||||
Match host.*
|
||||
Host elasticsearch-master
|
||||
Logstash_Format On
|
||||
Logstash_Prefix node
|
||||
Retry_Limit False
|
||||
|
||||
## https://docs.fluentbit.io/manual/pipeline/parsers
|
||||
customParsers: |
|
||||
[PARSER]
|
||||
Name docker_no_time
|
||||
Format json
|
||||
Time_Keep Off
|
||||
Time_Key time
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%L
|
@ -193,105 +193,21 @@ fluent-bit:
|
||||
enabled: false
|
||||
|
||||
config:
|
||||
outputs: |
|
||||
[OUTPUT]
|
||||
Match *
|
||||
Name forward
|
||||
Host logging-fluentd
|
||||
Port 24224
|
||||
Shared_Key cloudbender
|
||||
Send_options true
|
||||
Require_ack_response true
|
||||
outputs: {}
|
||||
output:
|
||||
host: logging-fluentd
|
||||
sharedKey: cloudbender
|
||||
tls: false
|
||||
|
||||
customParsers: |
|
||||
[PARSER]
|
||||
Name cri-log
|
||||
Format regex
|
||||
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
|
||||
Time_Key time
|
||||
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
|
||||
input:
|
||||
memBufLimit: 16MB
|
||||
refreshInterval: 10
|
||||
|
||||
inputs: |
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /var/log/containers/*.log
|
||||
Parser cri-log
|
||||
Tag cri.*
|
||||
Mem_Buf_Limit 16MB
|
||||
Skip_Long_Lines On
|
||||
Refresh_Interval 10
|
||||
DB /var/log/flb_kube.db
|
||||
DB.Sync Normal
|
||||
logLevel: warn
|
||||
flushInterval: 1
|
||||
|
||||
filters: |
|
||||
[FILTER]
|
||||
Name lua
|
||||
Match cri.*
|
||||
script /fluent-bit/etc/functions.lua
|
||||
call reassemble_cri_logs
|
||||
|
||||
[FILTER]
|
||||
Name kubernetes
|
||||
Match cri.*
|
||||
Merge_Log On
|
||||
Merge_Log_Key kube
|
||||
Kube_Tag_Prefix cri.var.log.containers.
|
||||
Keep_Log Off
|
||||
K8S-Logging.Parser Off
|
||||
K8S-Logging.Exclude Off
|
||||
|
||||
[FILTER]
|
||||
Name rewrite_tag
|
||||
Match cri.*
|
||||
Emitter_Name kube_tag_rewriter
|
||||
Rule logtag F kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false
|
||||
|
||||
[FILTER]
|
||||
Name lua
|
||||
Match kube.*
|
||||
script /fluent-bit/etc/functions.lua
|
||||
call nest_k8s_ns
|
||||
|
||||
service: |
|
||||
[SERVICE]
|
||||
Flush 1
|
||||
Daemon Off
|
||||
Log_Level warn
|
||||
Parsers_File parsers.conf
|
||||
Parsers_File custom_parsers.conf
|
||||
HTTP_Server On
|
||||
HTTP_Listen 0.0.0.0
|
||||
HTTP_Port 2020
|
||||
|
||||
lua: |
|
||||
local reassemble_state = {}
|
||||
|
||||
function reassemble_cri_logs(tag, timestamp, record)
|
||||
local reassemble_key = tag
|
||||
if record.logtag == 'P' then
|
||||
reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or "" .. record.log
|
||||
return -1, 0, 0
|
||||
end
|
||||
record.log = reassemble_state[reassemble_key] or "" .. (record.log or "")
|
||||
reassemble_state[reassemble_key] = nil
|
||||
return 1, timestamp, record
|
||||
end
|
||||
|
||||
function nest_k8s_ns(tag, timestamp, record)
|
||||
if not record['kubernetes']['namespace_name'] then
|
||||
return 0, 0, 0
|
||||
end
|
||||
new_record = {}
|
||||
for key, val in pairs(record) do
|
||||
if key == 'kube' then
|
||||
new_record[key] = {}
|
||||
new_record[key][record['kubernetes']['namespace_name']] = record[key]
|
||||
else
|
||||
new_record[key] = record[key]
|
||||
end
|
||||
end
|
||||
return 1, timestamp, new_record
|
||||
end
|
||||
#extraRecords:
|
||||
# source.clustername: MyKubeCluster
|
||||
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
|
Loading…
Reference in New Issue
Block a user