2020-08-14 22:02:30 +00:00
|
|
|
# use this for backwards compatability
|
2020-08-13 18:44:50 +00:00
|
|
|
# fullnameOverride: ""
|
2020-08-11 14:09:48 +00:00
|
|
|
|
2020-11-21 12:24:57 +00:00
|
|
|
eck-operator:
|
2020-12-07 21:06:00 +00:00
|
|
|
enabled: false
|
2020-12-05 22:16:33 +00:00
|
|
|
installCRDs: false
|
2020-11-21 12:24:57 +00:00
|
|
|
tolerations:
|
|
|
|
- key: node-role.kubernetes.io/master
|
|
|
|
effect: NoSchedule
|
|
|
|
nodeSelector:
|
2021-12-08 16:29:53 +00:00
|
|
|
node-role.kubernetes.io/control-plane: ""
|
2020-11-21 12:24:57 +00:00
|
|
|
|
2020-08-11 14:09:48 +00:00
|
|
|
# Version for ElasticSearch and Kibana have to match so we define it at top-level
|
2022-04-22 20:49:27 +00:00
|
|
|
version: 7.17.3
|
2020-08-11 14:09:48 +00:00
|
|
|
|
2020-08-15 13:25:07 +00:00
|
|
|
elastic_password: "" # super_secret_elastic_password
|
|
|
|
|
2020-08-11 14:09:48 +00:00
|
|
|
es:
|
|
|
|
nodeSets: []
|
|
|
|
#- count: 2
|
|
|
|
# storage:
|
|
|
|
# size: 16Gi
|
|
|
|
# class: local-sc-xfs
|
|
|
|
# zone: us-west-2a
|
2021-03-19 15:15:58 +00:00
|
|
|
# nodeAffinity:
|
|
|
|
# key: node label name
|
|
|
|
# value: node label value
|
2020-08-13 18:44:50 +00:00
|
|
|
s3Snapshot:
|
|
|
|
enabled: false
|
|
|
|
iamrole: "" # INSERT_CLOUDFORMATION_OUTPUT_ElasticSearchSnapshots
|
2020-08-11 14:09:48 +00:00
|
|
|
|
2020-08-13 18:44:50 +00:00
|
|
|
prometheus: false
|
|
|
|
|
2020-08-11 14:09:48 +00:00
|
|
|
kibana:
|
2020-08-14 14:52:10 +00:00
|
|
|
count: 1
|
2020-08-11 14:09:48 +00:00
|
|
|
#servicename: kibana.example.com
|
2021-03-19 15:15:58 +00:00
|
|
|
#nodeAffinity:
|
|
|
|
# key: node label name
|
|
|
|
# value: node label value
|
2020-08-11 14:09:48 +00:00
|
|
|
istio:
|
|
|
|
enabled: false
|
|
|
|
gateway: "istio-system/ingressgateway"
|
|
|
|
url: "" # kibana.example.com
|
2020-08-18 10:34:34 +00:00
|
|
|
|
|
|
|
fluentd:
|
|
|
|
enabled: false
|
2020-08-23 16:41:37 +00:00
|
|
|
image:
|
2021-09-03 17:21:14 +00:00
|
|
|
repository: public.ecr.aws/zero-downtime/fluentd-concenter
|
2021-11-01 11:30:58 +00:00
|
|
|
tag: v1.14.2
|
2020-08-18 10:34:34 +00:00
|
|
|
istio:
|
|
|
|
enabled: false
|
|
|
|
|
2021-03-10 09:32:12 +00:00
|
|
|
kind: Deployment
|
|
|
|
replicaCount: 1
|
2020-08-18 10:34:34 +00:00
|
|
|
|
2021-03-10 09:32:12 +00:00
|
|
|
#plugins:
|
|
|
|
#- fluent-plugin-s3
|
2020-08-18 10:34:34 +00:00
|
|
|
|
|
|
|
service:
|
|
|
|
ports:
|
|
|
|
- name: tcp-forward
|
|
|
|
protocol: TCP
|
|
|
|
containerPort: 24224
|
|
|
|
- name: http-fluentd
|
|
|
|
protocol: TCP
|
|
|
|
containerPort: 9880
|
|
|
|
|
|
|
|
metrics:
|
|
|
|
serviceMonitor:
|
2021-03-10 09:32:12 +00:00
|
|
|
enabled: false
|
2020-08-18 10:34:34 +00:00
|
|
|
additionalLabels:
|
|
|
|
release: metrics
|
|
|
|
|
2021-03-10 09:32:12 +00:00
|
|
|
dashboards:
|
|
|
|
enabled: false
|
|
|
|
|
2021-08-30 09:35:52 +00:00
|
|
|
podSecurityPolicy:
|
|
|
|
enabled: false
|
|
|
|
|
2021-03-10 09:32:12 +00:00
|
|
|
# No need for docker nor /var/log
|
|
|
|
volumes:
|
|
|
|
- name: etcfluentd-main
|
|
|
|
configMap:
|
|
|
|
name: fluentd-main
|
|
|
|
defaultMode: 0777
|
|
|
|
- name: etcfluentd-config
|
|
|
|
configMap:
|
|
|
|
name: fluentd-config
|
|
|
|
defaultMode: 0777
|
|
|
|
volumeMounts:
|
|
|
|
- name: etcfluentd-main
|
|
|
|
mountPath: /etc/fluent
|
|
|
|
- name: etcfluentd-config
|
|
|
|
mountPath: /etc/fluent/config.d/
|
|
|
|
|
|
|
|
resources:
|
|
|
|
requests:
|
2021-03-19 15:15:58 +00:00
|
|
|
cpu: 200m
|
2021-03-10 09:32:12 +00:00
|
|
|
memory: 256Mi
|
|
|
|
limits:
|
2021-03-19 15:15:58 +00:00
|
|
|
memory: 512Mi
|
2021-03-10 09:32:12 +00:00
|
|
|
|
2020-08-18 10:34:34 +00:00
|
|
|
output:
|
2020-09-18 15:12:52 +00:00
|
|
|
# Default should be "logging-kubezero-logging-es-http" if fullnameOverride is NOT used
|
2020-08-18 10:34:34 +00:00
|
|
|
host: logging-es-http
|
|
|
|
|
2020-08-18 10:58:37 +00:00
|
|
|
shared_key: "cloudbender"
|
|
|
|
|
2021-03-10 09:32:12 +00:00
|
|
|
# OUTPUT_USER: elastic
|
|
|
|
# OUTPUT_SSL_VERIFY: "false"
|
2020-08-18 10:34:34 +00:00
|
|
|
|
2021-03-10 09:32:12 +00:00
|
|
|
env:
|
|
|
|
- name: "FLUENTD_CONF"
|
|
|
|
value: "../../etc/fluent/fluent.conf"
|
2020-09-18 15:12:52 +00:00
|
|
|
# Same here the secret names change if fullnameOverride is not used !!
|
2020-08-18 10:34:34 +00:00
|
|
|
- name: OUTPUT_PASSWORD
|
|
|
|
valueFrom:
|
|
|
|
secretKeyRef:
|
|
|
|
name: logging-es-elastic-user
|
|
|
|
key: elastic
|
|
|
|
|
2021-03-10 09:32:12 +00:00
|
|
|
fileConfigs:
|
|
|
|
00_system.conf: |-
|
|
|
|
<system>
|
2021-09-03 17:21:14 +00:00
|
|
|
root_dir /fluentd/log
|
2021-07-14 15:38:32 +00:00
|
|
|
log_level info
|
|
|
|
ignore_repeated_log_interval 60s
|
|
|
|
ignore_same_log_interval 60s
|
2021-08-29 22:28:32 +00:00
|
|
|
workers 1
|
2021-03-10 09:32:12 +00:00
|
|
|
</system>
|
|
|
|
01_sources.conf: |-
|
2020-10-02 21:41:40 +00:00
|
|
|
<source>
|
|
|
|
@type http
|
2021-03-10 09:32:12 +00:00
|
|
|
@label @KUBERNETES
|
2020-10-02 21:41:40 +00:00
|
|
|
port 9880
|
|
|
|
bind 0.0.0.0
|
|
|
|
keepalive_timeout 30
|
|
|
|
</source>
|
|
|
|
|
2020-08-18 10:34:34 +00:00
|
|
|
<source>
|
|
|
|
@type forward
|
2021-03-10 09:32:12 +00:00
|
|
|
@label @KUBERNETES
|
2020-08-18 10:34:34 +00:00
|
|
|
port 24224
|
|
|
|
bind 0.0.0.0
|
2021-03-10 09:32:12 +00:00
|
|
|
# skip_invalid_event true
|
2020-10-05 10:50:23 +00:00
|
|
|
send_keepalive_packet true
|
2020-08-18 10:34:34 +00:00
|
|
|
<security>
|
|
|
|
self_hostname "#{ENV['HOSTNAME']}"
|
2021-03-10 09:32:12 +00:00
|
|
|
shared_key {{ .Values.shared_key }}
|
2020-08-18 10:34:34 +00:00
|
|
|
</security>
|
|
|
|
</source>
|
|
|
|
|
2021-03-10 09:32:12 +00:00
|
|
|
02_filters.conf: |-
|
|
|
|
<label @KUBERNETES>
|
|
|
|
# prevent log feedback loops eg. ES has issues etc.
|
|
|
|
# discard logs from our own pods
|
|
|
|
<match kube.logging.fluentd>
|
|
|
|
@type relabel
|
|
|
|
@label @FLUENT_LOG
|
|
|
|
</match>
|
|
|
|
|
2021-08-12 19:47:26 +00:00
|
|
|
# Exclude current fluent-bit multiline noise
|
|
|
|
<filter kube.logging.fluent-bit>
|
|
|
|
@type grep
|
|
|
|
<exclude>
|
|
|
|
key log
|
|
|
|
pattern /could not append content to multiline context/
|
|
|
|
</exclude>
|
|
|
|
</filter>
|
|
|
|
|
2021-08-29 22:28:32 +00:00
|
|
|
# Generate Hash ID to break endless loop for already ingested events during retries
|
|
|
|
<filter **>
|
|
|
|
@type elasticsearch_genid
|
|
|
|
use_entire_record true
|
|
|
|
</filter>
|
|
|
|
|
|
|
|
# Route through DISPATCH for Prometheus metrics
|
2021-03-10 09:32:12 +00:00
|
|
|
<match **>
|
|
|
|
@type relabel
|
|
|
|
@label @DISPATCH
|
|
|
|
</match>
|
|
|
|
</label>
|
|
|
|
|
|
|
|
04_outputs.conf: |-
|
|
|
|
<label @OUTPUT>
|
|
|
|
<match **>
|
2021-05-17 10:04:46 +00:00
|
|
|
@id out_es
|
2021-03-10 09:32:12 +00:00
|
|
|
@type elasticsearch
|
2021-07-14 15:38:32 +00:00
|
|
|
# @log_level debug
|
2021-03-10 09:32:12 +00:00
|
|
|
include_tag_key true
|
2021-09-03 17:21:14 +00:00
|
|
|
|
2021-08-29 22:28:32 +00:00
|
|
|
id_key _hash
|
|
|
|
remove_keys _hash
|
2021-09-03 17:21:14 +00:00
|
|
|
write_operation create
|
2021-03-10 09:32:12 +00:00
|
|
|
|
|
|
|
# KubeZero pipeline incl. GeoIP etc.
|
|
|
|
pipeline fluentd
|
|
|
|
|
|
|
|
hosts "{{ .Values.output.host }}"
|
|
|
|
port 9200
|
|
|
|
scheme http
|
|
|
|
user elastic
|
|
|
|
password "#{ENV['OUTPUT_PASSWORD']}"
|
|
|
|
|
|
|
|
log_es_400_reason
|
|
|
|
logstash_format true
|
|
|
|
reconnect_on_error true
|
|
|
|
reload_on_failure true
|
2021-09-03 17:21:14 +00:00
|
|
|
request_timeout 300s
|
2021-07-14 15:38:32 +00:00
|
|
|
slow_flush_log_threshold 55.0
|
2021-06-15 14:14:29 +00:00
|
|
|
|
2021-09-03 17:21:14 +00:00
|
|
|
#with_transporter_log true
|
|
|
|
|
|
|
|
verify_es_version_at_startup false
|
|
|
|
default_elasticsearch_version 7
|
|
|
|
suppress_type_name true
|
|
|
|
|
2021-06-15 14:14:29 +00:00
|
|
|
# Retry failed bulk requests
|
|
|
|
# https://github.com/uken/fluent-plugin-elasticsearch#unrecoverable-error-types
|
|
|
|
unrecoverable_error_types ["out_of_memory_error"]
|
2021-09-03 17:21:14 +00:00
|
|
|
bulk_message_request_threshold 1048576
|
2021-03-10 09:32:12 +00:00
|
|
|
|
2021-06-30 10:32:37 +00:00
|
|
|
<buffer>
|
2021-06-15 14:14:29 +00:00
|
|
|
@type file
|
2021-06-30 10:32:37 +00:00
|
|
|
|
2021-03-10 09:32:12 +00:00
|
|
|
flush_mode interval
|
2021-08-29 22:28:32 +00:00
|
|
|
flush_thread_count 2
|
2021-09-03 17:21:14 +00:00
|
|
|
flush_interval 10s
|
2021-06-30 10:32:37 +00:00
|
|
|
|
2021-09-03 17:21:14 +00:00
|
|
|
chunk_limit_size 2MB
|
|
|
|
total_limit_size 1GB
|
2021-06-30 10:32:37 +00:00
|
|
|
|
2021-03-10 09:32:12 +00:00
|
|
|
flush_at_shutdown true
|
|
|
|
retry_type exponential_backoff
|
2021-07-14 15:38:32 +00:00
|
|
|
retry_timeout 6h
|
2021-03-10 09:32:12 +00:00
|
|
|
overflow_action drop_oldest_chunk
|
|
|
|
disable_chunk_backup true
|
|
|
|
</buffer>
|
|
|
|
</match>
|
|
|
|
</label>
|
2020-09-08 14:05:31 +00:00
|
|
|
|
2020-08-22 17:27:18 +00:00
|
|
|
fluent-bit:
|
2020-08-23 16:41:37 +00:00
|
|
|
enabled: false
|
2021-03-05 15:53:02 +00:00
|
|
|
|
2021-09-01 11:37:02 +00:00
|
|
|
image:
|
2021-07-29 11:53:40 +00:00
|
|
|
#repository: public.ecr.aws/zero-downtime/fluent-bit
|
2022-04-21 18:57:04 +00:00
|
|
|
tag: 1.9.2
|
2021-07-20 11:28:37 +00:00
|
|
|
|
2021-03-05 15:53:02 +00:00
|
|
|
serviceMonitor:
|
2021-04-25 20:19:06 +00:00
|
|
|
enabled: false
|
2021-03-05 15:53:02 +00:00
|
|
|
selector:
|
|
|
|
release: metrics
|
|
|
|
|
2021-08-27 10:14:46 +00:00
|
|
|
#rbac:
|
|
|
|
# nodeAccess: true
|
|
|
|
#hostNetwork: true
|
|
|
|
#dnsPolicy: ClusterFirstWithHostNet
|
|
|
|
|
2021-03-05 15:53:02 +00:00
|
|
|
tolerations:
|
|
|
|
- key: node-role.kubernetes.io/master
|
|
|
|
effect: NoSchedule
|
2021-07-19 14:20:25 +00:00
|
|
|
- key: kubezero-workergroup
|
|
|
|
effect: NoSchedule
|
2021-07-19 14:38:32 +00:00
|
|
|
operator: Exists
|
2020-08-22 17:27:18 +00:00
|
|
|
|
2021-03-19 15:15:58 +00:00
|
|
|
resources:
|
|
|
|
requests:
|
|
|
|
cpu: 20m
|
2021-04-25 20:19:06 +00:00
|
|
|
memory: 32Mi
|
2021-03-19 15:15:58 +00:00
|
|
|
limits:
|
|
|
|
memory: 64Mi
|
|
|
|
|
2020-08-22 17:27:18 +00:00
|
|
|
config:
|
2020-12-08 15:05:25 +00:00
|
|
|
output:
|
|
|
|
host: logging-fluentd
|
|
|
|
sharedKey: cloudbender
|
|
|
|
tls: false
|
2020-10-05 16:01:50 +00:00
|
|
|
|
2020-12-08 15:05:25 +00:00
|
|
|
input:
|
2021-03-26 09:56:17 +00:00
|
|
|
memBufLimit: 4MB
|
2020-12-08 15:05:25 +00:00
|
|
|
refreshInterval: 10
|
2020-08-22 17:27:18 +00:00
|
|
|
|
2021-07-20 11:28:37 +00:00
|
|
|
logLevel: info
|
2021-03-10 09:32:12 +00:00
|
|
|
flushInterval: 5
|
2020-08-22 17:27:18 +00:00
|
|
|
|
2020-12-08 15:05:25 +00:00
|
|
|
#extraRecords:
|
|
|
|
# source.clustername: MyKubeCluster
|
2020-10-07 16:09:24 +00:00
|
|
|
|
2021-03-05 15:53:02 +00:00
|
|
|
service: |
|
|
|
|
[SERVICE]
|
|
|
|
Flush {{ .Values.config.flushInterval }}
|
|
|
|
Daemon Off
|
|
|
|
Log_Level {{ .Values.config.logLevel }}
|
|
|
|
Parsers_File parsers.conf
|
|
|
|
Parsers_File custom_parsers.conf
|
|
|
|
HTTP_Server On
|
|
|
|
HTTP_Listen 0.0.0.0
|
|
|
|
HTTP_Port {{ .Values.service.port }}
|
2021-08-27 10:14:46 +00:00
|
|
|
Health_Check On
|
2021-03-05 15:53:02 +00:00
|
|
|
inputs: |
|
|
|
|
[INPUT]
|
|
|
|
Name tail
|
|
|
|
Path /var/log/containers/*.log
|
2021-07-29 11:53:40 +00:00
|
|
|
multiline.parser cri
|
2021-03-05 15:53:02 +00:00
|
|
|
Tag cri.*
|
|
|
|
Skip_Long_Lines On
|
|
|
|
DB /var/log/flb_kube.db
|
|
|
|
DB.Sync Normal
|
2021-07-20 11:28:37 +00:00
|
|
|
DB.locking true
|
2021-07-29 11:50:16 +00:00
|
|
|
# Buffer_Max_Size 1M
|
2021-03-05 15:53:02 +00:00
|
|
|
{{- with .Values.config.input }}
|
2021-03-26 09:56:17 +00:00
|
|
|
Mem_Buf_Limit {{ default "4MB" .memBufLimit }}
|
2021-03-05 15:53:02 +00:00
|
|
|
Refresh_Interval {{ default 10 .refreshInterval }}
|
|
|
|
{{- end }}
|
2020-08-22 17:27:18 +00:00
|
|
|
|
2021-03-05 15:53:02 +00:00
|
|
|
filters: |
|
|
|
|
[FILTER]
|
2021-07-29 11:50:16 +00:00
|
|
|
Name parser
|
|
|
|
Match cri.*
|
|
|
|
Parser cri-log
|
|
|
|
Key_Name log
|
2021-03-05 15:53:02 +00:00
|
|
|
|
|
|
|
[FILTER]
|
|
|
|
Name kubernetes
|
|
|
|
Match cri.*
|
|
|
|
Merge_Log On
|
|
|
|
Merge_Log_Key kube
|
|
|
|
Kube_Tag_Prefix cri.var.log.containers.
|
|
|
|
Keep_Log Off
|
|
|
|
K8S-Logging.Parser Off
|
|
|
|
K8S-Logging.Exclude Off
|
2021-07-20 11:28:37 +00:00
|
|
|
Kube_Meta_Cache_TTL 3600s
|
|
|
|
Buffer_Size 0
|
2021-03-09 09:33:40 +00:00
|
|
|
#Use_Kubelet true
|
2021-03-05 15:53:02 +00:00
|
|
|
|
|
|
|
{{- if index .Values "config" "extraRecords" }}
|
|
|
|
|
|
|
|
[FILTER]
|
|
|
|
Name record_modifier
|
|
|
|
Match cri.*
|
|
|
|
{{- range $k,$v := index .Values "config" "extraRecords" }}
|
|
|
|
Record {{ $k }} {{ $v }}
|
|
|
|
{{- end }}
|
|
|
|
{{- end }}
|
|
|
|
|
|
|
|
[FILTER]
|
|
|
|
Name rewrite_tag
|
|
|
|
Match cri.*
|
|
|
|
Emitter_Name kube_tag_rewriter
|
2021-07-29 11:50:16 +00:00
|
|
|
Rule $kubernetes['pod_id'] .* kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false
|
2021-03-05 15:53:02 +00:00
|
|
|
|
|
|
|
[FILTER]
|
|
|
|
Name lua
|
|
|
|
Match kube.*
|
|
|
|
script /fluent-bit/scripts/kubezero.lua
|
|
|
|
call nest_k8s_ns
|
|
|
|
|
|
|
|
outputs: |
|
|
|
|
[OUTPUT]
|
|
|
|
Match *
|
|
|
|
Name forward
|
|
|
|
Host {{ .Values.config.output.host }}
|
|
|
|
Port 24224
|
|
|
|
Shared_Key {{ .Values.config.output.sharedKey }}
|
|
|
|
tls {{ ternary "on" "off" .Values.config.output.tls }}
|
|
|
|
Send_options true
|
|
|
|
Require_ack_response true
|
|
|
|
|
|
|
|
customParsers: |
|
|
|
|
[PARSER]
|
|
|
|
Name cri-log
|
|
|
|
Format regex
|
2021-07-20 11:28:37 +00:00
|
|
|
Regex ^(?<time>.+) (?<stream>stdout|stderr) (?<logtag>F|P) (?<log>.*)$
|
2021-03-05 15:53:02 +00:00
|
|
|
Time_Key time
|
|
|
|
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
|
|
|
|
|
|
|
|
luaScripts:
|
|
|
|
kubezero.lua: |
|
|
|
|
function nest_k8s_ns(tag, timestamp, record)
|
|
|
|
if not record['kubernetes']['namespace_name'] then
|
|
|
|
return 0, 0, 0
|
|
|
|
end
|
|
|
|
new_record = {}
|
|
|
|
for key, val in pairs(record) do
|
|
|
|
if key == 'kube' then
|
|
|
|
new_record[key] = {}
|
|
|
|
new_record[key][record['kubernetes']['namespace_name']] = record[key]
|
|
|
|
else
|
|
|
|
new_record[key] = record[key]
|
|
|
|
end
|
|
|
|
end
|
|
|
|
return 1, timestamp, new_record
|
|
|
|
end
|
2021-04-07 10:00:53 +00:00
|
|
|
|
|
|
|
daemonSetVolumes:
|
|
|
|
- name: varlog
|
|
|
|
hostPath:
|
|
|
|
path: /var/log
|
|
|
|
|
|
|
|
daemonSetVolumeMounts:
|
|
|
|
- name: varlog
|
|
|
|
mountPath: /var/log
|