ZeroDownTime/KubeZero
3
0
Fork 0
Code Issues 1 Pull Requests 23 Packages Projects Releases Wiki Activity

Add fluent-bit support to kuberzero-logging, istio fixes

Browse Source
This commit is contained in:
Stefan Reimer 2020-08-22 18:27:18 +01:00
parent 123d7ce946
commit 7310235fa2
9 changed files with 144 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
charts/fluent-bit Symbolic link
View File

@ -0,0 +1 @@
../../helm-charts/charts/fluent-bit

8
charts/kubezero-istio/templates/istio-private-ingress.yaml
View File

@ -45,11 +45,11 @@ spec:
{{- end }}
resources:
limits:
cpu: 2000m
# cpu: 2000m
memory: 1024Mi
requests:
cpu: 100m
memory: 128Mi
memory: 64Mi
strategy:
rollingUpdate:
maxSurge: 100%
@ -115,11 +115,11 @@ spec:
image: node-agent-k8s
resources:
limits:
#cpu: 2000m
cpu: 2000m
memory: 1024Mi
requests:
cpu: 100m
memory: 64Mi
memory: 128Mi
secretVolumes:
- mountPath: /etc/istio/ingressgateway-certs
name: ingressgateway-certs

2
charts/kubezero-lib/templates/_helpers.tpl
View File

@ -2,7 +2,7 @@
Common set of labels
*/ -}}
{{- define "kubezero-lib.labels" -}}
helm.sh/chart: {{ include "kubezero-lib.chart . }}
helm.sh/chart: {{ include "kubezero-lib.chart" . }}
app.kubernetes.io/name: {{ include "kubezero-lib.name" . }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
app.kubernetes.io/part-of: kubezero

7
charts/kubezero-logging/Chart.yaml
View File

@ -2,7 +2,7 @@ apiVersion: v2
name: kubezero-logging
description: KubeZero Umbrella Chart for complete EFK stack
type: application
version: 0.2.0
version: 0.3.1
appVersion: 1.2.1
home: https://kubezero.com
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@ -22,4 +22,9 @@ dependencies:
version: 2.5.1
repository: https://kubernetes-charts.storage.googleapis.com/
condition: fluentd.enabled
- name: fluent-bit
version: 0.6.3
repository: https://zero-down-time.github.io/kubezero/
# repository: https://fluent.github.io/helm-charts
condition: fluent-bit.enabled
kubeVersion: ">= 1.16.0"

8
charts/kubezero-logging/values-fluentbit.yaml Normal file
View File

@ -0,0 +1,8 @@
fluent-bit:
enabled: true
metrics:
enabled: true
url: fluentd.example.com

108
charts/kubezero-logging/values.yaml
View File

@ -125,7 +125,7 @@ fluentd:
id_key id
remove_keys id
# This pipeline incl. eg. GeoIP
# KubeZero pipeline incl. GeoIP etc.
pipeline fluentd
host "#{ENV['OUTPUT_HOST']}"
@ -193,3 +193,109 @@ fluentd:
# </grok>
# </parse>
# </filter>
fluent-bit:
enabled: true
test:
enabled: false
config:
outputs: |
[OUTPUT]
Match *
Name forward
Host fluentd
Port 24224
tls on
tls.verify off
Shared_Key cloudbender
inputs: |
[INPUT]
Name tail
Path /var/log/containers/*.log
Parser cri
Tag kube.*
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 10
DB /var/log/flb_kube.db
DB.Sync Normal
filters: |
[FILTER]
Name kubernetes
Match kube.*
Merge_Log On
Keep_Log Off
K8S-Logging.Parser On
K8S-Logging.Exclude On
[FILTER]
Name lua
Match kube.*
script /fluent-bit/etc/functions.lua
call dedot
customParsers: |
[PARSER]
# http://rubular.com/r/tjUt3Awgg4
Name cri
Format regex
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
# Decode_Field_As json log
service: |
[SERVICE]
Flush 5
Daemon Off
Log_Level warn
Parsers_File parsers.conf
Parsers_File custom_parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020
lua: |
function dedot(tag, timestamp, record)
if record["kubernetes"] == nil then
return 0, 0, 0
end
dedot_keys(record["kubernetes"]["annotations"])
dedot_keys(record["kubernetes"]["labels"])
return 1, timestamp, record
end
function dedot_keys(map)
if map == nil then
return
end
local new_map = {}
local changed_keys = {}
for k, v in pairs(map) do
local dedotted = string.gsub(k, "%.", "_")
if dedotted ~= k then
new_map[dedotted] = v
changed_keys[k] = true
end
end
for k in pairs(changed_keys) do
map[k] = nil
end
for k, v in pairs(new_map) do
map[k] = v
end
end
serviceMonitor:
enabled: true
namespace: monitoring
selector:
release: metrics
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule

12
deploy/templates/values.yaml
View File

@ -129,7 +129,9 @@ kubezero:
logging:
enabled: {{ .Values.logging.enabled }}
values:
{{- if .Values.logging.elastic_password }}
elastic_password: {{ .Values.logging.elastic_password }}
{{- end }}
{{- if .Values.logging.version }}
version: {{ .Values.logging.version }}
@ -176,6 +178,16 @@ kubezero:
{{- end }}
{{- end }}
fluent-bit:
enabled: {{ index .Values.logging "fluent-bit" "enabled" }}
metrics:
enabled: {{ .Values.metrics.enabled }}
{{- if index .Values.logging "fluent-bit" "config" }}
config:
{{- with index .Values.logging "fluent-bit" "config" }}
{{- toYaml . | nindent 10 }}
{{- end }}
{{- end }}
argo-cd:
controller:

2
deploy/values.yaml
View File

@ -37,6 +37,8 @@ logging:
enabled: false
fluentd:
enabled: false
fluent-bit:
enabled: false
argo-cd:
server: {}

5
scripts/publish.sh
View File

@ -1,6 +1,7 @@
#!/bin/bash
set -eux
CHARTS=${1:-'.*'}
# all credits go to the argoproj Helm guys https://github.com/argoproj/argo-helm
SRCROOT="$(cd "$(dirname "$0")/.." && pwd)"
@ -16,10 +17,10 @@ helm repo add jetstack https://charts.jetstack.io
helm repo add uswitch https://uswitch.github.io/kiam-helm-charts/charts/
helm repo update
for dir in $(find $SRCROOT/charts -mindepth 1 -maxdepth 1 -type d);
for dir in $(find -L $SRCROOT/charts -mindepth 1 -maxdepth 1 -type d);
do
name=$(basename $dir)
[[ $name =~ $CHARTS ]] || continue
if [ $(helm dep list $dir 2>/dev/null| wc -l) -gt 1 ]
then
echo "Processing chart dependencies"