feat: version bump logging for 1.21

This commit is contained in:
Stefan Reimer 2022-01-28 17:20:03 +01:00
parent c9d3c3b27e
commit 6bb90cfed0
23 changed files with 375 additions and 25 deletions

View File

@ -2,7 +2,7 @@ apiVersion: v2
name: kubezero-logging
description: KubeZero Umbrella Chart for complete EFK stack
type: application
version: 0.7.16
version: 0.7.17
appVersion: 1.6.0
home: https://kubezero.com
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@ -23,9 +23,9 @@ dependencies:
# repository: https://helm.elastic.co
condition: eck-operator.enabled
- name: fluentd
version: 0.3.0
version: 0.3.5
condition: fluentd.enabled
- name: fluent-bit
version: 0.19.2
version: 0.19.18
condition: fluent-bit.enabled
kubeVersion: ">= 1.18.0"

View File

@ -1,6 +1,6 @@
# kubezero-logging
![Version: 0.7.16](https://img.shields.io/badge/Version-0.7.16-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6.0](https://img.shields.io/badge/AppVersion-1.6.0-informational?style=flat-square)
![Version: 0.7.17](https://img.shields.io/badge/Version-0.7.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6.0](https://img.shields.io/badge/AppVersion-1.6.0-informational?style=flat-square)
KubeZero Umbrella Chart for complete EFK stack
@ -19,8 +19,8 @@ Kubernetes: `>= 1.18.0`
| Repository | Name | Version |
|------------|------|---------|
| | eck-operator | 1.6.0 |
| | fluent-bit | 0.19.2 |
| | fluentd | 0.3.0 |
| | fluent-bit | 0.19.18 |
| | fluentd | 0.3.5 |
| https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.3 |
## Changes from upstream
@ -59,7 +59,7 @@ Kubernetes: `>= 1.18.0`
|-----|------|---------|-------------|
| eck-operator.enabled | bool | `false` | |
| eck-operator.installCRDs | bool | `false` | |
| eck-operator.nodeSelector."node-role.kubernetes.io/master" | string | `""` | |
| eck-operator.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | |
| eck-operator.tolerations[0].effect | string | `"NoSchedule"` | |
| eck-operator.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | |
| elastic_password | string | `""` | |
@ -146,7 +146,7 @@ Kubernetes: `>= 1.18.0`
| kibana.istio.enabled | bool | `false` | |
| kibana.istio.gateway | string | `"istio-system/ingressgateway"` | |
| kibana.istio.url | string | `""` | |
| version | string | `"7.13.4"` | |
| version | string | `"7.16.3"` | |
## Resources:

View File

@ -1,9 +1,9 @@
annotations:
artifacthub.io/changes: |
- kind: changed
description: "Update Fluent Bit image to v1.8.9."
description: "Update fluent-bit image to 1.8.12."
apiVersion: v1
appVersion: 1.8.9
appVersion: 1.8.12
description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD
family operating systems.
home: https://fluentbit.io/
@ -19,7 +19,9 @@ maintainers:
name: naseemkullah
- email: towmeykaw@gmail.com
name: Towmeykaw
- email: steve.hipwell@gmail.com
name: stevehipwell
name: fluent-bit
sources:
- https://github.com/fluent/fluent-bit/
version: 0.19.2
version: 0.19.18

View File

@ -21,3 +21,37 @@ helm install fluent-bit fluent/fluent-bit
```sh
helm show values fluent/fluent-bit
```
## Using Lua scripts
Fluent Bit allows us to build filter to modify the incoming records using custom [Lua scripts.](https://docs.fluentbit.io/manual/pipeline/filters/lua)
### How to use Lua scripts with this Chart
First, you should add your Lua scripts to `luaScripts` in values.yaml, for example:
```yaml
luaScripts:
filter_example.lua: |
function filter_name(tag, timestamp, record)
-- put your lua code here.
end
```
After that, the Lua scripts will be ready to be used as filters. So next step is to add your Fluent bit [filter](https://docs.fluentbit.io/manual/concepts/data-pipeline/filter) to `config.filters` in values.yaml, for example:
```yaml
config:
filters: |
[FILTER]
Name lua
Match <your-tag>
script /fluent-bit/scripts/filter_example.lua
call filter_name
```
Under the hood, the chart will:
- Create a configmap using `luaScripts`.
- Add a volumeMounts for each Lua scripts using the path `/fluent-bit/scripts/<script>`.
- Add the Lua script's configmap as volume to the pod.
### Note
Remember to set the `script` attribute in the filter using `/fluent-bit/scripts/`, otherwise the file will not be found by fluent bit.

View File

@ -1,5 +1,6 @@
Get Fluent Bit build information by running these commands:
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "fluent-bit.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
echo "curl http://127.0.0.1:2020 for Fluent Bit build information"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 2020:2020
curl http://127.0.0.1:2020

View File

@ -61,3 +61,46 @@ Create the name of the service account to use
{{ default "default" .Values.serviceAccount.name }}
{{- end -}}
{{- end -}}
{{/*
Ingress ApiVersion according k8s version
*/}}
{{- define "fluent-bit.ingress.apiVersion" -}}
{{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion) -}}
networking.k8s.io/v1
{{- else if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1") (semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion) -}}
networking.k8s.io/v1beta1
{{- else -}}
extensions/v1beta1
{{- end }}
{{- end }}
{{/*
Return if ingress is stable.
*/}}
{{- define "fluent-bit.ingress.isStable" -}}
{{- eq (include "fluent-bit.ingress.apiVersion" .) "networking.k8s.io/v1" -}}
{{- end -}}
{{/*
Return if ingress supports ingressClassName.
*/}}
{{- define "fluent-bit.ingress.supportsIngressClassName" -}}
{{- or (eq (include "fluent-bit.ingress.isStable" .) "true") (and (eq (include "fluent-bit.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}}
{{- end -}}
{{/*
Return if ingress supports pathType.
*/}}
{{- define "fluent-bit.ingress.supportsPathType" -}}
{{- or (eq (include "fluent-bit.ingress.isStable" .) "true") (and (eq (include "fluent-bit.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}}
{{- end -}}
{{/*
Pdb apiVersion according k8s version and capabilities
*/}}
{{- define "fluent-bit.pdb.apiVersion" -}}
{{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">=1.21-0" .Capabilities.KubeVersion.GitVersion) -}}
policy/v1
{{- else -}}
policy/v1beta1
{{- end }}
{{- end -}}

View File

@ -37,9 +37,15 @@ containers:
{{- end }}
image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
{{- if .Values.env }}
{{- if or .Values.env .Values.envWithTpl }}
env:
{{- toYaml .Values.env | nindent 6 }}
{{- with .Values.env }}
{{- toYaml . | nindent 6 }}
{{- end }}
{{- range $item := .Values.envWithTpl }}
- name: {{ $item.name }}
value: {{ tpl $item.value $ | quote }}
{{- end }}
{{- end }}
{{- if .Values.envFrom }}
envFrom:
@ -55,7 +61,7 @@ containers:
{{- end }}
ports:
- name: http
containerPort: 2020
containerPort: {{ .Values.metricsPort }}
protocol: TCP
{{- if .Values.extraPorts }}
{{- range .Values.extraPorts }}
@ -64,6 +70,10 @@ containers:
protocol: {{ .protocol }}
{{- end }}
{{- end }}
{{- with .Values.lifecycle }}
lifecycle:
{{- toYaml . | nindent 6 }}
{{- end }}
livenessProbe:
{{- toYaml .Values.livenessProbe | nindent 6 }}
readinessProbe:

View File

@ -5,6 +5,9 @@ apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "fluent-bit.fullname" $ }}-dashboard-{{ trimSuffix ".json" (base $path) }}
{{- with $.Values.dashboards.namespace }}
namespace: {{ . }}
{{- end }}
{{- with $.Values.dashboards.annotations }}
annotations:
{{- toYaml . | nindent 4 -}}

View File

@ -15,6 +15,6 @@ data:
{{- (tpl .Values.config.outputs $) | nindent 4 }}
{{- range $key, $val := .Values.config.extraFiles }}
{{ $key }}: |
{{- $val | nindent 4 }}
{{- (tpl $val $) | nindent 4 }}
{{- end }}
{{- end -}}

View File

@ -13,7 +13,9 @@ metadata:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
{{- end }}
{{- with .Values.updateStrategy }}
strategy:
{{- toYaml . | nindent 4 }}

View File

@ -0,0 +1,39 @@
{{- if and ( eq .Values.kind "Deployment" ) .Values.autoscaling.enabled }}
apiVersion: autoscaling/v2beta2
kind: HorizontalPodAutoscaler
metadata:
name: {{ include "fluent-bit.fullname" . }}
labels:
{{- include "fluent-bit.labels" . | nindent 4 }}
spec:
{{- if .Values.autoscaling.behavior }}
behavior:
{{- toYaml .Values.autoscaling.behavior | nindent 4 }}
{{- end }}
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ include "fluent-bit.fullname" . }}
minReplicas: {{ .Values.autoscaling.minReplicas }}
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
metrics:
{{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
- type: Resource
resource:
name: cpu
target:
averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
type: Utilization
{{- end }}
{{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
- type: Resource
resource:
name: memory
target:
averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
type: Utilization
{{- end }}
{{- if .Values.autoscaling.customRules -}}
{{- toYaml .Values.autoscaling.customRules | nindent 4}}
{{- end -}}
{{- end }}

View File

@ -0,0 +1,62 @@
{{- $ingressApiIsStable := eq (include "fluent-bit.ingress.isStable" .) "true" -}}
{{- $ingressSupportsIngressClassName := eq (include "fluent-bit.ingress.supportsIngressClassName" .) "true" -}}
{{- $ingressSupportsPathType := eq (include "fluent-bit.ingress.supportsPathType" .) "true" -}}
{{- $fullName := include "fluent-bit.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- if and ( eq .Values.kind "Deployment" ) .Values.ingress.enabled }}
apiVersion: {{ include "fluent-bit.ingress.apiVersion" . }}
kind: Ingress
metadata:
name: {{ $fullName }}
labels:
{{- include "fluent-bit.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }}
ingressClassName: {{ .Values.ingress.ingressClassName }}
{{- end -}}
{{- if .Values.ingress.tls }}
tls:
{{- range .Values.ingress.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
{{- with .secretName }}
secretName: {{ . }}
{{- end }}
{{- end }}
{{- end }}
rules:
{{- range concat .Values.ingress.hosts .Values.ingress.extraHosts }}
- host: {{ .host | quote }}
http:
paths:
- path: /
{{- if $ingressSupportsPathType }}
pathType: Prefix
{{- end }}
backend:
{{- if $ingressApiIsStable }}
service:
name: {{ $fullName }}
port:
{{- if .port }}
number: {{ .port }}
{{- else }}
number: {{ $svcPort }}
{{- end }}
{{- else }}
serviceName: {{ $fullName }}
{{- if .port }}
servicePort: {{ .port }}
{{- else }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}

View File

@ -0,0 +1,20 @@
{{- if and ( eq .Values.kind "Deployment" ) .Values.podDisruptionBudget.enabled }}
apiVersion: {{ include "fluent-bit.pdb.apiVersion" . }}
kind: PodDisruptionBudget
metadata:
name: {{ include "fluent-bit.fullname" . }}
labels:
{{- include "fluent-bit.labels" . | nindent 4 }}
{{- with .Values.podDisruptionBudget.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }}
selector:
matchLabels:
{{- include "fluent-bit.selectorLabels" . | nindent 6 }}
{{- with .Values.labels }}
{{- toYaml . | nindent 6 }}
{{- end }}
{{- end }}

View File

@ -12,6 +12,9 @@ metadata:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if .Values.serviceMonitor.jobLabel }}
jobLabel: {{ .Values.serviceMonitor.jobLabel }}
{{- end }}
endpoints:
- port: http
path: /api/v1/metrics/prometheus
@ -21,6 +24,18 @@ spec:
{{- with .Values.serviceMonitor.scrapeTimeout }}
scrapeTimeout: {{ . }}
{{- end }}
{{- with .Values.serviceMonitor.metricRelabelings }}
metricRelabelings:
{{- if kindIs "string" . }}
{{- tpl . $ | nindent 8 }}
{{- else }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- with .Values.serviceMonitor.relabelings }}
relabelings:
{{- toYaml . | nindent 8 }}
{{- end }}
namespaceSelector:
matchNames:
- {{ .Release.Namespace }}

View File

@ -82,13 +82,31 @@ serviceMonitor:
# namespace: monitoring
# interval: 10s
# scrapeTimeout: 10s
# jobLabel: fluentbit
# selector:
# prometheus: my-prometheus
# ## metric relabel configs to apply to samples before ingestion.
# ##
# metricRelabelings:
# - sourceLabels: [__meta_kubernetes_service_label_cluster]
# targetLabel: cluster
# regex: (.*)
# replacement: ${1}
# action: replace
# ## relabel configs to apply to samples after ingestion.
# ##
# relabelings:
# - sourceLabels: [__meta_kubernetes_pod_node_name]
# separator: ;
# regex: ^(.*)$
# targetLabel: nodename
# replacement: $1
# action: replace
prometheusRule:
enabled: false
# namespace: ""
# additionnalLabels: {}
# additionalLabels: {}
# rules:
# - alert: NoOutputBytesProcessed
# expr: rate(fluentbit_output_proc_bytes_total[5m]) == 0
@ -105,6 +123,12 @@ dashboards:
enabled: false
labelKey: grafana_dashboard
annotations: {}
namespace: ""
lifecycle: {}
# preStop:
# exec:
# command: ["/bin/sh", "-c", "sleep 20"]
livenessProbe:
httpGet:
@ -124,6 +148,57 @@ resources: {}
# cpu: 100m
# memory: 128Mi
## only available if kind is Deployment
ingress:
enabled: false
className: ""
annotations: {}
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
hosts: []
# - host: fluent-bit.example.tld
extraHosts: []
# - host: fluent-bit-extra.example.tld
## specify extraPort number
# port: 5170
tls: []
# - secretName: fluent-bit-example-tld
# hosts:
# - fluent-bit.example.tld
## only available if kind is Deployment
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 3
targetCPUUtilizationPercentage: 75
# targetMemoryUtilizationPercentage: 75
## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics
customRules: []
# - type: Pods
# pods:
# metric:
# name: packets-per-second
# target:
# type: AverageValue
# averageValue: 1k
## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior
behavior: {}
# scaleDown:
# policies:
# - type: Pods
# value: 4
# periodSeconds: 60
# - type: Percent
# value: 10
# periodSeconds: 60
## only available if kind is Deployment
podDisruptionBudget:
enabled: false
annotations: {}
maxUnavailable: "30%"
nodeSelector: {}
tolerations: []
@ -141,6 +216,17 @@ podLabels: {}
priorityClassName: ""
env: []
# - name: FOO
# value: "bar"
# The envWithTpl array below has the same usage as "env", but is using the tpl function to support templatable string.
# This can be useful when you want to pass dynamic values to the Chart using the helm argument "--set <variable>=<value>"
# https://helm.sh/docs/howto/charts_tips_and_tricks/#using-the-tpl-function
envWithTpl: []
# - name: FOO_2
# value: "{{ .Values.foo2 }}"
#
# foo2: bar2
envFrom: []
@ -149,6 +235,8 @@ extraContainers: []
# image: busybox
# command: ['do', 'something']
metricsPort: 2020
extraPorts: []
# - port: 5170
# containerPort: 5170
@ -186,7 +274,7 @@ config:
Parsers_File custom_parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port {{ .Values.service.port }}
HTTP_Port {{ .Values.metricsPort }}
Health_Check On
## https://docs.fluentbit.io/manual/pipeline/inputs

View File

@ -1,5 +1,5 @@
apiVersion: v2
appVersion: v1.12.0
appVersion: v1.12.4
description: A Helm chart for Kubernetes
home: https://www.fluentd.org/
icon: https://www.fluentd.org/images/miscellany/fluentd-logo_2x.png
@ -12,4 +12,4 @@ name: fluentd
sources:
- https://github.com/fluent/fluentd/
- https://github.com/fluent/fluentd-kubernetes-daemonset
version: 0.3.0
version: 0.3.5

View File

@ -1,5 +1,5 @@
Get Fluentd build information by running these commands:
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "fluentd.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
echo "curl http://127.0.0.1:24231/metrics for Fluentd metrics"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 24231:24231
curl http://127.0.0.1:24231/metrics

View File

@ -10,6 +10,9 @@ priorityClassName: {{ .Values.priorityClassName }}
serviceAccountName: {{ include "fluentd.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 2 }}
{{- with .Values.terminationGracePeriodSeconds }}
terminationGracePeriodSeconds: {{ . }}
{{- end }}
containers:
- name: {{ .Chart.Name }}
securityContext:
@ -43,6 +46,10 @@ containers:
containerPort: {{ $port.containerPort }}
protocol: {{ $port.protocol }}
{{- end }}
{{- with .Values.lifecycle }}
lifecycle:
{{- toYaml . | nindent 6 }}
{{- end }}
livenessProbe:
{{- toYaml .Values.livenessProbe | nindent 6 }}
readinessProbe:

View File

@ -20,6 +20,9 @@ spec:
updateStrategy:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.minReadySeconds }}
minReadySeconds: {{ . }}
{{- end }}
template:
metadata:
annotations:

View File

@ -21,6 +21,9 @@ spec:
selector:
matchLabels:
{{- include "fluentd.selectorLabels" . | nindent 6 }}
{{- with .Values.minReadySeconds }}
minReadySeconds: {{ . }}
{{- end }}
template:
metadata:
annotations:

View File

@ -16,12 +16,15 @@ spec:
replicas: {{ .Values.replicaCount }}
serviceName: {{ include "fluentd.fullname" . }}
{{- with .Values.updateStrategy }}
strategy:
updateStrategy:
{{- toYaml . | nindent 4 }}
{{- end }}
selector:
matchLabels:
{{- include "fluentd.selectorLabels" . | nindent 6 }}
{{- with .Values.minReadySeconds }}
minReadySeconds: {{ . }}
{{- end }}
template:
metadata:
annotations:

View File

@ -49,7 +49,14 @@ securityContext: {}
# runAsNonRoot: true
# runAsUser: 1000
# Configure the livessProbe
# Configure the livecycle
# Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
lifecycle: {}
# preStop:
# exec:
# command: ["/bin/sh", "-c", "sleep 20"]
# Configure the livenessProbe
# Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
livenessProbe:
httpGet:
@ -140,6 +147,14 @@ podAnnotations: {}
##
podLabels: {}
## How long (in seconds) a pods needs to be stable before progressing the deployment
##
minReadySeconds:
## How long (in seconds) a pod may take to exit (useful with lifecycle hooks to ensure lb deregistration is done)
##
terminationGracePeriodSeconds:
## Deployment strategy / DaemonSet updateStrategy
##
updateStrategy: {}
@ -150,7 +165,7 @@ updateStrategy: {}
## Additional environment variables to set for fluentd pods
env:
- name: "FLUENTD_CONF"
value: "../../etc/fluent/fluent.conf"
value: "../../../etc/fluent/fluent.conf"
# - name: FLUENT_ELASTICSEARCH_HOST
# value: "elasticsearch-master"
# - name: FLUENT_ELASTICSEARCH_PORT

View File

@ -11,7 +11,7 @@ eck-operator:
node-role.kubernetes.io/control-plane: ""
# Version for ElasticSearch and Kibana have to match so we define it at top-level
version: 7.13.4
version: 7.16.3
elastic_password: "" # super_secret_elastic_password