feat: first step to migrate the logging pipeline into Telemetry
This commit is contained in:
parent
6a0df650a3
commit
71c0e96491
@ -1,6 +1,6 @@
|
||||
# kubezero-logging
|
||||
|
||||
  
|
||||
  
|
||||
|
||||
KubeZero Umbrella Chart for complete EFK stack
|
||||
|
||||
@ -19,7 +19,7 @@ Kubernetes: `>= 1.26.0`
|
||||
| Repository | Name | Version |
|
||||
|------------|------|---------|
|
||||
| https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
|
||||
| https://fluent.github.io/helm-charts | fluent-bit | 0.46.0 |
|
||||
| https://fluent.github.io/helm-charts | fluent-bit | 0.46.2 |
|
||||
| https://fluent.github.io/helm-charts | fluentd | 0.5.2 |
|
||||
|
||||
## Changes from upstream
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
annotations:
|
||||
artifacthub.io/changes: |
|
||||
- kind: changed
|
||||
description: "Updated _Fluent Bit_ OCI image to [v3.0.0](https://github.com/fluent/fluent-bit/releases/tag/v3.0.0)."
|
||||
description: "Updated _Fluent Bit_ OCI image to [v3.0.2](https://github.com/fluent/fluent-bit/releases/tag/v3.0.2)."
|
||||
apiVersion: v1
|
||||
appVersion: 3.0.1
|
||||
appVersion: 3.0.2
|
||||
description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD
|
||||
family operating systems.
|
||||
home: https://fluentbit.io/
|
||||
@ -24,4 +24,4 @@ maintainers:
|
||||
name: fluent-bit
|
||||
sources:
|
||||
- https://github.com/fluent/fluent-bit/
|
||||
version: 0.46.0
|
||||
version: 0.46.2
|
||||
|
||||
@ -25,4 +25,8 @@ dependencies:
|
||||
version: 3.0.3
|
||||
repository: https://jaegertracing.github.io/helm-charts
|
||||
condition: jaeger.enabled
|
||||
- name: fluentd
|
||||
version: 0.5.2
|
||||
repository: https://fluent.github.io/helm-charts
|
||||
condition: fluentd.enabled
|
||||
kubeVersion: ">= 1.26.0"
|
||||
|
||||
@ -19,6 +19,7 @@ Kubernetes: `>= 1.26.0`
|
||||
| Repository | Name | Version |
|
||||
|------------|------|---------|
|
||||
| https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
|
||||
| https://fluent.github.io/helm-charts | fluentd | 0.5.2 |
|
||||
| https://jaegertracing.github.io/helm-charts | jaeger | 3.0.3 |
|
||||
| https://open-telemetry.github.io/opentelemetry-helm-charts | opentelemetry-collector | 0.89.0 |
|
||||
|
||||
|
||||
23
charts/kubezero-telemetry/charts/fluentd/.helmignore
Normal file
23
charts/kubezero-telemetry/charts/fluentd/.helmignore
Normal file
@ -0,0 +1,23 @@
|
||||
# Patterns to ignore when building packages.
|
||||
# This supports shell glob matching, relative path matching, and
|
||||
# negation (prefixed with !). Only one pattern per line.
|
||||
.DS_Store
|
||||
# Common VCS dirs
|
||||
.git/
|
||||
.gitignore
|
||||
.bzr/
|
||||
.bzrignore
|
||||
.hg/
|
||||
.hgignore
|
||||
.svn/
|
||||
# Common backup files
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*.orig
|
||||
*~
|
||||
# Various IDEs
|
||||
.project
|
||||
.idea/
|
||||
*.tmproj
|
||||
.vscode/
|
||||
15
charts/kubezero-telemetry/charts/fluentd/Chart.yaml
Normal file
15
charts/kubezero-telemetry/charts/fluentd/Chart.yaml
Normal file
@ -0,0 +1,15 @@
|
||||
apiVersion: v2
|
||||
appVersion: v1.16.2
|
||||
description: A Helm chart for Kubernetes
|
||||
home: https://www.fluentd.org/
|
||||
icon: https://www.fluentd.org/images/miscellany/fluentd-logo_2x.png
|
||||
maintainers:
|
||||
- email: eduardo@treasure-data.com
|
||||
name: edsiper
|
||||
- email: diogo.filipe.tomas.guerra@cern.ch
|
||||
name: dioguerra
|
||||
name: fluentd
|
||||
sources:
|
||||
- https://github.com/fluent/fluentd/
|
||||
- https://github.com/fluent/fluentd-kubernetes-daemonset
|
||||
version: 0.5.2
|
||||
187
charts/kubezero-telemetry/charts/fluentd/README.md
Normal file
187
charts/kubezero-telemetry/charts/fluentd/README.md
Normal file
@ -0,0 +1,187 @@
|
||||
# Fluentd Helm Chart
|
||||
|
||||
[Fluentd](https://www.fluentd.org/) is an open source data collector for unified logging layer. Fluentd allows you to unify data collection and consumption for a better use and understanding of data.
|
||||
|
||||
## Installation
|
||||
|
||||
To add the `fluent` helm repo, run:
|
||||
|
||||
```sh
|
||||
helm repo add fluent https://fluent.github.io/helm-charts
|
||||
helm repo update
|
||||
```
|
||||
|
||||
To install a release named `fluentd`, run:
|
||||
|
||||
```sh
|
||||
helm install fluentd fluent/fluentd
|
||||
```
|
||||
## Upgrading
|
||||
|
||||
### To 0.4.0
|
||||
|
||||
Although the services will deploy and generally work, version 0.4.0 introduces some changes that are considered _breaking changes_. To upgrade, you should do the following to avoid any potential conflicts or problems:
|
||||
|
||||
- Add the `mountVarLogDirectory` and `mountDockerContainersDirectory` values and set them to the values you need; to follow the previous setup where these were mounted by default, set the values to `true`, e.g. `mountVarLogDirectory: true`
|
||||
- If you have the `varlog` mount point defined and enabled under both `volumes` and `volumeMounts`, set `mountVarLogDirectory` to true
|
||||
- If you have the `varlibdockercontainers` mount point defined and enabled under both `volumes` and `volumeMounts`, set `mountDockerContainersDirectory` to true
|
||||
- Remove the previous default volume and volume mount definitions - `etcfluentd-main`, `etcfluentd-config`, `varlog`, and `varlibdockercontainers`
|
||||
- Remove the `FLUENTD_CONF` entry from the `env:` list
|
||||
|
||||
## Chart Values
|
||||
|
||||
```sh
|
||||
helm show values fluent/fluentd
|
||||
```
|
||||
|
||||
## Value Details
|
||||
|
||||
### default-volumes
|
||||
|
||||
The default configurations bellow are required for the fluentd pod to be able to read the hosts container logs. The second section is responsible for allowing the user to load the "extra" configMaps either defined by the `fileConfigs` contained objects or, in addition, loaded externally and indicated by `configMapConfigs`.
|
||||
|
||||
```yaml
|
||||
- name: varlog
|
||||
hostPath:
|
||||
path: /var/log
|
||||
- name: varlibdockercontainers
|
||||
hostPath:
|
||||
path: /var/lib/docker/containers
|
||||
---
|
||||
- name: etcfluentd-main
|
||||
configMap:
|
||||
name: fluentd-main
|
||||
defaultMode: 0777
|
||||
- name: etcfluentd-config
|
||||
configMap:
|
||||
name: fluentd-config
|
||||
defaultMode: 0777
|
||||
```
|
||||
|
||||
### default-volumeMounts
|
||||
|
||||
The default configurations bellow are required for the fluentd pod to be able to read the hosts container logs. They should not be removed unless for some reason your container logs are accessible through a different path
|
||||
|
||||
```yaml
|
||||
- name: varlog
|
||||
mountPath: /var/log
|
||||
- name: varlibdockercontainers
|
||||
mountPath: /var/lib/docker/containers
|
||||
readOnly: true
|
||||
```
|
||||
|
||||
The section bellow is responsible for allowing the user to load the "extra" configMaps either defined by the `fileConfigs` contained objects or otherwise load externally and indicated by `configMapConfigs`.
|
||||
|
||||
```yaml
|
||||
- name: etcfluentd-main
|
||||
mountPath: /etc/fluent
|
||||
- name: etcfluentd-config
|
||||
mountPath: /etc/fluent/config.d/
|
||||
```
|
||||
|
||||
### default-fluentdConfig
|
||||
|
||||
The `fileConfigs` section is organized by sources -> filters -> destinations. Flow control must be configured using fluentd routing with tags or labels to guarantee that the configurations are executed as intended. Alternatively you can use numeration on your files to control the configurations loading order.
|
||||
|
||||
```yaml
|
||||
01_sources.conf: |-
|
||||
<source>
|
||||
@type tail
|
||||
@id in_tail_container_logs
|
||||
@label @KUBERNETES
|
||||
path /var/log/containers/*.log
|
||||
pos_file /var/log/fluentd-containers.log.pos
|
||||
tag kubernetes.*
|
||||
read_from_head true
|
||||
<parse>
|
||||
@type multi_format
|
||||
<pattern>
|
||||
format json
|
||||
time_key time
|
||||
time_type string
|
||||
time_format "%Y-%m-%dT%H:%M:%S.%NZ"
|
||||
keep_time_key false
|
||||
</pattern>
|
||||
<pattern>
|
||||
format regexp
|
||||
expression /^(?<time>.+) (?<stream>stdout|stderr)( (.))? (?<log>.*)$/
|
||||
time_format '%Y-%m-%dT%H:%M:%S.%NZ'
|
||||
keep_time_key false
|
||||
</pattern>
|
||||
</parse>
|
||||
emit_unmatched_lines true
|
||||
</source>
|
||||
|
||||
02_filters.conf: |-
|
||||
<label @KUBERNETES>
|
||||
<match kubernetes.var.log.containers.fluentd**>
|
||||
@type relabel
|
||||
@label @FLUENT_LOG
|
||||
</match>
|
||||
|
||||
# <match kubernetes.var.log.containers.**_kube-system_**>
|
||||
# @type null
|
||||
# @id ignore_kube_system_logs
|
||||
# </match>
|
||||
|
||||
<filter kubernetes.**>
|
||||
@type record_transformer
|
||||
enable_ruby
|
||||
<record>
|
||||
hostname ${record["kubernetes"]["host"]}
|
||||
raw ${record["log"]}
|
||||
</record>
|
||||
remove_keys $.kubernetes.host,log
|
||||
</filter>
|
||||
|
||||
<match **>
|
||||
@type relabel
|
||||
@label @DISPATCH
|
||||
</match>
|
||||
</label>
|
||||
|
||||
03_dispatch.conf: |-
|
||||
<label @DISPATCH>
|
||||
<filter **>
|
||||
@type prometheus
|
||||
<metric>
|
||||
name fluentd_input_status_num_records_total
|
||||
type counter
|
||||
desc The total number of incoming records
|
||||
<labels>
|
||||
tag ${tag}
|
||||
hostname ${hostname}
|
||||
</labels>
|
||||
</metric>
|
||||
</filter>
|
||||
|
||||
<match **>
|
||||
@type relabel
|
||||
@label @OUTPUT
|
||||
</match>
|
||||
</label>
|
||||
|
||||
04_outputs.conf: |-
|
||||
<label @OUTPUT>
|
||||
<match **>
|
||||
@type elasticsearch
|
||||
host "elasticsearch-master"
|
||||
port 9200
|
||||
path ""
|
||||
user elastic
|
||||
password changeme
|
||||
</match>
|
||||
</label>
|
||||
```
|
||||
|
||||
## Backwards Compatibility - v0.1.x
|
||||
|
||||
The old fluentd chart used the ENV variables and the default fluentd container definitions to set-up automatically many aspects of fluentd. It is still possible to trigger this behaviour by removing this charts current `.Values.env` configuration and replace by:
|
||||
|
||||
```yaml
|
||||
env:
|
||||
- name: FLUENT_ELASTICSEARCH_HOST
|
||||
value: "elasticsearch-master"
|
||||
- name: FLUENT_ELASTICSEARCH_PORT
|
||||
value: "9200"
|
||||
```
|
||||
3185
charts/kubezero-telemetry/charts/fluentd/dashboards/fluentd.json
Normal file
3185
charts/kubezero-telemetry/charts/fluentd/dashboards/fluentd.json
Normal file
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,5 @@
|
||||
Get Fluentd build information by running these commands:
|
||||
|
||||
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "fluentd.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
|
||||
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 24231:24231
|
||||
curl http://127.0.0.1:24231/metrics
|
||||
104
charts/kubezero-telemetry/charts/fluentd/templates/_helpers.tpl
Normal file
104
charts/kubezero-telemetry/charts/fluentd/templates/_helpers.tpl
Normal file
@ -0,0 +1,104 @@
|
||||
{{/* vim: set filetype=mustache: */}}
|
||||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "fluentd.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "fluentd.fullname" -}}
|
||||
{{- if .Values.fullnameOverride -}}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||
{{- if contains $name .Release.Name -}}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
{{- define "fluentd.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "fluentd.labels" -}}
|
||||
helm.sh/chart: {{ include "fluentd.chart" . }}
|
||||
{{ include "fluentd.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Selector labels
|
||||
*/}}
|
||||
{{- define "fluentd.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "fluentd.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create the name of the service account to use
|
||||
*/}}
|
||||
{{- define "fluentd.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccount.create -}}
|
||||
{{ default (include "fluentd.fullname" .) .Values.serviceAccount.name }}
|
||||
{{- else -}}
|
||||
{{ default "default" .Values.serviceAccount.name }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Shortened version of the releaseName, applied as a suffix to numerous resources.
|
||||
*/}}
|
||||
{{- define "fluentd.shortReleaseName" -}}
|
||||
{{- .Release.Name | trunc 35 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Name of the configMap used for the fluentd.conf configuration file; allows users to override the default.
|
||||
*/}}
|
||||
{{- define "fluentd.mainConfigMapName" -}}
|
||||
{{- if .Values.mainConfigMapNameOverride -}}
|
||||
{{ .Values.mainConfigMapNameOverride }}
|
||||
{{- else -}}
|
||||
{{ printf "%s-%s" "fluentd-main" ( include "fluentd.shortReleaseName" . ) }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Name of the configMap used for additional configuration files; allows users to override the default.
|
||||
*/}}
|
||||
{{- define "fluentd.extraFilesConfigMapName" -}}
|
||||
{{- if .Values.extraFilesConfigMapNameOverride -}}
|
||||
{{ printf "%s" .Values.extraFilesConfigMapNameOverride }}
|
||||
{{- else -}}
|
||||
{{ printf "%s-%s" "fluentd-config" ( include "fluentd.shortReleaseName" . ) }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
HPA ApiVersion according k8s version
|
||||
Check legacy first so helm template / kustomize will default to latest version
|
||||
*/}}
|
||||
{{- define "fluentd.hpa.apiVersion" -}}
|
||||
{{- if and (.Capabilities.APIVersions.Has "autoscaling/v2beta2") (semverCompare "<1.23-0" .Capabilities.KubeVersion.GitVersion) -}}
|
||||
autoscaling/v2beta2
|
||||
{{- else -}}
|
||||
autoscaling/v2
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
130
charts/kubezero-telemetry/charts/fluentd/templates/_pod.tpl
Normal file
130
charts/kubezero-telemetry/charts/fluentd/templates/_pod.tpl
Normal file
@ -0,0 +1,130 @@
|
||||
{{- define "fluentd.pod" -}}
|
||||
{{- $defaultTag := printf "%s-debian-%s-1.0" (.Chart.AppVersion) (.Values.variant) -}}
|
||||
{{- with .Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- if .Values.priorityClassName }}
|
||||
priorityClassName: {{ .Values.priorityClassName }}
|
||||
{{- end }}
|
||||
serviceAccountName: {{ include "fluentd.serviceAccountName" . }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.podSecurityContext | nindent 2 }}
|
||||
{{- with .Values.terminationGracePeriodSeconds }}
|
||||
terminationGracePeriodSeconds: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.initContainers }}
|
||||
initContainers:
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: {{ .Chart.Name }}
|
||||
securityContext:
|
||||
{{- toYaml .Values.securityContext | nindent 6 }}
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default $defaultTag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
{{- if .Values.plugins }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-c"
|
||||
- |
|
||||
{{- range $plugin := .Values.plugins }}
|
||||
{{- print "fluent-gem install " $plugin | nindent 6 }}
|
||||
{{- end }}
|
||||
exec /fluentd/entrypoint.sh
|
||||
{{- end }}
|
||||
env:
|
||||
- name: FLUENTD_CONF
|
||||
value: "../../../etc/fluent/fluent.conf"
|
||||
{{- if .Values.env }}
|
||||
{{- toYaml .Values.env | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- if .Values.envFrom }}
|
||||
envFrom:
|
||||
{{- toYaml .Values.envFrom | nindent 4 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 24231
|
||||
protocol: TCP
|
||||
{{- range $port := .Values.service.ports }}
|
||||
- name: {{ $port.name }}
|
||||
containerPort: {{ $port.containerPort }}
|
||||
protocol: {{ $port.protocol }}
|
||||
{{- end }}
|
||||
{{- with .Values.lifecycle }}
|
||||
lifecycle:
|
||||
{{- toYaml . | nindent 6 }}
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
{{- toYaml .Values.livenessProbe | nindent 6 }}
|
||||
readinessProbe:
|
||||
{{- toYaml .Values.readinessProbe | nindent 6 }}
|
||||
resources:
|
||||
{{- toYaml .Values.resources | nindent 8 }}
|
||||
volumeMounts:
|
||||
- name: etcfluentd-main
|
||||
mountPath: /etc/fluent
|
||||
- name: etcfluentd-config
|
||||
mountPath: /etc/fluent/config.d/
|
||||
{{- if .Values.mountVarLogDirectory }}
|
||||
- name: varlog
|
||||
mountPath: /var/log
|
||||
{{- end }}
|
||||
{{- if .Values.mountDockerContainersDirectory }}
|
||||
- name: varlibdockercontainers
|
||||
mountPath: /var/lib/docker/containers
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.volumeMounts -}}
|
||||
{{- toYaml .Values.volumeMounts | nindent 4 }}
|
||||
{{- end -}}
|
||||
{{- range $key := .Values.configMapConfigs }}
|
||||
{{- print "- name: " $key | nindent 4 }}
|
||||
{{- print "mountPath: /etc/fluent/" $key ".d" | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.persistence.enabled }}
|
||||
- mountPath: /var/log/fluent
|
||||
name: {{ include "fluentd.fullname" . }}-buffer
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: etcfluentd-main
|
||||
configMap:
|
||||
name: {{ include "fluentd.mainConfigMapName" . }}
|
||||
defaultMode: 0777
|
||||
- name: etcfluentd-config
|
||||
configMap:
|
||||
name: {{ include "fluentd.extraFilesConfigMapName" . }}
|
||||
defaultMode: 0777
|
||||
{{- if .Values.mountVarLogDirectory }}
|
||||
- name: varlog
|
||||
hostPath:
|
||||
path: /var/log
|
||||
{{- end }}
|
||||
{{- if .Values.mountDockerContainersDirectory }}
|
||||
- name: varlibdockercontainers
|
||||
hostPath:
|
||||
path: /var/lib/docker/containers
|
||||
{{- end }}
|
||||
{{- if .Values.volumes -}}
|
||||
{{- toYaml .Values.volumes | nindent 0 }}
|
||||
{{- end -}}
|
||||
{{- range $key := .Values.configMapConfigs }}
|
||||
{{- print "- name: " $key | nindent 0 }}
|
||||
configMap:
|
||||
{{- print "name: " $key "-" ( include "fluentd.shortReleaseName" $ ) | nindent 4 }}
|
||||
defaultMode: 0777
|
||||
{{- end }}
|
||||
{{- with .Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- with .Values.affinity }}
|
||||
affinity:
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- with .Values.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
@ -0,0 +1,28 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: {{ include "fluentd.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
rules:
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
- namespaces
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
- watch
|
||||
{{- if and .Values.podSecurityPolicy.enabled (semverCompare "<1.25-0" .Capabilities.KubeVersion.GitVersion) }}
|
||||
- apiGroups:
|
||||
- policy
|
||||
resourceNames:
|
||||
- {{ include "fluentd.fullname" . }}
|
||||
resources:
|
||||
- podsecuritypolicies
|
||||
verbs:
|
||||
- use
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
@ -0,0 +1,16 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: {{ include "fluentd.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: {{ include "fluentd.fullname" . }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "fluentd.serviceAccountName" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end -}}
|
||||
@ -0,0 +1,18 @@
|
||||
{{- if .Values.dashboards.enabled -}}
|
||||
{{- range $path, $_ := .Files.Glob "dashboards/*.json" }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: dashboard-{{ trimSuffix ".json" (base $path) }}-{{ include "fluentd.shortReleaseName" $ }}
|
||||
namespace: {{ $.Values.dashboards.namespace | default $.Release.Namespace }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" $ | nindent 4 }}
|
||||
{{- range $key, $val := $.Values.dashboards.labels }}
|
||||
{{ $key }}: {{ $val }}
|
||||
{{- end }}
|
||||
data:
|
||||
{{ base $path }}: |-
|
||||
{{- $.Files.Get $path | nindent 4 }}
|
||||
---
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
@ -0,0 +1,40 @@
|
||||
{{- if eq .Values.kind "DaemonSet" }}
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: {{ include "fluentd.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "fluentd.selectorLabels" . | nindent 6 }}
|
||||
{{- with .Values.updateStrategy }}
|
||||
updateStrategy:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.minReadySeconds }}
|
||||
minReadySeconds: {{ . }}
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
checksum/config: {{ include (print $.Template.BasePath "/fluentd-configurations-cm.yaml") . | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "fluentd.selectorLabels" . | nindent 8 }}
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- include "fluentd.pod" . | nindent 6 }}
|
||||
{{- end }}
|
||||
@ -0,0 +1,41 @@
|
||||
{{- if eq .Values.kind "Deployment" }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "fluentd.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
{{- with .Values.updateStrategy }}
|
||||
strategy:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "fluentd.selectorLabels" . | nindent 6 }}
|
||||
{{- with .Values.minReadySeconds }}
|
||||
minReadySeconds: {{ . }}
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
checksum/config: {{ include (print $.Template.BasePath "/fluentd-configurations-cm.yaml") . | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "fluentd.selectorLabels" . | nindent 8 }}
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- include "fluentd.pod" . | nindent 6 }}
|
||||
{{- end }}
|
||||
@ -0,0 +1,25 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
name: fluentd-prometheus-conf-{{ include "fluentd.shortReleaseName" . }}
|
||||
data:
|
||||
prometheus.conf: |-
|
||||
<source>
|
||||
@type prometheus
|
||||
@id in_prometheus
|
||||
bind "0.0.0.0"
|
||||
port 24231
|
||||
metrics_path "/metrics"
|
||||
</source>
|
||||
|
||||
<source>
|
||||
@type prometheus_monitor
|
||||
@id in_prometheus_monitor
|
||||
</source>
|
||||
|
||||
<source>
|
||||
@type prometheus_output_monitor
|
||||
@id in_prometheus_output_monitor
|
||||
</source>
|
||||
@ -0,0 +1,38 @@
|
||||
{{- if not .Values.extraFilesConfigMapNameOverride }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: fluentd-config-{{ include "fluentd.shortReleaseName" . }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
data:
|
||||
{{- range $key, $value := .Values.fileConfigs }}
|
||||
{{$key }}: |-
|
||||
{{- $value | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if not .Values.mainConfigMapNameOverride }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: fluentd-main-{{ include "fluentd.shortReleaseName" . }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
data:
|
||||
fluent.conf: |-
|
||||
# do not collect fluentd logs to avoid infinite loops.
|
||||
<label @FLUENT_LOG>
|
||||
<match **>
|
||||
@type null
|
||||
@id ignore_fluent_logs
|
||||
</match>
|
||||
</label>
|
||||
|
||||
@include config.d/*.conf
|
||||
{{- range $key := .Values.configMapConfigs }}
|
||||
{{- print "@include " $key ".d/*" | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
39
charts/kubezero-telemetry/charts/fluentd/templates/hpa.yaml
Normal file
39
charts/kubezero-telemetry/charts/fluentd/templates/hpa.yaml
Normal file
@ -0,0 +1,39 @@
|
||||
{{- if and ( eq .Values.kind "Deployment" ) .Values.autoscaling.enabled }}
|
||||
apiVersion: {{ include "fluentd.hpa.apiVersion" . }}
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: {{ include "fluentd.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
spec:
|
||||
{{- if .Values.autoscaling.behavior }}
|
||||
behavior:
|
||||
{{- toYaml .Values.autoscaling.behavior | nindent 4 }}
|
||||
{{- end }}
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: {{ include "fluentd.fullname" . }}
|
||||
minReplicas: {{ .Values.autoscaling.minReplicas }}
|
||||
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
|
||||
metrics:
|
||||
{{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
|
||||
type: Utilization
|
||||
{{- end }}
|
||||
{{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
target:
|
||||
averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
|
||||
type: Utilization
|
||||
{{- end }}
|
||||
{{- if .Values.autoscaling.customRules -}}
|
||||
{{- toYaml .Values.autoscaling.customRules | nindent 4}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
@ -0,0 +1,44 @@
|
||||
{{- if .Values.ingress.enabled -}}
|
||||
{{- $fullName := include "fluentd.fullname" . -}}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "fluentd.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.ingress.tls }}
|
||||
tls:
|
||||
{{- range .Values.ingress.tls }}
|
||||
- hosts:
|
||||
{{- range .hosts }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
{{- with .secretName }}
|
||||
secretName: {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.ingress.hosts }}
|
||||
- http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ $fullName }}
|
||||
port:
|
||||
number: {{ .port }}
|
||||
{{ if .host -}}
|
||||
host: {{ .host | quote }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
@ -0,0 +1,42 @@
|
||||
{{- if and .Values.podSecurityPolicy.enabled (semverCompare "<1.25-0" .Capabilities.KubeVersion.GitVersion) -}}
|
||||
apiVersion: policy/v1beta1
|
||||
kind: PodSecurityPolicy
|
||||
metadata:
|
||||
name: {{ include "fluentd.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
{{- if .Values.podSecurityPolicy.annotations }}
|
||||
annotations:
|
||||
{{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
privileged: false
|
||||
allowPrivilegeEscalation: false
|
||||
requiredDropCapabilities:
|
||||
- ALL
|
||||
hostNetwork: false
|
||||
hostIPC: false
|
||||
hostPID: false
|
||||
volumes:
|
||||
- 'configMap'
|
||||
- 'secret'
|
||||
- 'hostPath'
|
||||
{{- if .Values.persistence.enabled }}
|
||||
- 'persistentVolumeClaim'
|
||||
{{- end }}
|
||||
runAsUser:
|
||||
rule: 'RunAsAny'
|
||||
seLinux:
|
||||
rule: 'RunAsAny'
|
||||
supplementalGroups:
|
||||
rule: 'MustRunAs'
|
||||
ranges:
|
||||
- min: 1
|
||||
max: 65535
|
||||
fsGroup:
|
||||
rule: 'MustRunAs'
|
||||
ranges:
|
||||
- min: 1
|
||||
max: 65535
|
||||
readOnlyRootFilesystem: false
|
||||
{{- end }}
|
||||
@ -0,0 +1,21 @@
|
||||
{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.metrics.prometheusRule.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: {{ template "fluentd.fullname" . }}
|
||||
{{- if .Values.metrics.prometheusRule.namespace }}
|
||||
namespace: {{ .Values.metrics.prometheusRule.namespace }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
{{- with .Values.metrics.prometheusRule.additionalLabels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- with .Values.metrics.prometheusRule.rules }}
|
||||
groups:
|
||||
- name: {{ template "fluentd.fullname" $ }}
|
||||
rules:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@ -0,0 +1,35 @@
|
||||
{{- if .Values.service.enabled -}}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "fluentd.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
{{- with .Values.service.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
type: {{ .Values.service.type }}
|
||||
{{- if .Values.service.loadBalancerIP }}
|
||||
loadBalancerIP: {{ .Values.service.loadBalancerIP }}
|
||||
{{- end }}
|
||||
{{- if .Values.service.externalTrafficPolicy }}
|
||||
externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- port: 24231
|
||||
targetPort: metrics
|
||||
protocol: TCP
|
||||
name: metrics
|
||||
{{- if .Values.service.ports }}
|
||||
{{- range $port := .Values.service.ports }}
|
||||
- name: {{ $port.name }}
|
||||
port: {{ $port.containerPort }}
|
||||
targetPort: {{ $port.containerPort }}
|
||||
protocol: {{ $port.protocol }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
selector:
|
||||
{{- include "fluentd.selectorLabels" . | nindent 4 }}
|
||||
{{- end -}}
|
||||
@ -0,0 +1,12 @@
|
||||
{{- if .Values.serviceAccount.create -}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "fluentd.serviceAccountName" . }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
{{- with .Values.serviceAccount.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
@ -0,0 +1,44 @@
|
||||
{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) .Values.metrics.serviceMonitor.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: {{ template "fluentd.fullname" . }}
|
||||
{{- with .Values.metrics.serviceMonitor.namespace }}
|
||||
namespace: {{ . }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
{{- with .Values.metrics.serviceMonitor.additionalLabels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
jobLabel: {{ .Values.metrics.serviceMonitor.jobLabel | default .Release.Name }}
|
||||
endpoints:
|
||||
- port: metrics
|
||||
path: /metrics
|
||||
{{- with .Values.metrics.serviceMonitor.interval }}
|
||||
interval: {{ . }}
|
||||
{{- end }}
|
||||
{{- with .Values.metrics.serviceMonitor.scrapeTimeout }}
|
||||
scrapeTimeout: {{ . }}
|
||||
{{- end }}
|
||||
{{- if .Values.metrics.serviceMonitor.metricRelabelings }}
|
||||
metricRelabelings:
|
||||
{{ tpl (toYaml .Values.metrics.serviceMonitor.metricRelabelings | indent 6) . }}
|
||||
{{- end }}
|
||||
{{- if .Values.metrics.serviceMonitor.relabelings }}
|
||||
relabelings:
|
||||
{{ toYaml .Values.metrics.serviceMonitor.relabelings | indent 6 }}
|
||||
{{- end }}
|
||||
{{- if .Values.metrics.serviceMonitor.namespaceSelector }}
|
||||
namespaceSelector:
|
||||
{{ toYaml .Values.metrics.serviceMonitor.namespaceSelector | indent 4 -}}
|
||||
{{ else }}
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- {{ .Release.Namespace }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "fluentd.selectorLabels" . | nindent 6 }}
|
||||
{{- end }}
|
||||
@ -0,0 +1,55 @@
|
||||
{{- if eq .Values.kind "StatefulSet" }}
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: {{ include "fluentd.fullname" . }}
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
{{- with .Values.labels }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- with .Values.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
serviceName: {{ include "fluentd.fullname" . }}
|
||||
{{- with .Values.updateStrategy }}
|
||||
updateStrategy:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "fluentd.selectorLabels" . | nindent 6 }}
|
||||
{{- with .Values.minReadySeconds }}
|
||||
minReadySeconds: {{ . }}
|
||||
{{- end }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
checksum/config: {{ include (print $.Template.BasePath "/fluentd-configurations-cm.yaml") . | sha256sum }}
|
||||
{{- with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "fluentd.selectorLabels" . | nindent 8 }}
|
||||
{{- with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- include "fluentd.pod" . | nindent 6 }}
|
||||
{{- if or .Values.persistence.enabled }}
|
||||
volumeClaimTemplates:
|
||||
{{- if or .Values.persistence.enabled }}
|
||||
- metadata:
|
||||
name: {{ include "fluentd.fullname" . }}-buffer
|
||||
spec:
|
||||
accessModes: [{{ .Values.persistence.accessMode }}]
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ .Values.persistence.size }}
|
||||
storageClassName: {{ .Values.persistence.storageClass }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@ -0,0 +1,29 @@
|
||||
{{/*
|
||||
Target the very simple case where
|
||||
fluentd is deployed with the default values
|
||||
If the fluentd config is overriden and the metrics server removed
|
||||
this will fail.
|
||||
*/}}
|
||||
{{ if empty .Values.service.ports }}
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: "{{ include "fluentd.fullname" . }}-test-connection"
|
||||
labels:
|
||||
{{- include "fluentd.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/hook": test-success
|
||||
spec:
|
||||
containers:
|
||||
- name: wget
|
||||
image: busybox
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
# Give fluentd some time to start up
|
||||
while :; do nc -vz {{ include "fluentd.fullname" . }}:24231 && break; sleep 1; done
|
||||
wget '{{ include "fluentd.fullname" . }}:24231/metrics'
|
||||
restartPolicy: Never
|
||||
{{ end }}
|
||||
403
charts/kubezero-telemetry/charts/fluentd/values.yaml
Normal file
403
charts/kubezero-telemetry/charts/fluentd/values.yaml
Normal file
@ -0,0 +1,403 @@
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
# DaemonSet, Deployment or StatefulSet
|
||||
kind: "DaemonSet"
|
||||
# azureblob, cloudwatch, elasticsearch7, elasticsearch8, gcs, graylog , kafka, kafka2, kinesis, opensearch
|
||||
variant: elasticsearch7
|
||||
# # Only applicable for Deployment or StatefulSet
|
||||
# replicaCount: 1
|
||||
|
||||
image:
|
||||
repository: "fluent/fluentd-kubernetes-daemonset"
|
||||
pullPolicy: "IfNotPresent"
|
||||
tag: ""
|
||||
|
||||
## Optional array of imagePullSecrets containing private registry credentials
|
||||
## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
|
||||
imagePullSecrets: []
|
||||
|
||||
serviceAccount:
|
||||
create: true
|
||||
annotations: {}
|
||||
name: null
|
||||
|
||||
rbac:
|
||||
create: true
|
||||
|
||||
# from Kubernetes 1.25, PSP is deprecated
|
||||
# See: https://kubernetes.io/blog/2022/08/23/kubernetes-v1-25-release/#pod-security-changes
|
||||
# We automatically disable PSP if Kubernetes version is 1.25 or higher
|
||||
podSecurityPolicy:
|
||||
enabled: true
|
||||
annotations: {}
|
||||
|
||||
## Security Context policies for controller pods
|
||||
## See https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/ for
|
||||
## notes on enabling and using sysctls
|
||||
##
|
||||
podSecurityContext: {}
|
||||
# seLinuxOptions:
|
||||
# type: "spc_t"
|
||||
|
||||
securityContext: {}
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# readOnlyRootFilesystem: true
|
||||
# runAsNonRoot: true
|
||||
# runAsUser: 1000
|
||||
|
||||
# Configure the livecycle
|
||||
# Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
|
||||
lifecycle: {}
|
||||
# preStop:
|
||||
# exec:
|
||||
# command: ["/bin/sh", "-c", "sleep 20"]
|
||||
|
||||
# Configure the livenessProbe
|
||||
# Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /metrics
|
||||
port: metrics
|
||||
# initialDelaySeconds: 0
|
||||
# periodSeconds: 10
|
||||
# timeoutSeconds: 1
|
||||
# successThreshold: 1
|
||||
# failureThreshold: 3
|
||||
|
||||
# Configure the readinessProbe
|
||||
# Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /metrics
|
||||
port: metrics
|
||||
# initialDelaySeconds: 0
|
||||
# periodSeconds: 10
|
||||
# timeoutSeconds: 1
|
||||
# successThreshold: 1
|
||||
# failureThreshold: 3
|
||||
|
||||
resources: {}
|
||||
# requests:
|
||||
# cpu: 10m
|
||||
# memory: 128Mi
|
||||
# limits:
|
||||
# memory: 128Mi
|
||||
|
||||
## only available if kind is Deployment
|
||||
autoscaling:
|
||||
enabled: false
|
||||
minReplicas: 1
|
||||
maxReplicas: 100
|
||||
targetCPUUtilizationPercentage: 80
|
||||
# targetMemoryUtilizationPercentage: 80
|
||||
## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics
|
||||
customRules: []
|
||||
# - type: Pods
|
||||
# pods:
|
||||
# metric:
|
||||
# name: packets-per-second
|
||||
# target:
|
||||
# type: AverageValue
|
||||
# averageValue: 1k
|
||||
## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior
|
||||
# behavior:
|
||||
# scaleDown:
|
||||
# policies:
|
||||
# - type: Pods
|
||||
# value: 4
|
||||
# periodSeconds: 60
|
||||
# - type: Percent
|
||||
# value: 10
|
||||
# periodSeconds: 60
|
||||
|
||||
# priorityClassName: "system-node-critical"
|
||||
|
||||
nodeSelector: {}
|
||||
|
||||
## Node tolerations for server scheduling to nodes with taints
|
||||
## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
|
||||
##
|
||||
tolerations: []
|
||||
# - key: null
|
||||
# operator: Exists
|
||||
# effect: "NoSchedule"
|
||||
|
||||
## Affinity and anti-affinity
|
||||
## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
|
||||
##
|
||||
affinity: {}
|
||||
|
||||
## Annotations to be added to fluentd DaemonSet/Deployment
|
||||
##
|
||||
annotations: {}
|
||||
|
||||
## Labels to be added to fluentd DaemonSet/Deployment
|
||||
##
|
||||
labels: {}
|
||||
|
||||
## Annotations to be added to fluentd pods
|
||||
##
|
||||
podAnnotations: {}
|
||||
|
||||
## Labels to be added to fluentd pods
|
||||
##
|
||||
podLabels: {}
|
||||
|
||||
## How long (in seconds) a pods needs to be stable before progressing the deployment
|
||||
##
|
||||
minReadySeconds:
|
||||
|
||||
## How long (in seconds) a pod may take to exit (useful with lifecycle hooks to ensure lb deregistration is done)
|
||||
##
|
||||
terminationGracePeriodSeconds:
|
||||
|
||||
## Deployment strategy / DaemonSet updateStrategy
|
||||
##
|
||||
updateStrategy: {}
|
||||
# type: RollingUpdate
|
||||
# rollingUpdate:
|
||||
# maxUnavailable: 1
|
||||
|
||||
## Additional environment variables to set for fluentd pods
|
||||
env: []
|
||||
# - name: "FLUENTD_CONF"
|
||||
# value: "../../../etc/fluent/fluent.conf"
|
||||
# - name: FLUENT_ELASTICSEARCH_HOST
|
||||
# value: "elasticsearch-master"
|
||||
# - name: FLUENT_ELASTICSEARCH_PORT
|
||||
# value: "9200"
|
||||
|
||||
envFrom: []
|
||||
|
||||
initContainers: []
|
||||
|
||||
## Name of the configMap containing a custom fluentd.conf configuration file to use instead of the default.
|
||||
# mainConfigMapNameOverride: ""
|
||||
|
||||
## Name of the configMap containing files to be placed under /etc/fluent/config.d/
|
||||
## NOTE: This will replace ALL default files in the aforementioned path!
|
||||
# extraFilesConfigMapNameOverride: ""
|
||||
|
||||
mountVarLogDirectory: true
|
||||
mountDockerContainersDirectory: true
|
||||
|
||||
volumes: []
|
||||
|
||||
volumeMounts: []
|
||||
|
||||
## Only available if kind is StatefulSet
|
||||
## Fluentd persistence
|
||||
##
|
||||
persistence:
|
||||
enabled: false
|
||||
storageClass: ""
|
||||
accessMode: ReadWriteOnce
|
||||
size: 10Gi
|
||||
|
||||
## Fluentd service
|
||||
##
|
||||
service:
|
||||
enabled: true
|
||||
type: "ClusterIP"
|
||||
annotations: {}
|
||||
# loadBalancerIP:
|
||||
# externalTrafficPolicy: Local
|
||||
ports: []
|
||||
# - name: "forwarder"
|
||||
# protocol: TCP
|
||||
# containerPort: 24224
|
||||
|
||||
## Prometheus Monitoring
|
||||
##
|
||||
metrics:
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
additionalLabels:
|
||||
release: prometheus-operator
|
||||
namespace: ""
|
||||
namespaceSelector: {}
|
||||
## metric relabel configs to apply to samples before ingestion.
|
||||
##
|
||||
metricRelabelings: []
|
||||
# - sourceLabels: [__name__]
|
||||
# separator: ;
|
||||
# regex: ^fluentd_output_status_buffer_(oldest|newest)_.+
|
||||
# replacement: $1
|
||||
# action: drop
|
||||
## relabel configs to apply to samples after ingestion.
|
||||
##
|
||||
relabelings: []
|
||||
# - sourceLabels: [__meta_kubernetes_pod_node_name]
|
||||
# separator: ;
|
||||
# regex: ^(.*)$
|
||||
# targetLabel: nodename
|
||||
# replacement: $1
|
||||
# action: replace
|
||||
## Additional serviceMonitor config
|
||||
##
|
||||
# jobLabel: fluentd
|
||||
# scrapeInterval: 30s
|
||||
# scrapeTimeout: 5s
|
||||
# honorLabels: true
|
||||
|
||||
prometheusRule:
|
||||
enabled: false
|
||||
additionalLabels: {}
|
||||
namespace: ""
|
||||
rules: []
|
||||
# - alert: FluentdDown
|
||||
# expr: up{job="fluentd"} == 0
|
||||
# for: 5m
|
||||
# labels:
|
||||
# context: fluentd
|
||||
# severity: warning
|
||||
# annotations:
|
||||
# summary: "Fluentd Down"
|
||||
# description: "{{ $labels.pod }} on {{ $labels.nodename }} is down"
|
||||
# - alert: FluentdScrapeMissing
|
||||
# expr: absent(up{job="fluentd"} == 1)
|
||||
# for: 15m
|
||||
# labels:
|
||||
# context: fluentd
|
||||
# severity: warning
|
||||
# annotations:
|
||||
# summary: "Fluentd Scrape Missing"
|
||||
# description: "Fluentd instance has disappeared from Prometheus target discovery"
|
||||
|
||||
## Grafana Monitoring Dashboard
|
||||
##
|
||||
dashboards:
|
||||
enabled: "true"
|
||||
namespace: ""
|
||||
labels:
|
||||
grafana_dashboard: '"1"'
|
||||
|
||||
## Fluentd list of plugins to install
|
||||
##
|
||||
plugins: []
|
||||
# - fluent-plugin-out-http
|
||||
|
||||
## Add fluentd config files from K8s configMaps
|
||||
##
|
||||
configMapConfigs: []
|
||||
# - fluentd-prometheus-conf
|
||||
# - fluentd-systemd-conf
|
||||
|
||||
## Fluentd configurations:
|
||||
##
|
||||
fileConfigs:
|
||||
01_sources.conf: |-
|
||||
## logs from podman
|
||||
<source>
|
||||
@type tail
|
||||
@id in_tail_container_logs
|
||||
@label @KUBERNETES
|
||||
path /var/log/containers/*.log
|
||||
pos_file /var/log/fluentd-containers.log.pos
|
||||
tag kubernetes.*
|
||||
read_from_head true
|
||||
<parse>
|
||||
@type multi_format
|
||||
<pattern>
|
||||
format json
|
||||
time_key time
|
||||
time_type string
|
||||
time_format "%Y-%m-%dT%H:%M:%S.%NZ"
|
||||
keep_time_key false
|
||||
</pattern>
|
||||
<pattern>
|
||||
format regexp
|
||||
expression /^(?<time>.+) (?<stream>stdout|stderr)( (.))? (?<log>.*)$/
|
||||
time_format '%Y-%m-%dT%H:%M:%S.%NZ'
|
||||
keep_time_key false
|
||||
</pattern>
|
||||
</parse>
|
||||
emit_unmatched_lines true
|
||||
</source>
|
||||
|
||||
# expose metrics in prometheus format
|
||||
<source>
|
||||
@type prometheus
|
||||
bind 0.0.0.0
|
||||
port 24231
|
||||
metrics_path /metrics
|
||||
</source>
|
||||
|
||||
02_filters.conf: |-
|
||||
<label @KUBERNETES>
|
||||
<match kubernetes.var.log.containers.fluentd**>
|
||||
@type relabel
|
||||
@label @FLUENT_LOG
|
||||
</match>
|
||||
|
||||
# <match kubernetes.var.log.containers.**_kube-system_**>
|
||||
# @type null
|
||||
# @id ignore_kube_system_logs
|
||||
# </match>
|
||||
|
||||
<filter kubernetes.**>
|
||||
@type kubernetes_metadata
|
||||
@id filter_kube_metadata
|
||||
skip_labels false
|
||||
skip_container_metadata false
|
||||
skip_namespace_metadata true
|
||||
skip_master_url true
|
||||
</filter>
|
||||
|
||||
<match **>
|
||||
@type relabel
|
||||
@label @DISPATCH
|
||||
</match>
|
||||
</label>
|
||||
|
||||
03_dispatch.conf: |-
|
||||
<label @DISPATCH>
|
||||
<filter **>
|
||||
@type prometheus
|
||||
<metric>
|
||||
name fluentd_input_status_num_records_total
|
||||
type counter
|
||||
desc The total number of incoming records
|
||||
<labels>
|
||||
tag ${tag}
|
||||
hostname ${hostname}
|
||||
</labels>
|
||||
</metric>
|
||||
</filter>
|
||||
|
||||
<match **>
|
||||
@type relabel
|
||||
@label @OUTPUT
|
||||
</match>
|
||||
</label>
|
||||
|
||||
04_outputs.conf: |-
|
||||
<label @OUTPUT>
|
||||
<match **>
|
||||
@type elasticsearch
|
||||
host "elasticsearch-master"
|
||||
port 9200
|
||||
path ""
|
||||
user elastic
|
||||
password changeme
|
||||
# Don't wait for elastic to start up.
|
||||
verify_es_version_at_startup false
|
||||
</match>
|
||||
</label>
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
annotations: {}
|
||||
# kubernetes.io/ingress.class: nginx
|
||||
# kubernetes.io/tls-acme: "true"
|
||||
hosts:
|
||||
# - host: fluentd.example.tld
|
||||
- port: 9880
|
||||
|
||||
tls: []
|
||||
# - secretName: fluentd-tls
|
||||
# hosts:
|
||||
# - fluentd.example.tld
|
||||
@ -12,3 +12,10 @@ dashboards:
|
||||
tags:
|
||||
- OpenSearch
|
||||
- Telemetry
|
||||
- name: fluent-logging
|
||||
url: https://grafana.com/api/dashboards/7752/revisions/6/download
|
||||
#url: https://grafana.com/api/dashboards/13042/revisions/2/download
|
||||
tags:
|
||||
- fluentd
|
||||
- fluent-bit
|
||||
- Telemetry
|
||||
|
||||
File diff suppressed because one or more lines are too long
@ -3,9 +3,20 @@ set -ex
|
||||
|
||||
. ../../scripts/lib-update.sh
|
||||
|
||||
../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/grafana-dashboards.yaml
|
||||
|
||||
#login_ecr_public
|
||||
update_helm
|
||||
|
||||
#FLUENT_BIT_VERSION=$(yq eval '.dependencies[] | select(.name=="fluent-bit") | .version' Chart.yaml)
|
||||
FLUENTD_VERSION=$(yq eval '.dependencies[] | select(.name=="fluentd") | .version' Chart.yaml)
|
||||
|
||||
# fluent-bit
|
||||
#patch_chart fluent-bit
|
||||
|
||||
# FluentD
|
||||
patch_chart fluentd
|
||||
rm -f charts/fluentd/templates/files.conf/systemd.yaml
|
||||
|
||||
# Fetch dashboards from Grafana.com and update ZDT CM
|
||||
../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/grafana-dashboards.yaml
|
||||
|
||||
update_docs
|
||||
|
||||
Loading…
Reference in New Issue
Block a user