From 1c076eab61fcd267034bebf748148def4da5bc12 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 28 Nov 2023 18:42:00 +0000 Subject: [PATCH] Logging version bump, metrics fixes --- admin/migrate_argo_values.py | 7 +++ charts/kubeadm/templates/_helpers.tpl | 1 + charts/kubezero-logging/README.md | 26 ++++------- .../charts/fluent-bit/Chart.yaml | 8 ++-- .../fluent-bit/dashboards/fluent-bit.json | 20 ++++----- .../charts/fluentd/Chart.yaml | 4 +- .../charts/fluentd/templates/_pod.tpl | 2 +- .../charts/fluentd/templates/ingress.yaml | 44 +++++++++++++++++++ .../charts/fluentd/templates/service.yaml | 3 +- .../charts/fluentd/values.yaml | 18 +++++++- charts/kubezero-logging/update.sh | 2 + charts/kubezero-logging/values.yaml | 9 ---- charts/kubezero-metrics/values.yaml | 22 +++++++--- charts/kubezero/values.yaml | 2 +- docs/v1.27.md | 6 +++ 15 files changed, 122 insertions(+), 52 deletions(-) create mode 100644 charts/kubezero-logging/charts/fluentd/templates/ingress.yaml diff --git a/admin/migrate_argo_values.py b/admin/migrate_argo_values.py index 7f03e85..75a20b6 100755 --- a/admin/migrate_argo_values.py +++ b/admin/migrate_argo_values.py @@ -15,6 +15,13 @@ def migrate(values): if not values["addons"]: values.pop("addons") + # fix argoCD CM + try: + if not values["argocd"]["configs"]["cm"]["url"].startswith("http"): + values["argocd"]["configs"]["cm"]["url"] = "https://" + values["argocd"]["configs"]["cm"]["url"] + except KeyError: + pass + # migrate eck operator to new operator module try: if values["logging"]["eck-operator"]["enabled"]: diff --git a/charts/kubeadm/templates/_helpers.tpl b/charts/kubeadm/templates/_helpers.tpl index 2b3ac87..f78c39f 100644 --- a/charts/kubeadm/templates/_helpers.tpl +++ b/charts/kubeadm/templates/_helpers.tpl @@ -1,4 +1,5 @@ {{- /* Feature gates for all control plane components */ -}} +{{- /* ToAdd: "PodAndContainerStatsFromCRI" */ -}} {{- define "kubeadm.featuregates" }} {{- $gates := list "CustomCPUCFSQuotaPeriod" "MemoryQoS" }} {{- if eq .return "csv" }} diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md index 79697ca..7e5e6d7 100644 --- a/charts/kubezero-logging/README.md +++ b/charts/kubezero-logging/README.md @@ -1,6 +1,6 @@ # kubezero-logging -![Version: 0.8.6](https://img.shields.io/badge/Version-0.8.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6.0](https://img.shields.io/badge/AppVersion-1.6.0-informational?style=flat-square) +![Version: 0.8.9](https://img.shields.io/badge/Version-0.8.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6.0](https://img.shields.io/badge/AppVersion-1.6.0-informational?style=flat-square) KubeZero Umbrella Chart for complete EFK stack @@ -14,14 +14,13 @@ KubeZero Umbrella Chart for complete EFK stack ## Requirements -Kubernetes: `>= 1.24.0` +Kubernetes: `>= 1.26.0` | Repository | Name | Version | |------------|------|---------| -| | eck-operator | 2.4.0 | -| | fluent-bit | 0.24.0 | -| | fluentd | 0.3.9 | | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 | +| https://fluent.github.io/helm-charts | fluent-bit | 0.40.0 | +| https://fluent.github.io/helm-charts | fluentd | 0.5.0 | ## Changes from upstream ### ECK @@ -88,9 +87,9 @@ Kubernetes: `>= 1.24.0` | fluent-bit.daemonSetVolumes[1].hostPath.path | string | `"/var/lib/containers/logs"` | | | fluent-bit.daemonSetVolumes[1].name | string | `"newlog"` | | | fluent-bit.enabled | bool | `false` | | -| fluent-bit.image.tag | string | `"2.0.10"` | | +| fluent-bit.image | string | `nil` | | | fluent-bit.luaScripts."kubezero.lua" | string | `"function nest_k8s_ns(tag, timestamp, record)\n if not record['kubernetes']['namespace_name'] then\n return 0, 0, 0\n end\n new_record = {}\n for key, val in pairs(record) do\n if key == 'kube' then\n new_record[key] = {}\n new_record[key][record['kubernetes']['namespace_name']] = record[key]\n else\n new_record[key] = record[key]\n end\n end\n return 1, timestamp, new_record\nend\n"` | | -| fluent-bit.resources.limits.memory | string | `"64Mi"` | | +| fluent-bit.resources.limits.memory | string | `"128Mi"` | | | fluent-bit.resources.requests.cpu | string | `"20m"` | | | fluent-bit.resources.requests.memory | string | `"32Mi"` | | | fluent-bit.serviceMonitor.enabled | bool | `false` | | @@ -98,6 +97,7 @@ Kubernetes: `>= 1.24.0` | fluent-bit.testFramework.enabled | bool | `false` | | | fluent-bit.tolerations[0].effect | string | `"NoSchedule"` | | | fluent-bit.tolerations[0].operator | string | `"Exists"` | | +| fluentd.configMapConfigs[0] | string | `"fluentd-prometheus-conf"` | | | fluentd.dashboards.enabled | bool | `false` | | | fluentd.enabled | bool | `false` | | | fluentd.env[0].name | string | `"FLUENTD_CONF"` | | @@ -115,6 +115,8 @@ Kubernetes: `>= 1.24.0` | fluentd.kind | string | `"Deployment"` | | | fluentd.metrics.serviceMonitor.additionalLabels.release | string | `"metrics"` | | | fluentd.metrics.serviceMonitor.enabled | bool | `false` | | +| fluentd.mountDockerContainersDirectory | bool | `false` | | +| fluentd.mountVarLogDirectory | bool | `false` | | | fluentd.output.host | string | `"logging-es-http"` | | | fluentd.podSecurityPolicy.enabled | bool | `false` | | | fluentd.replicaCount | int | `1` | | @@ -128,16 +130,6 @@ Kubernetes: `>= 1.24.0` | fluentd.service.ports[1].name | string | `"http-fluentd"` | | | fluentd.service.ports[1].protocol | string | `"TCP"` | | | fluentd.shared_key | string | `"cloudbender"` | | -| fluentd.volumeMounts[0].mountPath | string | `"/etc/fluent"` | | -| fluentd.volumeMounts[0].name | string | `"etcfluentd-main"` | | -| fluentd.volumeMounts[1].mountPath | string | `"/etc/fluent/config.d/"` | | -| fluentd.volumeMounts[1].name | string | `"etcfluentd-config"` | | -| fluentd.volumes[0].configMap.defaultMode | int | `511` | | -| fluentd.volumes[0].configMap.name | string | `"fluentd-main"` | | -| fluentd.volumes[0].name | string | `"etcfluentd-main"` | | -| fluentd.volumes[1].configMap.defaultMode | int | `511` | | -| fluentd.volumes[1].configMap.name | string | `"fluentd-config"` | | -| fluentd.volumes[1].name | string | `"etcfluentd-config"` | | | kibana.count | int | `1` | | | kibana.istio.enabled | bool | `false` | | | kibana.istio.gateway | string | `"istio-system/ingressgateway"` | | diff --git a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml index 6c060a1..3b29f94 100644 --- a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml +++ b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml @@ -1,9 +1,9 @@ annotations: artifacthub.io/changes: | - - kind: added - description: "Added events permission to ClusteRole" + - kind: changed + description: "Updated Fluent Bit OCI image to v2.2.0." apiVersion: v1 -appVersion: 2.1.8 +appVersion: 2.2.0 description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems. home: https://fluentbit.io/ @@ -24,4 +24,4 @@ maintainers: name: fluent-bit sources: - https://github.com/fluent/fluent-bit/ -version: 0.37.1 +version: 0.40.0 diff --git a/charts/kubezero-logging/charts/fluent-bit/dashboards/fluent-bit.json b/charts/kubezero-logging/charts/fluent-bit/dashboards/fluent-bit.json index 87e3271..b0024d3 100644 --- a/charts/kubezero-logging/charts/fluent-bit/dashboards/fluent-bit.json +++ b/charts/kubezero-logging/charts/fluent-bit/dashboards/fluent-bit.json @@ -5,7 +5,7 @@ "builtIn": 1, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$DS_PROMETHEUS" }, "enable": true, "hide": true, @@ -28,7 +28,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$DS_PROMETHEUS" }, "gridPos": { "h": 1, @@ -42,7 +42,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$DS_PROMETHEUS" }, "refId": "A" } @@ -144,7 +144,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$DS_PROMETHEUS" }, "gridPos": { "h": 1, @@ -158,7 +158,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$DS_PROMETHEUS" }, "refId": "A" } @@ -1171,7 +1171,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$DS_PROMETHEUS" }, "gridPos": { "h": 1, @@ -1185,7 +1185,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$DS_PROMETHEUS" }, "refId": "A" } @@ -1321,7 +1321,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$DS_PROMETHEUS" }, "fieldConfig": { "defaults": { @@ -1420,7 +1420,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$DS_PROMETHEUS" }, "editorMode": "code", "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\",pod=~\"$pod\",container=\"fluent-bit\"}) by (pod)", @@ -1432,7 +1432,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "$DS_PROMETHEUS" }, "editorMode": "code", "expr": "avg(kube_pod_container_resource_requests{job=\"kube-state-metrics\",namespace=\"$namespace\",pod=~\"$pod\",container=\"fluent-bit\",resource=\"cpu\"})", diff --git a/charts/kubezero-logging/charts/fluentd/Chart.yaml b/charts/kubezero-logging/charts/fluentd/Chart.yaml index b0f99ac..eccd60f 100644 --- a/charts/kubezero-logging/charts/fluentd/Chart.yaml +++ b/charts/kubezero-logging/charts/fluentd/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: v1.15.2 +appVersion: v1.16.2 description: A Helm chart for Kubernetes home: https://www.fluentd.org/ icon: https://www.fluentd.org/images/miscellany/fluentd-logo_2x.png @@ -12,4 +12,4 @@ name: fluentd sources: - https://github.com/fluent/fluentd/ - https://github.com/fluent/fluentd-kubernetes-daemonset -version: 0.4.3 +version: 0.5.0 diff --git a/charts/kubezero-logging/charts/fluentd/templates/_pod.tpl b/charts/kubezero-logging/charts/fluentd/templates/_pod.tpl index f77fb2f..e285e3a 100644 --- a/charts/kubezero-logging/charts/fluentd/templates/_pod.tpl +++ b/charts/kubezero-logging/charts/fluentd/templates/_pod.tpl @@ -1,5 +1,5 @@ {{- define "fluentd.pod" -}} -{{- $defaultTag := printf "%s-debian-elasticsearch7-1.0" (.Chart.AppVersion) -}} +{{- $defaultTag := printf "%s-debian-%s-1.0" (.Chart.AppVersion) (.Values.variant) -}} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 2 }} diff --git a/charts/kubezero-logging/charts/fluentd/templates/ingress.yaml b/charts/kubezero-logging/charts/fluentd/templates/ingress.yaml new file mode 100644 index 0000000..e837713 --- /dev/null +++ b/charts/kubezero-logging/charts/fluentd/templates/ingress.yaml @@ -0,0 +1,44 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "fluentd.fullname" . -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "fluentd.fullname" . }} + labels: + {{- include "fluentd.labels" . | nindent 4 }} + {{- with .Values.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + {{- with .secretName }} + secretName: {{ . }} + {{- end }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ $fullName }} + port: + number: {{ .port }} + {{ if .host -}} + host: {{ .host | quote }} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/kubezero-logging/charts/fluentd/templates/service.yaml b/charts/kubezero-logging/charts/fluentd/templates/service.yaml index 357a85c..6cf58e6 100644 --- a/charts/kubezero-logging/charts/fluentd/templates/service.yaml +++ b/charts/kubezero-logging/charts/fluentd/templates/service.yaml @@ -1,4 +1,4 @@ - +{{- if .Values.service.enabled -}} apiVersion: v1 kind: Service metadata: @@ -32,3 +32,4 @@ spec: {{- end }} selector: {{- include "fluentd.selectorLabels" . | nindent 4 }} +{{- end -}} diff --git a/charts/kubezero-logging/charts/fluentd/values.yaml b/charts/kubezero-logging/charts/fluentd/values.yaml index 9f47d78..9fa66e9 100644 --- a/charts/kubezero-logging/charts/fluentd/values.yaml +++ b/charts/kubezero-logging/charts/fluentd/values.yaml @@ -3,7 +3,8 @@ fullnameOverride: "" # DaemonSet, Deployment or StatefulSet kind: "DaemonSet" - +# azureblob, cloudwatch, elasticsearch7, elasticsearch8, gcs, graylog , kafka, kafka2, kinesis, opensearch +variant: elasticsearch7 # # Only applicable for Deployment or StatefulSet # replicaCount: 1 @@ -202,6 +203,7 @@ persistence: ## Fluentd service ## service: + enabled: true type: "ClusterIP" annotations: {} # loadBalancerIP: @@ -378,3 +380,17 @@ fileConfigs: password changeme + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + # - host: fluentd.example.tld + - port: 9880 + + tls: [] + # - secretName: fluentd-tls + # hosts: + # - fluentd.example.tld diff --git a/charts/kubezero-logging/update.sh b/charts/kubezero-logging/update.sh index d74930b..8633929 100755 --- a/charts/kubezero-logging/update.sh +++ b/charts/kubezero-logging/update.sh @@ -18,3 +18,5 @@ rm -f charts/fluentd/templates/files.conf/systemd.yaml # Fetch dashboards from Grafana.com and update ZDT CM ../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/fluent-bit/grafana-dashboards.yaml ../kubezero-metrics/sync_grafana_dashboards.py dashboards-es.yaml templates/eck/grafana-dashboards.yaml + +update_docs diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index c5429d8..06eb687 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -1,15 +1,6 @@ # use this for backwards compatability # fullnameOverride: "" -eck-operator: - enabled: false - installCRDs: false - tolerations: - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - nodeSelector: - node-role.kubernetes.io/control-plane: "" - # Version for ElasticSearch and Kibana have to match so we define it at top-level version: 7.17.3 diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 7092c4f..147fb06 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -331,11 +331,17 @@ prometheus-adapter: default: false resource: cpu: - containerQuery: sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}[5m])) by (<<.GroupBy>>) - nodeQuery: sum(1 - irate(node_cpu_seconds_total{mode="idle"}[5m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>) + containerQuery: | + sum by (<<.GroupBy>>) ( + irate ( + container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="",pod!=""}[60s] + ) + ) + nodeQuery: | + sum(1 - irate(node_cpu_seconds_total{<<.LabelMatchers>>, mode="idle"}[60s])) by (<<.GroupBy>>) resources: overrides: - node: + instance: resource: node namespace: resource: namespace @@ -343,11 +349,15 @@ prometheus-adapter: resource: pod containerLabel: container memory: - containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}) by (<<.GroupBy>>) - nodeQuery: sum(node_memory_MemTotal_bytes{job="node-exporter",<<.LabelMatchers>>} - node_memory_MemAvailable_bytes{job="node-exporter",<<.LabelMatchers>>}) by (<<.GroupBy>>) + containerQuery: | + sum by (<<.GroupBy>>) ( + container_memory_working_set_bytes{<<.LabelMatchers>>,container!="",pod!="",container!="POD"} + ) + nodeQuery: | + sum(node_memory_MemTotal_bytes{<<.LabelMatchers>>} - node_memory_MemAvailable_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) resources: overrides: - node: + instance: resource: node namespace: resource: namespace diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 10b12e7..2400b14 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -108,7 +108,7 @@ metrics: logging: enabled: false namespace: logging - targetRevision: 0.8.8 + targetRevision: 0.8.9 argocd: enabled: false diff --git a/docs/v1.27.md b/docs/v1.27.md index d6abc0b..4dbec09 100644 --- a/docs/v1.27.md +++ b/docs/v1.27.md @@ -9,9 +9,15 @@ - new, optional, OpenSearch operator - all instances now enforce IMDSv2 +## Fixes +- `kubectl top nodes` works now using node-exporter metrics rather than cadvisor + ## Version upgrades - cilium 1.14.4 - istio 1.19.4 +- fluent-bit 2.2.0 +- ArgoCD 2.9 +- Prometheus / Grafana ### FeatureGates - CustomCPUCFSQuotaPeriod