feat(metrics): Version bump to support 1.22

charts/kubezero-metrics/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-metrics
 description: KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all Kubernetes integrations.
 type: application
-version: 0.7.4
+version: 0.8.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -18,15 +18,15 @@ dependencies:
     version: ">= 0.1.4"
     repository: https://cdn.zero-downtime.net/charts/
   - name: kube-prometheus-stack
-    version: 30.2.0
+    version: 34.9.0
     # Switch back to upstream once all alerts are fixed eg. etcd gpcr
     # repository: https://prometheus-community.github.io/helm-charts
   - name: prometheus-adapter
-    version: 3.0.1
+    version: 3.2.0
     repository: https://prometheus-community.github.io/helm-charts
     condition: prometheus-adapter.enabled
   - name: prometheus-pushgateway
-    version: 1.14.0
+    version: 1.16.1
     # Switch back to upstream once namespaces are supported
     # repository: https://prometheus-community.github.io/helm-charts
     condition: prometheus-pushgateway.enabled

charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml

@@ -6,20 +6,20 @@ annotations:
       url: https://github.com/prometheus-operator/kube-prometheus
   artifacthub.io/operator: "true"
 apiVersion: v2
-appVersion: 0.53.1
+appVersion: 0.55.0
 dependencies:
 - condition: kubeStateMetrics.enabled
   name: kube-state-metrics
   repository: https://prometheus-community.github.io/helm-charts
-  version: 4.4.*
+  version: 4.7.*
 - condition: nodeExporter.enabled
   name: prometheus-node-exporter
   repository: https://prometheus-community.github.io/helm-charts
-  version: 2.5.*
+  version: 3.1.*
 - condition: grafana.enabled
   name: grafana
   repository: https://grafana.github.io/helm-charts
-  version: 6.21.*
+  version: 6.26.*
 description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards,
   and Prometheus rules combined with documentation and scripts to provide easy to
   operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus
@@ -41,6 +41,8 @@ maintainers:
   name: gianrubio
 - email: github.gkarthiks@gmail.com
   name: gkarthiks
+- email: kube-prometheus-stack@sisti.pt
+  name: GMartinez-Sisti
 - email: scott@r6by.com
   name: scottrigby
 - email: miroslav.hadzhiev@gmail.com
@@ -50,4 +52,4 @@ sources:
 - https://github.com/prometheus-community/helm-charts
 - https://github.com/prometheus-operator/kube-prometheus
 type: application
-version: 30.2.0
+version: 34.9.0

charts/kubezero-metrics/charts/kube-prometheus-stack/README.md

@@ -83,6 +83,47 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen
 
 A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions.
 
+### From 33.x to 34.x
+This upgrades to prometheus-operator to v0.55.0 and prometheus to v2.33.5.
+
+Run these commands to update the CRDs before applying the upgrade.
+```console
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
+```
+
+
+### From 32.x to 33.x
+This upgrades the node exporter Chart to v3.0.0. Please review the changes to this subchart if you make customizations to hostMountPropagation.
+
+### From 31.x to 32.x
+This upgrades to prometheus-operator to v0.54.0 and prometheus to v2.33.1. It also changes the default for `grafana.serviceMonitor.enabled` to `true.
+
+Run these commands to update the CRDs before applying the upgrade.
+```console
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.54.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
+```
+
+
+### From 30.x to 31.x
+
+This version removes the built-in grafana ServiceMonitor and instead relies on the ServiceMonitor of the sub-chart.
+`grafana.serviceMonitor.enabled` must be set instead of `grafana.serviceMonitor.selfMonitor` and the old ServiceMonitor may
+need to be manually cleaned up after deploying the new release.
+
 ### From 29.x to 30.x
 
 This version updates kube-state-metrics to 4.3.0 and uses the new option `kube-state-metrics.releaseLabel=true` which adds the "release" label to kube-state-metrics labels, making scraping of the metrics by kube-prometheus-stack work out of the box again, independent of the used kube-prometheus-stack release name. If you already set the "release" label via `kube-state-metrics.customLabels` you might have to remove that and use it via the new option.

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 8.3.4
+appVersion: 8.4.5
 description: The leading tool for querying and visualizing time series and metrics.
 home: https://grafana.net
 icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png
@@ -19,4 +19,4 @@ name: grafana
 sources:
 - https://github.com/grafana/grafana
 type: application
-version: 6.21.0
+version: 6.26.0

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md

@@ -114,8 +114,10 @@ This version requires Helm >= 3.1.0.
 | `initChownData.resources`                 | init-chown-data pod resource requests & limits | `{}`                                                   |
 | `schedulerName`                           | Alternate scheduler name                      | `nil`                                                   |
 | `env`                                     | Extra environment variables passed to pods    | `{}`                                                    |
-| `envValueFrom`                            | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details.  | `{}` |
+| `envValueFrom`                            | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` |
 | `envFromSecret`                           | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` |
+| `envFromSecrets`                          | List of Kubernetes secrets (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` |
+| `envFromConfigMaps`                       | List of Kubernetes ConfigMaps (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `[]` |
 | `envRenderSecret`                         | Sensible environment variables passed to pods and stored as secret | `{}`                               |
 | `enableServiceLinks`                      | Inject Kubernetes services as environment variables. | `true`                                           |
 | `extraSecretMounts`                       | Additional grafana server secret mounts       | `[]`                                                    |
@@ -138,7 +140,7 @@ This version requires Helm >= 3.1.0.
 | `podLabels`                               | Pod labels                                    | `{}`                                                    |
 | `podPortName`                             | Name of the grafana port on the pod           | `grafana`                                               |
 | `sidecar.image.repository`                | Sidecar image repository                      | `quay.io/kiwigrid/k8s-sidecar`                          |
-| `sidecar.image.tag`                       | Sidecar image tag                             | `1.15.1`                                                |
+| `sidecar.image.tag`                       | Sidecar image tag                             | `1.15.6`                                                |
 | `sidecar.image.sha`                       | Sidecar image sha (optional)                  | `""`                                                    |
 | `sidecar.imagePullPolicy`                 | Sidecar image pull policy                     | `IfNotPresent`                                          |
 | `sidecar.resources`                       | Sidecar resources                             | `{}`                                                    |

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl

@@ -16,7 +16,7 @@ hostAliases:
 {{- if .Values.priorityClassName }}
 priorityClassName: {{ .Values.priorityClassName }}
 {{- end }}
-{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.sidecar.notifiers.enabled .Values.extraInitContainers) }}
+{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.sidecar.notifiers.enabled .Values.extraInitContainers (and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources)) }}
 initContainers:
 {{- end }}
 {{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }}
@@ -77,6 +77,49 @@ initContainers:
         readOnly: {{ .readOnly }}
     {{- end }}
 {{- end }}
+{{- if and .Values.sidecar.datasources.enabled .Values.sidecar.datasources.initDatasources }}
+  - name: {{ template "grafana.name" . }}-init-sc-datasources
+    {{- if .Values.sidecar.image.sha }}
+    image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}"
+    {{- else }}
+    image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}"
+    {{- end }}
+    imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }}
+    env:
+      - name: METHOD
+        value: "LIST"
+      - name: LABEL
+        value: "{{ .Values.sidecar.datasources.label }}"
+      {{- if .Values.sidecar.datasources.labelValue }}
+      - name: LABEL_VALUE
+        value: {{ quote .Values.sidecar.datasources.labelValue }}
+      {{- end }}
+      - name: FOLDER
+        value: "/etc/grafana/provisioning/datasources"
+      - name: RESOURCE
+        value: {{ quote .Values.sidecar.datasources.resource }}
+      {{- if .Values.sidecar.enableUniqueFilenames }}
+      - name: UNIQUE_FILENAMES
+        value: "{{ .Values.sidecar.enableUniqueFilenames }}"
+      {{- end }}
+      {{- if .Values.sidecar.datasources.searchNamespace }}
+      - name: NAMESPACE
+        value: "{{ .Values.sidecar.datasources.searchNamespace | join "," }}"
+      {{- end }}
+      {{- if .Values.sidecar.skipTlsVerify }}
+      - name: SKIP_TLS_VERIFY
+        value: "{{ .Values.sidecar.skipTlsVerify }}"
+      {{- end }}
+    resources:
+{{ toYaml .Values.sidecar.resources | indent 6 }}
+{{- if .Values.sidecar.securityContext }}
+    securityContext:
+{{- toYaml .Values.sidecar.securityContext | nindent 6 }}
+{{- end }}
+    volumeMounts:
+      - name: sc-datasources-volume
+        mountPath: "/etc/grafana/provisioning/datasources"
+{{- end }}
 {{- if .Values.sidecar.notifiers.enabled }}
   - name: {{ template "grafana.name" . }}-sc-notifiers
     {{- if .Values.sidecar.image.sha }}
@@ -117,7 +160,7 @@ initContainers:
         mountPath: "/etc/grafana/provisioning/notifiers"
 {{- end}}
 {{- if .Values.extraInitContainers }}
-{{ toYaml .Values.extraInitContainers | indent 2 }}
+{{ tpl (toYaml .Values.extraInitContainers) . | indent 2 }}
 {{- end }}
 {{- if .Values.image.pullSecrets }}
 imagePullSecrets:
@@ -170,6 +213,14 @@ containers:
       - name: SCRIPT
         value: "{{ .Values.sidecar.dashboards.script }}"
       {{- end }}
+      {{- if .Values.sidecar.dashboards.watchServerTimeout }}
+      - name: WATCH_SERVER_TIMEOUT
+        value: "{{ .Values.sidecar.dashboards.watchServerTimeout }}"
+      {{- end }}
+      {{- if .Values.sidecar.dashboards.watchClientTimeout }}
+      - name: WATCH_CLIENT_TIMEOUT
+        value: "{{ .Values.sidecar.dashboards.watchClientTimeout }}"
+      {{- end }}
     resources:
 {{ toYaml .Values.sidecar.resources | indent 6 }}
 {{- if .Values.sidecar.securityContext }}
@@ -245,6 +296,69 @@ containers:
     volumeMounts:
       - name: sc-datasources-volume
         mountPath: "/etc/grafana/provisioning/datasources"
+{{- end}}
+{{- if .Values.sidecar.plugins.enabled }}
+  - name: {{ template "grafana.name" . }}-sc-plugins
+    {{- if .Values.sidecar.image.sha }}
+    image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}"
+    {{- else }}
+    image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}"
+    {{- end }}
+    imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }}
+    env:
+      - name: METHOD
+        value: {{ .Values.sidecar.plugins.watchMethod }}
+      - name: LABEL
+        value: "{{ .Values.sidecar.plugins.label }}"
+      {{- if .Values.sidecar.plugins.labelValue }}
+      - name: LABEL_VALUE
+        value: {{ quote .Values.sidecar.plugins.labelValue }}
+      {{- end }}
+      - name: FOLDER
+        value: "/etc/grafana/provisioning/plugins"
+      - name: RESOURCE
+        value: {{ quote .Values.sidecar.plugins.resource }}
+      {{- if .Values.sidecar.enableUniqueFilenames }}
+      - name: UNIQUE_FILENAMES
+        value: "{{ .Values.sidecar.enableUniqueFilenames }}"
+      {{- end }}
+      {{- if .Values.sidecar.plugins.searchNamespace }}
+      - name: NAMESPACE
+        value: "{{ .Values.sidecar.plugins.searchNamespace | join "," }}"
+      {{- end }}
+      {{- if .Values.sidecar.skipTlsVerify }}
+      - name: SKIP_TLS_VERIFY
+        value: "{{ .Values.sidecar.skipTlsVerify }}"
+      {{- end }}
+      {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }}
+      - name: REQ_USERNAME
+        valueFrom:
+          secretKeyRef:
+            name: {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }}
+            key: {{ .Values.admin.userKey | default "admin-user" }}
+      {{- end }}
+      {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }}
+      - name: REQ_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }}
+            key: {{ .Values.admin.passwordKey | default "admin-password" }}
+      {{- end }}
+      {{- if not .Values.sidecar.plugins.skipReload }}
+      - name: REQ_URL
+        value: {{ .Values.sidecar.plugins.reloadURL }}
+      - name: REQ_METHOD
+        value: POST
+      {{- end }}
+    resources:
+{{ toYaml .Values.sidecar.resources | indent 6 }}
+{{- if .Values.sidecar.securityContext }}
+    securityContext:
+{{- toYaml .Values.sidecar.securityContext | nindent 6 }}
+{{- end }}
+    volumeMounts:
+      - name: sc-plugins-volume
+        mountPath: "/etc/grafana/provisioning/plugins"
 {{- end}}
   - name: {{ .Chart.Name }}
     {{- if .Values.image.sha }}
@@ -334,6 +448,10 @@ containers:
       - name: sc-datasources-volume
         mountPath: "/etc/grafana/provisioning/datasources"
 {{- end}}
+{{- if .Values.sidecar.plugins.enabled }}
+      - name: sc-plugins-volume
+        mountPath: "/etc/grafana/provisioning/plugins"
+{{- end}}
 {{- if .Values.sidecar.notifiers.enabled }}
       - name: sc-notifiers-volume
         mountPath: "/etc/grafana/provisioning/notifiers"
@@ -412,13 +530,13 @@ containers:
     {{- range $key, $value := .Values.envValueFrom }}
       - name: {{ $key | quote }}
         valueFrom:
-{{ toYaml $value | indent 10 }}
+{{ tpl (toYaml $value) $ | indent 10 }}
     {{- end }}
 {{- range $key, $value := .Values.env }}
       - name: "{{ tpl $key $ }}"
         value: "{{ tpl (print $value) $ }}"
 {{- end }}
-    {{- if or .Values.envFromSecret (or .Values.envRenderSecret .Values.envFromSecrets) }}
+    {{- if or .Values.envFromSecret (or .Values.envRenderSecret .Values.envFromSecrets) .Values.envFromConfigMaps }}
     envFrom:
     {{- if .Values.envFromSecret }}
       - secretRef:
@@ -430,7 +548,12 @@ containers:
     {{- end }}
     {{- range .Values.envFromSecrets }}
       - secretRef:
-          name: {{ .name }}
+          name: {{ tpl .name $ }}
+          optional: {{ .optional | default false }}
+    {{- end }}
+    {{- range .Values.envFromConfigMaps }}
+      - configMapRef:
+          name: {{ tpl .name $ }}
           optional: {{ .optional | default false }}
     {{- end }}
     {{- end }}
@@ -522,6 +645,10 @@ volumes:
   - name: sc-datasources-volume
     emptyDir: {}
 {{- end -}}
+{{- if .Values.sidecar.plugins.enabled }}
+  - name: sc-plugins-volume
+    emptyDir: {}
+{{- end -}}
 {{- if .Values.sidecar.notifiers.enabled }}
   - name: sc-notifiers-volume
     emptyDir: {}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml

@@ -1,4 +1,4 @@
-{{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }}
+{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -12,7 +12,7 @@ metadata:
 {{- end }}
 type: Opaque
 data:
-  {{- if and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }}
+  {{- if and (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }}
   admin-user: {{ .Values.adminUser | b64enc | quote }}
   {{- if .Values.adminPassword }}
   admin-password: {{ .Values.adminPassword | b64enc | quote }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/servicemonitor.yaml

@@ -14,12 +14,14 @@ metadata:
     {{- end }}
 spec:
   endpoints:
-  - interval: {{ .Values.serviceMonitor.interval }}
+  - port: {{ .Values.service.portName }}
-    {{- if .Values.serviceMonitor.scrapeTimeout }}
+    {{- with .Values.serviceMonitor.interval }}
-    scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }}
+    interval: {{ . }}
+    {{- end }}
+    {{- with .Values.serviceMonitor.scrapeTimeout }}
+    scrapeTimeout: {{ . }}
     {{- end }}
     honorLabels: true
-    port: {{ .Values.service.portName }}
     path: {{ .Values.serviceMonitor.path }}
     scheme: {{ .Values.serviceMonitor.scheme }}
     {{- if .Values.serviceMonitor.tlsConfig }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml

@@ -73,7 +73,7 @@ livenessProbe:
 
 image:
   repository: grafana/grafana
-  tag: 8.3.4
+  tag: 8.4.5
   sha: ""
   pullPolicy: IfNotPresent
 
@@ -241,6 +241,9 @@ tolerations: []
 ##
 affinity: {}
 
+## Additional init containers (evaluated as template)
+## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/
+##
 extraInitContainers: []
 
 ## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod
@@ -369,8 +372,8 @@ admin:
 
 env: {}
 
-## "valueFrom" environment variable references that will be added to deployment pods
+## "valueFrom" environment variable references that will be added to deployment pods. Name is templated.
-## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core
+## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core
 ## Renders in container spec as:
 ##   env:
 ##     ...
@@ -378,6 +381,10 @@ env: {}
 ##       valueFrom:
 ##         <value rendered as YAML>
 envValueFrom: {}
+  #  ENV_NAME:
+  #    configMapKeyRef:
+  #      name: configmap-name
+  #      key: value_key
 
 ## The name of a secret in the same kubernetes namespace which contain values to be added to the environment
 ## This can be useful for auth tokens, etc. Value is templated.
@@ -389,10 +396,19 @@ envRenderSecret: {}
 
 ## The names of secrets in the same kubernetes namespace which contain values to be added to the environment
 ## Each entry should contain a name key, and can optionally specify whether the secret must be defined with an optional key.
+## Name is templated.
 envFromSecrets: []
 ## - name: secret-name
 ##   optional: true
 
+## The names of conifgmaps in the same kubernetes namespace which contain values to be added to the environment
+## Each entry should contain a name key, and can optionally specify whether the configmap must be defined with an optional key.
+## Name is templated.
+## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#configmapenvsource-v1-core
+envFromConfigMaps: []
+## - name: configmap-name
+##   optional: true
+
 # Inject Kubernetes services as environment variables.
 # See https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#environment-variables
 enableServiceLinks: true
@@ -618,7 +634,7 @@ smtp:
 sidecar:
   image:
     repository: quay.io/kiwigrid/k8s-sidecar
-    tag: 1.15.1
+    tag: 1.15.6
     sha: ""
   imagePullPolicy: IfNotPresent
   resources: {}
@@ -656,6 +672,16 @@ sidecar:
     folderAnnotation: null
     # Absolute path to shell script to execute after a configmap got reloaded
     script: null
+    # watchServerTimeout: request to the server, asking it to cleanly close the connection after that.
+    # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S
+    # watchServerTimeout: 3600
+    #
+    # watchClientTimeout: is a client-side timeout, configuring your local socket.
+    # If you have a network outage dropping all packets with no RST/FIN,
+    # this is how long your client waits before realizing & dropping the connection.
+    # defaults to 66sec (sic!)
+    # watchClientTimeout: 60
+    #
     # provider configuration that lets grafana manage the dashboards
     provider:
       # name of the provider, should be unique
@@ -691,6 +717,29 @@ sidecar:
     # Endpoint to send request to reload datasources
     reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload"
     skipReload: false
+    # Deploy the datasource sidecar as an initContainer in addition to a container.
+    # This is needed if skipReload is true, to load any datasources defined at startup time.
+    initDatasources: false
+  plugins:
+    enabled: false
+    # label that the configmaps with plugins are marked with
+    label: grafana_plugin
+    # value of label that the configmaps with plugins are set to
+    labelValue: null
+    # If specified, the sidecar will search for plugin config-maps inside this namespace.
+    # Otherwise the namespace in which the sidecar is running will be used.
+    # It's also possible to specify ALL to search in all namespaces
+    searchNamespace: null
+    # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds.
+    watchMethod: WATCH
+    # search in configmap, secret or both
+    resource: both
+    # Endpoint to send request to reload plugins
+    reloadURL: "http://localhost:3000/api/admin/provisioning/plugins/reload"
+    skipReload: false
+    # Deploy the datasource sidecar as an initContainer in addition to a container.
+    # This is needed if skipReload is true, to load any plugins defined at startup time.
+    initPlugins: false
   notifiers:
     enabled: false
     # label that the configmaps with notifiers are marked with

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 2.3.0
+appVersion: 2.4.1
 description: Install kube-state-metrics to generate and expose cluster-level metrics
 home: https://github.com/kubernetes/kube-state-metrics/
 keywords:
@@ -18,4 +18,4 @@ name: kube-state-metrics
 sources:
 - https://github.com/kubernetes/kube-state-metrics/
 type: application
-version: 4.4.1
+version: 4.7.0

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml

@@ -78,6 +78,9 @@ spec:
         {{- if .Values.namespaces }}
         - --namespaces={{ tpl (.Values.namespaces | join ",") $ }}
         {{- end }}
+        {{- if .Values.namespacesDenylist }}
+        - --namespaces-denylist={{ tpl (.Values.namespacesDenylist | join ",") $ }}
+        {{- end }}
         {{- if .Values.autosharding.enabled }}
         - --pod=$(POD_NAME)
         - --pod-namespace=$(POD_NAMESPACE)

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml

@@ -1,5 +1,5 @@
 {{- if and (eq .Values.rbac.create true) (not .Values.rbac.useExistingRole) -}}
-{{- range (split "," .Values.namespaces) }}
+{{- range (ternary (split "," .Values.namespaces) (list "") (eq $.Values.rbac.useClusterRole false)) }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 {{- if eq $.Values.rbac.useClusterRole false }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/service.yaml

@@ -30,6 +30,9 @@ spec:
   {{ end }}
 {{- if .Values.service.loadBalancerIP }}
   loadBalancerIP: "{{ .Values.service.loadBalancerIP }}"
+{{- end }}
+{{- if .Values.service.clusterIP }}
+  clusterIP: "{{ .Values.service.clusterIP }}"
 {{- end }}
   selector:
     {{- include "kube-state-metrics.selectorLabels" . | indent 4 }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml

@@ -13,8 +13,8 @@ spec:
   jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }}
   selector:
     matchLabels:
-    {{- if .Values.prometheus.monitor.selectorOverride }}
+    {{- if .Values.prometheus.monitor.selectorOverride -}}
-      {{ toYaml .Values.prometheus.monitor.selectorOverride | indent 6 }}
+      {{ toYaml .Values.prometheus.monitor.selectorOverride | nindent 6 }}
     {{ else }}
       {{- include "kube-state-metrics.selectorLabels" . | indent 6 }}
     {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml

@@ -2,7 +2,7 @@
 prometheusScrape: true
 image:
   repository: k8s.gcr.io/kube-state-metrics/kube-state-metrics
-  tag: v2.3.0
+  tag: v2.4.1
   pullPolicy: IfNotPresent
 
 imagePullSecrets: []
@@ -28,6 +28,7 @@ service:
   type: ClusterIP
   nodePort: 0
   loadBalancerIP: ""
+  clusterIP: ""
   annotations: {}
 
 ## Additional labels to add to all resources
@@ -195,6 +196,10 @@ kubeconfig:
 # Comma-separated list of namespaces to be enabled for collecting resources. By default all namespaces are collected.
 namespaces: ""
 
+# Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set,
+# only namespaces that are excluded in namespaces-denylist will be used.
+namespacesDenylist: ""
+
 ## Override the deployment namespace
 ##
 namespaceOverride: ""

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml

@@ -10,8 +10,10 @@ maintainers:
 - email: gianrubio@gmail.com
   name: gianrubio
 - name: bismarck
+- email: zanhsieh@gmail.com
+  name: zanhsieh
 name: prometheus-node-exporter
 sources:
 - https://github.com/prometheus/node_exporter/
 type: application
-version: 2.5.0
+version: 3.1.0

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/README.md

@@ -41,6 +41,22 @@ helm upgrade [RELEASE_NAME] [CHART] --install
 
 _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._
 
+### From 2.x to 3.x
+
+Change the following:
+
+```yaml
+hostRootFsMount: true
+```
+
+to:
+
+```yaml
+hostRootFsMount:
+  enabled: true
+  mountPropagation: HostToContainer
+```
+
 ## Configuring
 
 See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands:

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml

@@ -41,10 +41,10 @@ spec:
           args:
             - --path.procfs=/host/proc
             - --path.sysfs=/host/sys
-            {{- if .Values.hostRootFsMount }}
+            {{- if .Values.hostRootFsMount.enabled }}
             - --path.rootfs=/host/root
             {{- end }}
-            - --web.listen-address=$(HOST_IP):{{ .Values.service.port }}
+            - --web.listen-address=[$(HOST_IP)]:{{ .Values.service.port }}
 {{- if .Values.extraArgs }}
 {{ toYaml .Values.extraArgs | indent 12 }}
 {{- end }}
@@ -61,6 +61,10 @@ spec:
                 apiVersion: v1
                 fieldPath: status.hostIP
             {{- end }}
+          {{- range $key, $value := .Values.env }}
+          - name: {{ $key }}
+            value: {{ $value | quote }}
+          {{- end }}
           ports:
             - name: {{ .Values.service.portName }}
               containerPort: {{ .Values.service.port }}
@@ -82,10 +86,12 @@ spec:
             - name: sys
               mountPath: /host/sys
               readOnly: true
-            {{- if .Values.hostRootFsMount }}
+            {{- if .Values.hostRootFsMount.enabled }}
             - name: root
               mountPath: /host/root
-              mountPropagation: HostToContainer
+              {{- with .Values.hostRootFsMount.mountPropagation }}
+              mountPropagation: {{ . }}
+              {{- end }}
               readOnly: true
             {{- end }}
             {{- if .Values.extraHostVolumeMounts }}
@@ -153,7 +159,7 @@ spec:
         - name: sys
           hostPath:
             path: /sys
-        {{- if .Values.hostRootFsMount }}
+        {{- if .Values.hostRootFsMount.enabled }}
         - name: root
           hostPath:
             path: /

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml

@@ -17,6 +17,11 @@ service:
   annotations:
     prometheus.io/scrape: "true"
 
+# Additional environment variables that will be passed to the daemonset
+env: {}
+##  env:
+##    VARIABLE: value
+
 prometheus:
   monitor:
     enabled: false
@@ -100,9 +105,15 @@ hostNetwork: true
 # Share the host process ID namespace
 hostPID: true
 
-## If true, node-exporter pods mounts host / at /host/root
+# Mount the node's root file system (/) at /host/root in the container
-##
+hostRootFsMount:
-hostRootFsMount: true
+  enabled: true
+  # Defines how new mounts in existing mounts on the node or in the container
+  # are propagated to the container or node, respectively. Possible values are
+  # None, HostToContainer, and Bidirectional. If this field is omitted, then
+  # None is used. More information on:
+  # https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation
+  mountPropagation: HostToContainer
 
 ## Assign a group of affinity scheduling rules
 ##

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1
@@ -973,6 +973,20 @@ spec:
                       are ANDed.
                     type: object
                 type: object
+              alertmanagerConfiguration:
+                description: 'EXPERIMENTAL: alertmanagerConfiguration specifies the
+                  global Alertmanager configuration. If defined, it takes precedence
+                  over the `configSecret` field. This field may change in future releases.
+                  The specified global alertmanager config will not force add a namespace
+                  label in routes and inhibitRules.'
+                properties:
+                  name:
+                    description: The name of the AlertmanagerConfig resource which
+                      holds the global configuration. It must be in the same namespace
+                      as the Alertmanager.
+                    minLength: 1
+                    type: string
+                type: object
               baseImage:
                 description: 'Base image that is used to deploy pods, without tag.
                   Deprecated: use ''image'' instead'

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1
@@ -69,7 +69,7 @@ spec:
                       in contrast to a list restricting them.
                     type: boolean
                   matchNames:
-                    description: List of namespace names.
+                    description: List of namespace names to select from.
                     items:
                       type: string
                     type: array
@@ -170,6 +170,10 @@ spec:
                       required:
                       - key
                       type: object
+                    followRedirects:
+                      description: FollowRedirects configures whether scrape requests
+                        follow HTTP 3xx redirects.
+                      type: boolean
                     honorLabels:
                       description: HonorLabels chooses the metric's labels on collisions
                         with target labels.
@@ -191,8 +195,17 @@ spec:
                           configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
                         properties:
                           action:
+                            default: replace
                             description: Action to perform based on regex matching.
                               Default is 'replace'
+                            enum:
+                            - replace
+                            - keep
+                            - drop
+                            - hashmod
+                            - labelmap
+                            - labeldrop
+                            - labelkeep
                             type: string
                           modulus:
                             description: Modulus to take of the hash of the source
@@ -218,6 +231,10 @@ spec:
                               separator and matched against the configured regular
                               expression for the replace, keep, and drop actions.
                             items:
+                              description: LabelName is a valid Prometheus label name
+                                which may only contain ASCII letters, numbers, as
+                                well as underscores.
+                              pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$
                               type: string
                             type: array
                           targetLabel:
@@ -332,8 +349,9 @@ spec:
                     relabelings:
                       description: 'RelabelConfigs to apply to samples before scraping.
                         Prometheus Operator automatically adds relabelings for a few
-                        standard Kubernetes fields and replaces original scrape job
+                        standard Kubernetes fields. The original scrape job''s name
-                        name with __tmp_prometheus_job_name. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
+                        is available via the `__tmp_prometheus_job_name` label. More
+                        info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
                       items:
                         description: 'RelabelConfig allows dynamic rewriting of the
                           label set, being applied to samples before ingestion. It
@@ -341,8 +359,17 @@ spec:
                           configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
                         properties:
                           action:
+                            default: replace
                             description: Action to perform based on regex matching.
                               Default is 'replace'
+                            enum:
+                            - replace
+                            - keep
+                            - drop
+                            - hashmod
+                            - labelmap
+                            - labeldrop
+                            - labelkeep
                             type: string
                           modulus:
                             description: Modulus to take of the hash of the source
@@ -368,6 +395,10 @@ spec:
                               separator and matched against the configured regular
                               expression for the replace, keep, and drop actions.
                             items:
+                              description: LabelName is a valid Prometheus label name
+                                which may only contain ASCII letters, numbers, as
+                                well as underscores.
+                              pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$
                               type: string
                             type: array
                           targetLabel:

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1
@@ -161,8 +161,17 @@ spec:
                     of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
                   properties:
                     action:
+                      default: replace
                       description: Action to perform based on regex matching. Default
                         is 'replace'
+                      enum:
+                      - replace
+                      - keep
+                      - drop
+                      - hashmod
+                      - labelmap
+                      - labeldrop
+                      - labelkeep
                       type: string
                     modulus:
                       description: Modulus to take of the hash of the source label
@@ -188,6 +197,9 @@ spec:
                         and matched against the configured regular expression for
                         the replace, keep, and drop actions.
                       items:
+                        description: LabelName is a valid Prometheus label name which
+                          may only contain ASCII letters, numbers, as well as underscores.
+                        pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$
                         type: string
                       type: array
                     targetLabel:
@@ -317,29 +329,34 @@ spec:
                 format: int64
                 type: integer
               targets:
-                description: Targets defines a set of static and/or dynamically discovered
+                description: Targets defines a set of static or dynamically discovered
-                  targets to be probed using the prober.
+                  targets to probe.
                 properties:
                   ingress:
-                    description: Ingress defines the set of dynamically discovered
+                    description: ingress defines the Ingress objects to probe and
-                      ingress objects which hosts are considered for probing.
+                      the relabeling configuration. If `staticConfig` is also defined,
+                      `staticConfig` takes precedence.
                     properties:
                       namespaceSelector:
-                        description: Select Ingress objects by namespace.
+                        description: From which namespaces to select Ingress objects.
                         properties:
                           any:
                             description: Boolean describing whether all namespaces
                               are selected in contrast to a list restricting them.
                             type: boolean
                           matchNames:
-                            description: List of namespace names.
+                            description: List of namespace names to select from.
                             items:
                               type: string
                             type: array
                         type: object
                       relabelingConfigs:
-                        description: 'RelabelConfigs to apply to samples before ingestion.
+                        description: 'RelabelConfigs to apply to the label set of
-                          More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
+                          the target before it gets scraped. The original ingress
+                          address is available via the `__tmp_prometheus_ingress_address`
+                          label. It can be used to customize the probed URL. The original
+                          scrape job''s name is available via the `__tmp_prometheus_job_name`
+                          label. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
                         items:
                           description: 'RelabelConfig allows dynamic rewriting of
                             the label set, being applied to samples before ingestion.
@@ -347,8 +364,17 @@ spec:
                             configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
                           properties:
                             action:
+                              default: replace
                               description: Action to perform based on regex matching.
                                 Default is 'replace'
+                              enum:
+                              - replace
+                              - keep
+                              - drop
+                              - hashmod
+                              - labelmap
+                              - labeldrop
+                              - labelkeep
                               type: string
                             modulus:
                               description: Modulus to take of the hash of the source
@@ -374,6 +400,10 @@ spec:
                                 separator and matched against the configured regular
                                 expression for the replace, keep, and drop actions.
                               items:
+                                description: LabelName is a valid Prometheus label
+                                  name which may only contain ASCII letters, numbers,
+                                  as well as underscores.
+                                pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$
                                 type: string
                               type: array
                             targetLabel:
@@ -384,7 +414,7 @@ spec:
                           type: object
                         type: array
                       selector:
-                        description: Select Ingress objects by labels.
+                        description: Selector to select the Ingress objects.
                         properties:
                           matchExpressions:
                             description: matchExpressions is a list of label selector
@@ -430,8 +460,9 @@ spec:
                         type: object
                     type: object
                   staticConfig:
-                    description: 'StaticConfig defines static targets which are considers
+                    description: 'staticConfig defines the static list of targets
-                      for probing. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config.'
+                      to probe and the relabeling configuration. If `ingress` is also
+                      defined, `staticConfig` takes precedence. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#static_config.'
                     properties:
                       labels:
                         additionalProperties:
@@ -440,8 +471,8 @@ spec:
                           targets.
                         type: object
                       relabelingConfigs:
-                        description: 'RelabelConfigs to apply to samples before ingestion.
+                        description: 'RelabelConfigs to apply to the label set of
-                          More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
+                          the targets before it gets scraped. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
                         items:
                           description: 'RelabelConfig allows dynamic rewriting of
                             the label set, being applied to samples before ingestion.
@@ -449,8 +480,17 @@ spec:
                             configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
                           properties:
                             action:
+                              default: replace
                               description: Action to perform based on regex matching.
                                 Default is 'replace'
+                              enum:
+                              - replace
+                              - keep
+                              - drop
+                              - hashmod
+                              - labelmap
+                              - labeldrop
+                              - labelkeep
                               type: string
                             modulus:
                               description: Modulus to take of the hash of the source
@@ -476,6 +516,10 @@ spec:
                                 separator and matched against the configured regular
                                 expression for the replace, keep, and drop actions.
                               items:
+                                description: LabelName is a valid Prometheus label
+                                  name which may only contain ASCII letters, numbers,
+                                  as well as underscores.
+                                pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$
                                 type: string
                               type: array
                             targetLabel:
@@ -486,8 +530,7 @@ spec:
                           type: object
                         type: array
                       static:
-                        description: Targets is a list of URLs to probe using the
+                        description: The list of hosts to probe.
-                          configured prober.
                         items:
                           type: string
                         type: array

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1
@@ -2697,11 +2697,11 @@ spec:
               enforcedNamespaceLabel:
                 description: "EnforcedNamespaceLabel If set, a label will be added
                   to \n 1. all user-metrics (created by `ServiceMonitor`, `PodMonitor`
-                  and `ProbeConfig` object) and 2. in all `PrometheusRule` objects
+                  and `Probe` objects) and 2. in all `PrometheusRule` objects (except
-                  (except the ones excluded in `prometheusRulesExcludedFromEnforce`)
+                  the ones excluded in `prometheusRulesExcludedFromEnforce`) to    *
-                  to    * alerting & recording rules and    * the metrics used in
+                  alerting & recording rules and    * the metrics used in their expressions
-                  their expressions (`expr`). \n Label name is this field's value.
+                  (`expr`). \n Label name is this field's value. Label value is the
-                  Label value is the namespace of the created object (mentioned above)."
+                  namespace of the created object (mentioned above)."
                 type: string
               enforcedSampleLimit:
                 description: EnforcedSampleLimit defines global limit on number of
@@ -2739,9 +2739,9 @@ spec:
                 type: string
               ignoreNamespaceSelectors:
                 description: IgnoreNamespaceSelectors if set to true will ignore NamespaceSelector
-                  settings from the podmonitor and servicemonitor configs, and they
+                  settings from all PodMonitor, ServiceMonitor and Probe objects.
-                  will only discover endpoints within their current namespace.  Defaults
+                  They will only discover endpoints within their current namespace.
-                  to false.
+                  Defaults to false.
                 type: boolean
               image:
                 description: Image if specified has precedence over baseImage, tag
@@ -4014,13 +4014,15 @@ spec:
                 description: Define which Nodes the Pods are scheduled on.
                 type: object
               overrideHonorLabels:
-                description: OverrideHonorLabels if set to true overrides all user
+                description: When true, Prometheus resolves label conflicts by renaming
-                  configured honor_labels. If HonorLabels is set in ServiceMonitor
+                  the labels in the scraped data to "exported_<label value>" for all
-                  or PodMonitor to true, this overrides honor_labels to false.
+                  targets created from service and pod monitors. Otherwise the HonorLabels
+                  field of the service or pod monitor applies.
                 type: boolean
               overrideHonorTimestamps:
-                description: OverrideHonorTimestamps allows to globally enforce honoring
+                description: When true, Prometheus ignores the timestamps for all
-                  timestamps in all scrape configs.
+                  the targets created from service and pod monitors. Otherwise the
+                  HonorTimestamps field of the service or pod monitor applies.
                 type: boolean
               paused:
                 description: When a Prometheus deployment is paused, no actions except
@@ -4292,19 +4294,22 @@ spec:
                 type: object
               queryLogFile:
                 description: QueryLogFile specifies the file to which PromQL queries
-                  are logged. Note that this location must be writable, and can be
+                  are logged. If the filename has an empty path, e.g. 'query.log',
-                  persisted using an attached volume. Alternatively, the location
+                  prometheus-operator will mount the file into an emptyDir volume
-                  can be set to a stdout location such as `/dev/stdout` to log querie
+                  at `/var/log/prometheus`. If a full path is provided, e.g. /var/log/prometheus/query.log,
+                  you must mount a volume in the specified directory and it must be
+                  writable. This is because the prometheus container runs with a read-only
+                  root filesystem for security reasons. Alternatively, the location
+                  can be set to a stdout location such as `/dev/stdout` to log query
                   information to the default Prometheus log stream. This is only available
                   in versions of Prometheus >= 2.16.0. For more details, see the Prometheus
                   docs (https://prometheus.io/docs/guides/query-log/)
                 type: string
               remoteRead:
-                description: If specified, the remote_read spec. This is an experimental
+                description: remoteRead is the list of remote read configurations.
-                  feature, it may change in any upcoming release in a breaking way.
                 items:
-                  description: RemoteReadSpec defines the remote_read configuration
+                  description: RemoteReadSpec defines the configuration for Prometheus
-                    for prometheus.
+                    to read back samples from a remote endpoint.
                   properties:
                     authorization:
                       description: Authorization section for remote read
@@ -4394,7 +4399,7 @@ spec:
                         versions 2.26.0 and newer.
                       type: object
                     name:
-                      description: The name of the remote read queue, must be unique
+                      description: The name of the remote read queue, it must be unique
                         if specified. The name is used in metrics and logging in order
                         to differentiate read configurations.  Only valid in Prometheus
                         versions 2.15.0 and newer.
@@ -4484,7 +4489,7 @@ spec:
                       - tokenUrl
                       type: object
                     proxyUrl:
-                      description: Optional ProxyURL
+                      description: Optional ProxyURL.
                       type: string
                     readRecent:
                       description: Whether reads should be made for queries for time
@@ -4627,18 +4632,17 @@ spec:
                           type: string
                       type: object
                     url:
-                      description: The URL of the endpoint to send samples to.
+                      description: The URL of the endpoint to query from.
                       type: string
                   required:
                   - url
                   type: object
                 type: array
               remoteWrite:
-                description: If specified, the remote_write spec. This is an experimental
+                description: remoteWrite is the list of remote write configurations.
-                  feature, it may change in any upcoming release in a breaking way.
                 items:
-                  description: RemoteWriteSpec defines the remote_write configuration
+                  description: RemoteWriteSpec defines the configuration to write
-                    for prometheus.
+                    samples from Prometheus to a remote endpoint.
                   properties:
                     authorization:
                       description: Authorization section for remote write
@@ -4729,22 +4733,22 @@ spec:
                       type: object
                     metadataConfig:
                       description: MetadataConfig configures the sending of series
-                        metadata to remote storage.
+                        metadata to the remote storage.
                       properties:
                         send:
-                          description: Whether metric metadata is sent to remote storage
+                          description: Whether metric metadata is sent to the remote
-                            or not.
+                            storage or not.
                           type: boolean
                         sendInterval:
-                          description: How frequently metric metadata is sent to remote
+                          description: How frequently metric metadata is sent to the
-                            storage.
+                            remote storage.
                           type: string
                       type: object
                     name:
-                      description: The name of the remote write queue, must be unique
+                      description: The name of the remote write queue, it must be
-                        if specified. The name is used in metrics and logging in order
+                        unique if specified. The name is used in metrics and logging
-                        to differentiate queues. Only valid in Prometheus versions
+                        in order to differentiate queues. Only valid in Prometheus
-                        2.15.0 and newer.
+                        versions 2.15.0 and newer.
                       type: string
                     oauth2:
                       description: OAuth2 for the URL. Only valid in Prometheus versions
@@ -4831,7 +4835,7 @@ spec:
                       - tokenUrl
                       type: object
                     proxyUrl:
-                      description: Optional ProxyURL
+                      description: Optional ProxyURL.
                       type: string
                     queueConfig:
                       description: QueueConfig allows tuning of the remote write queue
@@ -5075,8 +5079,17 @@ spec:
                           configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
                         properties:
                           action:
+                            default: replace
                             description: Action to perform based on regex matching.
                               Default is 'replace'
+                            enum:
+                            - replace
+                            - keep
+                            - drop
+                            - hashmod
+                            - labelmap
+                            - labeldrop
+                            - labelkeep
                             type: string
                           modulus:
                             description: Modulus to take of the hash of the source
@@ -5102,6 +5115,10 @@ spec:
                               separator and matched against the configured regular
                               expression for the replace, keep, and drop actions.
                             items:
+                              description: LabelName is a valid Prometheus label name
+                                which may only contain ASCII letters, numbers, as
+                                well as underscores.
+                              pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$
                               type: string
                             type: array
                           targetLabel:
@@ -5154,8 +5171,9 @@ spec:
                 type: object
               retention:
                 description: Time duration Prometheus shall retain data for. Default
-                  is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)`
+                  is '24h' if retentionSize is not set, and must match the regular
-                  (milliseconds seconds minutes hours days weeks years).
+                  expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes
+                  hours days weeks years).
                 type: string
               retentionSize:
                 description: 'Maximum amount of disk space used by blocks. Supported

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1
@@ -139,6 +139,10 @@ spec:
                       required:
                       - key
                       type: object
+                    followRedirects:
+                      description: FollowRedirects configures whether scrape requests
+                        follow HTTP 3xx redirects.
+                      type: boolean
                     honorLabels:
                       description: HonorLabels chooses the metric's labels on collisions
                         with target labels.
@@ -160,8 +164,17 @@ spec:
                           configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
                         properties:
                           action:
+                            default: replace
                             description: Action to perform based on regex matching.
                               Default is 'replace'
+                            enum:
+                            - replace
+                            - keep
+                            - drop
+                            - hashmod
+                            - labelmap
+                            - labeldrop
+                            - labelkeep
                             type: string
                           modulus:
                             description: Modulus to take of the hash of the source
@@ -187,6 +200,10 @@ spec:
                               separator and matched against the configured regular
                               expression for the replace, keep, and drop actions.
                             items:
+                              description: LabelName is a valid Prometheus label name
+                                which may only contain ASCII letters, numbers, as
+                                well as underscores.
+                              pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$
                               type: string
                             type: array
                           targetLabel:
@@ -301,8 +318,9 @@ spec:
                     relabelings:
                       description: 'RelabelConfigs to apply to samples before scraping.
                         Prometheus Operator automatically adds relabelings for a few
-                        standard Kubernetes fields and replaces original scrape job
+                        standard Kubernetes fields. The original scrape job''s name
-                        name with __tmp_prometheus_job_name. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
+                        is available via the `__tmp_prometheus_job_name` label. More
+                        info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
                       items:
                         description: 'RelabelConfig allows dynamic rewriting of the
                           label set, being applied to samples before ingestion. It
@@ -310,8 +328,17 @@ spec:
                           configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
                         properties:
                           action:
+                            default: replace
                             description: Action to perform based on regex matching.
                               Default is 'replace'
+                            enum:
+                            - replace
+                            - keep
+                            - drop
+                            - hashmod
+                            - labelmap
+                            - labeldrop
+                            - labelkeep
                             type: string
                           modulus:
                             description: Modulus to take of the hash of the source
@@ -337,6 +364,10 @@ spec:
                               separator and matched against the configured regular
                               expression for the replace, keep, and drop actions.
                             items:
+                              description: LabelName is a valid Prometheus label name
+                                which may only contain ASCII letters, numbers, as
+                                well as underscores.
+                              pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$
                               type: string
                             type: array
                           targetLabel:
@@ -521,7 +552,7 @@ spec:
                       in contrast to a list restricting them.
                     type: boolean
                   matchNames:
-                    description: List of namespace names.
+                    description: List of namespace names to select from.
                     items:
                       type: string
                     type: array
@@ -583,8 +614,7 @@ spec:
                 type: object
               targetLabels:
                 description: TargetLabels transfers labels from the Kubernetes `Service`
-                  onto the created metrics. All labels set in `selector.matchLabels`
+                  onto the created metrics.
-                  are automatically transferred.
                 items:
                   type: string
                 type: array

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.55.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1
@@ -19,7 +19,15 @@ spec:
     singular: thanosruler
   scope: Namespaced
   versions:
-  - name: v1
+  - additionalPrinterColumns:
+    - description: The desired replicas number of Thanos Rulers
+      jsonPath: .spec.replicas
+      name: Replicas
+      type: integer
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1
     schema:
       openAPIV3Schema:
         description: ThanosRuler defines a ThanosRuler deployment.
@@ -6199,6 +6207,7 @@ spec:
         type: object
     served: true
     storage: true
+    subresources: {}
 status:
   acceptedNames:
     kind: ""

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl

@@ -48,7 +48,7 @@ The longest name that gets created adds and extra 37 characters, so truncation s
 {{- define "kube-prometheus-stack.labels" }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 app.kubernetes.io/instance: {{ .Release.Name }}
-app.kubernetes.io/version: "{{ .Chart.Version }}"
+app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}"
 app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }}
 chart: {{ template "kube-prometheus-stack.chartref" . }}
 release: {{ $.Release.Name | quote }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml

@@ -68,6 +68,10 @@ spec:
 {{ else }}
   alertmanagerConfigNamespaceSelector: {}
 {{- end }}
+{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration }}
+  alertmanagerConfiguration:
+{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration | indent 4 }}
+{{- end }}
 {{- if .Values.alertmanager.alertmanagerSpec.resources }}
   resources:
 {{ toYaml .Values.alertmanager.alertmanagerSpec.resources | indent 4 }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/service.yaml

@@ -31,6 +31,9 @@ spec:
   {{- range $cidr := .Values.alertmanager.service.loadBalancerSourceRanges }}
     - {{ $cidr }}
   {{- end }}
+{{- end }}
+{{- if ne .Values.alertmanager.service.type "ClusterIP" }}
+  externalTrafficPolicy: {{ .Values.alertmanager.service.externalTrafficPolicy }}
 {{- end }}
   ports:
   - name: {{ .Values.alertmanager.alertmanagerSpec.portName }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceperreplica.yaml

@@ -30,6 +30,9 @@ items:
         - {{ $cidr }}
       {{- end }}
       {{- end }}
+      {{- if ne $serviceValues.type "ClusterIP" }}
+      externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }}
+      {{- end }}
       ports:
         - name: {{ $.Values.alertmanager.alertmanagerSpec.portName }}
           {{- if eq $serviceValues.type "NodePort" }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmap-dashboards.yaml

@@ -13,7 +13,7 @@ items:
     namespace: {{ template "kube-prometheus-stack-grafana.namespace" $ }}
     labels:
       {{- if $.Values.grafana.sidecar.dashboards.label }}
-      {{ $.Values.grafana.sidecar.dashboards.label }}: "1"
+      {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }}
       {{- end }}
       app: {{ template "kube-prometheus-stack.name" $ }}-grafana
 {{ include "kube-prometheus-stack.labels" $ | indent 6 }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml

@@ -9,7 +9,7 @@ metadata:
 {{ toYaml .Values.grafana.sidecar.datasources.annotations | indent 4 }}
 {{- end }}
   labels:
-    {{ $.Values.grafana.sidecar.datasources.label }}: "1"
+    {{ $.Values.grafana.sidecar.datasources.label }}: {{ $.Values.grafana.sidecar.datasources.labelValue | quote }}
     app: {{ template "kube-prometheus-stack.name" $ }}-grafana
 {{ include "kube-prometheus-stack.labels" $ | indent 4 }}
 data:
@@ -24,6 +24,7 @@ data:
 {{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }}
     - name: Prometheus
       type: prometheus
+      uid: {{ .Values.grafana.sidecar.datasources.uid }}
       {{- if .Values.grafana.sidecar.datasources.url }}
       url: {{ .Values.grafana.sidecar.datasources.url }}
       {{- else }}
@@ -37,6 +38,7 @@ data:
 {{- range until (int .Values.prometheus.prometheusSpec.replicas) }}
     - name: Prometheus-{{ . }}
       type: prometheus
+      uid: {{ .Values.grafana.sidecar.datasources.uid }}-replica-{{ . }}
       url: http://prometheus-{{ template "kube-prometheus-stack.fullname" $ }}-prometheus-{{ . }}.prometheus-operated:9090/{{ trimPrefix "/" $.Values.prometheus.prometheusSpec.routePrefix }}
       access: proxy
       isDefault: false

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/servicemonitor.yaml

@@ -1,34 +0,0 @@
-{{- if and .Values.grafana.enabled .Values.grafana.serviceMonitor.selfMonitor }}
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: {{ template "kube-prometheus-stack.fullname" . }}-grafana
-  namespace: {{ template "kube-prometheus-stack.namespace" . }}
-  labels:
-    app: {{ template "kube-prometheus-stack.name" . }}-grafana
-{{ include "kube-prometheus-stack.labels" . | indent 4 }}
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: grafana
-      app.kubernetes.io/instance: {{ $.Release.Name | quote }}
-  {{- if ne (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack-grafana.namespace" .) }}
-  namespaceSelector:
-    matchNames:
-      - {{ printf "%s" (include "kube-prometheus-stack-grafana.namespace" .) | quote }}
-  {{- end }}
-  endpoints:
-  - port: {{ .Values.grafana.service.portName }}
-    {{- if .Values.grafana.serviceMonitor.interval }}
-    interval: {{ .Values.grafana.serviceMonitor.interval }}
-    {{- end }}
-    path: {{ .Values.grafana.serviceMonitor.path | quote }}
-{{- if .Values.grafana.serviceMonitor.metricRelabelings }}
-    metricRelabelings:
-{{ tpl (toYaml .Values.grafana.serviceMonitor.metricRelabelings | indent 6) . }}
-{{- end }}
-{{- if .Values.grafana.serviceMonitor.relabelings }}
-    relabelings:
-{{ toYaml .Values.grafana.serviceMonitor.relabelings | indent 6 }}
-{{- end }}
-{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/aggregate-clusterroles.yaml

@@ -0,0 +1,31 @@
+{{/* This file is based on https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/rbac-crd.md */}}
+{{- if and .Values.global.rbac.create .Values.global.rbac.createAggregateClusterRoles }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-crd-view
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+    app: {{ template "kube-prometheus-stack.name" . }}-operator
+    {{- include "kube-prometheus-stack.labels" . | nindent 4 }}
+rules:
+- apiGroups: ["monitoring.coreos.com"]
+  resources: ["alertmanagers", "alertmanagerconfigs", "prometheuses", "prometheusrules", "servicemonitors", "podmonitors", "probes"]
+  verbs: ["get", "list", "watch"]
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-crd-edit
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+    app: {{ template "kube-prometheus-stack.name" . }}-operator
+    {{- include "kube-prometheus-stack.labels" . | nindent 4 }}
+rules:
+- apiGroups: ["monitoring.coreos.com"]
+  resources: ["alertmanagers", "alertmanagerconfigs", "prometheuses", "prometheusrules", "servicemonitors", "podmonitors", "probes"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml

@@ -112,8 +112,7 @@ spec:
           resources:
 {{ toYaml .Values.prometheusOperator.resources | indent 12 }}
           securityContext:
-            allowPrivilegeEscalation: false
+{{ toYaml .Values.prometheusOperator.containerSecurityContext | indent 12 }}
-            readOnlyRootFilesystem: true
 {{- if .Values.prometheusOperator.tls.enabled }}
           volumeMounts:
             - name: tls-secret

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/service.yaml

@@ -30,6 +30,9 @@ spec:
   {{- range $cidr := .Values.prometheusOperator.service.loadBalancerSourceRanges }}
     - {{ $cidr }}
   {{- end }}
+{{- end }}
+{{- if ne .Values.prometheusOperator.service.type "ClusterIP" }}
+  externalTrafficPolicy: {{ .Values.prometheusOperator.service.externalTrafficPolicy }}
 {{- end }}
   ports:
   {{- if not .Values.prometheusOperator.tls.enabled }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/_rules.tpl

@@ -10,6 +10,8 @@ rules:
   - "general.rules"
   - "k8s.rules"
   - "kube-apiserver-availability.rules"
+  - "kube-apiserver-burnrate.rules"
+  - "kube-apiserver-histogram.rules"
   - "kube-apiserver-slos"
   - "kube-prometheus-general.rules"
   - "kube-prometheus-node-recording.rules"

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/additionalScrapeConfigs.yaml

@@ -12,5 +12,9 @@ metadata:
     app: {{ template "kube-prometheus-stack.name" . }}-prometheus-scrape-confg
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 data:
+{{- if  eq ( typeOf .Values.prometheus.prometheusSpec.additionalScrapeConfigs ) "string" }}
+  additional-scrape-configs.yaml: {{ tpl .Values.prometheus.prometheusSpec.additionalScrapeConfigs $ | b64enc | quote }}
+{{- else }}
   additional-scrape-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalScrapeConfigs) $ | b64enc | quote }}
 {{- end }}
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/service.yaml

@@ -31,6 +31,9 @@ spec:
   {{- range $cidr := .Values.prometheus.service.loadBalancerSourceRanges }}
     - {{ $cidr }}
   {{- end }}
+{{- end }}
+{{- if ne .Values.prometheus.service.type "ClusterIP" }}
+  externalTrafficPolicy: {{ .Values.prometheus.service.externalTrafficPolicy }}
 {{- end }}
   ports:
   - name: {{ .Values.prometheus.prometheusSpec.portName }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecar.yaml

@@ -17,6 +17,9 @@ metadata:
 spec:
   type: {{ .Values.prometheus.thanosService.type }}
   clusterIP: {{ .Values.prometheus.thanosService.clusterIP }}
+{{- if ne .Values.prometheus.thanosService.type "ClusterIP" }}
+  externalTrafficPolicy: {{ .Values.prometheus.thanosService.externalTrafficPolicy }}
+{{- end }}
   ports:
   - name: {{ .Values.prometheus.thanosService.portName }}
     port: {{ .Values.prometheus.thanosService.port }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecarExternal.yaml

@@ -23,6 +23,9 @@ spec:
   {{- range $cidr := .Values.prometheus.thanosServiceExternal.loadBalancerSourceRanges }}
     - {{ $cidr }}
   {{- end }}
+{{- end }}
+{{- if ne .Values.prometheus.thanosServiceExternal.type "ClusterIP" }}
+  externalTrafficPolicy: {{ .Values.prometheus.thanosServiceExternal.externalTrafficPolicy }}
 {{- end }}
   ports:
   - name: {{ .Values.prometheus.thanosServiceExternal.portName }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceperreplica.yaml

@@ -30,6 +30,9 @@ items:
         - {{ $cidr }}
       {{- end }}
       {{- end }}
+      {{- if ne $serviceValues.type "ClusterIP" }}
+      externalTrafficPolicy: {{ $serviceValues.externalTrafficPolicy }}
+      {{- end }}
       ports:
         - name: {{ $.Values.prometheus.prometheusSpec.portName }}
           {{- if eq $serviceValues.type "NodePort" }}

charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml

@@ -101,6 +101,10 @@ additionalPrometheusRulesMap: {}
 global:
   rbac:
     create: true
+
+    ## Create ClusterRoles that extend the existing view, edit and admin ClusterRoles to interact with prometheus-operator CRDs
+    ## Ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#aggregated-clusterroles
+    createAggregateClusterRoles: false
     pspEnabled: false
     pspAnnotations: {}
       ## Specify pod annotations
@@ -322,6 +326,11 @@ alertmanager:
     externalIPs: []
     loadBalancerIP: ""
     loadBalancerSourceRanges: []
+
+    ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints
+    ##
+    externalTrafficPolicy: Cluster
+
     ## Service type
     ##
     type: ClusterIP
@@ -347,6 +356,11 @@ alertmanager:
     ## Loadbalancer source IP ranges
     ## Only used if servicePerReplica.type is "LoadBalancer"
     loadBalancerSourceRanges: []
+
+    ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints
+    ##
+    externalTrafficPolicy: Cluster
+
     ## Service type
     ##
     type: ClusterIP
@@ -404,7 +418,7 @@ alertmanager:
     ##
     image:
       repository: quay.io/prometheus/alertmanager
-      tag: v0.23.0
+      tag: v0.24.0
       sha: ""
 
     ## If true then the user will be responsible to provide a secret with alertmanager configuration
@@ -463,6 +477,11 @@ alertmanager:
     #   matchLabels:
     #     alertmanagerconfig: enabled
 
+    ## AlermanagerConfig to be used as top level configuration
+    ##
+    alertmanagerConfiguration: {}
+    # - name: global-alertmanager-Configuration
+
     ## Define Log Format
     # Use logfmt (default) or json logging
     logFormat: logfmt
@@ -660,6 +679,11 @@ grafana:
     ##
     enabled: false
 
+    ## IngressClassName for Grafana Ingress.
+    ## Should be provided if Ingress is enable.
+    ##
+    # ingressClassName: nginx
+
     ## Annotations for Grafana Ingress
     ##
     annotations: {}
@@ -692,6 +716,7 @@ grafana:
     dashboards:
       enabled: true
       label: grafana_dashboard
+      labelValue: "1"
 
       ## Annotations for Grafana dashboard configmaps
       ##
@@ -707,6 +732,8 @@ grafana:
       enabled: true
       defaultDatasourceEnabled: true
 
+      uid: prometheus
+
       ## URL of prometheus datasource
       ##
       # url: http://prometheus-stack-prometheus:9090/
@@ -724,6 +751,7 @@ grafana:
       ## ref: https://git.io/fjaBS
       createPrometheusReplicasDatasources: false
       label: grafana_datasource
+      labelValue: "1"
 
   extraConfigmapMounts: []
   # - name: certs-configmap
@@ -756,26 +784,27 @@ grafana:
   service:
     portName: http-web
 
-  ## If true, create a serviceMonitor for grafana
-  ##
   serviceMonitor:
-    ## Scrape interval. If not set, the Prometheus default scrape interval is used.
+    # If true, a ServiceMonitor CRD is created for a prometheus operator
-    ##
+    # https://github.com/coreos/prometheus-operator
-    interval: ""
+    #
-    selfMonitor: true
+    enabled: true
 
     # Path to use for scraping metrics. Might be different if server.root_url is set
     # in grafana.ini
     path: "/metrics"
 
+    #  namespace: monitoring  (defaults to use the namespace this chart is deployed to)
 
-    ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion.
+    # labels for the ServiceMonitor
-    ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig
+    labels: {}
-    ##
+
-    metricRelabelings: []
+    # Scrape interval. If not set, the Prometheus default scrape interval is used.
-    # - action: keep
+    #
-    #   regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+'
+    interval: ""
-    #   sourceLabels: [__name__]
+    scheme: http
+    tlsConfig: {}
+    scrapeTimeout: 30s
 
     ## RelabelConfigs to apply to samples before scraping
     ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig
@@ -1376,8 +1405,8 @@ prometheus-node-exporter:
     ##
     jobLabel: node-exporter
   extraArgs:
-    - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/)
+    - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/)
-    - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$
+    - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$
   service:
     portName: http-metrics
   prometheus:
@@ -1453,8 +1482,8 @@ prometheusOperator:
       enabled: true
       image:
         repository: k8s.gcr.io/ingress-nginx/kube-webhook-certgen
-        tag: v1.0
+        tag: v1.1.1
-        sha: "f3b6b39a6062328c095337b4cadcefd1612348fdd5190b1dcbcb9b9e90bd8068"
+        sha: ""
         pullPolicy: IfNotPresent
       resources: {}
       ## Provide a priority class name to the webhook patching job
@@ -1542,6 +1571,10 @@ prometheusOperator:
     loadBalancerIP: ""
     loadBalancerSourceRanges: []
 
+    ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints
+    ##
+    externalTrafficPolicy: Cluster
+
   ## Service type
   ## NodePort, ClusterIP, LoadBalancer
   ##
@@ -1664,11 +1697,18 @@ prometheusOperator:
     runAsNonRoot: true
     runAsUser: 65534
 
+  ## Container-specific security context configuration
+  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+  ##
+  containerSecurityContext:
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+
   ## Prometheus-operator image
   ##
   image:
     repository: quay.io/prometheus-operator/prometheus-operator
-    tag: v0.53.1
+    tag: v0.55.0
     sha: ""
     pullPolicy: IfNotPresent
 
@@ -1686,23 +1726,23 @@ prometheusOperator:
     # image to use for config and rule reloading
     image:
       repository: quay.io/prometheus-operator/prometheus-config-reloader
-      tag: v0.53.1
+      tag: v0.55.0
       sha: ""
 
     # resource config for prometheusConfigReloader
     resources:
       requests:
-        cpu: 100m
+        cpu: 200m
         memory: 50Mi
       limits:
-        cpu: 100m
+        cpu: 200m
         memory: 50Mi
 
   ## Thanos side-car image when configured
   ##
   thanosImage:
     repository: quay.io/thanos/thanos
-    tag: v0.24.0
+    tag: v0.25.2
     sha: ""
 
   ## Set a Field Selector to filter watched secrets
@@ -1737,6 +1777,10 @@ prometheus:
     annotations: {}
     labels: {}
 
+    ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints
+    ##
+    externalTrafficPolicy: Cluster
+
     ## Service type
     ##
     type: ClusterIP
@@ -1800,6 +1844,10 @@ prometheus:
     httpPort: 10902
     targetHttpPort: "http"
 
+    ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints
+    ##
+    externalTrafficPolicy: Cluster
+
     ## Service type
     ##
     type: LoadBalancer
@@ -1837,6 +1885,11 @@ prometheus:
     ## Only use if service.type is "LoadBalancer"
     loadBalancerIP: ""
     loadBalancerSourceRanges: []
+
+    ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints
+    ##
+    externalTrafficPolicy: Cluster
+
     ## Service type
     ##
     type: ClusterIP
@@ -1871,6 +1924,11 @@ prometheus:
     ## Loadbalancer source IP ranges
     ## Only used if servicePerReplica.type is "LoadBalancer"
     loadBalancerSourceRanges: []
+
+    ## Denotes if this Service desires to route external traffic to node-local or cluster-wide endpoints
+    ##
+    externalTrafficPolicy: Cluster
+
     ## Service type
     ##
     type: ClusterIP
@@ -2102,7 +2160,7 @@ prometheus:
     ##
     image:
       repository: quay.io/prometheus/prometheus
-      tag: v2.32.1
+      tag: v2.34.0
       sha: ""
 
     ## Tolerations for use with node taints
@@ -2412,6 +2470,7 @@ prometheus:
     ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility
     ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible
     ## scrape configs are going to break Prometheus after the upgrade.
+    ## AdditionalScrapeConfigs can be defined as a list or as a templated string.
     ##
     ## The scrape configuration example below will find master nodes, provided they have the name .*mst.*, relabel the
     ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes
@@ -2444,6 +2503,20 @@ prometheus:
     #   metric_relabel_configs:
     #   - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone)
     #     action: labeldrop
+    #
+    ## If scrape config contains a repetitive section, you may want to use a template.
+    ## In the following example, you can see how to define `gce_sd_configs` for multiple zones
+    # additionalScrapeConfigs: |
+    #  - job_name: "node-exporter"
+    #    gce_sd_configs:
+    #    {{range $zone := .Values.gcp_zones}}
+    #    - project: "project1"
+    #      zone: "{{$zone}}"
+    #      port: 9100
+    #    {{end}}
+    #    relabel_configs:
+    #    ...
+
 
     ## If additional scrape configurations are already deployed in a single secret file you can use this section.
     ## Expected values are the secret name and key

charts/kubezero-metrics/charts/prometheus-pushgateway/Chart.yaml

@@ -13,4 +13,4 @@ maintainers:
 name: prometheus-pushgateway
 sources:
 - https://github.com/prometheus/pushgateway
-version: 1.14.0
+version: 1.16.1

charts/kubezero-metrics/charts/prometheus-pushgateway/templates/deployment.yaml

@@ -47,20 +47,20 @@ spec:
             - name: metrics
               containerPort: 9091
               protocol: TCP
+{{- if .Values.liveness.enabled }}
           livenessProbe:
-            httpGet:
+{{ toYaml .Values.liveness.probe | indent 12 }}
-              path: /-/healthy
+        {{- end }}
-              port: 9091
+{{- if .Values.readiness.enabled }}
-            initialDelaySeconds: 10
-            timeoutSeconds: 10
           readinessProbe:
-            httpGet:
+{{ toYaml .Values.readiness.probe | indent 12 }}
-              path: /-/ready
+        {{- end }}
-              port: 9091
-            initialDelaySeconds: 10
-            timeoutSeconds: 10
           resources:
 {{ toYaml .Values.resources | indent 12 }}
+        {{- if .Values.containerSecurityContext }}
+          securityContext:
+{{ toYaml .Values.containerSecurityContext | indent 12 }}
+        {{- end }}
           volumeMounts:
             - name: storage-volume
               mountPath: "{{ .Values.persistentVolume.mountPath }}"

charts/kubezero-metrics/charts/prometheus-pushgateway/values.yaml

@@ -68,6 +68,24 @@ resources: {}
   #   cpu: 100m
   #   memory: 30Mi
 
+liveness:
+  enabled: true
+  probe:
+    httpGet:
+      path: /-/ready
+      port: 9091
+    initialDelaySeconds: 10
+    timeoutSeconds: 10
+
+readiness:
+  enabled: true
+  probe:
+    httpGet:
+      path: /-/ready
+      port: 9091
+    initialDelaySeconds: 10
+    timeoutSeconds: 10
+
 serviceAccount:
   # Specifies whether a ServiceAccount should be created
   create: true
@@ -126,6 +144,14 @@ securityContext:
   runAsUser: 65534
   runAsNonRoot: true
 
+## Security context to be added to push-gateway containers
+## Having a separate variable as securityContext differs for pods and containers.
+containerSecurityContext: {}
+#  allowPrivilegeEscalation: false
+#  readOnlyRootFilesystem: true
+#  runAsUser: 65534
+#  runAsNonRoot: true
+
 ## Affinity for pod assignment
 ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
 affinity: {}

charts/kubezero-metrics/configs/rules.libsonnet

@@ -1,49 +0,0 @@
-local addMixin = (import 'kube-prometheus/lib/mixin.libsonnet');
-
-local etcdMixin = addMixin({
-  name: 'etcd',
-  mixin: (import 'github.com/etcd-io/etcd/contrib/mixin/mixin.libsonnet') +
-    {
-			_config+: {
-				etcd_instance_labels: 'instance, pod',
-      }
-		} +
-		// Remove both etcdHighNumberOfFailedGRPCRequests from etcd-mixin for now
-		{ 
-			prometheusAlerts+: {
-				groups: std.map(
-					function(group)
-						if group.name == 'etcd' then
-							group {
-								rules: std.filter(
-									function(rule)
-										rule.alert != 'etcdHighNumberOfFailedGRPCRequests',
-									group.rules
-								),
-							}
-						else
-							group,
-					super.groups
-				),
-			},
-		},
-  });
-
-local kp = (import 'kube-prometheus/main.libsonnet') +
-  {
-    values+:: {
-      common+: {
-        namespace: 'monitoring',
-      },
-    } 
-  };
-
-// We just want the Prometheus Rules
-{ 'prometheus-operator-prometheusRule': kp.prometheusOperator.prometheusRule } +
-{ 'kube-prometheus-prometheusRule': kp.kubePrometheus.prometheusRule } +
-{ 'alertmanager-prometheusRule': kp.alertmanager.prometheusRule } +
-{ 'kube-state-metrics-prometheusRule': kp.kubeStateMetrics.prometheusRule } +
-{ 'kubernetes-prometheusRule': kp.kubernetesControlPlane.prometheusRule } +
-{ 'node-exporter-prometheusRule': kp.nodeExporter.prometheusRule } +
-{ 'prometheus-prometheusRule': kp.prometheus.prometheusRule } +
-{ 'etcd-mixin-prometheusRule': etcdMixin.prometheusRules }

charts/kubezero-metrics/jsonnet/Makefile

@@ -0,0 +1,30 @@
+JSONNET_ARGS := -n 2 --max-blank-lines 2 --string-style s --comment-style s
+ifneq (,$(shell which jsonnetfmt))
+	JSONNET_FMT_CMD := jsonnetfmt
+else
+	JSONNET_FMT_CMD := jsonnet
+	JSONNET_FMT_ARGS := fmt $(JSONNET_ARGS)
+endif
+JSONNET_FMT := $(JSONNET_FMT_CMD) $(JSONNET_FMT_ARGS)
+
+dashboardsDirPath=dashboards
+rulessDirPath=rules
+
+all: fmt generate lint
+
+fmt:
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		xargs -n 1 -- $(JSONNET_FMT) -i
+
+generate: build.sh
+	bash build.sh $<
+
+lint: 
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		while read f; do \
+			$(JSONNET_FMT) "$$f" | diff -u "$$f" -; \
+		done
+
+clean:
+	rm -rf $(dashboardsDirPath) $(rulessDirPath)
+	mkdir -p $(dashboardsDirPath) $(rulessDirPath)

charts/kubezero-metrics/jsonnet/build.sh

@@ -13,9 +13,8 @@ else
   jb install github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus@release-0.9
 fi
 
-rm -rf dashboards && mkdir -p dashboards
+make clean
-jsonnet -J vendor -m dashboards -e '(import "mixin.libsonnet").grafanaDashboards'
+jsonnet -J vendor -m dashboards -e '(import "dashboards.libsonnet").grafanaDashboards'
 
-rm -rf rules && mkdir -p rules
 #jsonnet -J vendor -m rules rules.libsonnet | xargs -I{} sh -c 'cat {} | gojsontoyaml > {}.yaml' -- {}
 jsonnet -J vendor -m rules rules.libsonnet

charts/kubezero-metrics/jsonnet/dashboards.libsonnet

@@ -1,6 +1,6 @@
-# https://github.com/kubernetes-monitoring/kubernetes-mixin
+// https://github.com/kubernetes-monitoring/kubernetes-mixin
 
-local kubernetes = import "kubernetes-mixin/mixin.libsonnet";
+local kubernetes = import 'kubernetes-mixin/mixin.libsonnet';
 
 kubernetes {
   _config+:: {

charts/kubezero-metrics/jsonnet/jsonnetfile.lock.json

@@ -18,7 +18,7 @@
           "subdir": "contrib/mixin"
         }
       },
-      "version": "a62444690780fd1eb72615b0cd6d804ea54cc771",
+      "version": "3ace62279270ef83c48496ba1a01fb38abe7f76a",
       "sum": "W/Azptf1PoqjyMwJON96UY69MFugDA4IAYiKURscryc="
     },
     {
@@ -28,7 +28,7 @@
           "subdir": "grafonnet"
         }
       },
-      "version": "3626fc4dc2326931c530861ac5bebe39444f6cbf",
+      "version": "6db00c292d3a1c71661fc875f90e0ec7caa538c2",
       "sum": "gF8foHByYcB25jcUOBqP6jxk0OPifQMjPvKY0HaCk6w="
     },
     {
@@ -38,7 +38,7 @@
           "subdir": "grafana-builder"
         }
       },
-      "version": "5a128df878434da37969b811e99bb9cd0a3779e3",
+      "version": "318a65120918fe8e298ffc8e63b9e941677ef321",
       "sum": "0KkygBQd/AFzUvVzezE4qF/uDYgrwUXVpZfINBti0oc="
     },
     {
@@ -58,7 +58,7 @@
           "subdir": "lib/promgrafonnet"
         }
       },
-      "version": "c76b9378b86d28bd617d94a57c72b4770efed510",
+      "version": "f2b5aab16039e7aa78c1b39aad1ec63d87eca85e",
       "sum": "zv7hXGui6BfHzE9wPatHI/AGZa4A2WKo6pq7ZdqBsps="
     },
     {

charts/kubezero-metrics/jsonnet/rules.libsonnet

@@ -0,0 +1,49 @@
+local addMixin = (import 'kube-prometheus/lib/mixin.libsonnet');
+
+local etcdMixin = addMixin({
+  name: 'etcd',
+  mixin: (import 'github.com/etcd-io/etcd/contrib/mixin/mixin.libsonnet') +
+         {
+           _config+: {
+             etcd_instance_labels: 'instance, pod',
+           },
+         } +
+         // Remove both etcdHighNumberOfFailedGRPCRequests from etcd-mixin for now
+         {
+           prometheusAlerts+: {
+             groups: std.map(
+               function(group)
+                 if group.name == 'etcd' then
+                   group {
+                     rules: std.filter(
+                       function(rule)
+                         rule.alert != 'etcdHighNumberOfFailedGRPCRequests',
+                       group.rules
+                     ),
+                   }
+                 else
+                   group,
+               super.groups
+             ),
+           },
+         },
+});
+
+local kp = (import 'kube-prometheus/main.libsonnet') +
+           {
+             values+:: {
+               common+: {
+                 namespace: 'monitoring',
+               },
+             },
+           };
+
+// We just want the Prometheus Rules
+{ 'prometheus-operator-prometheusRule': kp.prometheusOperator.prometheusRule } +
+{ 'kube-prometheus-prometheusRule': kp.kubePrometheus.prometheusRule } +
+{ 'alertmanager-prometheusRule': kp.alertmanager.prometheusRule } +
+{ 'kube-state-metrics-prometheusRule': kp.kubeStateMetrics.prometheusRule } +
+{ 'kubernetes-prometheusRule': kp.kubernetesControlPlane.prometheusRule } +
+{ 'node-exporter-prometheusRule': kp.nodeExporter.prometheusRule } +
+{ 'prometheus-prometheusRule': kp.prometheus.prometheusRule } +
+{ 'etcd-mixin-prometheusRule': etcdMixin.prometheusRules }

charts/kubezero-metrics/sync_grafana_dashboards.py

@@ -30,24 +30,30 @@ def traverse_json(obj):
 
 
 # read config file
-with open(config_file, 'r') as yaml_contents:
+with open(config_file, "r") as yaml_contents:
     config = yaml.safe_load(yaml_contents.read())
 
 
-configmap = ''
+configmap = ""
-if 'condition' in config:
+if "condition" in config:
     # use index function to make go template happy if '-' in names
-    if '-' in config['condition'] and "index" not in config['condition']:
+    if "-" in config["condition"] and "index" not in config["condition"]:
-        tokens = config['condition'].split('.')
+        tokens = config["condition"].split(".")
-        configmap = '''{{- if index .Values %(condition)s }}
+        configmap = """{{- if index .Values %(condition)s }}
-''' % {'condition': ' '.join(f'"{w}"' for w in tokens[2:])}
+""" % {
+            "condition": " ".join(f'"{w}"' for w in tokens[2:])
+        }
 
     else:
-        configmap = '''{{- if %(condition)s }}
+        configmap = (
-''' % config
+            """{{- if %(condition)s }}
+"""
+            % config
+        )
 
 # Base configmap for KubeZero
-configmap += '''apiVersion: v1
+configmap += (
+    """apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ printf "%%s-%%s" (include "kubezero-lib.fullname" $) "%(configmap)s" | trunc 63 | trimSuffix "-" }}
@@ -55,31 +61,39 @@ metadata:
   labels:
     grafana_dashboard: "1"
     {{- include "kubezero-lib.labels" . | nindent 4 }}
-''' % config
+"""
+    % config
+)
 
 # Put all dashboards into a folder ?
-if 'folder' in config:
+if "folder" in config:
-    configmap += '''  annotations:
+    configmap += (
+        """  annotations:
     k8s-sidecar-target-directory: %(folder)s
-''' % config
+"""
+        % config
+    )
 
 # compress ?
-if 'gzip' in config and config['gzip']:
+if "gzip" in config and config["gzip"]:
-    configmap += '''binaryData:
+    configmap += """binaryData:
-'''
+"""
 else:
-    configmap += '''data:
+    configmap += """data:
-'''
+"""
 
-for b in config['dashboards']:
+for b in config["dashboards"]:
-    if not b['url'].startswith('file://'):
+    if not b["url"].startswith("file://"):
-        response = requests.get(b['url'])
+        response = requests.get(b["url"])
         if response.status_code != 200:
-            print('Skipping the file, response code %s not equals 200' % response.status_code)
+            print(
+                "Skipping the file, response code %s not equals 200"
+                % response.status_code
+            )
             continue
         raw_text = response.text
     else:
-        with open(b['url'].replace('file://', ''), 'r') as file_contents:
+        with open(b["url"].replace("file://", ""), "r") as file_contents:
             raw_text = file_contents.read()
 
     obj = json.loads(raw_text)
@@ -88,39 +102,50 @@ for b in config['dashboards']:
     traverse_json(obj)
 
     # Set default tim in all charts to 1h
-    obj['time'] = {"from": "now-1h", "to": "now"}
+    obj["time"] = {"from": "now-1h", "to": "now"}
-    obj['refresh'] = "30s"
+    obj["refresh"] = "30s"
 
     # set tags
-    if 'tags' in b:
+    if "tags" in b:
-        obj['tags'] = b['tags']
+        obj["tags"] = b["tags"]
 
     text = json.dumps(obj, indent=2)
 
-    if 'gzip' in config and config['gzip']:
+    if "gzip" in config and config["gzip"]:
         # compress and base64 encode
         buf = io.BytesIO()
-        f = gzip.GzipFile(mode='w', fileobj=buf, mtime=0)
+        f = gzip.GzipFile(mode="w", fileobj=buf, mtime=0)
         f.write(text.encode())
         f.close()
 
-        textb64 = base64.b64encode(buf.getvalue()).decode('utf-8')
+        textb64 = base64.b64encode(buf.getvalue()).decode("utf-8")
-        textb64 = textwrap.indent(textb64, ' '*4)
+        textb64 = textwrap.indent(textb64, " " * 4)
-        configmap += '''  %(name)s.json.gz:
+        configmap += (
-''' % b
+            """  %(name)s.json.gz:
-        configmap += textb64+'\n'
+"""
+            % b
+        )
+        configmap += textb64 + "\n"
     else:
         # encode otherwise helm will mess with raw json
-        text = text.replace("{{", "{{`{{").replace("}}", "}}`}}").replace("{{`{{", "{{`{{`}}").replace("}}`}}", "{{`}}`}}")
+        text = (
-        text = textwrap.indent(text, ' '*4)
+            text.replace("{{", "{{`{{")
-        configmap += '''  %(name)s.json:
+            .replace("}}", "}}`}}")
-''' % b
+            .replace("{{`{{", "{{`{{`}}")
-        configmap += text+'\n'
+            .replace("}}`}}", "{{`}}`}}")
+        )
+        text = textwrap.indent(text, " " * 4)
+        configmap += (
+            """  %(name)s.json:
+"""
+            % b
+        )
+        configmap += text + "\n"
 
 
-if 'condition' in config:
+if "condition" in config:
-    configmap += '{{- end }}'+'\n'
+    configmap += "{{- end }}" + "\n"
 
 # Write Configmap
-with open(configmap_file, 'w') as f:
+with open(configmap_file, "w") as f:
     f.write(configmap)

charts/kubezero-metrics/sync_prometheus_rules.py

@@ -12,63 +12,79 @@ configmap_folder = sys.argv[2]
 
 
 # read config file
-with open(config_file, 'r') as yaml_contents:
+with open(config_file, "r") as yaml_contents:
     config = yaml.safe_load(yaml_contents.read())
 
 
 def base_rule(config):
-    rule = ''
+    rule = ""
-    if 'condition' in config:
+    if "condition" in config:
         # use index function to make go template happy if '-' in names
-        if '-' in config['condition'] and "index" not in config['condition']:
+        if "-" in config["condition"] and "index" not in config["condition"]:
-            tokens = config['condition'].split('.')
+            tokens = config["condition"].split(".")
-            rule = '''{{- if index .Values %(condition)s }}
+            rule = """{{- if index .Values %(condition)s }}
-    ''' % {'condition': ' '.join(f'"{w}"' for w in tokens[2:])}
+    """ % {
+                "condition": " ".join(f'"{w}"' for w in tokens[2:])
+            }
 
         else:
-            rule = '''{{- if %(condition)s }}
+            rule = (
-    ''' % config
+                """{{- if %(condition)s }}
+"""
+                % config
+            )
 
     # Base rule for KubeZero
-    rule += '''apiVersion: monitoring.coreos.com/v1
+    rule += (
+        """apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
   name: {{ printf "%%s-%%s" (include "kubezero-lib.fullname" $) "%(name)s" | trunc 63 | trimSuffix "-" }}
   namespace: {{ .Release.Namespace }}
   labels:
     {{- include "kubezero-lib.labels" . | nindent 4 }}
-''' % config
+"""
+        % config
+    )
 
     return rule
 
 
-for r in config['rules']:
+for r in config["rules"]:
-    if not r['url'].startswith('file://'):
+    if not r["url"].startswith("file://"):
-        response = requests.get(r['url'])
+        response = requests.get(r["url"])
         if response.status_code != 200:
-            print('Skipping the file, response code %s not equals 200' % response.status_code)
+            print(
+                "Skipping the file, response code %s not equals 200"
+                % response.status_code
+            )
             continue
         raw_text = response.text
     else:
-        with open(r['url'].replace('file://', ''), 'r') as file_contents:
+        with open(r["url"].replace("file://", ""), "r") as file_contents:
             raw_text = file_contents.read()
 
     obj = json.loads(raw_text)
 
     rule = base_rule(r)
 
-    text = yaml.dump(obj['spec'], default_flow_style=False, width=1000, indent=2)
+    text = yaml.dump(obj["spec"], default_flow_style=False, width=1000, indent=2)
 
     # Encode {{ }} for helm
-    text = text.replace("{{", "{{`{{").replace("}}", "}}`}}").replace("{{`{{", "{{`{{`}}").replace("}}`}}", "{{`}}`}}")
+    text = (
+        text.replace("{{", "{{`{{")
+        .replace("}}", "}}`}}")
+        .replace("{{`{{", "{{`{{`}}")
+        .replace("}}`}}", "{{`}}`}}")
+    )
 
-    rule += '''spec:\n'''
+    rule += """spec:\n"""
-    rule += textwrap.indent(text, ' '*2)+'\n'
+    rule += textwrap.indent(text, " " * 2) + "\n"
 
-    if 'condition' in r:
+    if "condition" in r:
-        rule += '{{- end }}'+'\n'
+        rule += "{{- end }}" + "\n"
 
     # Write Configmap
-    configmap_file = os.path.join(configmap_folder, r['name'] + '.yaml')
+    configmap_file = os.path.join(configmap_folder, r["name"] + ".yaml")
-    with open(configmap_file, 'w') as f:
+    with open(configmap_file, "w") as f:
         f.write(rule)

charts/kubezero-metrics/update.sh

@@ -16,7 +16,7 @@ patch -p0 -i zdt.patch --no-backup-if-mismatch
 patch -p0 -i zdt-pushgateway.patch --no-backup-if-mismatch
 
 # Create ZDT dashboard, alerts etc configmaps
-cd configs
+cd jsonnet
 ./build.sh
 
 ../sync_grafana_dashboards.py metrics-dashboards.yaml ../templates/grafana-dashboards-metrics.yaml