Latest Prometheus stack

charts/kubezero-metrics/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-metrics
 description: KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all Kubernetes integrations.
 type: application
-version: 0.8.9
+version: 0.9.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -19,16 +19,16 @@ dependencies:
     repository: https://cdn.zero-downtime.net/charts/
     # https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack
   - name: kube-prometheus-stack
-    version: 43.2.0
+    version: 45.9.1
     # Switch back to upstream once all alerts are fixed eg. etcd gpcr
     # repository: https://prometheus-community.github.io/helm-charts
   - name: prometheus-adapter
-    version: 3.5.0
+    version: 4.1.1
     repository: https://prometheus-community.github.io/helm-charts
     condition: prometheus-adapter.enabled
   - name: prometheus-pushgateway
-    version: 2.0.2
+    version: 2.1.3
     # Switch back to upstream once namespaces are supported
     repository: https://prometheus-community.github.io/helm-charts
     condition: prometheus-pushgateway.enabled
-kubeVersion: ">= 1.24.0"
+kubeVersion: ">= 1.25.0"

charts/kubezero-metrics/README.md

@@ -1,6 +1,6 @@
 # kubezero-metrics
 
-![Version: 0.8.9](https://img.shields.io/badge/Version-0.8.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.9.0](https://img.shields.io/badge/Version-0.9.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all Kubernetes integrations.
 
@@ -14,14 +14,14 @@ KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all
 
 ## Requirements
 
-Kubernetes: `>= 1.24.0`
+Kubernetes: `>= 1.25.0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-|  | kube-prometheus-stack | 43.2.0 |
+|  | kube-prometheus-stack | 45.9.1 |
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
-| https://prometheus-community.github.io/helm-charts | prometheus-adapter | 3.5.0 |
+| https://prometheus-community.github.io/helm-charts | prometheus-adapter | 4.1.1 |
-| https://prometheus-community.github.io/helm-charts | prometheus-pushgateway | 2.0.2 |
+| https://prometheus-community.github.io/helm-charts | prometheus-pushgateway | 2.1.3 |
 
 ## Values
 

charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml

@@ -7,20 +7,20 @@ annotations:
       url: https://github.com/prometheus-operator/kube-prometheus
   artifacthub.io/operator: "true"
 apiVersion: v2
-appVersion: 0.61.1
+appVersion: v0.63.0
 dependencies:
 - condition: kubeStateMetrics.enabled
   name: kube-state-metrics
   repository: https://prometheus-community.github.io/helm-charts
-  version: 4.24.*
+  version: 5.0.*
 - condition: nodeExporter.enabled
   name: prometheus-node-exporter
   repository: https://prometheus-community.github.io/helm-charts
-  version: 4.8.*
+  version: 4.14.*
 - condition: grafana.enabled
   name: grafana
   repository: https://grafana.github.io/helm-charts
-  version: 6.48.*
+  version: 6.51.*
 description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards,
   and Prometheus rules combined with documentation and scripts to provide easy to
   operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus
@@ -52,4 +52,4 @@ sources:
 - https://github.com/prometheus-community/helm-charts
 - https://github.com/prometheus-operator/kube-prometheus
 type: application
-version: 43.2.0
+version: 45.9.1

charts/kubezero-metrics/charts/kube-prometheus-stack/README.md

@@ -80,6 +80,44 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen
 
 A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions.
 
+### From 44.x to 45.x
+
+This version upgrades Prometheus-Operator to v0.63.0, Prometheus to v2.43.0 and Thanos to v0.30.2.
+
+Run these commands to update the CRDs before applying the upgrade.
+
+```console
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
+```
+
+### From 43.x to 44.x
+
+This version upgrades Prometheus-Operator to v0.62.0, Prometheus to v2.41.0 and Thanos to v0.30.1.
+
+Run these commands to update the CRDs before applying the upgrade.
+
+```console
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
+```
+
+If you have explicitly set `prometheusOperator.admissionWebhooks.failurePolicy`, this value is now always used even when `.prometheusOperator.admissionWebhooks.patch.enabled` is `true` (the default).
+
+The values for `prometheusOperator.image.tag` & `prometheusOperator.prometheusConfigReloader.image.tag` are now empty by default and the Chart.yaml `appVersion` field is used instead.
+
 ### From 42.x to 43.x
 
 This version upgrades Prometheus-Operator to v0.61.1, Prometheus to v2.40.5 and Thanos to v0.29.0.

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 9.3.1
+appVersion: 9.3.8
 description: The leading tool for querying and visualizing time series and metrics.
 home: https://grafana.net
 icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png
@@ -19,4 +19,4 @@ name: grafana
 sources:
 - https://github.com/grafana/grafana
 type: application
-version: 6.48.0
+version: 6.51.5

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md

@@ -146,7 +146,7 @@ This version requires Helm >= 3.1.0.
 | `podPortName`                             | Name of the grafana port on the pod           | `grafana`                                               |
 | `lifecycleHooks`                          | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers)     | `{}`                                                    |
 | `sidecar.image.repository`                | Sidecar image repository                      | `quay.io/kiwigrid/k8s-sidecar`                          |
-| `sidecar.image.tag`                       | Sidecar image tag                             | `1.19.2`                                                |
+| `sidecar.image.tag`                       | Sidecar image tag                             | `1.22.0`                                                |
 | `sidecar.image.sha`                       | Sidecar image sha (optional)                  | `""`                                                    |
 | `sidecar.imagePullPolicy`                 | Sidecar image pull policy                     | `IfNotPresent`                                          |
 | `sidecar.resources`                       | Sidecar resources                             | `{}`                                                    |
@@ -220,7 +220,8 @@ This version requires Helm >= 3.1.0.
 | `rbac.pspUseAppArmor`                     | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`)  | `true`                    |
 | `rbac.extraRoleRules`                     | Additional rules to add to the Role           | []                                                      |
 | `rbac.extraClusterRoleRules`              | Additional rules to add to the ClusterRole    | []                                                      |
-| `command`                     | Define command to be executed by grafana container at startup  | `nil`                                              |
+| `command`                                 | Define command to be executed by grafana container at startup | `nil`                                   |
+| `args`                                    | Define additional args if command is used     | `nil`                                                   |
 | `testFramework.enabled`                   | Whether to create test-related resources      | `true`                                                  |
 | `testFramework.image`                     | `test-framework` image repository.            | `bats/bats`                                             |
 | `testFramework.tag`                       | `test-framework` image tag.                   | `v1.4.1`                                                |
@@ -276,11 +277,10 @@ This version requires Helm >= 3.1.0.
 | `networkPolicy.egress.ports`               | An array of ports to allow for the egress                    | `[]`    |
 | `enableKubeBackwardCompatibility`          | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option  | `false`     |
 
-
-
 ### Example ingress with path
 
 With grafana 6.3 and above
+
 ```yaml
 grafana.ini:
   server:
@@ -491,6 +491,51 @@ delete_notifiers:
     # default org_id: 1
 ```
 
+## Provision alert rules, contact points, notification policies and notification templates
+
+There are two methods to provision alerting configuration in Grafana. Below are some examples and explanations as to how to use each method:
+
+```yaml
+alerting:
+  team1-alert-rules.yaml:
+    file: alerting/team1/rules.yaml
+  team2-alert-rules.yaml:
+    file: alerting/team2/rules.yaml
+  team3-alert-rules.yaml:
+    file: alerting/team3/rules.yaml
+  notification-policies.yaml:
+    file: alerting/shared/notification-policies.yaml
+  notification-templates.yaml:
+    file: alerting/shared/notification-templates.yaml
+  contactpoints.yaml:
+    apiVersion: 1
+    contactPoints:
+      - orgId: 1
+        name: Slack channel
+        receivers:
+          - uid: default-receiver
+            type: slack
+            settings:
+              # Webhook URL to be filled in
+              url: ""
+              # We need to escape double curly braces for the tpl function.
+              text: '{{ `{{ template "default.message" . }}` }}'
+              title: '{{ `{{ template "default.title" . }}` }}'
+```
+
+There are two possibilities:
+
+* Inlining the file contents as described in the example `values.yaml` and the official [Grafana documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/).
+* Importing a file using a relative path starting from the chart root directory.
+
+### Important notes on file provisioning
+
+* The chart supports importing YAML and JSON files.
+* The filename must be unique, otherwise one volume mount will overwrite the other.
+* In case of inlining, double curly braces that arise from the Grafana configuration format and are not intended as templates for the chart must be escaped.
+* The number of total files under `alerting:` is not limited. Each file will end up as a volume mount in the corresponding provisioning folder of the deployed Grafana instance.
+* The file size for each import is limited by what the function `.Files.Get` can handle, which suffices for most cases.
+
 ## How to serve Grafana with a path prefix (/grafana)
 
 In order to serve Grafana with a prefix (e.g., <http://example.com/grafana>), add the following to your values.yaml.
@@ -598,6 +643,9 @@ grafana.ini:
   unified_alerting:
     enabled: true
     ha_peers: {{ Name }}-headless:9094
+    ha_listen_address: ${POD_IP}:9094
+    ha_advertise_address: ${POD_IP}:9094
+
   alerting:
     enabled: false
 ```

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/NOTES.txt

@@ -1,6 +1,7 @@
 1. Get your '{{ .Values.adminUser }}' user password by running:
 
-   kubectl get secret --namespace {{ include "grafana.namespace" . }} {{ include "grafana.fullname" . }} -o jsonpath="{.data.admin-password}" | base64 --decode ; echo
+   kubectl get secret --namespace {{ include "grafana.namespace" . }} {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} -o jsonpath="{.data.{{ .Values.admin.passwordKey | default "admin-password" }}}" | base64 --decode ; echo
+
 
 2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster:
 

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl

@@ -68,7 +68,7 @@ Common labels
 helm.sh/chart: {{ include "grafana.chart" . }}
 {{ include "grafana.selectorLabels" . }}
 {{- if or .Chart.AppVersion .Values.image.tag }}
-app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }}
+app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }}
 {{- end }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- with .Values.extraLabels }}
@@ -91,7 +91,7 @@ Common labels
 helm.sh/chart: {{ include "grafana.chart" . }}
 {{ include "grafana.imageRenderer.selectorLabels" . }}
 {{- if or .Chart.AppVersion .Values.image.tag }}
-app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }}
+app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }}
 {{- end }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end }}
@@ -145,10 +145,12 @@ Return the appropriate apiVersion for ingress.
 Return the appropriate apiVersion for Horizontal Pod Autoscaler.
 */}}
 {{- define "grafana.hpa.apiVersion" -}}
-{{- if semverCompare "<1.23-0" .Capabilities.KubeVersion.Version }}
+{{- if $.Capabilities.APIVersions.Has "autoscaling/v2/HorizontalPodAutoscaler" }}
-{{- print "autoscaling/v2beta1" }}
-{{- else }}
 {{- print "autoscaling/v2" }}
+{{- else if $.Capabilities.APIVersions.Has "autoscaling/v2beta2/HorizontalPodAutoscaler" }}
+{{- print "autoscaling/v2beta2" }}
+{{- else }}
+{{- print "autoscaling/v2beta1" }}
 {{- end }}
 {{- end }}
 

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl

@@ -763,7 +763,13 @@ containers:
     {{- range .Values.command }}
       - {{ . | quote }}
     {{- end }}
-    {{- end}}
+    {{- end }}
+    {{- if .Values.args }}
+    args:
+    {{- range .Values.args }}
+      - {{ . | quote }}
+    {{- end }}
+    {{- end }}
     {{- with .Values.containerSecurityContext }}
     securityContext:
       {{- toYaml . | nindent 6 }}
@@ -878,7 +884,17 @@ containers:
       - name: {{ .Values.podPortName }}
         containerPort: {{ .Values.service.targetPort }}
         protocol: TCP
+      - name: {{ .Values.gossipPortName }}-tcp
+        containerPort: 9094
+        protocol: TCP
+      - name: {{ .Values.gossipPortName }}-udp
+        containerPort: 9094
+        protocol: UDP
     env:
+      - name: POD_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.podIP
       {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }}
       - name: GF_SECURITY_ADMIN_USER
         valueFrom:
@@ -1128,8 +1144,7 @@ volumes:
       path: {{ .hostPath }}
     {{- else if .csi }}
     csi:
-      data:
+      {{- toYaml .data | nindent 6 }}
-        {{- toYaml .data | nindent 8 }}
     {{- else }}
     emptyDir: {}
     {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/clusterrole.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.rbac.create (not .Values.rbac.namespaced) (not .Values.rbac.useExistingRole) }}
+{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) (not .Values.rbac.useExistingRole) }}
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/clusterrolebinding.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.rbac.create (not .Values.rbac.namespaced) }}
+{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) }}
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.createConfigmap }}
+{{- $files := .Files }}
 {{- $root := . -}}
 apiVersion: v1
 kind: ConfigMap
@@ -53,9 +54,14 @@ data:
   {{- end }}
 
   {{- range $key, $value := .Values.alerting }}
+  {{- if (hasKey $value "file") }}
+  {{- $key | nindent 2 }}:
+  {{- toYaml ( $files.Get $value.file ) | nindent 4}}
+  {{- else }}
   {{- $key | nindent 2 }}: |
     {{- tpl (toYaml $value | nindent 4) $root }}
   {{- end }}
+  {{- end }}
 
   {{- range $key, $value := .Values.dashboardProviders }}
   {{- $key | nindent 2 }}: |
@@ -87,6 +93,9 @@ data:
         {{- end }}
         {{- if $value.bearerToken }}
     -H "Authorization: Bearer {{ $value.bearerToken }}" \
+        {{- end }}
+        {{- if $value.basic }}
+    -H "Basic: {{ $value.basic }}" \
         {{- end }}
         {{- if $value.gitlabToken }}
     -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/headless-service.yaml

@@ -17,7 +17,6 @@ spec:
     {{- include "grafana.selectorLabels" . | nindent 4 }}
   type: ClusterIP
   ports:
-  - protocol: TCP
+  - name: {{ .Values.gossipPortName }}-tcp
-    port: 3000
+    port: 9094
-    targetPort: {{ .Values.service.targetPort }}
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/hpa.yaml

@@ -26,7 +26,7 @@ spec:
     - type: Resource
       resource:
         name: memory
-        {{- if semverCompare "<1.23-0" .Capabilities.KubeVersion.Version }}
+        {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }}
         targetAverageUtilization: {{ .Values.autoscaling.targetMemory }}
         {{- else }}
         target:
@@ -38,7 +38,7 @@ spec:
     - type: Resource
       resource:
         name: cpu
-        {{- if semverCompare "<1.23-0" .Capabilities.KubeVersion.Version }}
+        {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }}
         targetAverageUtilization: {{ .Values.autoscaling.targetCPU }}
         {{- else }}
         target:
@@ -46,4 +46,7 @@ spec:
           averageUtilization: {{ .Values.autoscaling.targetCPU }}
         {{- end }}
     {{- end }}
+  {{- if .Values.autoscaling.behavior }}
+  behavior: {{ toYaml .Values.autoscaling.behavior | nindent 4 }}
+  {{- end }}
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml

@@ -15,7 +15,9 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
+  {{- if and (not .Values.imageRenderer.autoscaling.enabled) (.Values.imageRenderer.replicas) }}
   replicas: {{ .Values.imageRenderer.replicas }}
+  {{- end }}
   revisionHistoryLimit: {{ .Values.imageRenderer.revisionHistoryLimit }}
   selector:
     matchLabels:
@@ -86,6 +88,10 @@ spec:
           env:
             - name: HTTP_PORT
               value: {{ .Values.imageRenderer.service.targetPort | quote }}
+          {{- if .Values.imageRenderer.serviceMonitor.enabled }}
+            - name: ENABLE_METRICS
+              value: "true"
+          {{- end }}
           {{- range $key, $value := .Values.imageRenderer.env }}
             - name: {{ $key | quote }}
               value: {{ $value | quote }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-hpa.yaml

@@ -0,0 +1,47 @@
+{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.autoscaling.enabled }}
+apiVersion: {{ include "grafana.hpa.apiVersion" . }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "grafana.fullname" . }}-image-renderer
+  namespace: {{ include "grafana.namespace" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer
+    helm.sh/chart: {{ include "grafana.chart" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "grafana.fullname" . }}-image-renderer
+  minReplicas: {{ .Values.imageRenderer.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.imageRenderer.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.imageRenderer.autoscaling.targetMemory }}
+    - type: Resource
+      resource:
+        name: memory
+        {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }}
+        targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }}
+        {{- else }}
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }}
+        {{- end }}
+    {{- end }}
+    {{- if .Values.imageRenderer.autoscaling.targetCPU }}
+    - type: Resource
+      resource:
+        name: cpu
+        {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }}
+        targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }}
+        {{- else }}
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }}
+        {{- end }}
+    {{- end }}
+  {{- if .Values.imageRenderer.autoscaling.behavior }}
+  behavior: {{ toYaml .Values.imageRenderer.autoscaling.behavior | nindent 4 }}
+  {{- end }}
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-network-policy.yaml

@@ -24,13 +24,16 @@ spec:
       from:
         - namespaceSelector:
             matchLabels:
-              name: {{ include "grafana.namespace" . }}
+              kubernetes.io/metadata.name: {{ include "grafana.namespace" . }}
-        - podSelector:
+          podSelector:
             matchLabels:
               {{- include "grafana.selectorLabels" . | nindent 14 }}
               {{- with .Values.podLabels }}
               {{- toYaml . | nindent 14 }}
               {{- end }}
+        {{- with .Values.imageRenderer.networkPolicy.extraIngressSelectors -}}
+        {{ toYaml . | nindent 8 }}
+        {{- end }}
 {{- end }}
 
 {{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitEgress }}
@@ -61,10 +64,13 @@ spec:
           protocol: TCP
     # talk only to grafana
     - ports:
-        - port: {{ .Values.service.port }}
+        - port: {{ .Values.service.targetPort }}
           protocol: TCP
       to:
-        - podSelector:
+        - namespaceSelector:
+            matchLabels:
+              name: {{ include "grafana.namespace" . }}
+          podSelector:
             matchLabels:
               {{- include "grafana.selectorLabels" . | nindent 14 }}
               {{- with .Values.podLabels }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-servicemonitor.yaml

@@ -0,0 +1,48 @@
+{{- if .Values.imageRenderer.serviceMonitor.enabled }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "grafana.fullname" . }}-image-renderer
+  {{- if .Values.imageRenderer.serviceMonitor.namespace }}
+  namespace: {{ tpl .Values.imageRenderer.serviceMonitor.namespace . }}
+  {{- else }}
+  namespace: {{ include "grafana.namespace" . }}
+  {{- end }}
+  labels:
+    {{- include "grafana.imageRenderer.labels" . | nindent 4 }}
+    {{- with .Values.imageRenderer.serviceMonitor.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+  - port: {{ .Values.imageRenderer.service.portName }}
+    {{- with .Values.imageRenderer.serviceMonitor.interval }}
+    interval: {{ . }}
+    {{- end }}
+    {{- with .Values.imageRenderer.serviceMonitor.scrapeTimeout }}
+    scrapeTimeout: {{ . }}
+    {{- end }}
+    honorLabels: true
+    path: {{ .Values.imageRenderer.serviceMonitor.path }}
+    scheme: {{ .Values.imageRenderer.serviceMonitor.scheme }}
+    {{- with .Values.imageRenderer.serviceMonitor.tlsConfig }}
+    tlsConfig:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    {{- with .Values.imageRenderer.serviceMonitor.relabelings }}
+    relabelings:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+  jobLabel: "{{ .Release.Name }}-image-renderer"
+  selector:
+    matchLabels:
+      {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }}
+  namespaceSelector:
+    matchNames:
+      - {{ include "grafana.namespace" . }}
+  {{- with .Values.imageRenderer.serviceMonitor.targetLabels }}
+  targetLabels:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/role.yaml

@@ -12,7 +12,7 @@ metadata:
   {{- end }}
 {{- if or .Values.rbac.pspEnabled (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.rbac.extraRoleRules)) }}
 rules:
-  {{- if .Values.rbac.pspEnabled }}
+  {{- if and .Values.rbac.pspEnabled (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }}
   - apiGroups:      ['extensions']
     resources:      ['podsecuritypolicies']
     verbs:          ['use']

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/servicemonitor.yaml

@@ -41,4 +41,8 @@ spec:
   namespaceSelector:
     matchNames:
       - {{ include "grafana.namespace" . }}
+  {{- with .Values.serviceMonitor.targetLabels }}
+  targetLabels:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml

@@ -17,8 +17,8 @@ rbac:
   create: true
   ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true)
   # useExistingRole: name-of-some-(cluster)role
-  pspEnabled: true
+  pspEnabled: false
-  pspUseAppArmor: true
+  pspUseAppArmor: false
   namespaced: false
   extraRoleRules: []
   # - apiGroups: []
@@ -52,6 +52,7 @@ autoscaling:
   maxReplicas: 5
   targetCPU: "60"
   targetMemory: ""
+  behavior: {}
 
 ## See `kubectl explain poddisruptionbudget.spec` for more
 ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/
@@ -159,7 +160,7 @@ downloadDashboards:
 # podLabels: {}
 
 podPortName: grafana
-
+gossipPortName: gossip
 ## Deployment annotations
 # annotations: {}
 
@@ -193,6 +194,7 @@ serviceMonitor:
   tlsConfig: {}
   scrapeTimeout: 30s
   relabelings: []
+  targetLabels: []
 
 extraExposePorts: []
  # - name: keycloak
@@ -382,6 +384,14 @@ admin:
 # - "sh"
 # - "/run.sh"
 
+## Optionally define args if command is used
+## Needed if using `hashicorp/envconsul` to manage secrets
+## By default no arguments are set
+# args:
+# - "-secret"
+# - "secret/grafana"
+# - "./grafana"
+
 ## Extra environment variables that will be pass onto deployment pods
 ##
 ## to provide grafana with access to CloudWatch on AWS EKS:
@@ -663,6 +673,9 @@ dashboards: {}
   #   local-dashboard-bitbucket:
   #     url: https://example.com/repository/test-bitbucket.json
   #     bearerToken: ''
+  #   local-dashboard-azure:
+  #     url: https://example.com/repository/test-azure.json
+  #     basic: ''
 
 ## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value.
 ## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both.
@@ -754,7 +767,7 @@ smtp:
 sidecar:
   image:
     repository: quay.io/kiwigrid/k8s-sidecar
-    tag: 1.21.0
+    tag: 1.22.0
     sha: ""
   imagePullPolicy: IfNotPresent
   resources: {}
@@ -1008,6 +1021,13 @@ imageRenderer:
   # Enable the image-renderer deployment & service
   enabled: false
   replicas: 1
+  autoscaling:
+    enabled: false
+    minReplicas: 1
+    maxReplicas: 5
+    targetCPU: "60"
+    targetMemory: ""
+    behavior: {}
   image:
     # image-renderer Image repository
     repository: grafana/grafana-image-renderer
@@ -1047,6 +1067,23 @@ imageRenderer:
     targetPort: 8081
     # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp"
     appProtocol: ""
+  serviceMonitor:
+    ## If true, a ServiceMonitor CRD is created for a prometheus operator
+    ## https://github.com/coreos/prometheus-operator
+    ##
+    enabled: false
+    path: /metrics
+    #  namespace: monitoring  (defaults to use the namespace this chart is deployed to)
+    labels: {}
+    interval: 1m
+    scheme: http
+    tlsConfig: {}
+    scrapeTimeout: 30s
+    relabelings: []
+    # See: https://doc.crds.dev/github.com/prometheus-operator/kube-prometheus/monitoring.coreos.com/ServiceMonitor/v1@v0.11.0#spec-targetLabels
+    targetLabels: []
+      # - targetLabel1
+      # - targetLabel2
   # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana
   grafanaProtocol: http
   # In case a sub_path is used this needs to be added to the image renderer callback
@@ -1060,6 +1097,8 @@ imageRenderer:
     limitIngress: true
     # Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods
     limitEgress: false
+    # Allow additional services to access image-renderer (eg. Prometheus operator when ServiceMonitor is enabled)
+    extraIngressSelectors: []
   resources: {}
 #   limits:
 #     cpu: 100m

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 2.7.0
+appVersion: 2.8.2
 description: Install kube-state-metrics to generate and expose cluster-level metrics
 home: https://github.com/kubernetes/kube-state-metrics/
 keywords:
@@ -18,4 +18,4 @@ name: kube-state-metrics
 sources:
 - https://github.com/kubernetes/kube-state-metrics/
 type: application
-version: 4.24.0
+version: 5.0.1

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md

@@ -2,14 +2,15 @@
 
 Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics).
 
-## Get Repo Info
+## Get Repository Info
-
+<!-- textlint-disable -->
 ```console
 helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 helm repo update
 ```
 
 _See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._
+<!-- textlint-enable -->
 
 ## Install Chart
 
@@ -43,20 +44,19 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen
 
 You can upgrade in-place:
 
-1. [get repo info](#get-repo-info)
+1. [get repository info](#get-repository-info)
-1. [upgrade](#upgrading-chart) your existing release name using the new chart repo
+1. [upgrade](#upgrading-chart) your existing release name using the new chart repository
-
 
 ## Upgrading to v3.0.0
 
 v3.0.0 includes kube-state-metrics v2.0, see the [changelog](https://github.com/kubernetes/kube-state-metrics/blob/release-2.0/CHANGELOG.md) for major changes on the application-side.
 
 The upgraded chart now the following changes:
+
 * Dropped support for helm v2 (helm v3 or later is required)
 * collectors key was renamed to resources
 * namespace key was renamed to namespaces
 
-
 ## Configuration
 
 See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments:
@@ -65,4 +65,21 @@ See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_h
 helm show values prometheus-community/kube-state-metrics
 ```
 
-You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options.
+### kube-rbac-proxy
+
+You can enable `kube-state-metrics` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy one RBAC proxy container per endpoint (metrics & telemetry).
+To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as:
+
+```yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics-read
+rules:
+  - apiGroups: [ "" ]
+    resources: ["services/kube-state-metrics"]
+    verbs:
+      - get
+```
+
+See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details.

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/NOTES.txt

@@ -8,3 +8,16 @@ In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-s
 They are served either as plaintext or protobuf depending on the Accept header.
 They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint.
 
+{{- if .Values.kubeRBACProxy.enabled}}
+
+kube-rbac-proxy endpoint protections is enabled:
+- Metrics endpoints are now HTTPS
+- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions:
+```
+rules:
+  - apiGroups: [ "" ]
+    resources: ["services/{{ template "kube-state-metrics.fullname" . }}"]
+    verbs:
+      - get
+```
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/_helpers.tpl

@@ -77,9 +77,13 @@ release: {{ .Release.Name }}
 Selector labels
 */}}
 {{- define "kube-state-metrics.selectorLabels" }}
+{{- if .Values.selectorOverride }}
+{{ toYaml .Values.selectorOverride }}
+{{- else }}
 app.kubernetes.io/name: {{ include "kube-state-metrics.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
+{{- end }}
 
 {{/* Sets default scrape limits for servicemonitor */}}
 {{- define "servicemonitor.scrapeLimits" -}}
@@ -99,3 +103,54 @@ labelNameLengthLimit: {{ . }}
 labelValueLengthLimit: {{ . }}
 {{- end }}
 {{- end -}}
+
+{{/*
+Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets})
+*/}}
+{{- define "kube-state-metrics.imagePullSecrets" -}}
+{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }}
+  {{- if eq (typeOf .) "map[string]interface {}" }}
+- {{ toYaml . | trim }}
+  {{- else }}
+- name: {{ . }}
+  {{- end }}
+{{- end }}
+{{- end -}}
+
+{{/*
+The image to use for kube-state-metrics
+*/}}
+{{- define "kube-state-metrics.image" -}}
+{{- if .Values.image.sha }}
+{{- if .Values.global.imageRegistry }}
+{{- printf "%s/%s:%s@%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }}
+{{- else }}
+{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }}
+{{- end }}
+{{- else }}
+{{- if .Values.global.imageRegistry }}
+{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }}
+{{- else }}
+{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+The image to use for kubeRBACProxy
+*/}}
+{{- define "kubeRBACProxy.image" -}}
+{{- if .Values.kubeRBACProxy.image.sha }}
+{{- if .Values.global.imageRegistry }}
+{{- printf "%s/%s:%s@%s" .Values.global.imageRegistry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }}
+{{- else }}
+{{- printf "%s/%s:%s@%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }}
+{{- end }}
+{{- else }}
+{{- if .Values.global.imageRegistry }}
+{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }}
+{{- else }}
+{{- printf "%s/%s:%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }}
+{{- end }}
+{{- end }}
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml

@@ -40,6 +40,8 @@ spec:
       priorityClassName: {{ .Values.priorityClassName }}
     {{- end }}
       containers:
+      {{- $httpPort := ternary 9090 (.Values.service.port | default 8080) .Values.kubeRBACProxy.enabled}}
+      {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}}
       - name: {{ template "kube-state-metrics.name" . }}
         {{- if .Values.autosharding.enabled }}
         env:
@@ -56,9 +58,7 @@ spec:
         {{-  if .Values.extraArgs  }}
         {{- .Values.extraArgs | toYaml | nindent 8 }}
         {{-  end  }}
-        {{- if .Values.service.port }}
+        - --port={{ $httpPort }}
-        - --port={{ .Values.service.port | default 8080}}
-        {{-  end  }}
         {{-  if .Values.collectors  }}
         - --resources={{ .Values.collectors | join "," }}
         {{-  end  }}
@@ -96,11 +96,16 @@ spec:
         {{- if .Values.kubeconfig.enabled }}
         - --kubeconfig=/opt/k8s/.kube/config
         {{- end }}
+        {{- if .Values.kubeRBACProxy.enabled }}
+        - --telemetry-host=127.0.0.1
+        - --telemetry-port={{ $telemetryPort }}
+        {{- else }}
         {{- if .Values.selfMonitor.telemetryHost }}
         - --telemetry-host={{ .Values.selfMonitor.telemetryHost }}
         {{- end }}
         {{- if .Values.selfMonitor.telemetryPort }}
-        - --telemetry-port={{ .Values.selfMonitor.telemetryPort | default 8081 }}
+        - --telemetry-port={{ $telemetryPort }}
+        {{- end }}
         {{- end }}
         {{- if or (.Values.kubeconfig.enabled) (.Values.volumeMounts) }}
         volumeMounts:
@@ -114,28 +119,26 @@ spec:
         {{- end }}
         {{- end }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
-        {{- if .Values.image.sha }}
+        image: {{ include "kube-state-metrics.image" . }}
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}@sha256:{{ .Values.image.sha }}"
+        {{- if eq .Values.kubeRBACProxy.enabled false }}
-        {{- else }}
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-        {{- end }}
         ports:
         - containerPort: {{ .Values.service.port | default 8080}}
           name: "http"
         {{- if .Values.selfMonitor.enabled }}
-        - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }}
+        - containerPort: {{ $telemetryPort }}
           name: "metrics"
         {{- end }}
+        {{- end }}
         livenessProbe:
           httpGet:
             path: /healthz
-            port: {{ .Values.service.port | default 8080}}
+            port: {{ $httpPort }}
           initialDelaySeconds: 5
           timeoutSeconds: 5
         readinessProbe:
           httpGet:
             path: /
-            port: {{ .Values.service.port | default 8080}}
+            port: {{ $httpPort }}
           initialDelaySeconds: 5
           timeoutSeconds: 5
         {{- if .Values.resources }}
@@ -146,9 +149,81 @@ spec:
         securityContext:
 {{ toYaml .Values.containerSecurityContext | indent 10 }}
 {{- end }}
-{{- if .Values.imagePullSecrets }}
+        {{-  if .Values.kubeRBACProxy.enabled  }}
+      - name: kube-rbac-proxy-http
+        args:
+        {{-  if .Values.kubeRBACProxy.extraArgs  }}
+        {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }}
+        {{-  end  }}
+        - --secure-listen-address=:{{ .Values.service.port | default 8080}}
+        - --upstream=http://127.0.0.1:{{ $httpPort }}/
+        - --proxy-endpoints-port=8888
+        - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml
+        volumeMounts:
+          - name: kube-rbac-proxy-config
+            mountPath: /etc/kube-rbac-proxy-config
+        imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }}
+        image: {{ include "kubeRBACProxy.image" . }}
+        ports:
+          - containerPort: {{ .Values.service.port | default 8080}}
+            name: "http"
+          - containerPort: 8888
+            name: "http-healthz"
+        readinessProbe:
+          httpGet:
+            scheme: HTTPS
+            port: 8888
+            path: healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        {{- if .Values.kubeRBACProxy.resources }}
+        resources:
+{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }}
+{{- end }}
+{{- if .Values.kubeRBACProxy.containerSecurityContext }}
+        securityContext:
+{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }}
+{{- end }}
+      {{-  if .Values.selfMonitor.enabled  }}
+      - name: kube-rbac-proxy-telemetry
+        args:
+        {{-  if .Values.kubeRBACProxy.extraArgs  }}
+        {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }}
+        {{-  end  }}
+        - --secure-listen-address=:{{ .Values.selfMonitor.telemetryPort | default 8081 }}
+        - --upstream=http://127.0.0.1:{{ $telemetryPort }}/
+        - --proxy-endpoints-port=8889
+        - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml
+        volumeMounts:
+          - name: kube-rbac-proxy-config
+            mountPath: /etc/kube-rbac-proxy-config
+        imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }}
+        image: {{ include "kubeRBACProxy.image" . }}
+        ports:
+          - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }}
+            name: "metrics"
+          - containerPort: 8889
+            name: "metrics-healthz"
+        readinessProbe:
+          httpGet:
+            scheme: HTTPS
+            port: 8889
+            path: healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        {{- if .Values.kubeRBACProxy.resources }}
+        resources:
+{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }}
+{{- end }}
+{{- if .Values.kubeRBACProxy.containerSecurityContext }}
+        securityContext:
+{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }}
+{{- end }}
+      {{- end }}
+      {{- end }}
+{{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }}
       imagePullSecrets:
-{{ toYaml .Values.imagePullSecrets | indent 8 }}
+        {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }}
       {{- end }}
       {{- if .Values.affinity }}
       affinity:
@@ -166,13 +241,18 @@ spec:
       topologySpreadConstraints:
 {{ toYaml .Values.topologySpreadConstraints | indent 8 }}
       {{- end }}
-      {{- if or (.Values.kubeconfig.enabled) (.Values.volumes) }}
+      {{- if or (.Values.kubeconfig.enabled) (.Values.volumes) (.Values.kubeRBACProxy.enabled) }}
       volumes:
       {{- if .Values.kubeconfig.enabled}}
         - name: kubeconfig
           secret:
             secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig
       {{- end }}
+      {{- if .Values.kubeRBACProxy.enabled}}
+        - name: kube-rbac-proxy-config
+          configMap:
+            name: {{ template "kube-state-metrics.fullname" . }}-rbac-config
+      {{- end }}
       {{- if .Values.volumes }}
 {{ toYaml .Values.volumes | indent 8 }}
       {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/networkpolicy.yaml

@@ -0,0 +1,43 @@
+{{- if .Values.networkPolicy.enabled }}
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  labels:
+    {{- include "kube-state-metrics.labels" . | indent 4 }}
+  name: {{ template "kube-state-metrics.fullname" . }}
+  namespace: {{ template "kube-state-metrics.namespace" . }}
+  {{- if .Values.annotations }}
+  annotations:
+    {{ toYaml .Values.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.networkPolicy.egress }}
+  ## Deny all egress by default
+  egress:
+    {{- toYaml .Values.networkPolicy.egress | nindent 4 }}
+  {{- end }}
+  ingress:
+  {{- if .Values.networkPolicy.ingress }}
+    {{- toYaml .Values.networkPolicy.ingress | nindent 4 }}
+  {{- else }}
+    ## Allow ingress on default ports by default
+  - ports:
+    - port: {{ .Values.service.port | default 8080 }}
+      protocol: TCP
+    {{- if .Values.selfMonitor.enabled }}
+    {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}}
+    - port: {{ $telemetryPort }}
+      protocol: TCP
+    {{- end }}
+  {{- end }}
+  podSelector:
+    {{- if .Values.networkPolicy.podSelector }}
+    {{- toYaml .Values.networkPolicy.podSelector | nindent 4 }}
+    {{- else }}
+    matchLabels:
+      {{- include "kube-state-metrics.selectorLabels" . | indent 6 }}
+    {{- end }}
+  policyTypes:
+    - Ingress
+    - Egress
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rbac-configmap.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.kubeRBACProxy.enabled}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "kube-state-metrics.fullname" . }}-rbac-config
+data:
+  config-file.yaml: |+
+    authorization:
+      resourceAttributes:
+        namespace: {{ template "kube-state-metrics.namespace" . }}
+        apiVersion: v1
+        resource: services
+        subresource: {{ template "kube-state-metrics.fullname" . }}
+        name: {{ template "kube-state-metrics.fullname" . }}
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml

@@ -189,6 +189,16 @@ rules:
     - verticalpodautoscalers
   verbs: ["list", "watch"]
 {{ end -}}
+{{-  if $.Values.kubeRBACProxy.enabled  }}
+- apiGroups: ["authentication.k8s.io"]
+  resources:
+    - tokenreviews
+  verbs: ["create"]
+- apiGroups: ["authorization.k8s.io"]
+  resources:
+    - subjectaccessreviews
+  verbs: ["create"]
+{{- end }}
 {{ if $.Values.rbac.extraRules }}
 {{ toYaml $.Values.rbac.extraRules }}
 {{ end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/serviceaccount.yaml

@@ -11,5 +11,5 @@ metadata:
 {{ toYaml .Values.serviceAccount.annotations | indent 4 }}
 {{- end }}
 imagePullSecrets:
-{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }}
+  {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }}
 {{- end -}}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml

@@ -11,6 +11,14 @@ metadata:
   {{- end }}
 spec:
   jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }}
+  {{- with .Values.prometheus.monitor.targetLabels }}
+  targetLabels:
+    {{- toYaml . | trim | nindent 4 }}
+  {{- end }}
+  {{- with .Values.prometheus.monitor.podTargetLabels }}
+  podTargetLabels:
+    {{- toYaml . | trim | nindent 4 }}
+  {{- end }}
   {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | indent 2 }}
   selector:
     matchLabels:

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml

@@ -1,14 +1,33 @@
 # Default values for kube-state-metrics.
 prometheusScrape: true
 image:
-  repository: registry.k8s.io/kube-state-metrics/kube-state-metrics
+  registry: registry.k8s.io
-  tag: v2.7.0
+  repository: kube-state-metrics/kube-state-metrics
+  # If unset use v + .Charts.appVersion
+  tag: ""
   sha: ""
   pullPolicy: IfNotPresent
 
 imagePullSecrets: []
 # - name: "image-pull-secret"
 
+global:
+  # To help compatibility with other charts which use global.imagePullSecrets.
+  # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style).
+  # global:
+  #   imagePullSecrets:
+  #   - name: pullSecret1
+  #   - name: pullSecret2
+  # or
+  # global:
+  #   imagePullSecrets:
+  #   - pullSecret1
+  #   - pullSecret2
+  imagePullSecrets: []
+  #
+  # Allow parent charts to override registry hostname
+  imageRegistry: ""
+
 # If set to true, this will deploy kube-state-metrics as a StatefulSet and the data
 # will be automatically sharded across <.Values.replicas> pods using the built-in
 # autodiscovery feature: https://github.com/kubernetes/kube-state-metrics#automated-sharding
@@ -38,6 +57,9 @@ service:
 customLabels: {}
   # app: kube-state-metrics
 
+## Override selector labels
+selectorOverride: {}
+
 ## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box
 releaseLabel: false
 
@@ -60,6 +82,39 @@ rbac:
   #   verbs: ["list", "watch"]
   extraRules: []
 
+# Configure kube-rbac-proxy. When enabled, creates one kube-rbac-proxy container per exposed HTTP endpoint (metrics and telemetry if enabled).
+# The requests are served through the same service but requests are then HTTPS.
+kubeRBACProxy:
+  enabled: false
+  image:
+    registry: quay.io
+    repository: brancz/kube-rbac-proxy
+    tag: v0.14.0
+    sha: ""
+    pullPolicy: IfNotPresent
+
+  # List of additional cli arguments to configure kube-rbac-prxy
+  # for example: --tls-cipher-suites, --log-file, etc.
+  # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage
+  extraArgs: []
+
+  ## Specify security settings for a Container
+  ## Allows overrides and additional options compared to (Pod) securityContext
+  ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container
+  containerSecurityContext: {}
+
+  resources: {}
+    # We usually recommend not to specify default resources and to leave this as a conscious
+    # choice for the user. This also increases chances charts run on environments with little
+    # resources, such as Minikube. If you do want to specify resources, uncomment the following
+    # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+    # limits:
+    #  cpu: 100m
+    #  memory: 64Mi
+    # requests:
+    #  cpu: 10m
+    #  memory: 32Mi
+
 serviceAccount:
   # Specifies whether a ServiceAccount should be created, require rbac true
   create: true
@@ -80,6 +135,8 @@ prometheus:
     additionalLabels: {}
     namespace: ""
     jobLabel: ""
+    targetLabels: []
+    podTargetLabels: []
     interval: ""
     ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
     ##
@@ -126,6 +183,17 @@ podSecurityPolicy:
 
   additionalVolumes: []
 
+## Configure network policy for kube-state-metrics
+networkPolicy:
+  enabled: false
+  # egress:
+  # - {}
+  # ingress:
+  # - {}
+  # podSelector:
+  #   matchLabels:
+  #     app.kubernetes.io/name: kube-state-metrics
+
 securityContext:
   enabled: true
   runAsGroup: 65534

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml

@@ -15,4 +15,4 @@ name: prometheus-node-exporter
 sources:
 - https://github.com/prometheus/node_exporter/
 type: application
-version: 4.8.0
+version: 4.14.0

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/README.md

@@ -75,3 +75,22 @@ See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_h
 ```console
 helm show values prometheus-community/prometheus-node-exporter
 ```
+
+### kube-rbac-proxy
+
+You can enable `prometheus-node-exporter` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy a RBAC proxy container protecting the node-exporter endpoint.
+To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as:
+
+```yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus-node-exporter-read
+rules:
+  - apiGroups: [ "" ]
+    resources: ["services/node-exporter-prometheus-node-exporter"]
+    verbs:
+      - get
+```
+
+See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details.

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/NOTES.txt

@@ -13,3 +13,17 @@
   echo "Visit http://127.0.0.1:9100 to use your application"
   kubectl port-forward --namespace {{ template "prometheus-node-exporter.namespace" . }} $POD_NAME 9100
 {{- end }}
+
+{{- if .Values.kubeRBACProxy.enabled}}
+
+kube-rbac-proxy endpoint protections is enabled:
+- Metrics endpoints is now HTTPS
+- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions:
+```
+rules:
+  - apiGroups: [ "" ]
+    resources: ["services/{{ template "prometheus-node-exporter.fullname" . }}"]
+    verbs:
+      - get
+```
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/_helpers.tpl

@@ -76,9 +76,17 @@ The image to use
 */}}
 {{- define "prometheus-node-exporter.image" -}}
 {{- if .Values.image.sha }}
-{{- printf "%s:%s@%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }}
+{{- if .Values.global.imageRegistry }}
+{{- printf "%s/%s:%s@%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }}
 {{- else }}
-{{- printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }}
+{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }}
+{{- end }}
+{{- else }}
+{{- if .Values.global.imageRegistry }}
+{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }}
+{{- else }}
+{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }}
+{{- end }}
 {{- end }}
 {{- end }}
 
@@ -126,3 +134,50 @@ labelNameLengthLimit: {{ . }}
 labelValueLengthLimit: {{ . }}
 {{- end }}
 {{- end }}
+
+{{/*
+Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets})
+*/}}
+{{- define "prometheus-node-exporter.imagePullSecrets" -}}
+{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }}
+  {{- if eq (typeOf .) "map[string]interface {}" }}
+- {{ toYaml . | trim }}
+  {{- else }}
+- name: {{ . }}
+  {{- end }}
+{{- end }}
+{{- end -}}
+
+{{/*
+Create the namespace name of the pod monitor
+*/}}
+{{- define "prometheus-node-exporter.podmonitor-namespace" -}}
+{{- if .Values.namespaceOverride }}
+{{- .Values.namespaceOverride }}
+{{- else }}
+{{- if .Values.prometheus.podMonitor.namespace }}
+{{- .Values.prometheus.podMonitor.namespace }}
+{{- else }}
+{{- .Release.Namespace }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/* Sets default scrape limits for podmonitor */}}
+{{- define "podmonitor.scrapeLimits" -}}
+{{- with .sampleLimit }}
+sampleLimit: {{ . }}
+{{- end }}
+{{- with .targetLimit }}
+targetLimit: {{ . }}
+{{- end }}
+{{- with .labelLimit }}
+labelLimit: {{ . }}
+{{- end }}
+{{- with .labelNameLengthLimit }}
+labelNameLengthLimit: {{ . }}
+{{- end }}
+{{- with .labelValueLengthLimit }}
+labelValueLengthLimit: {{ . }}
+{{- end }}
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/clusterrole.yaml

@@ -0,0 +1,20 @@
+{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "prometheus-node-exporter.fullname" . }}
+  namespace: {{ include "prometheus-node-exporter.namespace" . }}
+  labels:
+    {{- include "prometheus-node-exporter.labels" . | nindent 4 }}
+rules:
+  {{-  if $.Values.kubeRBACProxy.enabled  }}
+  - apiGroups: [ "authentication.k8s.io" ]
+    resources:
+      - tokenreviews
+    verbs: [ "create" ]
+  - apiGroups: [ "authorization.k8s.io" ]
+    resources:
+      - subjectaccessreviews
+    verbs: [ "create" ]
+  {{- end }}
+{{- end -}}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/clusterrolebinding.yaml

@@ -0,0 +1,20 @@
+{{- if and (eq .Values.rbac.create true) (eq .Values.kubeRBACProxy.enabled true) -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    {{- include "prometheus-node-exporter.labels" . | nindent 4 }}
+  name: {{ template "prometheus-node-exporter.fullname" . }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+{{- if .Values.rbac.useExistingRole }}
+  name: {{ .Values.rbac.useExistingRole }}
+{{- else }}
+  name: {{ template "prometheus-node-exporter.fullname" . }}
+{{- end }}
+subjects:
+- kind: ServiceAccount
+  name: {{ template "prometheus-node-exporter.serviceAccountName" . }}
+  namespace: {{ template "prometheus-node-exporter.namespace" . }}
+{{- end -}}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml

@@ -26,7 +26,7 @@ spec:
       labels:
         {{- include "prometheus-node-exporter.labels" . | nindent 8 }}
     spec:
-      automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }}
+      automountServiceAccountToken: {{ ternary true false (or .Values.serviceAccount.automountServiceAccountToken .Values.kubeRBACProxy.enabled) }}
       {{- with .Values.securityContext }}
       securityContext:
         {{- toYaml . | nindent 8 }}
@@ -40,6 +40,7 @@ spec:
       {{- end }}
       serviceAccountName: {{ include "prometheus-node-exporter.serviceAccountName" . }}
       containers:
+        {{- $servicePort := ternary 8100 .Values.service.port .Values.kubeRBACProxy.enabled }}
         - name: node-exporter
           image: {{ include "prometheus-node-exporter.image" . }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
@@ -48,8 +49,11 @@ spec:
             - --path.sysfs=/host/sys
             {{- if .Values.hostRootFsMount.enabled }}
             - --path.rootfs=/host/root
+            {{- if semverCompare ">=1.4.0" (default .Chart.AppVersion .Values.image.tag) }}
+            - --path.udev.data=/host/root/run/udev/data
             {{- end }}
-            - --web.listen-address=[$(HOST_IP)]:{{ .Values.service.port }}
+            {{- end }}
+            - --web.listen-address=[$(HOST_IP)]:{{ $servicePort }}
             {{- with .Values.extraArgs }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -71,10 +75,12 @@ spec:
             - name: {{ $key }}
               value: {{ $value | quote }}
             {{- end }}
+          {{- if eq .Values.kubeRBACProxy.enabled false }}
           ports:
             - name: {{ .Values.service.portName }}
               containerPort: {{ .Values.service.port }}
               protocol: TCP
+          {{- end }}
           livenessProbe:
             failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
             httpGet:
@@ -84,7 +90,7 @@ spec:
                 value: {{ $header.value }}
               {{- end }}
               path: /
-              port: {{ .Values.service.port }}
+              port: {{ $servicePort }}
               scheme: {{ upper .Values.livenessProbe.httpGet.scheme }}
             initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
@@ -99,7 +105,7 @@ spec:
                 value: {{ $header.value }}
               {{- end }}
               path: /
-              port: {{ .Values.service.port }}
+              port: {{ $servicePort }}
               scheme: {{ upper .Values.readinessProbe.httpGet.scheme }}
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
@@ -147,14 +153,14 @@ spec:
             {{- end }}
         {{- with .Values.sidecars }}
         {{- toYaml . | nindent 8 }}
-          {{- if or .Values.sidecarVolumeMount .Values.sidecarHostVolumeMounts }}
+          {{- if or $.Values.sidecarVolumeMount $.Values.sidecarHostVolumeMounts }}
           volumeMounts:
-            {{- range $_, $mount := .Values.sidecarVolumeMount }}
+            {{- range $_, $mount := $.Values.sidecarVolumeMount }}
             - name: {{ $mount.name }}
               mountPath: {{ $mount.mountPath }}
               readOnly: {{ $mount.readOnly }}
             {{- end }}
-            {{- range $_, $mount := .Values.sidecarHostVolumeMounts }}
+            {{- range $_, $mount := $.Values.sidecarHostVolumeMounts }}
             - name: {{ $mount.name }}
               mountPath: {{ $mount.mountPath }}
               readOnly: {{ $mount.readOnly }}
@@ -164,9 +170,49 @@ spec:
             {{- end }}
           {{- end }}
         {{- end }}
-      {{- with .Values.imagePullSecrets }}
+        {{-  if .Values.kubeRBACProxy.enabled  }}
+        - name: kube-rbac-proxy
+          args:
+            {{-  if .Values.kubeRBACProxy.extraArgs  }}
+            {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 12 }}
+            {{-  end  }}
+            - --secure-listen-address=:{{ .Values.service.port}}
+            - --upstream=http://127.0.0.1:{{ $servicePort }}/
+            - --proxy-endpoints-port=8888
+            - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml
+          volumeMounts:
+            - name: kube-rbac-proxy-config
+              mountPath: /etc/kube-rbac-proxy-config
+          imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }}
+          {{- if .Values.kubeRBACProxy.image.sha }}
+          image: "{{ .Values.global.imageRegistry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}@sha256:{{ .Values.kubeRBACProxy.image.sha }}"
+          {{- else }}
+          image: "{{ .Values.global.imageRegistry | default .Values.kubeRBACProxy.image.registry}}/{{ .Values.kubeRBACProxy.image.repository }}:{{ .Values.kubeRBACProxy.image.tag }}"
+          {{- end }}
+          ports:
+            - containerPort: {{ .Values.service.port}}
+              name: "http"
+            - containerPort: 8888
+              name: "http-healthz"
+          readinessProbe:
+            httpGet:
+              scheme: HTTPS
+              port: 8888
+              path: healthz
+            initialDelaySeconds: 5
+            timeoutSeconds: 5
+          {{- if .Values.kubeRBACProxy.resources }}
+          resources:
+          {{ toYaml .Values.kubeRBACProxy.resources | nindent 12 }}
+          {{- end }}
+          {{- if .Values.kubeRBACProxy.containerSecurityContext }}
+          securityContext:
+          {{ toYaml .Values.kubeRBACProxy.containerSecurityContext | nindent 12 }}
+        {{- end }}
+        {{- end }}
+      {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }}
       imagePullSecrets:
-        {{ toYaml . | nindent 8 }}
+        {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }}
       {{- end }}
       hostNetwork: {{ .Values.hostNetwork }}
       hostPID: {{ .Values.hostPID }}
@@ -223,3 +269,8 @@ spec:
           secret:
             secretName: {{ $mount.name }}
         {{- end }}
+        {{- if .Values.kubeRBACProxy.enabled }}
+        - name: kube-rbac-proxy-config
+          configMap:
+            name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config
+        {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/podmonitor.yaml

@@ -0,0 +1,91 @@
+{{- if .Values.prometheus.podMonitor.enabled }}
+apiVersion: {{ .Values.prometheus.podMonitor.apiVersion | default "monitoring.coreos.com/v1" }}
+kind: PodMonitor
+metadata:
+  name: {{ include "prometheus-node-exporter.fullname" . }}
+  namespace: {{ include "prometheus-node-exporter.podmonitor-namespace" . }}
+  labels:
+    {{- include "prometheus-node-exporter.labels" . | nindent 4 }}
+    {{- with .Values.prometheus.podMonitor.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.podMonitor.jobLabel }}
+  {{- include "podmonitor.scrapeLimits" .Values.prometheus.podMonitor | nindent 2 }}
+  selector:
+    matchLabels:
+    {{- with .Values.prometheus.podMonitor.selectorOverride }}
+      {{- toYaml . | nindent 6 }}
+    {{- else }}
+      {{- include "prometheus-node-exporter.selectorLabels" . | nindent 6 }}
+    {{- end }}
+  namespaceSelector:
+    matchNames:
+      - {{ include "prometheus-node-exporter.namespace" . }}
+  {{- with .Values.prometheus.podMonitor.attachMetadata }}
+  attachMetadata:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.prometheus.podMonitor.podTargetLabels }}
+  podTargetLabels:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  podMetricsEndpoints:
+    - port: {{ .Values.service.portName }}
+      {{- with .Values.prometheus.podMonitor.scheme }}
+      scheme: {{ . }}
+      {{- end }}
+      {{- with .Values.prometheus.podMonitor.path }}
+      path: {{ . }}
+      {{- end }}
+      {{- with .Values.prometheus.podMonitor.basicAuth }}
+      basicAuth:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.prometheus.podMonitor.bearerTokenSecret }}
+      bearerTokenSecret:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.prometheus.podMonitor.tlsConfig }}
+      tlsConfig:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.prometheus.podMonitor.authorization }}
+      authorization:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.prometheus.podMonitor.oauth2 }}
+      oauth2:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.prometheus.podMonitor.proxyUrl }}
+      proxyUrl: {{ . }}
+      {{- end }}
+      {{- with .Values.prometheus.podMonitor.interval }}
+      interval: {{ . }}
+      {{- end }}
+      {{- with .Values.prometheus.podMonitor.honorTimestamps }}
+      honorTimestamps: {{ . }}
+      {{- end }}
+      {{- with .Values.prometheus.podMonitor.honorLabels }}
+      honorLabels: {{ . }}
+      {{- end }}
+      {{- with .Values.prometheus.podMonitor.scrapeTimeout }}
+      scrapeTimeout: {{ . }}
+      {{- end }}
+      {{- with .Values.prometheus.podMonitor.relabelings }}
+      relabelings:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.prometheus.podMonitor.metricRelabelings }}
+      metricRelabelings:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      enableHttp2: {{ default false .Values.prometheus.podMonitor.enableHttp2 }}
+      filterRunning: {{ default true .Values.prometheus.podMonitor.filterRunning }}
+      followRedirects: {{ default false .Values.prometheus.podMonitor.followRedirects }}
+      {{- with .Values.prometheus.podMonitor.params }}
+      params:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/rbac-configmap.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.kubeRBACProxy.enabled}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "prometheus-node-exporter.fullname" . }}-rbac-config
+data:
+  config-file.yaml: |+
+    authorization:
+      resourceAttributes:
+        namespace: {{ template "prometheus-node-exporter.namespace" . }}
+        apiVersion: v1
+        resource: services
+        subresource: {{ template "prometheus-node-exporter.fullname" . }}
+        name: {{ template "prometheus-node-exporter.fullname" . }}
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/serviceaccount.yaml

@@ -10,8 +10,8 @@ metadata:
   annotations:
     {{- toYaml . | nindent 4 }}
   {{- end }}
-{{- with .Values.serviceAccount.imagePullSecrets }}
+{{- if or .Values.serviceAccount.imagePullSecrets .Values.global.imagePullSecrets }}
 imagePullSecrets:
-  {{- toYaml . | nindent 2 }}
+  {{- include "prometheus-node-exporter.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.serviceAccount.imagePullSecrets) | indent 2 }}
 {{- end }}
 {{- end -}}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/servicemonitor.yaml

@@ -12,6 +12,10 @@ metadata:
 spec:
   jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }}
   {{- include "servicemonitor.scrapeLimits" .Values.prometheus.monitor | nindent 2 }}
+  {{- with .Values.prometheus.monitor.podTargetLabels }}
+  podTargetLabels:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
     {{- with .Values.prometheus.monitor.selectorOverride }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/verticalpodautoscaler.yaml

@@ -9,7 +9,7 @@ metadata:
 spec:
   resourcePolicy:
     containerPolicies:
-    - containerName: {{ include "prometheus-node-exporter.name" . }}
+    - containerName: node-exporter
       {{- with .Values.verticalPodAutoscaler.controlledResources }}
       controlledResources: {{ . }}
       {{- end }}
@@ -24,7 +24,7 @@ spec:
   targetRef:
     apiVersion: apps/v1
     kind: DaemonSet
-    name:  {{ include "prometheus-node-exporter.fullname" . }}
+    name: {{ include "prometheus-node-exporter.fullname" . }}
   {{- if .Values.verticalPodAutoscaler.updatePolicy }}
   updatePolicy:
     {{- with .Values.verticalPodAutoscaler.updatePolicy.updateMode }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml

@@ -2,7 +2,8 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 image:
-  repository: quay.io/prometheus/node-exporter
+  registry: quay.io
+  repository: prometheus/node-exporter
   # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }}
   tag: ""
   pullPolicy: IfNotPresent
@@ -11,6 +12,56 @@ image:
 imagePullSecrets: []
 # - name: "image-pull-secret"
 
+global:
+  # To help compatibility with other charts which use global.imagePullSecrets.
+  # Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style).
+  # global:
+  #   imagePullSecrets:
+  #   - name: pullSecret1
+  #   - name: pullSecret2
+  # or
+  # global:
+  #   imagePullSecrets:
+  #   - pullSecret1
+  #   - pullSecret2
+  imagePullSecrets: []
+  #
+  # Allow parent charts to override registry hostname
+  imageRegistry: ""
+
+# Configure kube-rbac-proxy. When enabled, creates a kube-rbac-proxy to protect the node-exporter http endpoint.
+# The requests are served through the same service but requests are HTTPS.
+kubeRBACProxy:
+  enabled: false
+  image:
+    registry: quay.io
+    repository: brancz/kube-rbac-proxy
+    tag: v0.14.0
+    sha: ""
+    pullPolicy: IfNotPresent
+
+  # List of additional cli arguments to configure kube-rbac-prxy
+  # for example: --tls-cipher-suites, --log-file, etc.
+  # all the possible args can be found here: https://github.com/brancz/kube-rbac-proxy#usage
+  extraArgs: []
+
+  ## Specify security settings for a Container
+  ## Allows overrides and additional options compared to (Pod) securityContext
+  ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container
+  containerSecurityContext: {}
+
+  resources: {}
+    # We usually recommend not to specify default resources and to leave this as a conscious
+    # choice for the user. This also increases chances charts run on environments with little
+    # resources, such as Minikube. If you do want to specify resources, uncomment the following
+    # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+    # limits:
+    #  cpu: 100m
+    #  memory: 64Mi
+  # requests:
+  #  cpu: 10m
+  #  memory: 32Mi
+
 service:
   type: ClusterIP
   port: 9100
@@ -34,6 +85,10 @@ prometheus:
 
     jobLabel: ""
 
+    # List of pod labels to add to node exporter metrics
+    # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor
+    podTargetLabels: []
+
     scheme: http
     basicAuth: {}
     bearerTokenFile:
@@ -74,6 +129,96 @@ prometheus:
     ##
     labelValueLengthLimit: 0
 
+  # PodMonitor defines monitoring for a set of pods.
+  # ref. https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor
+  # Using a PodMonitor may be preferred in some environments where there is very large number
+  # of Node Exporter endpoints (1000+) behind a single service.
+  # The PodMonitor is disabled by default. When switching from ServiceMonitor to PodMonitor,
+  # the time series resulting from the configuration through PodMonitor may have different labels.
+  # For instance, there will not be the service label any longer which might
+  # affect PromQL queries selecting that label.
+  podMonitor:
+    enabled: false
+    # Namespace in which to deploy the pod monitor. Defaults to the release namespace.
+    namespace: ""
+    # Additional labels, e.g. setting a label for pod monitor selector as set in prometheus
+    additionalLabels: {}
+    #  release: kube-prometheus-stack
+    # PodTargetLabels transfers labels of the Kubernetes Pod onto the target.
+    podTargetLabels: []
+    # apiVersion defaults to monitoring.coreos.com/v1.
+    apiVersion: ""
+    # Override pod selector to select pod objects.
+    selectorOverride: {}
+    # Attach node metadata to discovered targets. Requires Prometheus v2.35.0 and above.
+    attachMetadata:
+      node: false
+    # The label to use to retrieve the job name from. Defaults to label app.kubernetes.io/name.
+    jobLabel: ""
+
+    # Scheme/protocol to use for scraping.
+    scheme: "http"
+    # Path to scrape metrics at.
+    path: "/metrics"
+
+    # BasicAuth allow an endpoint to authenticate over basic authentication.
+    # More info: https://prometheus.io/docs/operating/configuration/#endpoint
+    basicAuth: {}
+    # Secret to mount to read bearer token for scraping targets.
+    # The secret needs to be in the same namespace as the pod monitor and accessible by the Prometheus Operator.
+    # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#secretkeyselector-v1-core
+    bearerTokenSecret: {}
+    # TLS configuration to use when scraping the endpoint.
+    tlsConfig: {}
+    # Authorization section for this endpoint.
+    # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.SafeAuthorization
+    authorization: {}
+    # OAuth2 for the URL. Only valid in Prometheus versions 2.27.0 and newer.
+    # https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.OAuth2
+    oauth2: {}
+
+    # ProxyURL eg http://proxyserver:2195. Directs scrapes through proxy to this endpoint.
+    proxyUrl: ""
+    # Interval at which endpoints should be scraped. If not specified Prometheus’ global scrape interval is used.
+    interval: ""
+    # Timeout after which the scrape is ended. If not specified, the Prometheus global scrape interval is used.
+    scrapeTimeout: ""
+    # HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data.
+    honorTimestamps: true
+    # HonorLabels chooses the metric’s labels on collisions with target labels.
+    honorLabels: true
+    # Whether to enable HTTP2. Default false.
+    enableHttp2: ""
+    # Drop pods that are not running. (Failed, Succeeded).
+    # Enabled by default. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
+    filterRunning: ""
+    # FollowRedirects configures whether scrape requests follow HTTP 3xx redirects. Default false.
+    followRedirects: ""
+    # Optional HTTP URL parameters
+    params: {}
+
+    # RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds
+    # relabelings for a few standard Kubernetes fields. The original scrape job’s name
+    # is available via the __tmp_prometheus_job_name label.
+    # More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
+    relabelings: []
+    # MetricRelabelConfigs to apply to samples before ingestion.
+    metricRelabelings: []
+
+    # SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+    sampleLimit: 0
+    # TargetLimit defines a limit on the number of scraped targets that will be accepted.
+    targetLimit: 0
+    # Per-scrape limit on number of labels that will be accepted for a sample.
+    # Only valid in Prometheus versions 2.27.0 and newer.
+    labelLimit: 0
+    # Per-scrape limit on length of labels name that will be accepted for a sample.
+    # Only valid in Prometheus versions 2.27.0 and newer.
+    labelNameLengthLimit: 0
+    # Per-scrape limit on length of labels value that will be accepted for a sample.
+    # Only valid in Prometheus versions 2.27.0 and newer.
+    labelValueLengthLimit: 0
+
 ## Customize the updateStrategy if set
 updateStrategy:
   type: RollingUpdate

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml

@@ -1,10 +1,10 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
+    controller-gen.kubebuilder.io/version: v0.11.1
   creationTimestamp: null
   name: alertmanagerconfigs.monitoring.coreos.com
 spec:
@@ -4383,6 +4383,12 @@ spec:
                   the resource's namespace. If present, it will be added to the generated
                   Alertmanager configuration as a first-level route.
                 properties:
+                  activeTimeIntervals:
+                    description: ActiveTimeIntervals is a list of MuteTimeInterval
+                      names when this route should be active.
+                    items:
+                      type: string
+                    type: array
                   continue:
                     description: Boolean indicating whether an alert should continue
                       matching subsequent sibling nodes. It will always be overridden

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml

@@ -1,10 +1,10 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
+    controller-gen.kubebuilder.io/version: v0.11.1
   creationTimestamp: null
   name: alertmanagers.monitoring.coreos.com
 spec:
@@ -29,6 +29,16 @@ spec:
       jsonPath: .spec.replicas
       name: Replicas
       type: integer
+    - description: The number of ready replicas
+      jsonPath: .status.availableReplicas
+      name: Ready
+      type: integer
+    - jsonPath: .status.conditions[?(@.type == 'Reconciled')].status
+      name: Reconciled
+      type: string
+    - jsonPath: .status.conditions[?(@.type == 'Available')].status
+      name: Available
+      type: string
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
@@ -2215,6 +2225,27 @@ spec:
                       description: 'Compute Resources required by this container.
                         Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
                       properties:
+                        claims:
+                          description: "Claims lists the names of resources, defined
+                            in spec.resourceClaims, that are used by this container.
+                            \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                            feature gate. \n This field is immutable."
+                          items:
+                            description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                            properties:
+                              name:
+                                description: Name must match the name of one entry
+                                  in pod.spec.resourceClaims of the Pod where this
+                                  field is used. It makes that resource available
+                                  inside a container.
+                                type: string
+                            required:
+                            - name
+                            type: object
+                          type: array
+                          x-kubernetes-list-map-keys:
+                          - name
+                          x-kubernetes-list-type: map
                         limits:
                           additionalProperties:
                             anyOf:
@@ -2721,6 +2752,16 @@ spec:
                   to ensure the Prometheus Operator knows what version of Alertmanager
                   is being configured.
                 type: string
+              imagePullPolicy:
+                description: Image pull policy for the 'alertmanager', 'init-config-reloader'
+                  and 'config-reloader' containers. See https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy
+                  for more details.
+                enum:
+                - ""
+                - Always
+                - Never
+                - IfNotPresent
+                type: string
               imagePullSecrets:
                 description: An optional list of references to secrets in the same
                   namespace to use for pulling prometheus and alertmanager images
@@ -3500,6 +3541,27 @@ spec:
                       description: 'Compute Resources required by this container.
                         Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
                       properties:
+                        claims:
+                          description: "Claims lists the names of resources, defined
+                            in spec.resourceClaims, that are used by this container.
+                            \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                            feature gate. \n This field is immutable."
+                          items:
+                            description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                            properties:
+                              name:
+                                description: Name must match the name of one entry
+                                  in pod.spec.resourceClaims of the Pod where this
+                                  field is used. It makes that resource available
+                                  inside a container.
+                                type: string
+                            required:
+                            - name
+                            type: object
+                          type: array
+                          x-kubernetes-list-map-keys:
+                          - name
+                          x-kubernetes-list-type: map
                         limits:
                           additionalProperties:
                             anyOf:
@@ -3992,8 +4054,9 @@ spec:
                 description: Minimum number of seconds for which a newly created pod
                   should be ready without any of its container crashing for it to
                   be considered available. Defaults to 0 (pod will be considered available
-                  as soon as it is ready) This is an alpha field and requires enabling
+                  as soon as it is ready) This is an alpha field from kubernetes 1.22
-                  StatefulSetMinReadySeconds feature gate.
+                  until 1.24 which requires enabling the StatefulSetMinReadySeconds
+                  feature gate.
                 format: int32
                 type: integer
               nodeSelector:
@@ -4049,6 +4112,26 @@ spec:
               resources:
                 description: Define resources requests and limits for single Pods.
                 properties:
+                  claims:
+                    description: "Claims lists the names of resources, defined in
+                      spec.resourceClaims, that are used by this container. \n This
+                      is an alpha field and requires enabling the DynamicResourceAllocation
+                      feature gate. \n This field is immutable."
+                    items:
+                      description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                      properties:
+                        name:
+                          description: Name must match the name of one entry in pod.spec.resourceClaims
+                            of the Pod where this field is used. It makes that resource
+                            available inside a container.
+                          type: string
+                      required:
+                      - name
+                      type: object
+                    type: array
+                    x-kubernetes-list-map-keys:
+                    - name
+                    x-kubernetes-list-type: map
                   limits:
                     additionalProperties:
                       anyOf:
@@ -4196,9 +4279,14 @@ spec:
                     type: object
                   supplementalGroups:
                     description: A list of groups applied to the first process run
-                      in each container, in addition to the container's primary GID.  If
+                      in each container, in addition to the container's primary GID,
-                      unspecified, no groups will be added to any container. Note
+                      the fsGroup (if specified), and group memberships defined in
-                      that this field cannot be set when spec.os.name is windows.
+                      the container image for the uid of the container process. If
+                      unspecified, no additional groups are added to any container.
+                      Note that group memberships defined in the container image for
+                      the uid of the container process are still effective, even if
+                      they are not included in this list. Note that this field cannot
+                      be set when spec.os.name is windows.
                     items:
                       format: int64
                       type: integer
@@ -4280,9 +4368,9 @@ spec:
                       allows to remove any subPath usage in volume mounts.'
                     type: boolean
                   emptyDir:
-                    description: 'EmptyDirVolumeSource to be used by the Prometheus
+                    description: 'EmptyDirVolumeSource to be used by the StatefulSet.
-                      StatefulSets. If specified, used in place of any volumeClaimTemplate.
+                      If specified, used in place of any volumeClaimTemplate. More
-                      More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir'
+                      info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir'
                     properties:
                       medium:
                         description: 'medium represents what type of storage medium
@@ -4305,9 +4393,9 @@ spec:
                         x-kubernetes-int-or-string: true
                     type: object
                   ephemeral:
-                    description: 'EphemeralVolumeSource to be used by the Prometheus
+                    description: 'EphemeralVolumeSource to be used by the StatefulSet.
-                      StatefulSets. This is a beta field in k8s 1.21, for lower versions,
+                      This is a beta field in k8s 1.21, for lower versions, starting
-                      starting with k8s 1.19, it requires enabling the GenericEphemeralVolume
+                      with k8s 1.19, it requires enabling the GenericEphemeralVolume
                       feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes'
                     properties:
                       volumeClaimTemplate:
@@ -4354,9 +4442,12 @@ spec:
                                   provisioner or an external controller can support
                                   the specified data source, it will create a new
                                   volume based on the contents of the specified data
-                                  source. If the AnyVolumeDataSource feature gate
+                                  source. When the AnyVolumeDataSource feature gate
-                                  is enabled, this field will always have the same
+                                  is enabled, dataSource contents will be copied to
-                                  contents as the DataSourceRef field.'
+                                  dataSourceRef, and dataSourceRef contents will be
+                                  copied to dataSource when dataSourceRef.namespace
+                                  is not specified. If the namespace is specified,
+                                  then dataSourceRef will not be copied to dataSource.'
                                 properties:
                                   apiGroup:
                                     description: APIGroup is the group for the resource
@@ -4381,27 +4472,33 @@ spec:
                               dataSourceRef:
                                 description: 'dataSourceRef specifies the object from
                                   which to populate the volume with data, if a non-empty
-                                  volume is desired. This may be any local object
+                                  volume is desired. This may be any object from a
-                                  from a non-empty API group (non core object) or
+                                  non-empty API group (non core object) or a PersistentVolumeClaim
-                                  a PersistentVolumeClaim object. When this field
+                                  object. When this field is specified, volume binding
-                                  is specified, volume binding will only succeed if
+                                  will only succeed if the type of the specified object
-                                  the type of the specified object matches some installed
+                                  matches some installed volume populator or dynamic
-                                  volume populator or dynamic provisioner. This field
+                                  provisioner. This field will replace the functionality
-                                  will replace the functionality of the DataSource
+                                  of the dataSource field and as such if both fields
-                                  field and as such if both fields are non-empty,
+                                  are non-empty, they must have the same value. For
-                                  they must have the same value. For backwards compatibility,
+                                  backwards compatibility, when namespace isn''t specified
-                                  both fields (DataSource and DataSourceRef) will
+                                  in dataSourceRef, both fields (dataSource and dataSourceRef)
-                                  be set to the same value automatically if one of
+                                  will be set to the same value automatically if one
-                                  them is empty and the other is non-empty. There
+                                  of them is empty and the other is non-empty. When
-                                  are two important differences between DataSource
+                                  namespace is specified in dataSourceRef, dataSource
-                                  and DataSourceRef: * While DataSource only allows
+                                  isn''t set to the same value and must be empty.
-                                  two specific types of objects, DataSourceRef allows
+                                  There are three important differences between dataSource
+                                  and dataSourceRef: * While dataSource only allows
+                                  two specific types of objects, dataSourceRef allows
                                   any non-core object, as well as PersistentVolumeClaim
-                                  objects. * While DataSource ignores disallowed values
+                                  objects. * While dataSource ignores disallowed values
-                                  (dropping them), DataSourceRef preserves all values,
+                                  (dropping them), dataSourceRef preserves all values,
                                   and generates an error if a disallowed value is
-                                  specified. (Beta) Using this field requires the
+                                  specified. * While dataSource only allows local
-                                  AnyVolumeDataSource feature gate to be enabled.'
+                                  objects, dataSourceRef allows objects in any namespaces.
+                                  (Beta) Using this field requires the AnyVolumeDataSource
+                                  feature gate to be enabled. (Alpha) Using the namespace
+                                  field of dataSourceRef requires the CrossNamespaceVolumeDataSource
+                                  feature gate to be enabled.'
                                 properties:
                                   apiGroup:
                                     description: APIGroup is the group for the resource
@@ -4418,11 +4515,21 @@ spec:
                                     description: Name is the name of resource being
                                       referenced
                                     type: string
+                                  namespace:
+                                    description: Namespace is the namespace of resource
+                                      being referenced Note that when a namespace
+                                      is specified, a gateway.networking.k8s.io/ReferenceGrant
+                                      object is required in the referent namespace
+                                      to allow that namespace's owner to accept the
+                                      reference. See the ReferenceGrant documentation
+                                      for details. (Alpha) This field requires the
+                                      CrossNamespaceVolumeDataSource feature gate
+                                      to be enabled.
+                                    type: string
                                 required:
                                 - kind
                                 - name
                                 type: object
-                                x-kubernetes-map-type: atomic
                               resources:
                                 description: 'resources represents the minimum resources
                                   the volume should have. If RecoverVolumeExpansionFailure
@@ -4431,6 +4538,29 @@ spec:
                                   value but must still be higher than capacity recorded
                                   in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
                                 properties:
+                                  claims:
+                                    description: "Claims lists the names of resources,
+                                      defined in spec.resourceClaims, that are used
+                                      by this container. \n This is an alpha field
+                                      and requires enabling the DynamicResourceAllocation
+                                      feature gate. \n This field is immutable."
+                                    items:
+                                      description: ResourceClaim references one entry
+                                        in PodSpec.ResourceClaims.
+                                      properties:
+                                        name:
+                                          description: Name must match the name of
+                                            one entry in pod.spec.resourceClaims of
+                                            the Pod where this field is used. It makes
+                                            that resource available inside a container.
+                                          type: string
+                                      required:
+                                      - name
+                                      type: object
+                                    type: array
+                                    x-kubernetes-list-map-keys:
+                                    - name
+                                    x-kubernetes-list-type: map
                                   limits:
                                     additionalProperties:
                                       anyOf:
@@ -4524,7 +4654,10 @@ spec:
                         type: object
                     type: object
                   volumeClaimTemplate:
-                    description: A PVC spec to be used by the Prometheus StatefulSets.
+                    description: A PVC spec to be used by the StatefulSet. The easiest
+                      way to use a volume that cannot be automatically provisioned
+                      (for whatever reason) is to use a label selector alongside manually
+                      created PersistentVolumes.
                     properties:
                       apiVersion:
                         description: 'APIVersion defines the versioned schema of this
@@ -4584,9 +4717,12 @@ spec:
                               * An existing PVC (PersistentVolumeClaim) If the provisioner
                               or an external controller can support the specified
                               data source, it will create a new volume based on the
-                              contents of the specified data source. If the AnyVolumeDataSource
+                              contents of the specified data source. When the AnyVolumeDataSource
-                              feature gate is enabled, this field will always have
+                              feature gate is enabled, dataSource contents will be
-                              the same contents as the DataSourceRef field.'
+                              copied to dataSourceRef, and dataSourceRef contents
+                              will be copied to dataSource when dataSourceRef.namespace
+                              is not specified. If the namespace is specified, then
+                              dataSourceRef will not be copied to dataSource.'
                             properties:
                               apiGroup:
                                 description: APIGroup is the group for the resource
@@ -4608,24 +4744,31 @@ spec:
                           dataSourceRef:
                             description: 'dataSourceRef specifies the object from
                               which to populate the volume with data, if a non-empty
-                              volume is desired. This may be any local object from
+                              volume is desired. This may be any object from a non-empty
-                              a non-empty API group (non core object) or a PersistentVolumeClaim
+                              API group (non core object) or a PersistentVolumeClaim
                               object. When this field is specified, volume binding
                               will only succeed if the type of the specified object
                               matches some installed volume populator or dynamic provisioner.
-                              This field will replace the functionality of the DataSource
+                              This field will replace the functionality of the dataSource
                               field and as such if both fields are non-empty, they
                               must have the same value. For backwards compatibility,
-                              both fields (DataSource and DataSourceRef) will be set
+                              when namespace isn''t specified in dataSourceRef, both
-                              to the same value automatically if one of them is empty
+                              fields (dataSource and dataSourceRef) will be set to
-                              and the other is non-empty. There are two important
+                              the same value automatically if one of them is empty
-                              differences between DataSource and DataSourceRef: *
+                              and the other is non-empty. When namespace is specified
-                              While DataSource only allows two specific types of objects,
+                              in dataSourceRef, dataSource isn''t set to the same
-                              DataSourceRef allows any non-core object, as well as
+                              value and must be empty. There are three important differences
-                              PersistentVolumeClaim objects. * While DataSource ignores
+                              between dataSource and dataSourceRef: * While dataSource
-                              disallowed values (dropping them), DataSourceRef preserves
+                              only allows two specific types of objects, dataSourceRef
-                              all values, and generates an error if a disallowed value
+                              allows any non-core object, as well as PersistentVolumeClaim
-                              is specified. (Beta) Using this field requires the AnyVolumeDataSource
+                              objects. * While dataSource ignores disallowed values
+                              (dropping them), dataSourceRef preserves all values,
+                              and generates an error if a disallowed value is specified.
+                              * While dataSource only allows local objects, dataSourceRef
+                              allows objects in any namespaces. (Beta) Using this
+                              field requires the AnyVolumeDataSource feature gate
+                              to be enabled. (Alpha) Using the namespace field of
+                              dataSourceRef requires the CrossNamespaceVolumeDataSource
                               feature gate to be enabled.'
                             properties:
                               apiGroup:
@@ -4640,11 +4783,20 @@ spec:
                               name:
                                 description: Name is the name of resource being referenced
                                 type: string
+                              namespace:
+                                description: Namespace is the namespace of resource
+                                  being referenced Note that when a namespace is specified,
+                                  a gateway.networking.k8s.io/ReferenceGrant object
+                                  is required in the referent namespace to allow that
+                                  namespace's owner to accept the reference. See the
+                                  ReferenceGrant documentation for details. (Alpha)
+                                  This field requires the CrossNamespaceVolumeDataSource
+                                  feature gate to be enabled.
+                                type: string
                             required:
                             - kind
                             - name
                             type: object
-                            x-kubernetes-map-type: atomic
                           resources:
                             description: 'resources represents the minimum resources
                               the volume should have. If RecoverVolumeExpansionFailure
@@ -4653,6 +4805,29 @@ spec:
                               must still be higher than capacity recorded in the status
                               field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
                             properties:
+                              claims:
+                                description: "Claims lists the names of resources,
+                                  defined in spec.resourceClaims, that are used by
+                                  this container. \n This is an alpha field and requires
+                                  enabling the DynamicResourceAllocation feature gate.
+                                  \n This field is immutable."
+                                items:
+                                  description: ResourceClaim references one entry
+                                    in PodSpec.ResourceClaims.
+                                  properties:
+                                    name:
+                                      description: Name must match the name of one
+                                        entry in pod.spec.resourceClaims of the Pod
+                                        where this field is used. It makes that resource
+                                        available inside a container.
+                                      type: string
+                                  required:
+                                  - name
+                                  type: object
+                                type: array
+                                x-kubernetes-list-map-keys:
+                                - name
+                                x-kubernetes-list-type: map
                               limits:
                                 additionalProperties:
                                   anyOf:
@@ -4996,8 +5171,8 @@ spec:
                         are included in the calculations. - Ignore: nodeAffinity/nodeSelector
                         are ignored. All nodes are included in the calculations. \n
                         If this value is nil, the behavior is equivalent to the Honor
-                        policy. This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread
+                        policy. This is a beta-level feature default enabled by the
-                        feature flag."
+                        NodeInclusionPolicyInPodTopologySpread feature flag."
                       type: string
                     nodeTaintsPolicy:
                       description: "NodeTaintsPolicy indicates how we will treat node
@@ -5006,8 +5181,8 @@ spec:
                         for which the incoming pod has a toleration, are included.
                         - Ignore: node taints are ignored. All nodes are included.
                         \n If this value is nil, the behavior is equivalent to the
-                        Ignore policy. This is a alpha-level feature enabled by the
+                        Ignore policy. This is a beta-level feature default enabled
-                        NodeInclusionPolicyInPodTopologySpread feature flag."
+                        by the NodeInclusionPolicyInPodTopologySpread feature flag."
                       type: string
                     topologyKey:
                       description: TopologyKey is the key of node labels. Nodes that
@@ -5548,9 +5723,12 @@ spec:
                                     provisioner or an external controller can support
                                     the specified data source, it will create a new
                                     volume based on the contents of the specified
-                                    data source. If the AnyVolumeDataSource feature
+                                    data source. When the AnyVolumeDataSource feature
-                                    gate is enabled, this field will always have the
+                                    gate is enabled, dataSource contents will be copied
-                                    same contents as the DataSourceRef field.'
+                                    to dataSourceRef, and dataSourceRef contents will
+                                    be copied to dataSource when dataSourceRef.namespace
+                                    is not specified. If the namespace is specified,
+                                    then dataSourceRef will not be copied to dataSource.'
                                   properties:
                                     apiGroup:
                                       description: APIGroup is the group for the resource
@@ -5576,27 +5754,35 @@ spec:
                                   description: 'dataSourceRef specifies the object
                                     from which to populate the volume with data, if
                                     a non-empty volume is desired. This may be any
-                                    local object from a non-empty API group (non core
+                                    object from a non-empty API group (non core object)
-                                    object) or a PersistentVolumeClaim object. When
+                                    or a PersistentVolumeClaim object. When this field
-                                    this field is specified, volume binding will only
+                                    is specified, volume binding will only succeed
-                                    succeed if the type of the specified object matches
+                                    if the type of the specified object matches some
-                                    some installed volume populator or dynamic provisioner.
+                                    installed volume populator or dynamic provisioner.
                                     This field will replace the functionality of the
-                                    DataSource field and as such if both fields are
+                                    dataSource field and as such if both fields are
                                     non-empty, they must have the same value. For
-                                    backwards compatibility, both fields (DataSource
+                                    backwards compatibility, when namespace isn''t
-                                    and DataSourceRef) will be set to the same value
+                                    specified in dataSourceRef, both fields (dataSource
+                                    and dataSourceRef) will be set to the same value
                                     automatically if one of them is empty and the
-                                    other is non-empty. There are two important differences
+                                    other is non-empty. When namespace is specified
-                                    between DataSource and DataSourceRef: * While
+                                    in dataSourceRef, dataSource isn''t set to the
-                                    DataSource only allows two specific types of objects,
+                                    same value and must be empty. There are three
-                                    DataSourceRef allows any non-core object, as well
+                                    important differences between dataSource and dataSourceRef:
-                                    as PersistentVolumeClaim objects. * While DataSource
+                                    * While dataSource only allows two specific types
-                                    ignores disallowed values (dropping them), DataSourceRef
+                                    of objects, dataSourceRef allows any non-core
-                                    preserves all values, and generates an error if
+                                    object, as well as PersistentVolumeClaim objects.
-                                    a disallowed value is specified. (Beta) Using
+                                    * While dataSource ignores disallowed values (dropping
-                                    this field requires the AnyVolumeDataSource feature
+                                    them), dataSourceRef preserves all values, and
-                                    gate to be enabled.'
+                                    generates an error if a disallowed value is specified.
+                                    * While dataSource only allows local objects,
+                                    dataSourceRef allows objects in any namespaces.
+                                    (Beta) Using this field requires the AnyVolumeDataSource
+                                    feature gate to be enabled. (Alpha) Using the
+                                    namespace field of dataSourceRef requires the
+                                    CrossNamespaceVolumeDataSource feature gate to
+                                    be enabled.'
                                   properties:
                                     apiGroup:
                                       description: APIGroup is the group for the resource
@@ -5613,11 +5799,21 @@ spec:
                                       description: Name is the name of resource being
                                         referenced
                                       type: string
+                                    namespace:
+                                      description: Namespace is the namespace of resource
+                                        being referenced Note that when a namespace
+                                        is specified, a gateway.networking.k8s.io/ReferenceGrant
+                                        object is required in the referent namespace
+                                        to allow that namespace's owner to accept
+                                        the reference. See the ReferenceGrant documentation
+                                        for details. (Alpha) This field requires the
+                                        CrossNamespaceVolumeDataSource feature gate
+                                        to be enabled.
+                                      type: string
                                   required:
                                   - kind
                                   - name
                                   type: object
-                                  x-kubernetes-map-type: atomic
                                 resources:
                                   description: 'resources represents the minimum resources
                                     the volume should have. If RecoverVolumeExpansionFailure
@@ -5626,6 +5822,30 @@ spec:
                                     value but must still be higher than capacity recorded
                                     in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
                                   properties:
+                                    claims:
+                                      description: "Claims lists the names of resources,
+                                        defined in spec.resourceClaims, that are used
+                                        by this container. \n This is an alpha field
+                                        and requires enabling the DynamicResourceAllocation
+                                        feature gate. \n This field is immutable."
+                                      items:
+                                        description: ResourceClaim references one
+                                          entry in PodSpec.ResourceClaims.
+                                        properties:
+                                          name:
+                                            description: Name must match the name
+                                              of one entry in pod.spec.resourceClaims
+                                              of the Pod where this field is used.
+                                              It makes that resource available inside
+                                              a container.
+                                            type: string
+                                        required:
+                                        - name
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-map-keys:
+                                      - name
+                                      x-kubernetes-list-type: map
                                     limits:
                                       additionalProperties:
                                         anyOf:
@@ -6809,31 +7029,71 @@ spec:
             type: object
           status:
             description: 'Most recent observed status of the Alertmanager cluster.
-              Read-only. Not included when requesting from the apiserver, only from
+              Read-only. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'
-              the Prometheus Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'
             properties:
               availableReplicas:
                 description: Total number of available pods (ready for at least minReadySeconds)
                   targeted by this Alertmanager cluster.
                 format: int32
                 type: integer
+              conditions:
+                description: The current state of the Alertmanager object.
+                items:
+                  description: Condition represents the state of the resources associated
+                    with the Prometheus or Alertmanager resource.
+                  properties:
+                    lastTransitionTime:
+                      description: lastTransitionTime is the time of the last update
+                        to the current status property.
+                      format: date-time
+                      type: string
+                    message:
+                      description: Human-readable message indicating details for the
+                        condition's last transition.
+                      type: string
+                    observedGeneration:
+                      description: ObservedGeneration represents the .metadata.generation
+                        that the condition was set based upon. For instance, if `.metadata.generation`
+                        is currently 12, but the `.status.conditions[].observedGeneration`
+                        is 9, the condition is out of date with respect to the current
+                        state of the instance.
+                      format: int64
+                      type: integer
+                    reason:
+                      description: Reason for the condition's last transition.
+                      type: string
+                    status:
+                      description: Status of the condition.
+                      type: string
+                    type:
+                      description: Type of the condition being reported.
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - status
+                  - type
+                  type: object
+                type: array
+                x-kubernetes-list-map-keys:
+                - type
+                x-kubernetes-list-type: map
               paused:
                 description: Represents whether any actions on the underlying managed
                   objects are being performed. Only delete actions will be performed.
                 type: boolean
               replicas:
                 description: Total number of non-terminated pods targeted by this
-                  Alertmanager cluster (their labels match the selector).
+                  Alertmanager object (their labels match the selector).
                 format: int32
                 type: integer
               unavailableReplicas:
                 description: Total number of unavailable pods targeted by this Alertmanager
-                  cluster.
+                  object.
                 format: int32
                 type: integer
               updatedReplicas:
                 description: Total number of non-terminated pods targeted by this
-                  Alertmanager cluster that have the desired version spec.
+                  Alertmanager object that have the desired version spec.
                 format: int32
                 type: integer
             required:
@@ -6848,4 +7108,5 @@ spec:
         type: object
     served: true
     storage: true
-    subresources: {}
+    subresources:
+      status: {}

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml

@@ -1,10 +1,10 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
+    controller-gen.kubebuilder.io/version: v0.11.1
   creationTimestamp: null
   name: podmonitors.monitoring.coreos.com
 spec:

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml

@@ -1,10 +1,10 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
+    controller-gen.kubebuilder.io/version: v0.11.1
   creationTimestamp: null
   name: probes.monitoring.coreos.com
 spec:

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml

@@ -1,10 +1,10 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
+    controller-gen.kubebuilder.io/version: v0.11.1
     argocd.argoproj.io/sync-options: Replace=true
   creationTimestamp: null
   name: prometheuses.monitoring.coreos.com
@@ -1051,6 +1051,53 @@ spec:
                                 Bearer, Basic will cause an error
                               type: string
                           type: object
+                        basicAuth:
+                          description: BasicAuth allow an endpoint to authenticate
+                            over basic authentication
+                          properties:
+                            password:
+                              description: The secret in the service monitor namespace
+                                that contains the password for authentication.
+                              properties:
+                                key:
+                                  description: The key of the secret to select from.  Must
+                                    be a valid secret key.
+                                  type: string
+                                name:
+                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                    TODO: Add other useful fields. apiVersion, kind,
+                                    uid?'
+                                  type: string
+                                optional:
+                                  description: Specify whether the Secret or its key
+                                    must be defined
+                                  type: boolean
+                              required:
+                              - key
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            username:
+                              description: The secret in the service monitor namespace
+                                that contains the username for authentication.
+                              properties:
+                                key:
+                                  description: The key of the secret to select from.  Must
+                                    be a valid secret key.
+                                  type: string
+                                name:
+                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                                    TODO: Add other useful fields. apiVersion, kind,
+                                    uid?'
+                                  type: string
+                                optional:
+                                  description: Specify whether the Secret or its key
+                                    must be defined
+                                  type: boolean
+                              required:
+                              - key
+                              type: object
+                              x-kubernetes-map-type: atomic
+                          type: object
                         bearerTokenFile:
                           description: BearerTokenFile to read from filesystem to
                             use when authenticating to Alertmanager.
@@ -2249,6 +2296,27 @@ spec:
                       description: 'Compute Resources required by this container.
                         Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
                       properties:
+                        claims:
+                          description: "Claims lists the names of resources, defined
+                            in spec.resourceClaims, that are used by this container.
+                            \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                            feature gate. \n This field is immutable."
+                          items:
+                            description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                            properties:
+                              name:
+                                description: Name must match the name of one entry
+                                  in pod.spec.resourceClaims of the Pod where this
+                                  field is used. It makes that resource available
+                                  inside a container.
+                                type: string
+                            required:
+                            - name
+                            type: object
+                          type: array
+                          x-kubernetes-list-map-keys:
+                          - name
+                          x-kubernetes-list-type: map
                         limits:
                           additionalProperties:
                             anyOf:
@@ -2910,6 +2978,16 @@ spec:
                   to ensure the Prometheus Operator knows what version of Prometheus
                   is being configured.
                 type: string
+              imagePullPolicy:
+                description: Image pull policy for the 'prometheus', 'init-config-reloader'
+                  and 'config-reloader' containers. See https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy
+                  for more details.
+                enum:
+                - ""
+                - Always
+                - Never
+                - IfNotPresent
+                type: string
               imagePullSecrets:
                 description: An optional list of references to secrets in the same
                   namespace to use for pulling prometheus and alertmanager images
@@ -3691,6 +3769,27 @@ spec:
                       description: 'Compute Resources required by this container.
                         Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
                       properties:
+                        claims:
+                          description: "Claims lists the names of resources, defined
+                            in spec.resourceClaims, that are used by this container.
+                            \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                            feature gate. \n This field is immutable."
+                          items:
+                            description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                            properties:
+                              name:
+                                description: Name must match the name of one entry
+                                  in pod.spec.resourceClaims of the Pod where this
+                                  field is used. It makes that resource available
+                                  inside a container.
+                                type: string
+                            required:
+                            - name
+                            type: object
+                          type: array
+                          x-kubernetes-list-map-keys:
+                          - name
+                          x-kubernetes-list-type: map
                         limits:
                           additionalProperties:
                             anyOf:
@@ -4182,8 +4281,9 @@ spec:
                 description: Minimum number of seconds for which a newly created pod
                   should be ready without any of its container crashing for it to
                   be considered available. Defaults to 0 (pod will be considered available
-                  as soon as it is ready) This is an alpha field and requires enabling
+                  as soon as it is ready) This is an alpha field from kubernetes 1.22
-                  StatefulSetMinReadySeconds feature gate.
+                  until 1.24 which requires enabling the StatefulSetMinReadySeconds
+                  feature gate.
                 format: int32
                 type: integer
               nodeSelector:
@@ -4281,9 +4381,15 @@ spec:
                 type: object
                 x-kubernetes-map-type: atomic
               podMonitorSelector:
-                description: '*Experimental* PodMonitors to be selected for target
+                description: "*Experimental* PodMonitors to be selected for target
-                  discovery. *Deprecated:* if neither this nor serviceMonitorSelector
+                  discovery. \n If `spec.serviceMonitorSelector`, `spec.podMonitorSelector`
-                  are specified, configuration is unmanaged.'
+                  and `spec.probeSelector` are null, the Prometheus configuration
+                  is unmanaged. The Prometheus operator will ensure that the Prometheus
+                  configuration's Secret exists, but it is the responsibility of the
+                  user to provide the raw gzipped Prometheus configuration under the
+                  `prometheus.yaml.gz` key. This behavior is deprecated and will be
+                  removed in the next major version of the custom resource definition.
+                  It is recommended to use `spec.additionalScrapeConfigs` instead."
                 properties:
                   matchExpressions:
                     description: matchExpressions is a list of label selector requirements.
@@ -4327,6 +4433,12 @@ spec:
                     type: object
                 type: object
                 x-kubernetes-map-type: atomic
+              podTargetLabels:
+                description: PodTargetLabels are added to all Pod/ServiceMonitors'
+                  podTargetLabels
+                items:
+                  type: string
+                type: array
               portName:
                 description: Port name used for the pods and governing service. This
                   defaults to web
@@ -4381,7 +4493,15 @@ spec:
                 type: object
                 x-kubernetes-map-type: atomic
               probeSelector:
-                description: '*Experimental* Probes to be selected for target discovery.'
+                description: "*Experimental* Probes to be selected for target discovery.
+                  \n If `spec.serviceMonitorSelector`, `spec.podMonitorSelector` and
+                  `spec.probeSelector` are null, the Prometheus configuration is unmanaged.
+                  The Prometheus operator will ensure that the Prometheus configuration's
+                  Secret exists, but it is the responsibility of the user to provide
+                  the raw gzipped Prometheus configuration under the `prometheus.yaml.gz`
+                  key. This behavior is deprecated and will be removed in the next
+                  major version of the custom resource definition. It is recommended
+                  to use `spec.additionalScrapeConfigs` instead."
                 properties:
                   matchExpressions:
                     description: matchExpressions is a list of label selector requirements.
@@ -4463,6 +4583,7 @@ spec:
                   maxConcurrency:
                     description: Number of concurrent queries that can be run at once.
                     format: int32
+                    minimum: 1
                     type: integer
                   maxSamples:
                     description: Maximum number of samples a single query can load
@@ -5372,6 +5493,26 @@ spec:
               resources:
                 description: Define resources requests and limits for single Pods.
                 properties:
+                  claims:
+                    description: "Claims lists the names of resources, defined in
+                      spec.resourceClaims, that are used by this container. \n This
+                      is an alpha field and requires enabling the DynamicResourceAllocation
+                      feature gate. \n This field is immutable."
+                    items:
+                      description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                      properties:
+                        name:
+                          description: Name must match the name of one entry in pod.spec.resourceClaims
+                            of the Pod where this field is used. It makes that resource
+                            available inside a container.
+                          type: string
+                      required:
+                      - name
+                      type: object
+                    type: array
+                    x-kubernetes-list-map-keys:
+                    - name
+                    x-kubernetes-list-type: map
                   limits:
                     additionalProperties:
                       anyOf:
@@ -5651,9 +5792,14 @@ spec:
                     type: object
                   supplementalGroups:
                     description: A list of groups applied to the first process run
-                      in each container, in addition to the container's primary GID.  If
+                      in each container, in addition to the container's primary GID,
-                      unspecified, no groups will be added to any container. Note
+                      the fsGroup (if specified), and group memberships defined in
-                      that this field cannot be set when spec.os.name is windows.
+                      the container image for the uid of the container process. If
+                      unspecified, no additional groups are added to any container.
+                      Note that group memberships defined in the container image for
+                      the uid of the container process are still effective, even if
+                      they are not included in this list. Note that this field cannot
+                      be set when spec.os.name is windows.
                     items:
                       format: int64
                       type: integer
@@ -5765,9 +5911,15 @@ spec:
                 type: object
                 x-kubernetes-map-type: atomic
               serviceMonitorSelector:
-                description: ServiceMonitors to be selected for target discovery.
+                description: "ServiceMonitors to be selected for target discovery.
-                  *Deprecated:* if neither this nor podMonitorSelector are specified,
+                  \n If `spec.serviceMonitorSelector`, `spec.podMonitorSelector` and
-                  configuration is unmanaged.
+                  `spec.probeSelector` are null, the Prometheus configuration is unmanaged.
+                  The Prometheus operator will ensure that the Prometheus configuration's
+                  Secret exists, but it is the responsibility of the user to provide
+                  the raw gzipped Prometheus configuration under the `prometheus.yaml.gz`
+                  key. This behavior is deprecated and will be removed in the next
+                  major version of the custom resource definition. It is recommended
+                  to use `spec.additionalScrapeConfigs` instead."
                 properties:
                   matchExpressions:
                     description: matchExpressions is a list of label selector requirements.
@@ -5838,9 +5990,9 @@ spec:
                       allows to remove any subPath usage in volume mounts.'
                     type: boolean
                   emptyDir:
-                    description: 'EmptyDirVolumeSource to be used by the Prometheus
+                    description: 'EmptyDirVolumeSource to be used by the StatefulSet.
-                      StatefulSets. If specified, used in place of any volumeClaimTemplate.
+                      If specified, used in place of any volumeClaimTemplate. More
-                      More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir'
+                      info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir'
                     properties:
                       medium:
                         description: 'medium represents what type of storage medium
@@ -5863,9 +6015,9 @@ spec:
                         x-kubernetes-int-or-string: true
                     type: object
                   ephemeral:
-                    description: 'EphemeralVolumeSource to be used by the Prometheus
+                    description: 'EphemeralVolumeSource to be used by the StatefulSet.
-                      StatefulSets. This is a beta field in k8s 1.21, for lower versions,
+                      This is a beta field in k8s 1.21, for lower versions, starting
-                      starting with k8s 1.19, it requires enabling the GenericEphemeralVolume
+                      with k8s 1.19, it requires enabling the GenericEphemeralVolume
                       feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes'
                     properties:
                       volumeClaimTemplate:
@@ -5912,9 +6064,12 @@ spec:
                                   provisioner or an external controller can support
                                   the specified data source, it will create a new
                                   volume based on the contents of the specified data
-                                  source. If the AnyVolumeDataSource feature gate
+                                  source. When the AnyVolumeDataSource feature gate
-                                  is enabled, this field will always have the same
+                                  is enabled, dataSource contents will be copied to
-                                  contents as the DataSourceRef field.'
+                                  dataSourceRef, and dataSourceRef contents will be
+                                  copied to dataSource when dataSourceRef.namespace
+                                  is not specified. If the namespace is specified,
+                                  then dataSourceRef will not be copied to dataSource.'
                                 properties:
                                   apiGroup:
                                     description: APIGroup is the group for the resource
@@ -5939,27 +6094,33 @@ spec:
                               dataSourceRef:
                                 description: 'dataSourceRef specifies the object from
                                   which to populate the volume with data, if a non-empty
-                                  volume is desired. This may be any local object
+                                  volume is desired. This may be any object from a
-                                  from a non-empty API group (non core object) or
+                                  non-empty API group (non core object) or a PersistentVolumeClaim
-                                  a PersistentVolumeClaim object. When this field
+                                  object. When this field is specified, volume binding
-                                  is specified, volume binding will only succeed if
+                                  will only succeed if the type of the specified object
-                                  the type of the specified object matches some installed
+                                  matches some installed volume populator or dynamic
-                                  volume populator or dynamic provisioner. This field
+                                  provisioner. This field will replace the functionality
-                                  will replace the functionality of the DataSource
+                                  of the dataSource field and as such if both fields
-                                  field and as such if both fields are non-empty,
+                                  are non-empty, they must have the same value. For
-                                  they must have the same value. For backwards compatibility,
+                                  backwards compatibility, when namespace isn''t specified
-                                  both fields (DataSource and DataSourceRef) will
+                                  in dataSourceRef, both fields (dataSource and dataSourceRef)
-                                  be set to the same value automatically if one of
+                                  will be set to the same value automatically if one
-                                  them is empty and the other is non-empty. There
+                                  of them is empty and the other is non-empty. When
-                                  are two important differences between DataSource
+                                  namespace is specified in dataSourceRef, dataSource
-                                  and DataSourceRef: * While DataSource only allows
+                                  isn''t set to the same value and must be empty.
-                                  two specific types of objects, DataSourceRef allows
+                                  There are three important differences between dataSource
+                                  and dataSourceRef: * While dataSource only allows
+                                  two specific types of objects, dataSourceRef allows
                                   any non-core object, as well as PersistentVolumeClaim
-                                  objects. * While DataSource ignores disallowed values
+                                  objects. * While dataSource ignores disallowed values
-                                  (dropping them), DataSourceRef preserves all values,
+                                  (dropping them), dataSourceRef preserves all values,
                                   and generates an error if a disallowed value is
-                                  specified. (Beta) Using this field requires the
+                                  specified. * While dataSource only allows local
-                                  AnyVolumeDataSource feature gate to be enabled.'
+                                  objects, dataSourceRef allows objects in any namespaces.
+                                  (Beta) Using this field requires the AnyVolumeDataSource
+                                  feature gate to be enabled. (Alpha) Using the namespace
+                                  field of dataSourceRef requires the CrossNamespaceVolumeDataSource
+                                  feature gate to be enabled.'
                                 properties:
                                   apiGroup:
                                     description: APIGroup is the group for the resource
@@ -5976,11 +6137,21 @@ spec:
                                     description: Name is the name of resource being
                                       referenced
                                     type: string
+                                  namespace:
+                                    description: Namespace is the namespace of resource
+                                      being referenced Note that when a namespace
+                                      is specified, a gateway.networking.k8s.io/ReferenceGrant
+                                      object is required in the referent namespace
+                                      to allow that namespace's owner to accept the
+                                      reference. See the ReferenceGrant documentation
+                                      for details. (Alpha) This field requires the
+                                      CrossNamespaceVolumeDataSource feature gate
+                                      to be enabled.
+                                    type: string
                                 required:
                                 - kind
                                 - name
                                 type: object
-                                x-kubernetes-map-type: atomic
                               resources:
                                 description: 'resources represents the minimum resources
                                   the volume should have. If RecoverVolumeExpansionFailure
@@ -5989,6 +6160,29 @@ spec:
                                   value but must still be higher than capacity recorded
                                   in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
                                 properties:
+                                  claims:
+                                    description: "Claims lists the names of resources,
+                                      defined in spec.resourceClaims, that are used
+                                      by this container. \n This is an alpha field
+                                      and requires enabling the DynamicResourceAllocation
+                                      feature gate. \n This field is immutable."
+                                    items:
+                                      description: ResourceClaim references one entry
+                                        in PodSpec.ResourceClaims.
+                                      properties:
+                                        name:
+                                          description: Name must match the name of
+                                            one entry in pod.spec.resourceClaims of
+                                            the Pod where this field is used. It makes
+                                            that resource available inside a container.
+                                          type: string
+                                      required:
+                                      - name
+                                      type: object
+                                    type: array
+                                    x-kubernetes-list-map-keys:
+                                    - name
+                                    x-kubernetes-list-type: map
                                   limits:
                                     additionalProperties:
                                       anyOf:
@@ -6082,7 +6276,10 @@ spec:
                         type: object
                     type: object
                   volumeClaimTemplate:
-                    description: A PVC spec to be used by the Prometheus StatefulSets.
+                    description: A PVC spec to be used by the StatefulSet. The easiest
+                      way to use a volume that cannot be automatically provisioned
+                      (for whatever reason) is to use a label selector alongside manually
+                      created PersistentVolumes.
                     properties:
                       apiVersion:
                         description: 'APIVersion defines the versioned schema of this
@@ -6142,9 +6339,12 @@ spec:
                               * An existing PVC (PersistentVolumeClaim) If the provisioner
                               or an external controller can support the specified
                               data source, it will create a new volume based on the
-                              contents of the specified data source. If the AnyVolumeDataSource
+                              contents of the specified data source. When the AnyVolumeDataSource
-                              feature gate is enabled, this field will always have
+                              feature gate is enabled, dataSource contents will be
-                              the same contents as the DataSourceRef field.'
+                              copied to dataSourceRef, and dataSourceRef contents
+                              will be copied to dataSource when dataSourceRef.namespace
+                              is not specified. If the namespace is specified, then
+                              dataSourceRef will not be copied to dataSource.'
                             properties:
                               apiGroup:
                                 description: APIGroup is the group for the resource
@@ -6166,24 +6366,31 @@ spec:
                           dataSourceRef:
                             description: 'dataSourceRef specifies the object from
                               which to populate the volume with data, if a non-empty
-                              volume is desired. This may be any local object from
+                              volume is desired. This may be any object from a non-empty
-                              a non-empty API group (non core object) or a PersistentVolumeClaim
+                              API group (non core object) or a PersistentVolumeClaim
                               object. When this field is specified, volume binding
                               will only succeed if the type of the specified object
                               matches some installed volume populator or dynamic provisioner.
-                              This field will replace the functionality of the DataSource
+                              This field will replace the functionality of the dataSource
                               field and as such if both fields are non-empty, they
                               must have the same value. For backwards compatibility,
-                              both fields (DataSource and DataSourceRef) will be set
+                              when namespace isn''t specified in dataSourceRef, both
-                              to the same value automatically if one of them is empty
+                              fields (dataSource and dataSourceRef) will be set to
-                              and the other is non-empty. There are two important
+                              the same value automatically if one of them is empty
-                              differences between DataSource and DataSourceRef: *
+                              and the other is non-empty. When namespace is specified
-                              While DataSource only allows two specific types of objects,
+                              in dataSourceRef, dataSource isn''t set to the same
-                              DataSourceRef allows any non-core object, as well as
+                              value and must be empty. There are three important differences
-                              PersistentVolumeClaim objects. * While DataSource ignores
+                              between dataSource and dataSourceRef: * While dataSource
-                              disallowed values (dropping them), DataSourceRef preserves
+                              only allows two specific types of objects, dataSourceRef
-                              all values, and generates an error if a disallowed value
+                              allows any non-core object, as well as PersistentVolumeClaim
-                              is specified. (Beta) Using this field requires the AnyVolumeDataSource
+                              objects. * While dataSource ignores disallowed values
+                              (dropping them), dataSourceRef preserves all values,
+                              and generates an error if a disallowed value is specified.
+                              * While dataSource only allows local objects, dataSourceRef
+                              allows objects in any namespaces. (Beta) Using this
+                              field requires the AnyVolumeDataSource feature gate
+                              to be enabled. (Alpha) Using the namespace field of
+                              dataSourceRef requires the CrossNamespaceVolumeDataSource
                               feature gate to be enabled.'
                             properties:
                               apiGroup:
@@ -6198,11 +6405,20 @@ spec:
                               name:
                                 description: Name is the name of resource being referenced
                                 type: string
+                              namespace:
+                                description: Namespace is the namespace of resource
+                                  being referenced Note that when a namespace is specified,
+                                  a gateway.networking.k8s.io/ReferenceGrant object
+                                  is required in the referent namespace to allow that
+                                  namespace's owner to accept the reference. See the
+                                  ReferenceGrant documentation for details. (Alpha)
+                                  This field requires the CrossNamespaceVolumeDataSource
+                                  feature gate to be enabled.
+                                type: string
                             required:
                             - kind
                             - name
                             type: object
-                            x-kubernetes-map-type: atomic
                           resources:
                             description: 'resources represents the minimum resources
                               the volume should have. If RecoverVolumeExpansionFailure
@@ -6211,6 +6427,29 @@ spec:
                               must still be higher than capacity recorded in the status
                               field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
                             properties:
+                              claims:
+                                description: "Claims lists the names of resources,
+                                  defined in spec.resourceClaims, that are used by
+                                  this container. \n This is an alpha field and requires
+                                  enabling the DynamicResourceAllocation feature gate.
+                                  \n This field is immutable."
+                                items:
+                                  description: ResourceClaim references one entry
+                                    in PodSpec.ResourceClaims.
+                                  properties:
+                                    name:
+                                      description: Name must match the name of one
+                                        entry in pod.spec.resourceClaims of the Pod
+                                        where this field is used. It makes that resource
+                                        available inside a container.
+                                      type: string
+                                  required:
+                                  - name
+                                  type: object
+                                type: array
+                                x-kubernetes-list-map-keys:
+                                - name
+                                x-kubernetes-list-type: map
                               limits:
                                 additionalProperties:
                                   anyOf:
@@ -6644,6 +6883,27 @@ spec:
                       Thanos sidecar. If not provided, no requests/limits will be
                       set
                     properties:
+                      claims:
+                        description: "Claims lists the names of resources, defined
+                          in spec.resourceClaims, that are used by this container.
+                          \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                          feature gate. \n This field is immutable."
+                        items:
+                          description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                          properties:
+                            name:
+                              description: Name must match the name of one entry in
+                                pod.spec.resourceClaims of the Pod where this field
+                                is used. It makes that resource available inside a
+                                container.
+                              type: string
+                          required:
+                          - name
+                          type: object
+                        type: array
+                        x-kubernetes-list-map-keys:
+                        - name
+                        x-kubernetes-list-type: map
                       limits:
                         additionalProperties:
                           anyOf:
@@ -6911,8 +7171,8 @@ spec:
                         are included in the calculations. - Ignore: nodeAffinity/nodeSelector
                         are ignored. All nodes are included in the calculations. \n
                         If this value is nil, the behavior is equivalent to the Honor
-                        policy. This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread
+                        policy. This is a beta-level feature default enabled by the
-                        feature flag."
+                        NodeInclusionPolicyInPodTopologySpread feature flag."
                       type: string
                     nodeTaintsPolicy:
                       description: "NodeTaintsPolicy indicates how we will treat node
@@ -6921,8 +7181,8 @@ spec:
                         for which the incoming pod has a toleration, are included.
                         - Ignore: node taints are ignored. All nodes are included.
                         \n If this value is nil, the behavior is equivalent to the
-                        Ignore policy. This is a alpha-level feature enabled by the
+                        Ignore policy. This is a beta-level feature default enabled
-                        NodeInclusionPolicyInPodTopologySpread feature flag."
+                        by the NodeInclusionPolicyInPodTopologySpread feature flag."
                       type: string
                     topologyKey:
                       description: TopologyKey is the key of node labels. Nodes that
@@ -7477,9 +7737,12 @@ spec:
                                     provisioner or an external controller can support
                                     the specified data source, it will create a new
                                     volume based on the contents of the specified
-                                    data source. If the AnyVolumeDataSource feature
+                                    data source. When the AnyVolumeDataSource feature
-                                    gate is enabled, this field will always have the
+                                    gate is enabled, dataSource contents will be copied
-                                    same contents as the DataSourceRef field.'
+                                    to dataSourceRef, and dataSourceRef contents will
+                                    be copied to dataSource when dataSourceRef.namespace
+                                    is not specified. If the namespace is specified,
+                                    then dataSourceRef will not be copied to dataSource.'
                                   properties:
                                     apiGroup:
                                       description: APIGroup is the group for the resource
@@ -7505,27 +7768,35 @@ spec:
                                   description: 'dataSourceRef specifies the object
                                     from which to populate the volume with data, if
                                     a non-empty volume is desired. This may be any
-                                    local object from a non-empty API group (non core
+                                    object from a non-empty API group (non core object)
-                                    object) or a PersistentVolumeClaim object. When
+                                    or a PersistentVolumeClaim object. When this field
-                                    this field is specified, volume binding will only
+                                    is specified, volume binding will only succeed
-                                    succeed if the type of the specified object matches
+                                    if the type of the specified object matches some
-                                    some installed volume populator or dynamic provisioner.
+                                    installed volume populator or dynamic provisioner.
                                     This field will replace the functionality of the
-                                    DataSource field and as such if both fields are
+                                    dataSource field and as such if both fields are
                                     non-empty, they must have the same value. For
-                                    backwards compatibility, both fields (DataSource
+                                    backwards compatibility, when namespace isn''t
-                                    and DataSourceRef) will be set to the same value
+                                    specified in dataSourceRef, both fields (dataSource
+                                    and dataSourceRef) will be set to the same value
                                     automatically if one of them is empty and the
-                                    other is non-empty. There are two important differences
+                                    other is non-empty. When namespace is specified
-                                    between DataSource and DataSourceRef: * While
+                                    in dataSourceRef, dataSource isn''t set to the
-                                    DataSource only allows two specific types of objects,
+                                    same value and must be empty. There are three
-                                    DataSourceRef allows any non-core object, as well
+                                    important differences between dataSource and dataSourceRef:
-                                    as PersistentVolumeClaim objects. * While DataSource
+                                    * While dataSource only allows two specific types
-                                    ignores disallowed values (dropping them), DataSourceRef
+                                    of objects, dataSourceRef allows any non-core
-                                    preserves all values, and generates an error if
+                                    object, as well as PersistentVolumeClaim objects.
-                                    a disallowed value is specified. (Beta) Using
+                                    * While dataSource ignores disallowed values (dropping
-                                    this field requires the AnyVolumeDataSource feature
+                                    them), dataSourceRef preserves all values, and
-                                    gate to be enabled.'
+                                    generates an error if a disallowed value is specified.
+                                    * While dataSource only allows local objects,
+                                    dataSourceRef allows objects in any namespaces.
+                                    (Beta) Using this field requires the AnyVolumeDataSource
+                                    feature gate to be enabled. (Alpha) Using the
+                                    namespace field of dataSourceRef requires the
+                                    CrossNamespaceVolumeDataSource feature gate to
+                                    be enabled.'
                                   properties:
                                     apiGroup:
                                       description: APIGroup is the group for the resource
@@ -7542,11 +7813,21 @@ spec:
                                       description: Name is the name of resource being
                                         referenced
                                       type: string
+                                    namespace:
+                                      description: Namespace is the namespace of resource
+                                        being referenced Note that when a namespace
+                                        is specified, a gateway.networking.k8s.io/ReferenceGrant
+                                        object is required in the referent namespace
+                                        to allow that namespace's owner to accept
+                                        the reference. See the ReferenceGrant documentation
+                                        for details. (Alpha) This field requires the
+                                        CrossNamespaceVolumeDataSource feature gate
+                                        to be enabled.
+                                      type: string
                                   required:
                                   - kind
                                   - name
                                   type: object
-                                  x-kubernetes-map-type: atomic
                                 resources:
                                   description: 'resources represents the minimum resources
                                     the volume should have. If RecoverVolumeExpansionFailure
@@ -7555,6 +7836,30 @@ spec:
                                     value but must still be higher than capacity recorded
                                     in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
                                   properties:
+                                    claims:
+                                      description: "Claims lists the names of resources,
+                                        defined in spec.resourceClaims, that are used
+                                        by this container. \n This is an alpha field
+                                        and requires enabling the DynamicResourceAllocation
+                                        feature gate. \n This field is immutable."
+                                      items:
+                                        description: ResourceClaim references one
+                                          entry in PodSpec.ResourceClaims.
+                                        properties:
+                                          name:
+                                            description: Name must match the name
+                                              of one entry in pod.spec.resourceClaims
+                                              of the Pod where this field is used.
+                                              It makes that resource available inside
+                                              a container.
+                                            type: string
+                                        required:
+                                        - name
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-map-keys:
+                                      - name
+                                      x-kubernetes-list-type: map
                                     limits:
                                       additionalProperties:
                                         anyOf:
@@ -8589,6 +8894,13 @@ spec:
                           a rolling update will be triggered.
                         type: boolean
                     type: object
+                  maxConnections:
+                    description: Defines the maximum number of simultaneous connections
+                      A zero value means that Prometheus doesn't accept any incoming
+                      connection.
+                    format: int32
+                    minimum: 0
+                    type: integer
                   pageTitle:
                     description: The prometheus web page title
                     type: string
@@ -8755,8 +9067,8 @@ spec:
               conditions:
                 description: The current state of the Prometheus deployment.
                 items:
-                  description: PrometheusCondition represents the state of the resources
+                  description: Condition represents the state of the resources associated
-                    associated with the Prometheus resource.
+                    with the Prometheus or Alertmanager resource.
                   properties:
                     lastTransitionTime:
                       description: lastTransitionTime is the time of the last update
@@ -8769,8 +9081,8 @@ spec:
                       type: string
                     observedGeneration:
                       description: ObservedGeneration represents the .metadata.generation
-                        that the condition was set based upon. For instance, if .metadata.generation
+                        that the condition was set based upon. For instance, if `.metadata.generation`
-                        is currently 12, but the .status.conditions[x].observedGeneration
+                        is currently 12, but the `.status.conditions[].observedGeneration`
                         is 9, the condition is out of date with respect to the current
                         state of the instance.
                       format: int64
@@ -8779,7 +9091,7 @@ spec:
                       description: Reason for the condition's last transition.
                       type: string
                     status:
-                      description: status of the condition.
+                      description: Status of the condition.
                       type: string
                     type:
                       description: Type of the condition being reported.

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml

@@ -1,10 +1,10 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
+    controller-gen.kubebuilder.io/version: v0.11.1
   creationTimestamp: null
   name: prometheusrules.monitoring.coreos.com
 spec:
@@ -57,7 +57,6 @@ spec:
                       minLength: 1
                       type: string
                     partial_response_strategy:
-                      default: ""
                       description: 'PartialResponseStrategy is only used by ThanosRuler
                         and will be ignored by Prometheus instances. More info: https://github.com/thanos-io/thanos/blob/main/docs/components/rule.md#partial-response'
                       pattern: ^(?i)(abort|warn)?$

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml

@@ -1,10 +1,10 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
+    controller-gen.kubebuilder.io/version: v0.11.1
   creationTimestamp: null
   name: servicemonitors.monitoring.coreos.com
 spec:

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml

@@ -1,10 +1,10 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.61.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.9.2
+    controller-gen.kubebuilder.io/version: v0.11.1
   creationTimestamp: null
   name: thanosrulers.monitoring.coreos.com
 spec:
@@ -54,6 +54,31 @@ spec:
             description: 'Specification of the desired behavior of the ThanosRuler
               cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status'
             properties:
+              additionalArgs:
+                description: AdditionalArgs allows setting additional arguments for
+                  the ThanosRuler container. It is intended for e.g. activating hidden
+                  flags which are not supported by the dedicated configuration options
+                  yet. The arguments are passed as-is to the ThanosRuler container
+                  which may cause issues if they are invalid or not supported by the
+                  given ThanosRuler version. In case of an argument conflict (e.g.
+                  an argument which is already set by the operator itself) or when
+                  providing an invalid argument the reconciliation will fail and an
+                  error will be logged.
+                items:
+                  description: Argument as part of the AdditionalArgs list.
+                  properties:
+                    name:
+                      description: Name of the argument, e.g. "scrape.discovery-reload-interval".
+                      minLength: 1
+                      type: string
+                    value:
+                      description: Argument value, e.g. 30s. Can be empty for name-only
+                        arguments (e.g. --storage.tsdb.no-lockfile)
+                      type: string
+                  required:
+                  - name
+                  type: object
+                type: array
               affinity:
                 description: If specified, the pod's scheduling constraints.
                 properties:
@@ -1713,6 +1738,27 @@ spec:
                       description: 'Compute Resources required by this container.
                         Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
                       properties:
+                        claims:
+                          description: "Claims lists the names of resources, defined
+                            in spec.resourceClaims, that are used by this container.
+                            \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                            feature gate. \n This field is immutable."
+                          items:
+                            description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                            properties:
+                              name:
+                                description: Name must match the name of one entry
+                                  in pod.spec.resourceClaims of the Pod where this
+                                  field is used. It makes that resource available
+                                  inside a container.
+                                type: string
+                            required:
+                            - name
+                            type: object
+                          type: array
+                          x-kubernetes-list-map-keys:
+                          - name
+                          x-kubernetes-list-type: map
                         limits:
                           additionalProperties:
                             anyOf:
@@ -2383,6 +2429,16 @@ spec:
               image:
                 description: Thanos container image URL.
                 type: string
+              imagePullPolicy:
+                description: Image pull policy for the 'thanos', 'init-config-reloader'
+                  and 'config-reloader' containers. See https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy
+                  for more details.
+                enum:
+                - ""
+                - Always
+                - Never
+                - IfNotPresent
+                type: string
               imagePullSecrets:
                 description: An optional list of references to secrets in the same
                   namespace to use for pulling thanos images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod
@@ -3161,6 +3217,27 @@ spec:
                       description: 'Compute Resources required by this container.
                         Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
                       properties:
+                        claims:
+                          description: "Claims lists the names of resources, defined
+                            in spec.resourceClaims, that are used by this container.
+                            \n This is an alpha field and requires enabling the DynamicResourceAllocation
+                            feature gate. \n This field is immutable."
+                          items:
+                            description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                            properties:
+                              name:
+                                description: Name must match the name of one entry
+                                  in pod.spec.resourceClaims of the Pod where this
+                                  field is used. It makes that resource available
+                                  inside a container.
+                                type: string
+                            required:
+                            - name
+                            type: object
+                          type: array
+                          x-kubernetes-list-map-keys:
+                          - name
+                          x-kubernetes-list-type: map
                         limits:
                           additionalProperties:
                             anyOf:
@@ -3660,8 +3737,9 @@ spec:
                 description: Minimum number of seconds for which a newly created pod
                   should be ready without any of its container crashing for it to
                   be considered available. Defaults to 0 (pod will be considered available
-                  as soon as it is ready) This is an alpha field and requires enabling
+                  as soon as it is ready) This is an alpha field from kubernetes 1.22
-                  StatefulSetMinReadySeconds feature gate.
+                  until 1.24 which requires enabling the StatefulSetMinReadySeconds
+                  feature gate.
                 format: int32
                 type: integer
               nodeSelector:
@@ -3789,6 +3867,26 @@ spec:
                 description: Resources defines the resource requirements for single
                   Pods. If not provided, no requests/limits will be set
                 properties:
+                  claims:
+                    description: "Claims lists the names of resources, defined in
+                      spec.resourceClaims, that are used by this container. \n This
+                      is an alpha field and requires enabling the DynamicResourceAllocation
+                      feature gate. \n This field is immutable."
+                    items:
+                      description: ResourceClaim references one entry in PodSpec.ResourceClaims.
+                      properties:
+                        name:
+                          description: Name must match the name of one entry in pod.spec.resourceClaims
+                            of the Pod where this field is used. It makes that resource
+                            available inside a container.
+                          type: string
+                      required:
+                      - name
+                      type: object
+                    type: array
+                    x-kubernetes-list-map-keys:
+                    - name
+                    x-kubernetes-list-type: map
                   limits:
                     additionalProperties:
                       anyOf:
@@ -4016,9 +4114,14 @@ spec:
                     type: object
                   supplementalGroups:
                     description: A list of groups applied to the first process run
-                      in each container, in addition to the container's primary GID.  If
+                      in each container, in addition to the container's primary GID,
-                      unspecified, no groups will be added to any container. Note
+                      the fsGroup (if specified), and group memberships defined in
-                      that this field cannot be set when spec.os.name is windows.
+                      the container image for the uid of the container process. If
+                      unspecified, no additional groups are added to any container.
+                      Note that group memberships defined in the container image for
+                      the uid of the container process are still effective, even if
+                      they are not included in this list. Note that this field cannot
+                      be set when spec.os.name is windows.
                     items:
                       format: int64
                       type: integer
@@ -4092,9 +4195,9 @@ spec:
                       allows to remove any subPath usage in volume mounts.'
                     type: boolean
                   emptyDir:
-                    description: 'EmptyDirVolumeSource to be used by the Prometheus
+                    description: 'EmptyDirVolumeSource to be used by the StatefulSet.
-                      StatefulSets. If specified, used in place of any volumeClaimTemplate.
+                      If specified, used in place of any volumeClaimTemplate. More
-                      More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir'
+                      info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir'
                     properties:
                       medium:
                         description: 'medium represents what type of storage medium
@@ -4117,9 +4220,9 @@ spec:
                         x-kubernetes-int-or-string: true
                     type: object
                   ephemeral:
-                    description: 'EphemeralVolumeSource to be used by the Prometheus
+                    description: 'EphemeralVolumeSource to be used by the StatefulSet.
-                      StatefulSets. This is a beta field in k8s 1.21, for lower versions,
+                      This is a beta field in k8s 1.21, for lower versions, starting
-                      starting with k8s 1.19, it requires enabling the GenericEphemeralVolume
+                      with k8s 1.19, it requires enabling the GenericEphemeralVolume
                       feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes'
                     properties:
                       volumeClaimTemplate:
@@ -4166,9 +4269,12 @@ spec:
                                   provisioner or an external controller can support
                                   the specified data source, it will create a new
                                   volume based on the contents of the specified data
-                                  source. If the AnyVolumeDataSource feature gate
+                                  source. When the AnyVolumeDataSource feature gate
-                                  is enabled, this field will always have the same
+                                  is enabled, dataSource contents will be copied to
-                                  contents as the DataSourceRef field.'
+                                  dataSourceRef, and dataSourceRef contents will be
+                                  copied to dataSource when dataSourceRef.namespace
+                                  is not specified. If the namespace is specified,
+                                  then dataSourceRef will not be copied to dataSource.'
                                 properties:
                                   apiGroup:
                                     description: APIGroup is the group for the resource
@@ -4193,27 +4299,33 @@ spec:
                               dataSourceRef:
                                 description: 'dataSourceRef specifies the object from
                                   which to populate the volume with data, if a non-empty
-                                  volume is desired. This may be any local object
+                                  volume is desired. This may be any object from a
-                                  from a non-empty API group (non core object) or
+                                  non-empty API group (non core object) or a PersistentVolumeClaim
-                                  a PersistentVolumeClaim object. When this field
+                                  object. When this field is specified, volume binding
-                                  is specified, volume binding will only succeed if
+                                  will only succeed if the type of the specified object
-                                  the type of the specified object matches some installed
+                                  matches some installed volume populator or dynamic
-                                  volume populator or dynamic provisioner. This field
+                                  provisioner. This field will replace the functionality
-                                  will replace the functionality of the DataSource
+                                  of the dataSource field and as such if both fields
-                                  field and as such if both fields are non-empty,
+                                  are non-empty, they must have the same value. For
-                                  they must have the same value. For backwards compatibility,
+                                  backwards compatibility, when namespace isn''t specified
-                                  both fields (DataSource and DataSourceRef) will
+                                  in dataSourceRef, both fields (dataSource and dataSourceRef)
-                                  be set to the same value automatically if one of
+                                  will be set to the same value automatically if one
-                                  them is empty and the other is non-empty. There
+                                  of them is empty and the other is non-empty. When
-                                  are two important differences between DataSource
+                                  namespace is specified in dataSourceRef, dataSource
-                                  and DataSourceRef: * While DataSource only allows
+                                  isn''t set to the same value and must be empty.
-                                  two specific types of objects, DataSourceRef allows
+                                  There are three important differences between dataSource
+                                  and dataSourceRef: * While dataSource only allows
+                                  two specific types of objects, dataSourceRef allows
                                   any non-core object, as well as PersistentVolumeClaim
-                                  objects. * While DataSource ignores disallowed values
+                                  objects. * While dataSource ignores disallowed values
-                                  (dropping them), DataSourceRef preserves all values,
+                                  (dropping them), dataSourceRef preserves all values,
                                   and generates an error if a disallowed value is
-                                  specified. (Beta) Using this field requires the
+                                  specified. * While dataSource only allows local
-                                  AnyVolumeDataSource feature gate to be enabled.'
+                                  objects, dataSourceRef allows objects in any namespaces.
+                                  (Beta) Using this field requires the AnyVolumeDataSource
+                                  feature gate to be enabled. (Alpha) Using the namespace
+                                  field of dataSourceRef requires the CrossNamespaceVolumeDataSource
+                                  feature gate to be enabled.'
                                 properties:
                                   apiGroup:
                                     description: APIGroup is the group for the resource
@@ -4230,11 +4342,21 @@ spec:
                                     description: Name is the name of resource being
                                       referenced
                                     type: string
+                                  namespace:
+                                    description: Namespace is the namespace of resource
+                                      being referenced Note that when a namespace
+                                      is specified, a gateway.networking.k8s.io/ReferenceGrant
+                                      object is required in the referent namespace
+                                      to allow that namespace's owner to accept the
+                                      reference. See the ReferenceGrant documentation
+                                      for details. (Alpha) This field requires the
+                                      CrossNamespaceVolumeDataSource feature gate
+                                      to be enabled.
+                                    type: string
                                 required:
                                 - kind
                                 - name
                                 type: object
-                                x-kubernetes-map-type: atomic
                               resources:
                                 description: 'resources represents the minimum resources
                                   the volume should have. If RecoverVolumeExpansionFailure
@@ -4243,6 +4365,29 @@ spec:
                                   value but must still be higher than capacity recorded
                                   in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
                                 properties:
+                                  claims:
+                                    description: "Claims lists the names of resources,
+                                      defined in spec.resourceClaims, that are used
+                                      by this container. \n This is an alpha field
+                                      and requires enabling the DynamicResourceAllocation
+                                      feature gate. \n This field is immutable."
+                                    items:
+                                      description: ResourceClaim references one entry
+                                        in PodSpec.ResourceClaims.
+                                      properties:
+                                        name:
+                                          description: Name must match the name of
+                                            one entry in pod.spec.resourceClaims of
+                                            the Pod where this field is used. It makes
+                                            that resource available inside a container.
+                                          type: string
+                                      required:
+                                      - name
+                                      type: object
+                                    type: array
+                                    x-kubernetes-list-map-keys:
+                                    - name
+                                    x-kubernetes-list-type: map
                                   limits:
                                     additionalProperties:
                                       anyOf:
@@ -4336,7 +4481,10 @@ spec:
                         type: object
                     type: object
                   volumeClaimTemplate:
-                    description: A PVC spec to be used by the Prometheus StatefulSets.
+                    description: A PVC spec to be used by the StatefulSet. The easiest
+                      way to use a volume that cannot be automatically provisioned
+                      (for whatever reason) is to use a label selector alongside manually
+                      created PersistentVolumes.
                     properties:
                       apiVersion:
                         description: 'APIVersion defines the versioned schema of this
@@ -4396,9 +4544,12 @@ spec:
                               * An existing PVC (PersistentVolumeClaim) If the provisioner
                               or an external controller can support the specified
                               data source, it will create a new volume based on the
-                              contents of the specified data source. If the AnyVolumeDataSource
+                              contents of the specified data source. When the AnyVolumeDataSource
-                              feature gate is enabled, this field will always have
+                              feature gate is enabled, dataSource contents will be
-                              the same contents as the DataSourceRef field.'
+                              copied to dataSourceRef, and dataSourceRef contents
+                              will be copied to dataSource when dataSourceRef.namespace
+                              is not specified. If the namespace is specified, then
+                              dataSourceRef will not be copied to dataSource.'
                             properties:
                               apiGroup:
                                 description: APIGroup is the group for the resource
@@ -4420,24 +4571,31 @@ spec:
                           dataSourceRef:
                             description: 'dataSourceRef specifies the object from
                               which to populate the volume with data, if a non-empty
-                              volume is desired. This may be any local object from
+                              volume is desired. This may be any object from a non-empty
-                              a non-empty API group (non core object) or a PersistentVolumeClaim
+                              API group (non core object) or a PersistentVolumeClaim
                               object. When this field is specified, volume binding
                               will only succeed if the type of the specified object
                               matches some installed volume populator or dynamic provisioner.
-                              This field will replace the functionality of the DataSource
+                              This field will replace the functionality of the dataSource
                               field and as such if both fields are non-empty, they
                               must have the same value. For backwards compatibility,
-                              both fields (DataSource and DataSourceRef) will be set
+                              when namespace isn''t specified in dataSourceRef, both
-                              to the same value automatically if one of them is empty
+                              fields (dataSource and dataSourceRef) will be set to
-                              and the other is non-empty. There are two important
+                              the same value automatically if one of them is empty
-                              differences between DataSource and DataSourceRef: *
+                              and the other is non-empty. When namespace is specified
-                              While DataSource only allows two specific types of objects,
+                              in dataSourceRef, dataSource isn''t set to the same
-                              DataSourceRef allows any non-core object, as well as
+                              value and must be empty. There are three important differences
-                              PersistentVolumeClaim objects. * While DataSource ignores
+                              between dataSource and dataSourceRef: * While dataSource
-                              disallowed values (dropping them), DataSourceRef preserves
+                              only allows two specific types of objects, dataSourceRef
-                              all values, and generates an error if a disallowed value
+                              allows any non-core object, as well as PersistentVolumeClaim
-                              is specified. (Beta) Using this field requires the AnyVolumeDataSource
+                              objects. * While dataSource ignores disallowed values
+                              (dropping them), dataSourceRef preserves all values,
+                              and generates an error if a disallowed value is specified.
+                              * While dataSource only allows local objects, dataSourceRef
+                              allows objects in any namespaces. (Beta) Using this
+                              field requires the AnyVolumeDataSource feature gate
+                              to be enabled. (Alpha) Using the namespace field of
+                              dataSourceRef requires the CrossNamespaceVolumeDataSource
                               feature gate to be enabled.'
                             properties:
                               apiGroup:
@@ -4452,11 +4610,20 @@ spec:
                               name:
                                 description: Name is the name of resource being referenced
                                 type: string
+                              namespace:
+                                description: Namespace is the namespace of resource
+                                  being referenced Note that when a namespace is specified,
+                                  a gateway.networking.k8s.io/ReferenceGrant object
+                                  is required in the referent namespace to allow that
+                                  namespace's owner to accept the reference. See the
+                                  ReferenceGrant documentation for details. (Alpha)
+                                  This field requires the CrossNamespaceVolumeDataSource
+                                  feature gate to be enabled.
+                                type: string
                             required:
                             - kind
                             - name
                             type: object
-                            x-kubernetes-map-type: atomic
                           resources:
                             description: 'resources represents the minimum resources
                               the volume should have. If RecoverVolumeExpansionFailure
@@ -4465,6 +4632,29 @@ spec:
                               must still be higher than capacity recorded in the status
                               field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
                             properties:
+                              claims:
+                                description: "Claims lists the names of resources,
+                                  defined in spec.resourceClaims, that are used by
+                                  this container. \n This is an alpha field and requires
+                                  enabling the DynamicResourceAllocation feature gate.
+                                  \n This field is immutable."
+                                items:
+                                  description: ResourceClaim references one entry
+                                    in PodSpec.ResourceClaims.
+                                  properties:
+                                    name:
+                                      description: Name must match the name of one
+                                        entry in pod.spec.resourceClaims of the Pod
+                                        where this field is used. It makes that resource
+                                        available inside a container.
+                                      type: string
+                                  required:
+                                  - name
+                                  type: object
+                                type: array
+                                x-kubernetes-list-map-keys:
+                                - name
+                                x-kubernetes-list-type: map
                               limits:
                                 additionalProperties:
                                   anyOf:
@@ -4802,8 +4992,8 @@ spec:
                         are included in the calculations. - Ignore: nodeAffinity/nodeSelector
                         are ignored. All nodes are included in the calculations. \n
                         If this value is nil, the behavior is equivalent to the Honor
-                        policy. This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread
+                        policy. This is a beta-level feature default enabled by the
-                        feature flag."
+                        NodeInclusionPolicyInPodTopologySpread feature flag."
                       type: string
                     nodeTaintsPolicy:
                       description: "NodeTaintsPolicy indicates how we will treat node
@@ -4812,8 +5002,8 @@ spec:
                         for which the incoming pod has a toleration, are included.
                         - Ignore: node taints are ignored. All nodes are included.
                         \n If this value is nil, the behavior is equivalent to the
-                        Ignore policy. This is a alpha-level feature enabled by the
+                        Ignore policy. This is a beta-level feature default enabled
-                        NodeInclusionPolicyInPodTopologySpread feature flag."
+                        by the NodeInclusionPolicyInPodTopologySpread feature flag."
                       type: string
                     topologyKey:
                       description: TopologyKey is the key of node labels. Nodes that
@@ -4877,6 +5067,9 @@ spec:
                   file. When used alongside with TracingConfig, TracingConfigFile
                   takes precedence.
                 type: string
+              version:
+                description: Version of Thanos to be deployed.
+                type: string
               volumes:
                 description: Volumes allows configuration of additional volumes on
                   the output StatefulSet definition. Volumes specified will be appended
@@ -5334,9 +5527,12 @@ spec:
                                     provisioner or an external controller can support
                                     the specified data source, it will create a new
                                     volume based on the contents of the specified
-                                    data source. If the AnyVolumeDataSource feature
+                                    data source. When the AnyVolumeDataSource feature
-                                    gate is enabled, this field will always have the
+                                    gate is enabled, dataSource contents will be copied
-                                    same contents as the DataSourceRef field.'
+                                    to dataSourceRef, and dataSourceRef contents will
+                                    be copied to dataSource when dataSourceRef.namespace
+                                    is not specified. If the namespace is specified,
+                                    then dataSourceRef will not be copied to dataSource.'
                                   properties:
                                     apiGroup:
                                       description: APIGroup is the group for the resource
@@ -5362,27 +5558,35 @@ spec:
                                   description: 'dataSourceRef specifies the object
                                     from which to populate the volume with data, if
                                     a non-empty volume is desired. This may be any
-                                    local object from a non-empty API group (non core
+                                    object from a non-empty API group (non core object)
-                                    object) or a PersistentVolumeClaim object. When
+                                    or a PersistentVolumeClaim object. When this field
-                                    this field is specified, volume binding will only
+                                    is specified, volume binding will only succeed
-                                    succeed if the type of the specified object matches
+                                    if the type of the specified object matches some
-                                    some installed volume populator or dynamic provisioner.
+                                    installed volume populator or dynamic provisioner.
                                     This field will replace the functionality of the
-                                    DataSource field and as such if both fields are
+                                    dataSource field and as such if both fields are
                                     non-empty, they must have the same value. For
-                                    backwards compatibility, both fields (DataSource
+                                    backwards compatibility, when namespace isn''t
-                                    and DataSourceRef) will be set to the same value
+                                    specified in dataSourceRef, both fields (dataSource
+                                    and dataSourceRef) will be set to the same value
                                     automatically if one of them is empty and the
-                                    other is non-empty. There are two important differences
+                                    other is non-empty. When namespace is specified
-                                    between DataSource and DataSourceRef: * While
+                                    in dataSourceRef, dataSource isn''t set to the
-                                    DataSource only allows two specific types of objects,
+                                    same value and must be empty. There are three
-                                    DataSourceRef allows any non-core object, as well
+                                    important differences between dataSource and dataSourceRef:
-                                    as PersistentVolumeClaim objects. * While DataSource
+                                    * While dataSource only allows two specific types
-                                    ignores disallowed values (dropping them), DataSourceRef
+                                    of objects, dataSourceRef allows any non-core
-                                    preserves all values, and generates an error if
+                                    object, as well as PersistentVolumeClaim objects.
-                                    a disallowed value is specified. (Beta) Using
+                                    * While dataSource ignores disallowed values (dropping
-                                    this field requires the AnyVolumeDataSource feature
+                                    them), dataSourceRef preserves all values, and
-                                    gate to be enabled.'
+                                    generates an error if a disallowed value is specified.
+                                    * While dataSource only allows local objects,
+                                    dataSourceRef allows objects in any namespaces.
+                                    (Beta) Using this field requires the AnyVolumeDataSource
+                                    feature gate to be enabled. (Alpha) Using the
+                                    namespace field of dataSourceRef requires the
+                                    CrossNamespaceVolumeDataSource feature gate to
+                                    be enabled.'
                                   properties:
                                     apiGroup:
                                       description: APIGroup is the group for the resource
@@ -5399,11 +5603,21 @@ spec:
                                       description: Name is the name of resource being
                                         referenced
                                       type: string
+                                    namespace:
+                                      description: Namespace is the namespace of resource
+                                        being referenced Note that when a namespace
+                                        is specified, a gateway.networking.k8s.io/ReferenceGrant
+                                        object is required in the referent namespace
+                                        to allow that namespace's owner to accept
+                                        the reference. See the ReferenceGrant documentation
+                                        for details. (Alpha) This field requires the
+                                        CrossNamespaceVolumeDataSource feature gate
+                                        to be enabled.
+                                      type: string
                                   required:
                                   - kind
                                   - name
                                   type: object
-                                  x-kubernetes-map-type: atomic
                                 resources:
                                   description: 'resources represents the minimum resources
                                     the volume should have. If RecoverVolumeExpansionFailure
@@ -5412,6 +5626,30 @@ spec:
                                     value but must still be higher than capacity recorded
                                     in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources'
                                   properties:
+                                    claims:
+                                      description: "Claims lists the names of resources,
+                                        defined in spec.resourceClaims, that are used
+                                        by this container. \n This is an alpha field
+                                        and requires enabling the DynamicResourceAllocation
+                                        feature gate. \n This field is immutable."
+                                      items:
+                                        description: ResourceClaim references one
+                                          entry in PodSpec.ResourceClaims.
+                                        properties:
+                                          name:
+                                            description: Name must match the name
+                                              of one entry in pod.spec.resourceClaims
+                                              of the Pod where this field is used.
+                                              It makes that resource available inside
+                                              a container.
+                                            type: string
+                                        required:
+                                        - name
+                                        type: object
+                                      type: array
+                                      x-kubernetes-list-map-keys:
+                                      - name
+                                      x-kubernetes-list-type: map
                                     limits:
                                       additionalProperties:
                                         anyOf:

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl

@@ -57,6 +57,12 @@ The longest name that gets created adds and extra 37 characters, so truncation s
 {{- printf "%s-thanos-ruler" (include "kube-prometheus-stack.fullname" .) -}}
 {{- end }}
 
+{{/* Shortened name suffixed with thanos-ruler */}}
+{{- define "kube-prometheus-stack.thanosRuler.name" -}}
+{{- default (printf "%s-thanos-ruler" (include "kube-prometheus-stack.name" .)) .Values.thanosRuler.name -}}
+{{- end }}
+
+
 {{/* Create chart name and version as used by the chart label. */}}
 {{- define "kube-prometheus-stack.chartref" -}}
 {{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}}
@@ -106,7 +112,7 @@ heritage: {{ $.Release.Service | quote }}
 {{/* Create the name of thanosRuler service account to use */}}
 {{- define "kube-prometheus-stack.thanosRuler.serviceAccountName" -}}
 {{- if .Values.thanosRuler.serviceAccount.create -}}
-    {{ default (include "kube-prometheus-stack.thanosRuler.fullname" .) .Values.thanosRuler.serviceAccount.name }}
+    {{ default (include "kube-prometheus-stack.thanosRuler.name" .) .Values.thanosRuler.serviceAccount.name }}
 {{- else -}}
     {{ default "default" .Values.thanosRuler.serviceAccount.name }}
 {{- end -}}
@@ -228,6 +234,25 @@ Use the prometheus-node-exporter namespace override for multi-namespace deployme
   {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.23-0" $insecure $secure $userValue) -}}
 {{- end -}}
 
+{{/* Sets default scrape limits for servicemonitor */}}
+{{- define "servicemonitor.scrapeLimits" -}}
+{{- with .sampleLimit }}
+sampleLimit: {{ . }}
+{{- end }}
+{{- with .targetLimit }}
+targetLimit: {{ . }}
+{{- end }}
+{{- with .labelLimit }}
+labelLimit: {{ . }}
+{{- end }}
+{{- with .labelNameLengthLimit }}
+labelNameLengthLimit: {{ . }}
+{{- end }}
+{{- with .labelValueLengthLimit }}
+labelValueLengthLimit: {{ . }}
+{{- end }}
+{{- end -}}
+
 {{/*
 To help compatibility with other charts which use global.imagePullSecrets.
 Allow either an array of {name: pullSecret} maps (k8s-style), or an array of strings (more common helm-style).

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml

@@ -77,6 +77,10 @@ spec:
   alertmanagerConfiguration:
 {{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfiguration | indent 4 }}
 {{- end }}
+{{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy }}
+  alertmanagerConfigMatcherStrategy:
+{{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigMatcherStrategy | indent 4 }}
+{{- end }}
 {{- if .Values.alertmanager.alertmanagerSpec.resources }}
   resources:
 {{ toYaml .Values.alertmanager.alertmanagerSpec.resources | indent 4 }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/secret.yaml

@@ -13,14 +13,16 @@ metadata:
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 data:
 {{- if .Values.alertmanager.tplConfig }}
-{{- if eq (typeOf .Values.alertmanager.config) "string" }}
+{{- if .Values.alertmanager.stringConfig }}
+  alertmanager.yaml: {{ tpl (.Values.alertmanager.stringConfig) . | b64enc | quote }}
+{{- else if eq (typeOf .Values.alertmanager.config) "string" }}
   alertmanager.yaml: {{ tpl (.Values.alertmanager.config) . | b64enc | quote }}
 {{- else }}
   alertmanager.yaml: {{ tpl (toYaml .Values.alertmanager.config) . | b64enc | quote }}
 {{- end }}
 {{- else }}
   alertmanager.yaml: {{ toYaml .Values.alertmanager.config | b64enc | quote }}
-{{- end}}
+{{- end }}
 {{- range $key, $val := .Values.alertmanager.templateFiles }}
   {{ $key }}: {{ $val | b64enc | quote }}
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/servicemonitor.yaml

@@ -7,7 +7,11 @@ metadata:
   labels:
     app: {{ template "kube-prometheus-stack.name" . }}-alertmanager
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
+{{- with .Values.alertmanager.serviceMonitor.additionalLabels }}
+{{- toYaml . | nindent 4 }}
+{{- end }}
 spec:
+  {{- include "servicemonitor.scrapeLimits" .Values.alertmanager.serviceMonitor | nindent 2 }}
   selector:
     matchLabels:
       app: {{ template "kube-prometheus-stack.name" . }}-alertmanager

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/core-dns/servicemonitor.yaml

@@ -12,6 +12,7 @@ metadata:
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 spec:
   jobLabel: jobLabel
+  {{- include "servicemonitor.scrapeLimits" .Values.coreDns.serviceMonitor | nindent 2 }}
   selector:
     matchLabels:
       app: {{ template "kube-prometheus-stack.name" . }}-coredns

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-api-server/servicemonitor.yaml

@@ -11,13 +11,14 @@ metadata:
   {{- end }}
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 spec:
+  {{- include "servicemonitor.scrapeLimits" .Values.kubeApiServer.serviceMonitor | nindent 2 }}
   endpoints:
   - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
     {{- if .Values.kubeApiServer.serviceMonitor.interval }}
     interval: {{ .Values.kubeApiServer.serviceMonitor.interval }}
     {{- end }}
     {{- if .Values.kubeApiServer.serviceMonitor.proxyUrl }}
-    proxyUrl: {{ .Values.kubeApiServer.serviceMonitor.proxyUrl}}
+    proxyUrl: {{ .Values.kubeApiServer.serviceMonitor.proxyUrl }}
     {{- end }}
     port: https
     scheme: https

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-controller-manager/servicemonitor.yaml

@@ -12,6 +12,7 @@ metadata:
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 spec:
   jobLabel: jobLabel
+  {{- include "servicemonitor.scrapeLimits" .Values.kubeControllerManager.serviceMonitor | nindent 2 }}
   selector:
     matchLabels:
       app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-dns/servicemonitor.yaml

@@ -12,6 +12,7 @@ metadata:
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 spec:
   jobLabel: jobLabel
+  {{- include "servicemonitor.scrapeLimits" .Values.kubeDns.serviceMonitor | nindent 2 }}
   selector:
     matchLabels:
       app: {{ template "kube-prometheus-stack.name" . }}-kube-dns

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-etcd/servicemonitor.yaml

@@ -12,6 +12,7 @@ metadata:
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 spec:
   jobLabel: jobLabel
+  {{- include "servicemonitor.scrapeLimits" .Values.kubeEtcd.serviceMonitor | nindent 4 }}
   selector:
     matchLabels:
       app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-proxy/servicemonitor.yaml

@@ -12,6 +12,7 @@ metadata:
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 spec:
   jobLabel: jobLabel
+  {{- include "servicemonitor.scrapeLimits" .Values.kubeProxy.serviceMonitor | nindent 2 }}
   selector:
     matchLabels:
       app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-scheduler/servicemonitor.yaml

@@ -12,6 +12,7 @@ metadata:
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 spec:
   jobLabel: jobLabel
+  {{- include "servicemonitor.scrapeLimits" .Values.kubeScheduler.serviceMonitor | nindent 2 }}
   selector:
     matchLabels:
       app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml

@@ -11,6 +11,7 @@ metadata:
   {{- end }}
 {{- include "kube-prometheus-stack.labels" . | indent 4 }}
 spec:
+  {{- include "servicemonitor.scrapeLimits" .Values.kubelet.serviceMonitor | nindent 2 }}
   endpoints:
   {{- if .Values.kubelet.serviceMonitor.https }}
   - port: https-metrics

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }}
 {{- if .Values.grafana.sidecar.datasources.annotations }}
   annotations:
-{{ toYaml .Values.grafana.sidecar.datasources.annotations | indent 4 }}
+    {{- toYaml .Values.grafana.sidecar.datasources.annotations | nindent 4 }}
 {{- end }}
   labels:
     {{ $.Values.grafana.sidecar.datasources.label }}: {{ $.Values.grafana.sidecar.datasources.labelValue | quote }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml

@@ -13,10 +13,12 @@ metadata:
 {{- include "kube-prometheus-stack.labels" $ | indent 4 }}
 webhooks:
   - name: prometheusrulemutate.monitoring.coreos.com
-    {{- if .Values.prometheusOperator.admissionWebhooks.patch.enabled  }}
+    {{- if .Values.prometheusOperator.admissionWebhooks.failurePolicy  }}
+    failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }}
+    {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }}
     failurePolicy: Ignore
     {{- else }}
-    failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }}
+    failurePolicy: Fail
     {{- end }}
     rules:
       - apiGroups:

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml

@@ -13,10 +13,12 @@ metadata:
 {{- include "kube-prometheus-stack.labels" $ | indent 4 }}
 webhooks:
   - name: prometheusrulemutate.monitoring.coreos.com
-    {{- if .Values.prometheusOperator.admissionWebhooks.patch.enabled  }}
+    {{- if .Values.prometheusOperator.admissionWebhooks.failurePolicy  }}
+    failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }}
+    {{- else if .Values.prometheusOperator.admissionWebhooks.patch.enabled }}
     failurePolicy: Ignore
     {{- else }}
-    failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }}
+    failurePolicy: Fail
     {{- end }}
     rules:
       - apiGroups:

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/clusterrole.yaml

@@ -11,6 +11,7 @@ rules:
   - monitoring.coreos.com
   resources:
   - alertmanagers
+  - alertmanagers/status
   - alertmanagers/finalizers
   - alertmanagerconfigs
   - prometheuses
@@ -78,4 +79,14 @@ rules:
   - get
   - list
   - watch
+{{- if .Capabilities.APIVersions.Has "discovery.k8s.io/v1/EndpointSlice" }}
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+{{- end }}
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml

@@ -40,11 +40,13 @@ spec:
     {{- end }}
       containers:
         - name: {{ template "kube-prometheus-stack.name" . }}
-          {{- $registry := .Values.global.imageRegistry | default .Values.prometheusOperator.image.registry -}}
+          {{- $configReloaderRegistry := .Values.global.imageRegistry | default .Values.prometheusOperator.prometheusConfigReloader.image.registry -}}
+          {{- $operatorRegistry := .Values.global.imageRegistry | default .Values.prometheusOperator.image.registry -}}
+          {{- $thanosRegistry := .Values.global.imageRegistry | default .Values.prometheusOperator.thanosImage.registry -}}
           {{- if .Values.prometheusOperator.image.sha }}
-          image: "{{ $registry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag }}@sha256:{{ .Values.prometheusOperator.image.sha }}"
+          image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.image.sha }}"
           {{- else }}
-          image: "{{ $registry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag }}"
+          image: "{{ $operatorRegistry }}/{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag | default .Chart.AppVersion }}"
           {{- end }}
           imagePullPolicy: "{{ .Values.prometheusOperator.image.pullPolicy }}"
           args:
@@ -80,9 +82,9 @@ spec:
             - --alertmanager-default-base-image={{ .Values.global.imageRegistry | default .Values.prometheusOperator.alertmanagerDefaultBaseImageRegistry }}/{{ .Values.prometheusOperator.alertmanagerDefaultBaseImage }}
             {{- end }}
             {{- if .Values.prometheusOperator.prometheusConfigReloader.image.sha }}
-            - --prometheus-config-reloader={{ $registry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloader.image.sha }}
+            - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloader.image.sha }}
             {{- else }}
-            - --prometheus-config-reloader={{ $registry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag }}
+            - --prometheus-config-reloader={{ $configReloaderRegistry }}/{{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag | default .Chart.AppVersion }}
             {{- end }}
             - --config-reloader-cpu-request={{ .Values.prometheusOperator.prometheusConfigReloader.resources.requests.cpu }}
             - --config-reloader-cpu-limit={{ .Values.prometheusOperator.prometheusConfigReloader.resources.limits.cpu }}
@@ -98,9 +100,9 @@ spec:
             - --prometheus-instance-namespaces={{ .Values.prometheusOperator.prometheusInstanceNamespaces | join "," }}
             {{- end }}
             {{- if .Values.prometheusOperator.thanosImage.sha }}
-            - --thanos-default-base-image={{ $registry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}@sha256:{{ .Values.prometheusOperator.thanosImage.sha }}
+            - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}@sha256:{{ .Values.prometheusOperator.thanosImage.sha }}
             {{- else }}
-            - --thanos-default-base-image={{ $registry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}
+            - --thanos-default-base-image={{ $thanosRegistry }}/{{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}
             {{- end }} 
             {{- if .Values.prometheusOperator.thanosRulerInstanceNamespaces }}
             - --thanos-ruler-instance-namespaces={{ .Values.prometheusOperator.thanosRulerInstanceNamespaces | join "," }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/networkpolicy.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: {{ template "kube-prometheus-stack.namespace" . }}
   labels:
     app: {{ template "kube-prometheus-stack.name" . }}-operator
+    {{- include "kube-prometheus-stack.labels" . | nindent 4 }}
 spec:
   egress: 
     - {}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/servicemonitor.yaml

@@ -11,6 +11,7 @@ metadata:
 {{ toYaml . | indent 4 }}
 {{- end }}
 spec:
+  {{- include "servicemonitor.scrapeLimits" .Values.prometheusOperator.serviceMonitor | nindent 2 }}
   endpoints:
   {{- if .Values.prometheusOperator.tls.enabled }}
   - port: https

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/csi-secret.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.prometheus.prometheusSpec.thanos.secretProviderClass }}
+{{- if and .Values.prometheus.prometheusSpec.thanos .Values.prometheus.prometheusSpec.thanos.secretProviderClass }}
 ---
 apiVersion: secrets-store.csi.x-k8s.io/v1alpha1
 kind: SecretProviderClass

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/networkpolicy.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.prometheus.networkPolicy.enabled }}
+apiVersion: {{ template "kube-prometheus-stack.prometheus.networkPolicy.apiVersion" . }}
+kind: NetworkPolicy
+metadata:
+  labels:
+    app: {{ template "kube-prometheus-stack.name" . }}-prometheus
+    {{- include "kube-prometheus-stack.labels" . | nindent 4 }}
+  name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus
+  namespace: {{ template "kube-prometheus-stack.namespace" . }}
+spec:
+  {{- if .Values.prometheus.networkPolicy.egress }}
+  ## Deny all egress by default
+  egress:
+    {{- toYaml .Values.prometheus.networkPolicy.egress | nindent 4 }}
+  {{- end }}
+  {{- if .Values.prometheus.networkPolicy.ingress }}
+  # Deny all ingress by default (prometheus scrapes itself using localhost)
+  ingress:
+    {{- toYaml .Values.prometheus.networkPolicy.ingress | nindent 4 }}
+  {{- end }}
+  policyTypes:
+    - Egress
+    - Ingress
+  podSelector:
+    {{- if .Values.prometheus.networkPolicy.podSelector }}
+    {{- toYaml .Values.prometheus.networkPolicy.podSelector | nindent 4 }}
+    {{- else }}
+    matchExpressions:
+      - {key: app.kubernetes.io/name, operator: In, values: [prometheus]}
+      - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.prometheus.crname" . }}]}
+    {{- end }}
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/podmonitors.yaml

@@ -15,6 +15,7 @@ items:
 {{ toYaml .additionalLabels | indent 8 }}
         {{- end }}
     spec:
+      {{- include "servicemonitor.scrapeLimits" . | nindent 6 }}
       podMetricsEndpoints:
 {{ toYaml .podMetricsEndpoints | indent 8 }}
     {{- if .jobLabel }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml

@@ -117,6 +117,12 @@ spec:
 {{- if .Values.prometheus.prometheusSpec.retentionSize }}
   retentionSize: {{ .Values.prometheus.prometheusSpec.retentionSize | quote }}
 {{- end }}
+{{- if .Values.prometheus.prometheusSpec.tsdb }}
+  tsdb:
+    {{- if .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }}
+    outOfOrderTimeWindow: {{ .Values.prometheus.prometheusSpec.tsdb.outOfOrderTimeWindow }}
+    {{- end }}
+{{- end }}
 {{- if eq .Values.prometheus.prometheusSpec.walCompression false }}
   walCompression: false
 {{ else }}
@@ -391,4 +397,8 @@ spec:
   minReadySeconds: {{ .Values.prometheus.prometheusSpec.minReadySeconds }}
 {{- end }}
   hostNetwork: {{ .Values.prometheus.prometheusSpec.hostNetwork }}
+{{- if .Values.prometheus.prometheusSpec.hostAliases }}
+  hostAliases:
+{{ toYaml .Values.prometheus.prometheusSpec.hostAliases | indent 4 }}
+{{- end }}
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/servicemonitor.yaml

@@ -7,7 +7,11 @@ metadata:
   labels:
     app: {{ template "kube-prometheus-stack.name" . }}-prometheus
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
+{{- with .Values.prometheus.serviceMonitor.additionalLabels }}
+{{- toYaml . | nindent 4 }}
+{{- end }}
 spec:
+  {{- include "servicemonitor.scrapeLimits" .Values.prometheus.serviceMonitor | nindent 2 }}
   selector:
     matchLabels:
       app: {{ template "kube-prometheus-stack.name" . }}-prometheus

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/servicemonitorThanosSidecar.yaml

@@ -7,7 +7,11 @@ metadata:
   labels:
     app: {{ template "kube-prometheus-stack.name" . }}-thanos-sidecar
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
+{{- with .Values.prometheus.thanosServiceMonitor.additionalLabels }}
+{{- toYaml . | nindent 4 }}
+{{- end }}
 spec:
+  {{- include "servicemonitor.scrapeLimits" .Values.prometheus.thanosServiceMonitor | nindent 2 }}
   selector:
     matchLabels:
       app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/servicemonitors.yaml

@@ -15,6 +15,7 @@ items:
 {{ toYaml .additionalLabels | indent 8 }}
         {{- end }}
     spec:
+      {{- include "servicemonitor.scrapeLimits" . | nindent 6 }}
       endpoints:
 {{ toYaml .endpoints | indent 8 }}
     {{- if .jobLabel }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/thanos-ruler/extrasecret.yaml

@@ -1,5 +1,5 @@
 {{- if .Values.thanosRuler.extraSecret.data -}}
-{{- $secretName := printf "thanos-ruler-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}}
+{{- $secretName := printf "%s-extra" (include "kube-prometheus-stack.thanosRuler.name" . ) -}}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -10,7 +10,7 @@ metadata:
 {{ toYaml .Values.thanosRuler.extraSecret.annotations | indent 4 }}
 {{- end }}
   labels:
-    app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler
+    app: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
     app.kubernetes.io/component: thanos-ruler
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 data:

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/thanos-ruler/ingress.yaml

@@ -1,6 +1,6 @@
 {{- if and .Values.thanosRuler.enabled .Values.thanosRuler.ingress.enabled }}
 {{- $pathType := .Values.thanosRuler.ingress.pathType | default "ImplementationSpecific" }}
-{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "thanos-ruler" }}
+{{- $serviceName := include "kube-prometheus-stack.thanosRuler.name" . }}
 {{- $servicePort := .Values.thanosRuler.service.port -}}
 {{- $routePrefix := list .Values.thanosRuler.thanosRulerSpec.routePrefix }}
 {{- $paths := .Values.thanosRuler.ingress.paths | default $routePrefix -}}
@@ -16,7 +16,7 @@ metadata:
 {{ toYaml .Values.thanosRuler.ingress.annotations | indent 4 }}
 {{- end }}
   labels:
-    app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler
+    app: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
 {{- if .Values.thanosRuler.ingress.labels }}
 {{ toYaml .Values.thanosRuler.ingress.labels | indent 4 }}
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/thanos-ruler/podDisruptionBudget.yaml

@@ -2,10 +2,10 @@
 apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }}
 kind: PodDisruptionBudget
 metadata:
-  name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler
+  name: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
   namespace: {{ template "kube-prometheus-stack.namespace" . }}
   labels:
-    app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler
+    app: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 spec:
   {{- if .Values.thanosRuler.podDisruptionBudget.minAvailable }}
@@ -17,5 +17,5 @@ spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: thanos-ruler
-      thanos-ruler: {{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler
+      thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/thanos-ruler/ruler.yaml

@@ -2,11 +2,11 @@
 apiVersion: monitoring.coreos.com/v1
 kind: ThanosRuler
 metadata:
-  name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler
+  name: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
   namespace: {{ template "kube-prometheus-stack.namespace" . }}
   labels:
-    app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler
+    app: {{ include "kube-prometheus-stack.thanosRuler.name" . }}
-{{ include "kube-prometheus-stack.labels" . | indent 4 }}
+{{- include "kube-prometheus-stack.labels" . | indent 4 -}}
 {{- if .Values.thanosRuler.annotations }}
   annotations:
 {{ toYaml .Values.thanosRuler.annotations | indent 4 }}
@@ -35,7 +35,7 @@ spec:
 {{- else if and .Values.thanosRuler.ingress.enabled .Values.thanosRuler.ingress.hosts }}
   externalPrefix: "http://{{ tpl (index .Values.thanosRuler.ingress.hosts 0) . }}{{ .Values.thanosRuler.thanosRulerSpec.routePrefix }}"
 {{- else }}
-  externalPrefix: http://{{ template "kube-prometheus-stack.fullname" . }}-thanosRuler.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.thanosRuler.service.port }}
+  externalPrefix: http://{{ template "kube-prometheus-stack.thanosRuler.name" . }}.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.thanosRuler.service.port }}
 {{- end }}
 {{- if .Values.thanosRuler.thanosRulerSpec.nodeSelector }}
   nodeSelector:
@@ -126,7 +126,7 @@ spec:
         labelSelector:
           matchExpressions:
             - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]}
-            - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler]}
+            - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]}
 {{- else if eq .Values.thanosRuler.thanosRulerSpec.podAntiAffinity "soft" }}
     podAntiAffinity:
       preferredDuringSchedulingIgnoredDuringExecution:
@@ -136,7 +136,7 @@ spec:
           labelSelector:
             matchExpressions:
               - {key: app.kubernetes.io/name, operator: In, values: [thanos-ruler]}
-              - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler]}
+              - {key: thanos-ruler, operator: In, values: [{{ template "kube-prometheus-stack.thanosRuler.name" . }}]}
 {{- end }}
 {{- if .Values.thanosRuler.thanosRulerSpec.tolerations }}
   tolerations:

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/thanos-ruler/service.yaml

@@ -2,12 +2,12 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler
+  name: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
   namespace: {{ template "kube-prometheus-stack.namespace" . }}
   labels:
-    app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler
+    app: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
     self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }}
-{{ include "kube-prometheus-stack.labels" . | indent 4 }}
+{{- include "kube-prometheus-stack.labels" . | indent 4 -}}
 {{- if .Values.thanosRuler.service.labels }}
 {{ toYaml .Values.thanosRuler.service.labels | indent 4 }}
 {{- end }}
@@ -48,6 +48,6 @@ spec:
 {{- end }}
   selector:
     app.kubernetes.io/name: thanos-ruler
-    thanos-ruler: {{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler
+    thanos-ruler: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
   type: "{{ .Values.thanosRuler.service.type }}"
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/thanos-ruler/serviceaccount.yaml

@@ -5,10 +5,10 @@ metadata:
   name: {{ template "kube-prometheus-stack.thanosRuler.serviceAccountName" . }}
   namespace: {{ template "kube-prometheus-stack.namespace" . }}
   labels:
-    app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler
+    app: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
-    app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler
+    app.kubernetes.io/name: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
     app.kubernetes.io/component: thanos-ruler
-{{ include "kube-prometheus-stack.labels" . | indent 4 }}
+{{- include "kube-prometheus-stack.labels" . | indent 4 -}}
 {{- if .Values.thanosRuler.serviceAccount.annotations }}
   annotations:
 {{ toYaml .Values.thanosRuler.serviceAccount.annotations | indent 4 }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/thanos-ruler/servicemonitor.yaml

@@ -2,15 +2,19 @@
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
-  name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-ruler
+  name: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
   namespace: {{ template "kube-prometheus-stack.namespace" . }}
   labels:
-    app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler
+    app: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
+{{- with .Values.thanosRuler.serviceMonitor.additionalLabels }}
+{{- toYaml . | nindent 4 }}
+{{- end }}
 spec:
+  {{- include "servicemonitor.scrapeLimits" .Values.thanosRuler.serviceMonitor | nindent 2 }}
   selector:
     matchLabels:
-      app: {{ template "kube-prometheus-stack.name" . }}-thanos-ruler
+      app: {{ template "kube-prometheus-stack.thanosRuler.name" . }}
       release: {{ $.Release.Name | quote }}
       self-monitor: {{ .Values.thanosRuler.serviceMonitor.selfMonitor | quote }}
   namespaceSelector:

charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml

@@ -212,6 +212,13 @@ alertmanager:
     templates:
     - '/etc/alertmanager/config/*.tmpl'
 
+  ## Alertmanager configuration directives (as string type, preferred over the config hash map)
+  ## stringConfig will be used only, if tplConfig is true
+  ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file
+  ##      https://prometheus.io/webtools/alerting/routing-tree-editor/
+  ##
+  stringConfig: ""
+
   ## Pass the Alertmanager configuration directives through Helm's templating
   ## engine. If the Alertmanager configuration contains Alertmanager templates,
   ## they'll need to be properly escaped so that they are not interpreted by
@@ -413,6 +420,30 @@ alertmanager:
     interval: ""
     selfMonitor: true
 
+    ## Additional labels
+    ##
+    additionalLabels: {}
+
+    ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+    ##
+    sampleLimit: 0
+
+    ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+    ##
+    targetLimit: 0
+
+    ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelLimit: 0
+
+    ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelNameLengthLimit: 0
+
+    ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelValueLengthLimit: 0
+
     ## proxyUrl: URL of a proxy that should be used for scraping.
     ##
     proxyUrl: ""
@@ -533,6 +564,13 @@ alertmanager:
     # alertmanagerConfiguration:
     #   name: global-alertmanager-Configuration
 
+    ## Defines the strategy used by AlertmanagerConfig objects to match alerts. eg:
+    ##
+    alertmanagerConfigMatcherStrategy: {}
+    ## Example with use OnNamespace strategy
+    # alertmanagerConfigMatcherStrategy:
+    #   type: OnNamespace
+
     ## Define Log Format
     # Use logfmt (default) or json logging
     logFormat: logfmt
@@ -902,6 +940,27 @@ kubeApiServer:
     ## Scrape interval. If not set, the Prometheus default scrape interval is used.
     ##
     interval: ""
+
+    ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+    ##
+    sampleLimit: 0
+
+    ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+    ##
+    targetLimit: 0
+
+    ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelLimit: 0
+
+    ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelNameLengthLimit: 0
+
+    ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelValueLengthLimit: 0
+
     ## proxyUrl: URL of a proxy that should be used for scraping.
     ##
     proxyUrl: ""
@@ -955,6 +1014,26 @@ kubelet:
     ##
     interval: ""
 
+    ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+    ##
+    sampleLimit: 0
+
+    ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+    ##
+    targetLimit: 0
+
+    ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelLimit: 0
+
+    ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelNameLengthLimit: 0
+
+    ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelValueLengthLimit: 0
+
     ## proxyUrl: URL of a proxy that should be used for scraping.
     ##
     proxyUrl: ""
@@ -1141,6 +1220,26 @@ kubeControllerManager:
     ##
     interval: ""
 
+    ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+    ##
+    sampleLimit: 0
+
+    ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+    ##
+    targetLimit: 0
+
+    ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelLimit: 0
+
+    ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelNameLengthLimit: 0
+
+    ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelValueLengthLimit: 0
+
     ## proxyUrl: URL of a proxy that should be used for scraping.
     ##
     proxyUrl: ""
@@ -1195,6 +1294,26 @@ coreDns:
     ##
     interval: ""
 
+    ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+    ##
+    sampleLimit: 0
+
+    ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+    ##
+    targetLimit: 0
+
+    ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelLimit: 0
+
+    ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelNameLengthLimit: 0
+
+    ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelValueLengthLimit: 0
+
     ## proxyUrl: URL of a proxy that should be used for scraping.
     ##
     proxyUrl: ""
@@ -1241,6 +1360,26 @@ kubeDns:
     ##
     interval: ""
 
+    ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+    ##
+    sampleLimit: 0
+
+    ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+    ##
+    targetLimit: 0
+
+    ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelLimit: 0
+
+    ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelNameLengthLimit: 0
+
+    ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelValueLengthLimit: 0
+
     ## proxyUrl: URL of a proxy that should be used for scraping.
     ##
     proxyUrl: ""
@@ -1325,6 +1464,27 @@ kubeEtcd:
     ## Scrape interval. If not set, the Prometheus default scrape interval is used.
     ##
     interval: ""
+
+    ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+    ##
+    sampleLimit: 0
+
+    ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+    ##
+    targetLimit: 0
+
+    ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelLimit: 0
+
+    ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelNameLengthLimit: 0
+
+    ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelValueLengthLimit: 0
+
     ## proxyUrl: URL of a proxy that should be used for scraping.
     ##
     proxyUrl: ""
@@ -1388,6 +1548,27 @@ kubeScheduler:
     ## Scrape interval. If not set, the Prometheus default scrape interval is used.
     ##
     interval: ""
+
+    ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+    ##
+    sampleLimit: 0
+
+    ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+    ##
+    targetLimit: 0
+
+    ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelLimit: 0
+
+    ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelNameLengthLimit: 0
+
+    ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelValueLengthLimit: 0
+
     ## proxyUrl: URL of a proxy that should be used for scraping.
     ##
     proxyUrl: ""
@@ -1452,6 +1633,26 @@ kubeProxy:
     ##
     interval: ""
 
+    ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+    ##
+    sampleLimit: 0
+
+    ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+    ##
+    targetLimit: 0
+
+    ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelLimit: 0
+
+    ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelNameLengthLimit: 0
+
+    ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelValueLengthLimit: 0
+
     ## proxyUrl: URL of a proxy that should be used for scraping.
     ##
     proxyUrl: ""
@@ -1502,6 +1703,26 @@ kube-state-metrics:
       ##
       interval: ""
 
+      ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+      ##
+      sampleLimit: 0
+
+      ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+      ##
+      targetLimit: 0
+
+      ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+      ##
+      labelLimit: 0
+
+      ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+      ##
+      labelNameLengthLimit: 0
+
+      ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+      ##
+      labelValueLengthLimit: 0
+
       ## Scrape Timeout. If not set, the Prometheus default scrape timeout is used.
       ##
       scrapeTimeout: ""
@@ -1565,6 +1786,26 @@ prometheus-node-exporter:
       ##
       interval: ""
 
+      ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+      ##
+      sampleLimit: 0
+
+      ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+      ##
+      targetLimit: 0
+
+      ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+      ##
+      labelLimit: 0
+
+      ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+      ##
+      labelNameLengthLimit: 0
+
+      ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+      ##
+      labelValueLengthLimit: 0
+
       ## How long until a scrape request times out. If not set, the Prometheus default scape timeout is used.
       ##
       scrapeTimeout: ""
@@ -1615,7 +1856,7 @@ prometheusOperator:
   ## Admission webhook support for PrometheusRules resources added in Prometheus Operator 0.30 can be enabled to prevent incorrectly formatted
   ## rules from making their way into prometheus and potentially preventing the container from starting
   admissionWebhooks:
-    failurePolicy: Fail
+    failurePolicy:
     ## The default timeoutSeconds is 10 and the maximum value is 30.
     timeoutSeconds: 10
     enabled: true
@@ -1632,9 +1873,9 @@ prometheusOperator:
     patch:
       enabled: true
       image:
-        registry: k8s.gcr.io
+        registry: registry.k8s.io
         repository: ingress-nginx/kube-webhook-certgen
-        tag: v1.3.0
+        tag: v20221220-controller-v1.5.1-58-g787ea74b6
         sha: ""
         pullPolicy: IfNotPresent
       resources: {}
@@ -1798,6 +2039,27 @@ prometheusOperator:
     ## Scrape interval. If not set, the Prometheus default scrape interval is used.
     ##
     interval: ""
+
+    ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+    ##
+    sampleLimit: 0
+
+    ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+    ##
+    targetLimit: 0
+
+    ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelLimit: 0
+
+    ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelNameLengthLimit: 0
+
+    ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelValueLengthLimit: 0
+
     ## Scrape timeout. If not set, the Prometheus default scrape timeout is used.
     scrapeTimeout: ""
     selfMonitor: true
@@ -1909,7 +2171,8 @@ prometheusOperator:
   image:
     registry: quay.io
     repository: prometheus-operator/prometheus-operator
-    tag: v0.61.1
+    # if not set appVersion field from Chart.yaml is used
+    tag: ""
     sha: ""
     pullPolicy: IfNotPresent
 
@@ -1935,7 +2198,8 @@ prometheusOperator:
     image:
       registry: quay.io
       repository: prometheus-operator/prometheus-config-reloader
-      tag: v0.61.1
+      # if not set appVersion field from Chart.yaml is used
+      tag: ""
       sha: ""
 
     # resource config for prometheusConfigReloader
@@ -1952,7 +2216,7 @@ prometheusOperator:
   thanosImage:
     registry: quay.io
     repository: thanos/thanos
-    tag: v0.29.0
+    tag: v0.30.2
     sha: ""
 
   ## Set a Field Selector to filter watched secrets
@@ -1962,13 +2226,23 @@ prometheusOperator:
 ## Deploy a Prometheus instance
 ##
 prometheus:
-
   enabled: true
 
   ## Annotations for Prometheus
   ##
   annotations: {}
 
+  ## Configure network policy for the prometheus
+  networkPolicy:
+    enabled: false
+    # egress:
+    # - {}
+    # ingress:
+    # - {}
+    # podSelector:
+    #   matchLabels:
+    #     app: prometheus
+
   ## Service account for Prometheuses to use.
   ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
   ##
@@ -2020,6 +2294,10 @@ prometheus:
     enabled: false
     interval: ""
 
+    ## Additional labels
+    ##
+    additionalLabels: {}
+
     ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS.
     scheme: ""
 
@@ -2302,6 +2580,30 @@ prometheus:
     interval: ""
     selfMonitor: true
 
+    ## Additional labels
+    ##
+    additionalLabels: {}
+
+    ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+    ##
+    sampleLimit: 0
+
+    ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+    ##
+    targetLimit: 0
+
+    ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelLimit: 0
+
+    ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelNameLengthLimit: 0
+
+    ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelValueLengthLimit: 0
+
     ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS.
     scheme: ""
 
@@ -2390,7 +2692,7 @@ prometheus:
     image:
       registry: quay.io
       repository: prometheus/prometheus
-      tag: v2.40.5
+      tag: v2.42.0
       sha: ""
 
     ## Tolerations for use with node taints
@@ -2580,6 +2882,11 @@ prometheus:
     ##
     retentionSize: ""
 
+    ## Allow out-of-order/out-of-bounds samples ingested into Prometheus for a specified duration
+    ## See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#tsdb
+    tsdb:
+      outOfOrderTimeWindow: 0s
+
     ## Enable compression of the write-ahead log using Snappy.
     ##
     walCompression: true
@@ -2940,6 +3247,14 @@ prometheus:
     # When hostNetwork is enabled, this will set dnsPolicy to ClusterFirstWithHostNet automatically.
     hostNetwork: false
 
+    # HostAlias holds the mapping between IP and hostnames that will be injected
+    # as an entry in the pod’s hosts file.
+    hostAliases: []
+    #  - ip: 10.10.0.100
+    #    hostnames:
+    #      - a1.app.local
+    #      - b1.app.local
+
   additionalRulesForClusterRole: []
   #  - apiGroups: [ "" ]
   #    resources:
@@ -3186,6 +3501,30 @@ thanosRuler:
     interval: ""
     selfMonitor: true
 
+    ## Additional labels
+    ##
+    additionalLabels: {}
+
+    ## SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
+    ##
+    sampleLimit: 0
+
+    ## TargetLimit defines a limit on the number of scraped targets that will be accepted.
+    ##
+    targetLimit: 0
+
+    ## Per-scrape limit on number of labels that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelLimit: 0
+
+    ## Per-scrape limit on length of labels name that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelNameLengthLimit: 0
+
+    ## Per-scrape limit on length of labels value that will be accepted for a sample. Only valid in Prometheus versions 2.27.0 and newer.
+    ##
+    labelValueLengthLimit: 0
+
     ## proxyUrl: URL of a proxy that should be used for scraping.
     ##
     proxyUrl: ""
@@ -3232,7 +3571,7 @@ thanosRuler:
     image:
       registry: quay.io
       repository: thanos/thanos
-      tag: v0.29.0
+      tag: v0.30.2
       sha: ""
 
     ## Namespaces to be selected for PrometheusRules discovery.

charts/kubezero-metrics/jsonnet/build.sh

@@ -9,6 +9,6 @@ which jb > /dev/null || { echo "Required jb ( json-bundler ) not found!"; exit 1
 if [ -r jsonnetfile.lock.json ]; then
   jb update
 else
-  #jb install github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus@main
+  jb install github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus@main
-  jb install github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus@release-0.11
+  #jb install github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus@release-0.11
 fi

charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-cluster.json

@@ -2231,7 +2231,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", container!=\"\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))",
+                     "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", container!=\"\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{namespace}}",
@@ -2310,7 +2310,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", container!=\"\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
+                     "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", container!=\"\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{namespace}}",
@@ -2529,7 +2529,7 @@
                ],
                "targets": [
                   {
-                     "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
+                     "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -2538,7 +2538,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
+                     "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -2547,7 +2547,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
+                     "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -2556,7 +2556,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
+                     "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -2565,7 +2565,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
+                     "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -2574,7 +2574,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
+                     "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,

charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-namespace.json

@@ -1957,7 +1957,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))",
+                     "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{pod}}",
@@ -2036,7 +2036,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                     "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "{{pod}}",
@@ -2255,7 +2255,7 @@
                ],
                "targets": [
                   {
-                     "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                     "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -2264,7 +2264,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                     "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -2273,7 +2273,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                     "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -2282,7 +2282,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                     "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -2291,7 +2291,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                     "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -2300,7 +2300,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                     "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,

charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-pod.json

@@ -1461,7 +1461,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))",
+                     "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "Reads",
@@ -1469,7 +1469,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))",
+                     "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "Writes",
@@ -1548,7 +1548,7 @@
                "steppedLine": false,
                "targets": [
                   {
-                     "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))",
+                     "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "Reads",
@@ -1556,7 +1556,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))",
+                     "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))",
                      "format": "time_series",
                      "intervalFactor": 2,
                      "legendFormat": "Writes",
@@ -1946,7 +1946,7 @@
                ],
                "targets": [
                   {
-                     "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+                     "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -1955,7 +1955,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(container) (rate(container_fs_writes_total{job=\"kubelet\",device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+                     "expr": "sum by(container) (rate(container_fs_writes_total{job=\"kubelet\",device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -1964,7 +1964,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+                     "expr": "sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -1973,7 +1973,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+                     "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -1982,7 +1982,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+                     "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,
@@ -1991,7 +1991,7 @@
                      "step": 10
                   },
                   {
-                     "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
+                     "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", device=~\"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": 2,

charts/kubezero-metrics/jsonnet/dashboards/kubelet.json

@@ -694,7 +694,7 @@
          "steppedLine": false,
          "targets": [
             {
-               "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))",
+               "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))",
                "format": "time_series",
                "intervalFactor": 2,
                "legendFormat": "{{instance}} pod",

charts/kubezero-metrics/jsonnet/jsonnetfile.json

@@ -8,7 +8,7 @@
           "subdir": "jsonnet/kube-prometheus"
         }
       },
-      "version": "release-0.11"
+      "version": "main"
     }
   ],
   "legacyImports": true

charts/kubezero-metrics/jsonnet/jsonnetfile.lock.json

@@ -18,8 +18,8 @@
           "subdir": "contrib/mixin"
         }
       },
-      "version": "9e3966fbce6dccd2271b7ade588fefeb4ca7b247",
+      "version": "22f3e50adafd9d4cf9dd29dd5837483a6417238c",
-      "sum": "W/Azptf1PoqjyMwJON96UY69MFugDA4IAYiKURscryc="
+      "sum": "QTzBqwjnM6cGGVBhOiVJyA+ZVTkmCTuH6C6YW7XKRFw="
     },
     {
       "source": {
@@ -28,7 +28,7 @@
           "subdir": "grafana-mixin"
         }
       },
-      "version": "3eed09056849ab873b867b561b7ce580ef2c75ba",
+      "version": "1120f9e255760a3c104b57871fcb91801e934382",
       "sum": "MkjR7zCgq6MUZgjDzop574tFKoTX2OBr7DTwm1K+Ofs="
     },
     {
@@ -38,9 +38,19 @@
           "subdir": "grafonnet"
         }
       },
-      "version": "30280196507e0fe6fa978a3e0eaca3a62844f817",
+      "version": "f0b70307b8e5f12236b277883d998af129a8211f",
       "sum": "342u++/7rViR/zj2jeJOjshzglkZ1SY+hFNuyCBFMdc="
     },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/grafonnet-lib.git",
+          "subdir": "grafonnet-7.0"
+        }
+      },
+      "version": "f0b70307b8e5f12236b277883d998af129a8211f",
+      "sum": "gCtR9s/4D5fxU9aKXg0Bru+/njZhA0YjLjPiASc61FM="
+    },
     {
       "source": {
         "git": {
@@ -48,7 +58,7 @@
           "subdir": "grafana-builder"
         }
       },
-      "version": "d68f9a6e0b1af7c4c4056dc2b43fb8f3bac01f43",
+      "version": "e0b90a4435817ad642d8d049e7dd975264cb960e",
       "sum": "tDR6yT2GVfw0wTU12iZH+m01HrbIr6g/xN+/8nzNkU0="
     },
     {
@@ -58,18 +68,8 @@
           "subdir": ""
         }
       },
-      "version": "b8f44bb7be728423836bef0e904ec7166895a34b",
+      "version": "eed459199703c969afc318ea55b9361ae48180a7",
-      "sum": "LCgSosxceeYuoau5fYSPtE5eXOFe46DxexfkrctUv7c="
+      "sum": "iKDOR7+jXw3Rctog6Z1ofweIK5BLjuGeguIZjXLP8ls="
-    },
-    {
-      "source": {
-        "git": {
-          "remote": "https://github.com/kubernetes-monitoring/kubernetes-mixin.git",
-          "subdir": "lib/promgrafonnet"
-        }
-      },
-      "version": "3c386687c1f8ceb6b79ff887c4a934e9cee1b90a",
-      "sum": "zv7hXGui6BfHzE9wPatHI/AGZa4A2WKo6pq7ZdqBsps="
     },
     {
       "source": {
@@ -78,8 +78,8 @@
           "subdir": "jsonnet/kube-state-metrics"
         }
       },
-      "version": "0567e1e1b981755e563d2244fa1659563f2cddbc",
+      "version": "e3d99ba7cf690b28ab2df9cf8d38c88afa630474",
-      "sum": "P0dCnbzyPScQGNXwXRcwiPkMLeTq0IPNbSTysDbySnM="
+      "sum": "+dOzAK+fwsFf97uZpjcjTcEJEC1H8hh/j8f5uIQK/5g="
     },
     {
       "source": {
@@ -88,7 +88,7 @@
           "subdir": "jsonnet/kube-state-metrics-mixin"
         }
       },
-      "version": "0567e1e1b981755e563d2244fa1659563f2cddbc",
+      "version": "e3d99ba7cf690b28ab2df9cf8d38c88afa630474",
       "sum": "u8gaydJoxEjzizQ8jY8xSjYgWooPmxw+wIWdDxifMAk="
     },
     {
@@ -98,8 +98,8 @@
           "subdir": "jsonnet/kube-prometheus"
         }
       },
-      "version": "e3066575dc8be21f578f12887563bda3ee7a2eff",
+      "version": "a209d48272a0726260784bcb74dca5c8ab7d4591",
-      "sum": "nNEMDrb5sQDOxJ20ITDvldyfIbbiGcVr8Bq46PH2ww8="
+      "sum": "iiIzgEw2EKncbZWzdPGrek+0b0VPwMa5plbW/R1hLPw="
     },
     {
       "source": {
@@ -108,7 +108,7 @@
           "subdir": "jsonnet/mixin"
         }
       },
-      "version": "5db6996d3ca995e66301c53c33959fd64c3f6ae6",
+      "version": "1448496c52158c0c286a696605818a0f5de87892",
       "sum": "GQmaVFJwKMiD/P4n3N2LrAZVcwutriWrP8joclDtBYQ=",
       "name": "prometheus-operator-mixin"
     },
@@ -119,8 +119,8 @@
           "subdir": "jsonnet/prometheus-operator"
         }
       },
-      "version": "5db6996d3ca995e66301c53c33959fd64c3f6ae6",
+      "version": "1448496c52158c0c286a696605818a0f5de87892",
-      "sum": "pUggCYwO/3Y/p6Vgryx8Y4KO3QkJ+GqimrZtn/luzzI="
+      "sum": "Ynpnbz195OTwY1DDpGRWlxmDI+tdwxjIXAphN9VIEkU="
     },
     {
       "source": {
@@ -129,8 +129,8 @@
           "subdir": "doc/alertmanager-mixin"
         }
       },
-      "version": "14b01e6a34dd3155768c7e9bd5c4376055de9419",
+      "version": "9a8d1f976e12b325ec47b84987a78b7845738be6",
-      "sum": "f3iZDUXQ/YWB5yDCY7VLD5bs442+3CdJgXJhJyWhNf8=",
+      "sum": "PsK+V7oETCPKu2gLoPfqY0wwPKH9TzhNj6o2xezjjXc=",
       "name": "alertmanager"
     },
     {
@@ -140,8 +140,8 @@
           "subdir": "docs/node-mixin"
         }
       },
-      "version": "a2321e7b940ddcff26873612bccdf7cd4c42b6b6",
+      "version": "b87c6a8826d41a242182f798e3e5688c870a9b12",
-      "sum": "MlWDAKGZ+JArozRKdKEvewHeWn8j2DNBzesJfLVd0dk="
+      "sum": "TwdaTm0Z++diiLyaKAAimmC6hBL7XbrJc0RHhBCpAdU="
     },
     {
       "source": {
@@ -150,8 +150,8 @@
           "subdir": "documentation/prometheus-mixin"
         }
       },
-      "version": "d7e7b8e04b5ecdc1dd153534ba376a622b72741b",
+      "version": "136b48855a974ce16e3bf591f1452d41d55eefa9",
-      "sum": "APXOIP3B3dZ3Tyh7L2UhyWR8Vbf5+9adTLz/ya7n6uU=",
+      "sum": "LRx0tbMnoE1p8KEn+i81j2YsA5Sgt3itE5Y6jBf5eOQ=",
       "name": "prometheus"
     },
     {
@@ -161,8 +161,8 @@
           "subdir": "config/crd/bases"
         }
       },
-      "version": "3738a607a42a0c9566587a49cec7587cc92d61bd",
+      "version": "ffb5f03ca7a99a31be783472e3411df2c1d09ab7",
-      "sum": "GQ0GFKGdIWKx1b78VRs6jtC4SMqkBjT5jl65QUjPKK4="
+      "sum": "bY/Pcrrbynguq8/HaI88cQ3B2hLv/xc+76QILY7IL+g="
     },
     {
       "source": {
@@ -171,8 +171,8 @@
           "subdir": "mixin"
         }
       },
-      "version": "17c576472d80972bfd3705e1e0a08e6f8da8e04b",
+      "version": "f8d401d92c1c59b88a203b71e975395271444212",
-      "sum": "dBm9ML50quhu6dwTIgfNmVruMqfaUeQVCO/6EKtQLxE=",
+      "sum": "zSLNV/0bN4DcVKojzCqjmhfjtzTY4pDKZXqbAUzw5R0=",
       "name": "thanos-mixin"
     }
   ],

charts/kubezero-metrics/jsonnet/rules/alertmanager-prometheusRule

@@ -7,7 +7,7 @@
          "app.kubernetes.io/instance": "main",
          "app.kubernetes.io/name": "alertmanager",
          "app.kubernetes.io/part-of": "kube-prometheus",
-         "app.kubernetes.io/version": "0.24.0",
+         "app.kubernetes.io/version": "0.25.0",
          "prometheus": "k8s",
          "role": "alert-rules"
       },

charts/kubezero-metrics/jsonnet/rules/etcd-mixin-prometheusRule

@@ -64,10 +64,10 @@
                {
                   "alert": "etcdGRPCRequestsSlow",
                   "annotations": {
-                     "description": "etcd cluster \"{{ $labels.job }}\": gRPC requests to {{ $labels.grpc_method }} are taking {{ $value }}s on etcd instance {{ $labels.instance }}.",
+                     "description": "etcd cluster \"{{ $labels.job }}\": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method }} method.",
                      "summary": "etcd grpc requests are slow"
                   },
-                  "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~\".*etcd.*\", grpc_type=\"unary\"}[5m])) without(grpc_type))\n> 0.15\n",
+                  "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~\".*etcd.*\", grpc_method!=\"Defragment\", grpc_type=\"unary\"}[5m])) without(grpc_type))\n> 0.15\n",
                   "for": "10m",
                   "labels": {
                      "severity": "critical"
@@ -112,7 +112,8 @@
                {
                   "alert": "etcdHighFsyncDurations",
                   "annotations": {
-                     "message": "etcd cluster \"{{ $labels.job }}\": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}."
+                     "description": "etcd cluster \"{{ $labels.job }}\": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.",
+                     "summary": "etcd cluster 99th percentile fsync durations are too high."
                   },
                   "expr": "histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~\".*etcd.*\"}[5m]))\n> 1\n",
                   "for": "10m",
@@ -133,11 +134,12 @@
                   }
                },
                {
-                  "alert": "etcdBackendQuotaLowSpace",
+                  "alert": "etcdDatabaseQuotaLowSpace",
                   "annotations": {
-                     "message": "etcd cluster \"{{ $labels.job }}\": database size exceeds the defined quota on etcd instance {{ $labels.instance }}, please defrag or increase the quota as the writes to etcd will be disabled when it is full."
+                     "description": "etcd cluster \"{{ $labels.job }}\": database size exceeds the defined quota on etcd instance {{ $labels.instance }}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.",
+                     "summary": "etcd cluster database is running full."
                   },
-                  "expr": "(etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100 > 95\n",
+                  "expr": "(last_over_time(etcd_mvcc_db_total_size_in_bytes[5m]) / last_over_time(etcd_server_quota_backend_bytes[5m]))*100 > 95\n",
                   "for": "10m",
                   "labels": {
                      "severity": "critical"
@@ -146,9 +148,23 @@
                {
                   "alert": "etcdExcessiveDatabaseGrowth",
                   "annotations": {
-                     "message": "etcd cluster \"{{ $labels.job }}\": Observed surge in etcd writes leading to 50% increase in database size over the past four hours on etcd instance {{ $labels.instance }}, please check as it might be disruptive."
+                     "description": "etcd cluster \"{{ $labels.job }}\": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{ $labels.instance }}, please check as it might be disruptive.",
+                     "summary": "etcd cluster database growing very fast."
                   },
-                  "expr": "increase(((etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100)[240m:1m]) > 50\n",
+                  "expr": "predict_linear(etcd_mvcc_db_total_size_in_bytes[4h], 4*60*60) > etcd_server_quota_backend_bytes\n",
+                  "for": "10m",
+                  "labels": {
+                     "severity": "warning"
+                  }
+               },
+               {
+                  "alert": "etcdDatabaseHighFragmentationRatio",
+                  "annotations": {
+                     "description": "etcd cluster \"{{ $labels.job }}\": database size in use on instance {{ $labels.instance }} is {{ $value | humanizePercentage }} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.",
+                     "runbook_url": "https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation",
+                     "summary": "etcd database size in use is less than 50% of the actual allocated storage."
+                  },
+                  "expr": "(last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes > 104857600\n",
                   "for": "10m",
                   "labels": {
                      "severity": "warning"

charts/kubezero-metrics/jsonnet/rules/kube-state-metrics-prometheusRule

@@ -6,7 +6,7 @@
          "app.kubernetes.io/component": "exporter",
          "app.kubernetes.io/name": "kube-state-metrics",
          "app.kubernetes.io/part-of": "kube-prometheus",
-         "app.kubernetes.io/version": "2.5.0",
+         "app.kubernetes.io/version": "2.8.2",
          "prometheus": "k8s",
          "role": "alert-rules"
       },

charts/kubezero-metrics/jsonnet/rules/kubernetes-prometheusRule

@@ -36,7 +36,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready",
                      "summary": "Pod has been in a non-ready state for more than 15 minutes."
                   },
-                  "expr": "sum by (namespace, pod, cluster) (\n  max by(namespace, pod, cluster) (\n    kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Pending|Unknown\"}\n  ) * on(namespace, pod, cluster) group_left(owner_kind) topk by(namespace, pod, cluster) (\n    1, max by(namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!=\"Job\"})\n  )\n) > 0\n",
+                  "expr": "sum by (namespace, pod, cluster) (\n  max by(namespace, pod, cluster) (\n    kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Pending|Unknown|Failed\"}\n  ) * on(namespace, pod, cluster) group_left(owner_kind) topk by(namespace, pod, cluster) (\n    1, max by(namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!=\"Job\"})\n  )\n) > 0\n",
                   "for": "15m",
                   "labels": {
                      "severity": "warning"
@@ -189,7 +189,7 @@
                   "annotations": {
                      "description": "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }} has not matched the desired number of replicas for longer than 15 minutes.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpareplicasmismatch",
-                     "summary": "HPA has not matched descired number of replicas."
+                     "summary": "HPA has not matched desired number of replicas."
                   },
                   "expr": "(kube_horizontalpodautoscaler_status_desired_replicas{job=\"kube-state-metrics\"}\n  !=\nkube_horizontalpodautoscaler_status_current_replicas{job=\"kube-state-metrics\"})\n  and\n(kube_horizontalpodautoscaler_status_current_replicas{job=\"kube-state-metrics\"}\n  >\nkube_horizontalpodautoscaler_spec_min_replicas{job=\"kube-state-metrics\"})\n  and\n(kube_horizontalpodautoscaler_status_current_replicas{job=\"kube-state-metrics\"}\n  <\nkube_horizontalpodautoscaler_spec_max_replicas{job=\"kube-state-metrics\"})\n  and\nchanges(kube_horizontalpodautoscaler_status_current_replicas{job=\"kube-state-metrics\"}[15m]) == 0\n",
                   "for": "15m",
@@ -222,7 +222,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuovercommit",
                      "summary": "Cluster has overcommitted CPU resource requests."
                   },
-                  "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource=\"cpu\"}) - max(kube_node_status_allocatable{resource=\"cpu\"})) > 0\nand\n(sum(kube_node_status_allocatable{resource=\"cpu\"}) - max(kube_node_status_allocatable{resource=\"cpu\"})) > 0\n",
+                  "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource=\"cpu\", job=\"kube-state-metrics\"}) - max(kube_node_status_allocatable{resource=\"cpu\", job=\"kube-state-metrics\"})) > 0\nand\n(sum(kube_node_status_allocatable{resource=\"cpu\", job=\"kube-state-metrics\"}) - max(kube_node_status_allocatable{resource=\"cpu\", job=\"kube-state-metrics\"})) > 0\n",
                   "for": "10m",
                   "labels": {
                      "severity": "warning"
@@ -235,7 +235,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit",
                      "summary": "Cluster has overcommitted memory resource requests."
                   },
-                  "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource=\"memory\"}) - max(kube_node_status_allocatable{resource=\"memory\"})) > 0\nand\n(sum(kube_node_status_allocatable{resource=\"memory\"}) - max(kube_node_status_allocatable{resource=\"memory\"})) > 0\n",
+                  "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource=\"memory\", job=\"kube-state-metrics\"}) - max(kube_node_status_allocatable{resource=\"memory\", job=\"kube-state-metrics\"})) > 0\nand\n(sum(kube_node_status_allocatable{resource=\"memory\", job=\"kube-state-metrics\"}) - max(kube_node_status_allocatable{resource=\"memory\", job=\"kube-state-metrics\"})) > 0\n",
                   "for": "10m",
                   "labels": {
                      "severity": "warning"
@@ -414,7 +414,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclienterrors",
                      "summary": "Kubernetes API server client is experiencing errors."
                   },
-                  "expr": "(sum(rate(rest_client_requests_total{code=~\"5..\"}[5m])) by (cluster, instance, job, namespace)\n  /\nsum(rate(rest_client_requests_total[5m])) by (cluster, instance, job, namespace))\n> 0.01\n",
+                  "expr": "(sum(rate(rest_client_requests_total{job=\"apiserver\",code=~\"5..\"}[5m])) by (cluster, instance, job, namespace)\n  /\nsum(rate(rest_client_requests_total{job=\"apiserver\"}[5m])) by (cluster, instance, job, namespace))\n> 0.01\n",
                   "for": "15m",
                   "labels": {
                      "severity": "warning"
@@ -498,6 +498,7 @@
                      "summary": "Client certificate is about to expire."
                   },
                   "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 604800\n",
+                  "for": "5m",
                   "labels": {
                      "severity": "warning"
                   }
@@ -510,6 +511,7 @@
                      "summary": "Client certificate is about to expire."
                   },
                   "expr": "apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 86400\n",
+                  "for": "5m",
                   "labels": {
                      "severity": "critical"
                   }
@@ -616,7 +618,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodereadinessflapping",
                      "summary": "Node readiness status is flapping."
                   },
-                  "expr": "sum(changes(kube_node_status_condition{status=\"true\",condition=\"Ready\"}[15m])) by (cluster, node) > 2\n",
+                  "expr": "sum(changes(kube_node_status_condition{job=\"kube-state-metrics\",status=\"true\",condition=\"Ready\"}[15m])) by (cluster, node) > 2\n",
                   "for": "15m",
                   "labels": {
                      "severity": "warning"
@@ -996,19 +998,19 @@
                   "record": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate"
                },
                {
-                  "expr": "container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n  max by(namespace, pod, node) (kube_pod_info{node!=\"\"})\n)\n",
+                  "expr": "container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", image!=\"\"}\n* on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1,\n  max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"})\n)\n",
                   "record": "node_namespace_pod_container:container_memory_working_set_bytes"
                },
                {
-                  "expr": "container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n  max by(namespace, pod, node) (kube_pod_info{node!=\"\"})\n)\n",
+                  "expr": "container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", image!=\"\"}\n* on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1,\n  max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"})\n)\n",
                   "record": "node_namespace_pod_container:container_memory_rss"
                },
                {
-                  "expr": "container_memory_cache{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n  max by(namespace, pod, node) (kube_pod_info{node!=\"\"})\n)\n",
+                  "expr": "container_memory_cache{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", image!=\"\"}\n* on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1,\n  max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"})\n)\n",
                   "record": "node_namespace_pod_container:container_memory_cache"
                },
                {
-                  "expr": "container_memory_swap{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n  max by(namespace, pod, node) (kube_pod_info{node!=\"\"})\n)\n",
+                  "expr": "container_memory_swap{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", image!=\"\"}\n* on (cluster, namespace, pod) group_left(node) topk by(cluster, namespace, pod) (1,\n  max by(cluster, namespace, pod, node) (kube_pod_info{node!=\"\"})\n)\n",
                   "record": "node_namespace_pod_container:container_memory_swap"
                },
                {
@@ -1149,7 +1151,7 @@
                   "record": "node_namespace_pod:kube_pod_info:"
                },
                {
-                  "expr": "count by (cluster, node) (sum by (node, cpu) (\n  node_cpu_seconds_total{job=\"node-exporter\"}\n* on (namespace, pod) group_left(node)\n  topk by(namespace, pod) (1, node_namespace_pod:kube_pod_info:)\n))\n",
+                  "expr": "count by (cluster, node) (\n  node_cpu_seconds_total{mode=\"idle\",job=\"node-exporter\"}\n  * on (namespace, pod) group_left(node)\n  topk by(namespace, pod) (1, node_namespace_pod:kube_pod_info:)\n)\n",
                   "record": "node:node_num_cpu:sum"
                },
                {
@@ -1157,7 +1159,11 @@
                   "record": ":node_memory_MemAvailable_bytes:sum"
                },
                {
-                  "expr": "sum(rate(node_cpu_seconds_total{job=\"node-exporter\",mode!=\"idle\",mode!=\"iowait\",mode!=\"steal\"}[5m])) /\ncount(sum(node_cpu_seconds_total{job=\"node-exporter\"}) by (cluster, instance, cpu))\n",
+                  "expr": "avg by (cluster, node) (\n  sum without (mode) (\n    rate(node_cpu_seconds_total{mode!=\"idle\",mode!=\"iowait\",mode!=\"steal\",job=\"node-exporter\"}[5m])\n  )\n)\n",
+                  "record": "node:node_cpu_utilization:ratio_rate5m"
+               },
+               {
+                  "expr": "avg by (cluster) (\n  node:node_cpu_utilization:ratio_rate5m\n)\n",
                   "record": "cluster:node_cpu:ratio_rate5m"
                }
             ]
@@ -1166,21 +1172,21 @@
             "name": "kubelet.rules",
             "rules": [
                {
-                  "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"})\n",
+                  "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job=\"kubelet\", metrics_path=\"/metrics\"}[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"})\n",
                   "labels": {
                      "quantile": "0.99"
                   },
                   "record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile"
                },
                {
-                  "expr": "histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"})\n",
+                  "expr": "histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job=\"kubelet\", metrics_path=\"/metrics\"}[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"})\n",
                   "labels": {
                      "quantile": "0.9"
                   },
                   "record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile"
                },
                {
-                  "expr": "histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"})\n",
+                  "expr": "histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job=\"kubelet\", metrics_path=\"/metrics\"}[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"})\n",
                   "labels": {
                      "quantile": "0.5"
                   },

charts/kubezero-metrics/jsonnet/rules/node-exporter-prometheusRule

@@ -6,7 +6,7 @@
          "app.kubernetes.io/component": "exporter",
          "app.kubernetes.io/name": "node-exporter",
          "app.kubernetes.io/part-of": "kube-prometheus",
-         "app.kubernetes.io/version": "1.3.1",
+         "app.kubernetes.io/version": "1.5.0",
          "prometheus": "k8s",
          "role": "alert-rules"
       },
@@ -25,7 +25,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup",
                      "summary": "Filesystem is predicted to run out of space within the next 24 hours."
                   },
-                  "expr": "(\n  node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 15\nand\n  predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
+                  "expr": "(\n  node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} * 100 < 15\nand\n  predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"}[6h], 24*60*60) < 0\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} == 0\n)\n",
                   "for": "1h",
                   "labels": {
                      "severity": "warning"
@@ -38,7 +38,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup",
                      "summary": "Filesystem is predicted to run out of space within the next 4 hours."
                   },
-                  "expr": "(\n  node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 10\nand\n  predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 4*60*60) < 0\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
+                  "expr": "(\n  node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} * 100 < 10\nand\n  predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"}[6h], 4*60*60) < 0\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} == 0\n)\n",
                   "for": "1h",
                   "labels": {
                      "severity": "critical"
@@ -51,7 +51,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace",
                      "summary": "Filesystem has less than 5% space left."
                   },
-                  "expr": "(\n  node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 5\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
+                  "expr": "(\n  node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} * 100 < 5\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} == 0\n)\n",
                   "for": "30m",
                   "labels": {
                      "severity": "warning"
@@ -64,7 +64,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace",
                      "summary": "Filesystem has less than 3% space left."
                   },
-                  "expr": "(\n  node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 3\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
+                  "expr": "(\n  node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} * 100 < 3\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} == 0\n)\n",
                   "for": "30m",
                   "labels": {
                      "severity": "critical"
@@ -77,7 +77,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup",
                      "summary": "Filesystem is predicted to run out of inodes within the next 24 hours."
                   },
-                  "expr": "(\n  node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\"} * 100 < 40\nand\n  predict_linear(node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
+                  "expr": "(\n  node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} * 100 < 40\nand\n  predict_linear(node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"}[6h], 24*60*60) < 0\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} == 0\n)\n",
                   "for": "1h",
                   "labels": {
                      "severity": "warning"
@@ -90,7 +90,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup",
                      "summary": "Filesystem is predicted to run out of inodes within the next 4 hours."
                   },
-                  "expr": "(\n  node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\"} * 100 < 20\nand\n  predict_linear(node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"}[6h], 4*60*60) < 0\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
+                  "expr": "(\n  node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} * 100 < 20\nand\n  predict_linear(node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"}[6h], 4*60*60) < 0\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} == 0\n)\n",
                   "for": "1h",
                   "labels": {
                      "severity": "critical"
@@ -103,7 +103,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles",
                      "summary": "Filesystem has less than 5% inodes left."
                   },
-                  "expr": "(\n  node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\"} * 100 < 5\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
+                  "expr": "(\n  node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} * 100 < 5\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} == 0\n)\n",
                   "for": "1h",
                   "labels": {
                      "severity": "warning"
@@ -116,7 +116,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles",
                      "summary": "Filesystem has less than 3% inodes left."
                   },
-                  "expr": "(\n  node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\"} * 100 < 3\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
+                  "expr": "(\n  node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} * 100 < 3\nand\n  node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\",mountpoint!=\"\"} == 0\n)\n",
                   "for": "1h",
                   "labels": {
                      "severity": "critical"
@@ -179,7 +179,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodeclockskewdetected",
                      "summary": "Clock skew detected."
                   },
-                  "expr": "(\n  node_timex_offset_seconds > 0.05\nand\n  deriv(node_timex_offset_seconds[5m]) >= 0\n)\nor\n(\n  node_timex_offset_seconds < -0.05\nand\n  deriv(node_timex_offset_seconds[5m]) <= 0\n)\n",
+                  "expr": "(\n  node_timex_offset_seconds{job=\"node-exporter\"} > 0.05\nand\n  deriv(node_timex_offset_seconds{job=\"node-exporter\"}[5m]) >= 0\n)\nor\n(\n  node_timex_offset_seconds{job=\"node-exporter\"} < -0.05\nand\n  deriv(node_timex_offset_seconds{job=\"node-exporter\"}[5m]) <= 0\n)\n",
                   "for": "10m",
                   "labels": {
                      "severity": "warning"
@@ -192,7 +192,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodeclocknotsynchronising",
                      "summary": "Clock not synchronising."
                   },
-                  "expr": "min_over_time(node_timex_sync_status[5m]) == 0\nand\nnode_timex_maxerror_seconds >= 16\n",
+                  "expr": "min_over_time(node_timex_sync_status{job=\"node-exporter\"}[5m]) == 0\nand\nnode_timex_maxerror_seconds{job=\"node-exporter\"} >= 16\n",
                   "for": "10m",
                   "labels": {
                      "severity": "warning"
@@ -205,7 +205,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddegraded",
                      "summary": "RAID Array is degraded"
                   },
-                  "expr": "node_md_disks_required - ignoring (state) (node_md_disks{state=\"active\"}) > 0\n",
+                  "expr": "node_md_disks_required{job=\"node-exporter\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"} - ignoring (state) (node_md_disks{state=\"active\",job=\"node-exporter\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}) > 0\n",
                   "for": "15m",
                   "labels": {
                      "severity": "critical"
@@ -218,7 +218,7 @@
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddiskfailure",
                      "summary": "Failed device in RAID array"
                   },
-                  "expr": "node_md_disks{state=\"failed\"} > 0\n",
+                  "expr": "node_md_disks{state=\"failed\",job=\"node-exporter\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"} > 0\n",
                   "labels": {
                      "severity": "warning"
                   }
@@ -275,11 +275,11 @@
                   "record": "instance:node_vmstat_pgmajfault:rate5m"
                },
                {
-                  "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[5m])\n",
+                  "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[5m])\n",
                   "record": "instance_device:node_disk_io_time_seconds:rate5m"
                },
                {
-                  "expr": "rate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[5m])\n",
+                  "expr": "rate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[5m])\n",
                   "record": "instance_device:node_disk_io_time_weighted_seconds:rate5m"
                },
                {

charts/kubezero-metrics/jsonnet/rules/prometheus-operator-prometheusRule

@@ -6,7 +6,7 @@
          "app.kubernetes.io/component": "controller",
          "app.kubernetes.io/name": "prometheus-operator",
          "app.kubernetes.io/part-of": "kube-prometheus",
-         "app.kubernetes.io/version": "0.57.0",
+         "app.kubernetes.io/version": "0.64.0",
          "prometheus": "k8s",
          "role": "alert-rules"
       },