feat: kubezero-metrics version bump, new Grafana dashboard management tooling for KubeZero, add ability to for compressed dashboards in configmaps

charts/kubezero-metrics/.helmignore

@@ -21,3 +21,8 @@
 .idea/
 *.tmproj
 .vscode/
+
+README.md.gotmpl
+*.patch
+*.sh
+*.py

charts/kubezero-metrics/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-metrics
 description: KubeZero Umbrella Chart for prometheus-operator
 type: application
-version: 0.3.5
+version: 0.4.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -16,7 +16,7 @@ dependencies:
     version: ">= 0.1.3"
     repository: https://zero-down-time.github.io/kubezero/
   - name: kube-prometheus-stack
-    version: 14.9.0
+    version: 15.2.0
     # Switch back to upstream once all alerts are fixed eg. etcd gpcr
     # repository: https://prometheus-community.github.io/helm-charts
   - name: prometheus-adapter

charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml

@@ -6,7 +6,7 @@ annotations:
       url: https://github.com/prometheus-operator/kube-prometheus
   artifacthub.io/operator: "true"
 apiVersion: v2
-appVersion: 0.46.0
+appVersion: 0.47.0
 dependencies:
 - condition: kubeStateMetrics.enabled
   name: kube-state-metrics
@@ -19,7 +19,7 @@ dependencies:
 - condition: grafana.enabled
   name: grafana
   repository: https://grafana.github.io/helm-charts
-  version: 6.7.*
+  version: 6.8.*
 description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus Operator.
 home: https://github.com/prometheus-operator/kube-prometheus
 icon: https://raw.githubusercontent.com/prometheus/prometheus.github.io/master/assets/prometheus_logo-cb55bb5c346.png
@@ -44,4 +44,4 @@ sources:
 - https://github.com/prometheus-community/helm-charts
 - https://github.com/prometheus-operator/kube-prometheus
 type: application
-version: 14.9.0
+version: 15.2.0

charts/kubezero-metrics/charts/kube-prometheus-stack/README.md

@@ -83,6 +83,19 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen
 
 A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions.
 
+### From 14.x to 15.x
+Version 15 upgrades prometheus-operator from 0.46.x to 0.47.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating:
+
+```console
+kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
+```
+
 ### From 13.x to 14.x
 
 Version 14 upgrades prometheus-operator from 0.45.x to 0.46.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating:

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml

@@ -19,4 +19,4 @@ name: grafana
 sources:
 - https://github.com/grafana/grafana
 type: application
-version: 6.7.4
+version: 6.8.0

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md

@@ -63,6 +63,7 @@ This version requires Helm >= 3.1.0.
 | `image.sha`                               | Image sha (optional)                          | `2b56f6106ddc376bb46d974230d530754bf65a640dfbc5245191d72d3b49efc6` |
 | `image.pullPolicy`                        | Image pull policy                             | `IfNotPresent`                                          |
 | `image.pullSecrets`                       | Image pull secrets                            | `{}`                                                    |
+| `service.enabled`                         | Enable grafana service                        | `true`                                                  |
 | `service.type`                            | Kubernetes service type                       | `ClusterIP`                                             |
 | `service.port`                            | Kubernetes port where service is exposed      | `80`                                                    |
 | `service.portName`                        | Name of the port on the service               | `service`                                               |
@@ -215,6 +216,7 @@ This version requires Helm >= 3.1.0.
 | `imageRenderer.securityContext`            | image-renderer deployment securityContext                                          | `{}`                             |
 | `imageRenderer.hostAliases`                | image-renderer deployment Host Aliases                                             | `[]`                             |
 | `imageRenderer.priorityClassName`          | image-renderer deployment priority class                                           | `''`                             |
+| `imageRenderer.service.enabled`            | Enable the image-renderer service                                                  | `true`                           |
 | `imageRenderer.service.portName`           | image-renderer service port name                                                   | `'http'`                         |
 | `imageRenderer.service.port`               | image-renderer service port used by both service and deployment                    | `8081`                           |
 | `imageRenderer.grafanaSubPath`             | Grafana sub path to use for image renderer callback url                            | `''`                             |

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl

@@ -197,11 +197,16 @@ containers:
       - name: FOLDER_ANNOTATION
         value: "{{ .Values.sidecar.dashboards.folderAnnotation }}"
       {{- end }}
+      - name: SCRIPT
+        value: /opt/script.sh
     resources:
 {{ toYaml .Values.sidecar.resources | indent 6 }}
     volumeMounts:
       - name: sc-dashboard-volume
         mountPath: {{ .Values.sidecar.dashboards.folder | quote }}
+      - name: script-volume
+        mountPath: /opt/script.sh
+        subPath: script.sh
 {{- end}}
   - name: {{ .Chart.Name }}
     {{- if .Values.image.sha }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml

@@ -19,8 +19,10 @@ data:
     {{- range $elem, $elemVal := $value }}
     {{- if kindIs "invalid" $elemVal }}
     {{ $elem }} =
+    {{- else if kindIs "string" $elemVal }}
+    {{ $elem }} = {{ tpl $elemVal $ }}
     {{- else }}
-    {{ $elem }} = {{ tpl (toYaml $elemVal) $ }}
+    {{ $elem }} = {{ $elemVal }}
     {{- end }}
     {{- end }}
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-service.yaml

@@ -1,4 +1,5 @@
 {{ if .Values.imageRenderer.enabled }}
+{{ if .Values.imageRenderer.service.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -26,3 +27,4 @@ spec:
   selector:
     {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }}
 {{ end }}
+{{ end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/service.yaml

@@ -1,3 +1,4 @@
+{{ if .Values.service.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -47,4 +48,4 @@ spec:
   {{- end }}
   selector:
     {{- include "grafana.selectorLabels" . | nindent 4 }}
-
+{{ end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml

@@ -125,6 +125,7 @@ podPortName: grafana
 ## ref: http://kubernetes.io/docs/user-guide/services/
 ##
 service:
+  enabled: true
   type: ClusterIP
   port: 80
   targetPort: 3000
@@ -688,6 +689,8 @@ imageRenderer:
   # image-renderer deployment priority class
   priorityClassName: ''
   service:
+    # Enable the image-renderer service
+    enabled: true
     # image-renderer service port name
     portName: 'http'
     # image-renderer service port used by both service and deployment

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1
@@ -161,7 +161,7 @@ spec:
                       description: ProxyURL eg http://proxyserver:2195 Directs scrapes to proxy through this endpoint.
                       type: string
                     relabelings:
-                      description: 'RelabelConfigs to apply to samples before ingestion. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
+                      description: 'RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds relabelings for a few standard Kubernetes fields and replaces original scrape job name with __tmp_prometheus_job_name. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
                       items:
                         description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `<metric_relabel_configs>`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
                         properties:

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1
@@ -35,6 +35,55 @@ spec:
           spec:
             description: Specification of desired Ingress selection for target discovery by Prometheus.
             properties:
+              basicAuth:
+                description: 'BasicAuth allow an endpoint to authenticate over basic authentication. More info: https://prometheus.io/docs/operating/configuration/#endpoint'
+                properties:
+                  password:
+                    description: The secret in the service monitor namespace that contains the password for authentication.
+                    properties:
+                      key:
+                        description: The key of the secret to select from.  Must be a valid secret key.
+                        type: string
+                      name:
+                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
+                        type: string
+                      optional:
+                        description: Specify whether the Secret or its key must be defined
+                        type: boolean
+                    required:
+                    - key
+                    type: object
+                  username:
+                    description: The secret in the service monitor namespace that contains the username for authentication.
+                    properties:
+                      key:
+                        description: The key of the secret to select from.  Must be a valid secret key.
+                        type: string
+                      name:
+                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
+                        type: string
+                      optional:
+                        description: Specify whether the Secret or its key must be defined
+                        type: boolean
+                    required:
+                    - key
+                    type: object
+                type: object
+              bearerTokenSecret:
+                description: Secret to mount to read bearer token for scraping targets. The secret needs to be in the same namespace as the probe and accessible by the Prometheus Operator.
+                properties:
+                  key:
+                    description: The key of the secret to select from.  Must be a valid secret key.
+                    type: string
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
+                    type: string
+                  optional:
+                    description: Specify whether the Secret or its key must be defined
+                    type: boolean
+                required:
+                - key
+                type: object
               interval:
                 description: Interval at which targets are probed using the configured prober. If not specified Prometheus' global scrape interval is used.
                 type: string
@@ -188,6 +237,99 @@ spec:
                         type: array
                     type: object
                 type: object
+              tlsConfig:
+                description: TLS configuration to use when scraping the endpoint.
+                properties:
+                  ca:
+                    description: Struct containing the CA cert to use for the targets.
+                    properties:
+                      configMap:
+                        description: ConfigMap containing data to use for the targets.
+                        properties:
+                          key:
+                            description: The key to select.
+                            type: string
+                          name:
+                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
+                            type: string
+                          optional:
+                            description: Specify whether the ConfigMap or its key must be defined
+                            type: boolean
+                        required:
+                        - key
+                        type: object
+                      secret:
+                        description: Secret containing data to use for the targets.
+                        properties:
+                          key:
+                            description: The key of the secret to select from.  Must be a valid secret key.
+                            type: string
+                          name:
+                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
+                            type: string
+                          optional:
+                            description: Specify whether the Secret or its key must be defined
+                            type: boolean
+                        required:
+                        - key
+                        type: object
+                    type: object
+                  cert:
+                    description: Struct containing the client cert file for the targets.
+                    properties:
+                      configMap:
+                        description: ConfigMap containing data to use for the targets.
+                        properties:
+                          key:
+                            description: The key to select.
+                            type: string
+                          name:
+                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
+                            type: string
+                          optional:
+                            description: Specify whether the ConfigMap or its key must be defined
+                            type: boolean
+                        required:
+                        - key
+                        type: object
+                      secret:
+                        description: Secret containing data to use for the targets.
+                        properties:
+                          key:
+                            description: The key of the secret to select from.  Must be a valid secret key.
+                            type: string
+                          name:
+                            description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
+                            type: string
+                          optional:
+                            description: Specify whether the Secret or its key must be defined
+                            type: boolean
+                        required:
+                        - key
+                        type: object
+                    type: object
+                  insecureSkipVerify:
+                    description: Disable target certificate validation.
+                    type: boolean
+                  keySecret:
+                    description: Secret containing the client key file for the targets.
+                    properties:
+                      key:
+                        description: The key of the secret to select from.  Must be a valid secret key.
+                        type: string
+                      name:
+                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
+                        type: string
+                      optional:
+                        description: Specify whether the Secret or its key must be defined
+                        type: boolean
+                    required:
+                    - key
+                    type: object
+                  serverName:
+                    description: Used to verify the hostname for the targets.
+                    type: string
+                type: object
             type: object
         required:
         - spec

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1
@@ -1467,6 +1467,11 @@ spec:
               enableAdminAPI:
                 description: 'Enable access to prometheus web admin API. Defaults to the value of `false`. WARNING: Enabling the admin APIs enables mutating endpoints, to delete data, shutdown Prometheus, and more. Enabling this should be done with care and the user is advised to add additional authentication authorization via a proxy to ensure only clients authorized to perform these actions can do so. For more information see https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis'
                 type: boolean
+              enableFeatures:
+                description: Enable access to Prometheus disabled features. By default, no features are enabled. Enabling disabled features is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice. For more information see https://prometheus.io/docs/prometheus/latest/disabled_features/
+                items:
+                  type: string
+                type: array
               enforcedNamespaceLabel:
                 description: EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. The label value will always be the namespace of the object that is being created.
                 type: string
@@ -2465,7 +2470,7 @@ spec:
                           type: object
                       type: object
                     bearerToken:
-                      description: bearer token for remote read.
+                      description: Bearer token for remote read.
                       type: string
                     bearerTokenFile:
                       description: File to read bearer token for remote read.
@@ -2636,7 +2641,7 @@ spec:
                           type: object
                       type: object
                     bearerToken:
-                      description: File to read bearer token for remote write.
+                      description: Bearer token for remote write.
                       type: string
                     bearerTokenFile:
                       description: File to read bearer token for remote write.
@@ -2646,6 +2651,16 @@ spec:
                         type: string
                       description: Custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Prometheus itself can't be overwritten. Only valid in Prometheus versions 2.25.0 and newer.
                       type: object
+                    metadataConfig:
+                      description: MetadataConfig configures the sending of series metadata to remote storage.
+                      properties:
+                        send:
+                          description: Whether metric metadata is sent to remote storage or not.
+                          type: boolean
+                        sendInterval:
+                          description: How frequently metric metadata is sent to remote storage.
+                          type: string
+                      type: object
                     name:
                       description: The name of the remote write queue, must be unique if specified. The name is used in metrics and logging in order to differentiate queues. Only valid in Prometheus versions 2.15.0 and newer.
                       type: string

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1
@@ -149,7 +149,7 @@ spec:
                       description: ProxyURL eg http://proxyserver:2195 Directs scrapes to proxy through this endpoint.
                       type: string
                     relabelings:
-                      description: 'RelabelConfigs to apply to samples before scraping. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
+                      description: 'RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds relabelings for a few standard Kubernetes fields and replaces original scrape job name with __tmp_prometheus_job_name. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
                       items:
                         description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `<metric_relabel_configs>`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
                         properties:

charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml

@@ -1,4 +1,4 @@
-# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
+# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
 
 ---
 apiVersion: apiextensions.k8s.io/v1

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl

@@ -46,6 +46,10 @@ The longest name that gets created adds and extra 37 characters, so truncation s
 
 {{/* Generate basic labels */}}
 {{- define "kube-prometheus-stack.labels" }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/version: "{{ .Chart.Version }}"
+app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }}  
 chart: {{ template "kube-prometheus-stack.chartref" . }}
 release: {{ $.Release.Name | quote }}
 heritage: {{ $.Release.Service | quote }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml

@@ -91,7 +91,7 @@ spec:
         labelSelector:
           matchExpressions:
             - {key: app, operator: In, values: [alertmanager]}
-            - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-alertmanager]}
+            - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-alertmanager]}
 {{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }}
     podAntiAffinity:
       preferredDuringSchedulingIgnoredDuringExecution:

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml

@@ -4,11 +4,13 @@
 {{- $servicePort := .Values.alertmanager.service.port -}}
 {{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }}
 {{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}}
-{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
+{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" -}}
+apiVersion: networking.k8s.io/v1
+  {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}}
 apiVersion: networking.k8s.io/v1beta1
-{{ else }}
+  {{- else -}}
 apiVersion: extensions/v1beta1
-{{ end -}}
+  {{- end }}
 kind: Ingress
 metadata:
   name: {{ $serviceName }}
@@ -41,8 +43,15 @@ spec:
             pathType: {{ $pathType }}
             {{- end }}
             backend:
+              {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+              service:
+                name: {{ $serviceName }}
+                port:
+                  number: {{ $servicePort }}
+              {{- else }}
               serviceName: {{ $serviceName }}
               servicePort: {{ $servicePort }}
+    {{- end }}
     {{- end -}}
   {{- end -}}
   {{- else }}
@@ -54,8 +63,15 @@ spec:
             pathType: {{ $pathType }}
             {{- end }}
             backend:
+              {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+              service:
+                name: {{ $serviceName }}
+                port:
+                  number: {{ $servicePort }}
+              {{- else }}
               serviceName: {{ $serviceName }}
               servicePort: {{ $servicePort }}
+  {{- end }}
   {{- end -}}
   {{- end -}}
   {{- if .Values.alertmanager.ingress.tls }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml

@@ -11,11 +11,13 @@ metadata:
 items:
 {{ range $i, $e := until $count }}
   - kind: Ingress
-    {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
+    {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+    apiVersion: networking.k8s.io/v1
+    {{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
     apiVersion: networking.k8s.io/v1beta1
-    {{ else }}
+    {{- else }}
     apiVersion: extensions/v1beta1
-    {{ end -}}
+    {{- end }}
     metadata:
       name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }}
       namespace: {{ template "kube-prometheus-stack.namespace" $ }}
@@ -45,8 +47,15 @@ items:
                 pathType: {{ $pathType }}
                 {{- end }}
                 backend:
+                  {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+                  service:
+                    name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }}
+                    port:
+                      number: {{ $servicePort }}
+                  {{- else }}
                   serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }}
                   servicePort: {{ $servicePort }}
+      {{- end }}
       {{- end -}}
       {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }}
       tls:

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceaccount.yaml

@@ -6,6 +6,8 @@ metadata:
   namespace: {{ template "kube-prometheus-stack.namespace" . }}
   labels:
     app: {{ template "kube-prometheus-stack.name" . }}-alertmanager
+    app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-alertmanager
+    app.kubernetes.io/component: alertmanager
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 {{- if .Values.alertmanager.serviceAccount.annotations }}
   annotations:

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/serviceaccount.yaml

@@ -6,6 +6,8 @@ metadata:
   namespace: {{ template "kube-prometheus-stack.namespace" . }}
   labels:
     app: {{ template "kube-prometheus-stack.name" . }}-operator
+    app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator
+    app.kubernetes.io/component: prometheus-operator
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 imagePullSecrets:
 {{ toYaml .Values.global.imagePullSecrets | indent 2 }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml

@@ -1,14 +1,16 @@
-{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled }}
+{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled -}}
-{{- $pathType := .Values.prometheus.ingress.pathType | default "" }}
+  {{- $pathType := .Values.prometheus.ingress.pathType | default "" -}}
-{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }}
+  {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" -}}
   {{- $servicePort := .Values.prometheus.service.port -}}
-{{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }}
+  {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}}
   {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}}
-{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
+  {{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" -}}
+apiVersion: networking.k8s.io/v1
+  {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}}
 apiVersion: networking.k8s.io/v1beta1
-{{ else }}
+  {{- else -}}
 apiVersion: extensions/v1beta1
-{{ end -}}
+  {{- end }}
 kind: Ingress
 metadata:
 {{- if .Values.prometheus.ingress.annotations }}
@@ -41,8 +43,15 @@ spec:
             pathType: {{ $pathType }}
             {{- end }}
             backend:
+              {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+              service:
+                name: {{ $serviceName }}
+                port:
+                  number: {{ $servicePort }}
+              {{- else }}
               serviceName: {{ $serviceName }}
               servicePort: {{ $servicePort }}
+    {{- end }}
     {{- end -}}
   {{- end -}}
   {{- else }}
@@ -54,8 +63,15 @@ spec:
             pathType: {{ $pathType }}
             {{- end }}
             backend:
+              {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+              service:
+                name: {{ $serviceName }}
+                port:
+                  number: {{ $servicePort }}
+              {{- else }}
               serviceName: {{ $serviceName }}
               servicePort: {{ $servicePort }}
+  {{- end }}
   {{- end -}}
   {{- end -}}
   {{- if .Values.prometheus.ingress.tls }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml

@@ -4,11 +4,13 @@
 {{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}}
 {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }}
 {{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}}
-{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
+{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" -}}
+apiVersion: networking.k8s.io/v1
+  {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}}
 apiVersion: networking.k8s.io/v1beta1
-{{ else }}
+  {{- else -}}
 apiVersion: extensions/v1beta1
-{{ end -}}
+  {{- end }}
 kind: Ingress
 metadata:
 {{- if .Values.prometheus.thanosIngress.annotations }}
@@ -40,8 +42,15 @@ spec:
             pathType: {{ $pathType }}
             {{- end }}
             backend:
+              {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+              service:
+                name: {{ $serviceName }}
+                port:
+                  number: {{ $thanosPort }}
+              {{- else }}
               serviceName: {{ $serviceName }}
               servicePort: {{ $thanosPort }}
+    {{- end }}
     {{- end -}}
   {{- end -}}
   {{- else }}
@@ -53,8 +62,15 @@ spec:
             pathType: {{ $pathType }}
             {{- end }}
             backend:
+              {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+              service:
+                name: {{ $serviceName }}
+                port:
+                  number: {{ $thanosPort }}
+              {{- else }}
               serviceName: {{ $serviceName }}
               servicePort: {{ $thanosPort }}
+  {{- end }}
   {{- end -}}
   {{- end -}}
   {{- if .Values.prometheus.thanosIngress.tls }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml

@@ -11,11 +11,13 @@ metadata:
 items:
 {{ range $i, $e := until $count }}
   - kind: Ingress
-    {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
+    {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+    apiVersion: networking.k8s.io/v1
+    {{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }}
     apiVersion: networking.k8s.io/v1beta1
-    {{ else }}
+    {{- else }}
     apiVersion: extensions/v1beta1
-    {{ end -}}
+    {{- end }}
     metadata:
       name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }}
       namespace: {{ template "kube-prometheus-stack.namespace" $ }}
@@ -45,8 +47,15 @@ items:
                 pathType: {{ $pathType }}
                 {{- end }}
                 backend:
+                  {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }}
+                  service:
+                    name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }}
+                    port:
+                      number: {{ $servicePort }}
+                  {{- else }}
                   serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }}
                   servicePort: {{ $servicePort }}
+      {{- end }}
       {{- end -}}
       {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }}
       tls:

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml

@@ -70,6 +70,12 @@ spec:
   logFormat:  {{ .Values.prometheus.prometheusSpec.logFormat }}
   listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }}
   enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }}
+{{- if .Values.prometheus.prometheusSpec.enableFeatures }}
+  enableFeatures:
+{{- range $enableFeatures := .Values.prometheus.prometheusSpec.enableFeatures }}
+  - {{ tpl $enableFeatures $ }}
+{{- end }}
+{{- end }}
 {{- if .Values.prometheus.prometheusSpec.scrapeInterval }}
   scrapeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval }}
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceaccount.yaml

@@ -6,6 +6,8 @@ metadata:
   namespace: {{ template "kube-prometheus-stack.namespace" . }}
   labels:
     app: {{ template "kube-prometheus-stack.name" . }}-prometheus
+    app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus
+    app.kubernetes.io/component: prometheus
 {{ include "kube-prometheus-stack.labels" . | indent 4 }}
 {{- if .Values.prometheus.serviceAccount.annotations }}
   annotations:

charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml

@@ -1536,7 +1536,7 @@ prometheusOperator:
   ##
   image:
     repository: quay.io/prometheus-operator/prometheus-operator
-    tag: v0.46.0
+    tag: v0.47.0
     sha: ""
     pullPolicy: IfNotPresent
 
@@ -1552,7 +1552,7 @@ prometheusOperator:
   ##
   prometheusConfigReloaderImage:
     repository: quay.io/prometheus-operator/prometheus-config-reloader
-    tag: v0.46.0
+    tag: v0.47.0
     sha: ""
 
   ## Set the prometheus config reloader side-car CPU limit
@@ -1868,6 +1868,11 @@ prometheus:
     ##
     enableAdminAPI: false
 
+    # EnableFeatures API enables access to Prometheus disabled features.
+    # ref: https://prometheus.io/docs/prometheus/latest/disabled_features/
+    enableFeatures: []
+    # - exemplar-storage
+
     ## Image of Prometheus.
     ##
     image:

charts/kubezero-metrics/metrics-zdt.path

@@ -1,6 +1,26 @@
+diff -turN charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl charts/kube-prometheus-stack.zdt/charts/grafana/templates/_pod.tpl
+--- charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl	2021-04-22 00:36:35.000000000 +0200
++++ charts/kube-prometheus-stack.zdt/charts/grafana/templates/_pod.tpl	2021-04-25 11:16:06.807876896 +0200
+@@ -197,11 +197,16 @@
+       - name: FOLDER_ANNOTATION
+         value: "{{ .Values.sidecar.dashboards.folderAnnotation }}"
+       {{- end }}
++      - name: SCRIPT
++        value: /opt/script.sh
+     resources:
+ {{ toYaml .Values.sidecar.resources | indent 6 }}
+     volumeMounts:
+       - name: sc-dashboard-volume
+         mountPath: {{ .Values.sidecar.dashboards.folder | quote }}
++      - name: script-volume
++        mountPath: /opt/script.sh
++        subPath: script.sh
+ {{- end}}
+   - name: {{ .Chart.Name }}
+     {{- if .Values.image.sha }}
 diff -turN charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/etcd.yaml
---- charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml	2021-04-14 22:13:29.000000000 +0200
+--- charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml	2021-04-22 00:36:35.000000000 +0200
-+++ charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/etcd.yaml	2021-04-15 14:43:03.074281889 +0200
++++ charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/etcd.yaml	2021-04-25 11:10:33.021207900 +0200
 @@ -54,34 +54,6 @@
  {{- if .Values.defaultRules.additionalRuleLabels }}
  {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
@@ -37,8 +57,8 @@ diff -turN charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yam
        annotations:
          message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": gRPC requests to {{`{{`}} $labels.grpc_method {{`}}`}} are taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
 diff -turN charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/node-exporter.yaml
---- charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml	2021-04-14 22:13:29.000000000 +0200
+--- charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml	2021-04-22 00:36:35.000000000 +0200
-+++ charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/node-exporter.yaml	2021-04-15 14:49:41.614282790 +0200
++++ charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/node-exporter.yaml	2021-04-25 11:10:33.021207900 +0200
 @@ -30,7 +30,7 @@
          summary: Filesystem is predicted to run out of space within the next 24 hours.
        expr: |-

charts/kubezero-metrics/sync_grafana_dashboards.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+
+import sys
+import json
+import yaml
+import requests
+import textwrap
+import io
+import gzip
+import base64
+
+config_file = sys.argv[1]
+configmap_file = sys.argv[2]
+
+
+# read config file
+with open(config_file, 'r') as yaml_contents:
+    config = yaml.safe_load(yaml_contents.read())
+
+
+if 'condition' in config:
+    configmap = '''{{- if %(condition)s }}
+''' % config
+else:
+    configmap = ''
+
+# Base configmap for KubeZero
+configmap += '''apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ printf "%%s-%%s" (include "kubezero-lib.fullname" $) "%(configmap)s" | trunc 63 | trimSuffix "-" }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    grafana_dashboard: "1"
+{{ include "kubezero-lib.labels" . | indent 4 }}
+''' % config
+
+# Put all dashboards into a folder ?
+if 'folder' in config:
+    configmap += '''  annotations:
+    k8s-sidecar-target-directory: %(folder)s
+''' % config
+
+# compress ?
+if 'gzip' in config and config['gzip']:
+    configmap += '''binaryData:
+'''
+else:
+    configmap += '''data:
+'''
+
+for b in config['dashboards']:
+    response = requests.get(b['url'])
+    if response.status_code != 200:
+        print('Skipping the file, response code %s not equals 200' % response.status_code)
+        continue
+    raw_text = response.text
+
+    obj = json.loads(raw_text)
+
+    # Customize each dashboard
+   
+    # Set default tim in all charts to 1h
+    obj['time'] = { "from": "now-1h", "to": "now" }
+    obj['refresh'] = "30s"
+
+    if 'tags' in b:
+      obj['tags'] = b['tags']
+
+    text = json.dumps(obj, indent=2)
+    text = text.replace("{{", "{{`{{").replace("}}", "}}`}}").replace("{{`{{", "{{`{{`}}").replace("}}`}}", "{{`}}`}}")
+
+    if 'gzip' in config and config['gzip']:
+        # compress and base64 encode
+        buf = io.BytesIO()
+        f = gzip.GzipFile(mode='w', fileobj=buf, mtime=0)
+        f.write(text.encode())
+        f.close()
+
+        textb64 = base64.b64encode(buf.getvalue()).decode('utf-8')
+        textb64 = textwrap.indent(textb64, ' '*4)
+        configmap += '''  %(name)s.json.gz:
+''' % b
+        configmap += textb64+'\n'
+    else:
+        text = textwrap.indent(text, ' '*4)
+        configmap += '''  %(name)s.json:
+''' % b
+        configmap += text+'\n'
+
+
+if 'condition' in config:
+    configmap += '{{- end }}'+'\n'
+
+# Write Configmap
+with open(configmap_file, 'w') as f:
+    f.write(configmap)

charts/kubezero-metrics/templates/dashboard-gunzip-cm.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: script-configmap
+  namespace: {{ $.Release.Namespace }}
+  labels:
+{{ include "kubezero-lib.labels" $ | indent 4 }}
+data:
+  script.sh: |-
+     #!/bin/sh
+     /bin/gunzip -k /tmp/dashboards/*/*.gz

charts/kubezero-metrics/update.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
-VERSION=14.9.0
+VERSION=15.2.0
 
 rm -rf charts/kube-prometheus-stack
 curl -L -s -o - https://github.com/prometheus-community/helm-charts/releases/download/kube-prometheus-stack-${VERSION}/kube-prometheus-stack-${VERSION}.tgz | tar xfz - -C charts
 
 # The grpc alerts could be re-enabled with etcd 3.5
 # https://github.com/etcd-io/etcd/pull/12196
-patch -p0 -i adjust_alarms.patch --no-backup-if-mismatch
+patch -p0 -i metrics-zdt.path --no-backup-if-mismatch

charts/kubezero-metrics/values.yaml

@@ -1,4 +1,8 @@
 kube-prometheus-stack:
+  global:
+    rbac:
+      pspEnabled: false
+
   defaultRules:
     create: true
 
@@ -61,14 +65,6 @@ kube-prometheus-stack:
       limits:
         memory: 64Mi
 
-    namespaces:
-      releaseNamespace: true
-      additional:
-        - kube-system
-        - istio-system
-        - istio-ingress
-        - logging
-
     admissionWebhooks:
       patch:
         tolerations:
@@ -89,6 +85,8 @@ kube-prometheus-stack:
         action: replace
 
   prometheus-node-exporter:
+    rbac:
+      pspEnabled: false
     resources:
       requests:
         cpu: 20m
@@ -103,6 +101,10 @@ kube-prometheus-stack:
       logFormat: json
       # externalUrl:
 
+      # gather all monitors 
+      podMonitorSelectorNilUsesHelmValues: false
+      serviceMonitorSelectorNilUsesHelmValues: false
+
       resources:
         requests:
           memory: 512Mi
@@ -111,25 +113,66 @@ kube-prometheus-stack:
           memory: 3Gi
           # cpu: "1000m"
 
+      walCompression: true
       storageSpec:
         volumeClaimTemplate:
           spec:
-            storageClassName: ebs-sc-gp2-xfs
+            storageClassName: ebs-sc-gp3-xfs
             accessModes: ["ReadWriteOnce"]
             resources:
               requests:
                 storage: 16Gi
 
   # Custom Grafana tweaks
-  # - persistence, plugins, auth
   grafana:
     enabled: true
-    persistence:
+    rbac:
+      pspEnabled: false
+
+    grafana.ini:
+      server:
+        enable_gzip: true
+      analytics:
+        check_for_updates: false
+      security:
+        disable_gravatar: true
+        cookie_secure: true
+        strict_transport_security: true
+      #auth:
+      #  disable_login_form: true
+      #  disable_signout_menu: true
+      auth.anonymous:
         enabled: true
-      size: 4Gi
+      dashboards:
-      storageClassName: ebs-sc-gp2-xfs
+        min_refresh_interval: "30s"
-    deploymentStrategy:
+        # default_home_dashboard_path: 'dashboards/kubezero.json'
-      type: Recreate
+      alerting:
+        enabled: false
+      date_formats:
+        default_timezone: UTC
+
+    sidecar:
+      # We require at least 1.12.0 for the SCRIPT and relative folder names
+      image:
+        tag: 1.12.0
+      dashboards:
+        searchNamespace: ALL
+        provider:
+          foldersFromFilesStructure: true
+    # For the gunzip script
+    extraContainerVolumes:
+    - name: script-volume
+      configMap:
+        name: script-configmap
+        defaultMode: 0777
+
+    #persistence:
+    #  enabled: true
+    #  size: 4Gi
+    #  storageClassName: ebs-sc-gp2-xfs
+    #deploymentStrategy:
+    #  type: Recreate
+
     plugins:
     - grafana-piechart-panel
     service:
@@ -141,6 +184,8 @@ kube-prometheus-stack:
 
   # Assign state metrics to control plane
   kube-state-metrics:
+    podSecurityPolicy:
+      enabled: false
     tolerations:
     - key: node-role.kubernetes.io/master
       effect: NoSchedule