From 5c1643e7ef9f84443ed3583ac6b6edbe1d81f16f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sun, 25 Apr 2021 11:59:54 +0200 Subject: [PATCH] feat: kubezero-metrics version bump, new Grafana dashboard management tooling for KubeZero, add ability to for compressed dashboards in configmaps --- charts/kubezero-metrics/.helmignore | 5 + charts/kubezero-metrics/Chart.yaml | 4 +- .../charts/kube-prometheus-stack/Chart.yaml | 6 +- .../charts/kube-prometheus-stack/README.md | 13 ++ .../charts/grafana/Chart.yaml | 2 +- .../charts/grafana/README.md | 2 + .../charts/grafana/templates/_pod.tpl | 5 + .../charts/grafana/templates/configmap.yaml | 4 +- .../templates/image-renderer-service.yaml | 2 + .../charts/grafana/templates/service.yaml | 3 +- .../charts/grafana/values.yaml | 3 + .../crds/crd-alertmanagerconfigs.yaml | 2 +- .../crds/crd-alertmanagers.yaml | 2 +- .../crds/crd-podmonitors.yaml | 4 +- .../crds/crd-probes.yaml | 144 +++++++++++++++++- .../crds/crd-prometheuses.yaml | 21 ++- .../crds/crd-prometheusrules.yaml | 2 +- .../crds/crd-servicemonitors.yaml | 4 +- .../crds/crd-thanosrulers.yaml | 4 +- .../templates/_helpers.tpl | 4 + .../templates/alertmanager/alertmanager.yaml | 2 +- .../templates/alertmanager/ingress.yaml | 24 ++- .../alertmanager/ingressperreplica.yaml | 17 ++- .../alertmanager/serviceaccount.yaml | 2 + .../prometheus-operator/serviceaccount.yaml | 2 + .../templates/prometheus/ingress.yaml | 36 +++-- .../prometheus/ingressThanosSidecar.yaml | 24 ++- .../prometheus/ingressperreplica.yaml | 17 ++- .../templates/prometheus/prometheus.yaml | 6 + .../templates/prometheus/serviceaccount.yaml | 2 + .../charts/kube-prometheus-stack/values.yaml | 9 +- .../{adjust_alarms.patch => metrics-zdt.path} | 28 +++- .../sync_grafana_dashboards.py | 97 ++++++++++++ .../templates/dashboard-gunzip-cm.yaml | 11 ++ charts/kubezero-metrics/update.sh | 4 +- charts/kubezero-metrics/values.yaml | 77 ++++++++-- 36 files changed, 522 insertions(+), 72 deletions(-) rename charts/kubezero-metrics/{adjust_alarms.patch => metrics-zdt.path} (77%) create mode 100755 charts/kubezero-metrics/sync_grafana_dashboards.py create mode 100644 charts/kubezero-metrics/templates/dashboard-gunzip-cm.yaml diff --git a/charts/kubezero-metrics/.helmignore b/charts/kubezero-metrics/.helmignore index 0e8a0eb..5299ab4 100644 --- a/charts/kubezero-metrics/.helmignore +++ b/charts/kubezero-metrics/.helmignore @@ -21,3 +21,8 @@ .idea/ *.tmproj .vscode/ + +README.md.gotmpl +*.patch +*.sh +*.py diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index 316bea9..28f7890 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-metrics description: KubeZero Umbrella Chart for prometheus-operator type: application -version: 0.3.5 +version: 0.4.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -16,7 +16,7 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: kube-prometheus-stack - version: 14.9.0 + version: 15.2.0 # Switch back to upstream once all alerts are fixed eg. etcd gpcr # repository: https://prometheus-community.github.io/helm-charts - name: prometheus-adapter diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml index 3523fba..2973939 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml @@ -6,7 +6,7 @@ annotations: url: https://github.com/prometheus-operator/kube-prometheus artifacthub.io/operator: "true" apiVersion: v2 -appVersion: 0.46.0 +appVersion: 0.47.0 dependencies: - condition: kubeStateMetrics.enabled name: kube-state-metrics @@ -19,7 +19,7 @@ dependencies: - condition: grafana.enabled name: grafana repository: https://grafana.github.io/helm-charts - version: 6.7.* + version: 6.8.* description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. home: https://github.com/prometheus-operator/kube-prometheus icon: https://raw.githubusercontent.com/prometheus/prometheus.github.io/master/assets/prometheus_logo-cb55bb5c346.png @@ -44,4 +44,4 @@ sources: - https://github.com/prometheus-community/helm-charts - https://github.com/prometheus-operator/kube-prometheus type: application -version: 14.9.0 +version: 15.2.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md index 2d19cbb..93b23c0 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md @@ -83,6 +83,19 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. +### From 14.x to 15.x +Version 15 upgrades prometheus-operator from 0.46.x to 0.47.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + ### From 13.x to 14.x Version 14 upgrades prometheus-operator from 0.45.x to 0.46.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml index e9466a3..7b52c49 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml @@ -19,4 +19,4 @@ name: grafana sources: - https://github.com/grafana/grafana type: application -version: 6.7.4 +version: 6.8.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md index 76a4acb..c2513a8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md @@ -63,6 +63,7 @@ This version requires Helm >= 3.1.0. | `image.sha` | Image sha (optional) | `2b56f6106ddc376bb46d974230d530754bf65a640dfbc5245191d72d3b49efc6` | | `image.pullPolicy` | Image pull policy | `IfNotPresent` | | `image.pullSecrets` | Image pull secrets | `{}` | +| `service.enabled` | Enable grafana service | `true` | | `service.type` | Kubernetes service type | `ClusterIP` | | `service.port` | Kubernetes port where service is exposed | `80` | | `service.portName` | Name of the port on the service | `service` | @@ -215,6 +216,7 @@ This version requires Helm >= 3.1.0. | `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` | | `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | | `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | +| `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | | `imageRenderer.service.portName` | image-renderer service port name | `'http'` | | `imageRenderer.service.port` | image-renderer service port used by both service and deployment | `8081` | | `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl index 9af7717..4d55ac9 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl @@ -197,11 +197,16 @@ containers: - name: FOLDER_ANNOTATION value: "{{ .Values.sidecar.dashboards.folderAnnotation }}" {{- end }} + - name: SCRIPT + value: /opt/script.sh resources: {{ toYaml .Values.sidecar.resources | indent 6 }} volumeMounts: - name: sc-dashboard-volume mountPath: {{ .Values.sidecar.dashboards.folder | quote }} + - name: script-volume + mountPath: /opt/script.sh + subPath: script.sh {{- end}} - name: {{ .Chart.Name }} {{- if .Values.image.sha }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml index de32b7a..c72219f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml @@ -19,8 +19,10 @@ data: {{- range $elem, $elemVal := $value }} {{- if kindIs "invalid" $elemVal }} {{ $elem }} = + {{- else if kindIs "string" $elemVal }} + {{ $elem }} = {{ tpl $elemVal $ }} {{- else }} - {{ $elem }} = {{ tpl (toYaml $elemVal) $ }} + {{ $elem }} = {{ $elemVal }} {{- end }} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-service.yaml index f5d3eb0..f29586c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-service.yaml @@ -1,4 +1,5 @@ {{ if .Values.imageRenderer.enabled }} +{{ if .Values.imageRenderer.service.enabled }} apiVersion: v1 kind: Service metadata: @@ -26,3 +27,4 @@ spec: selector: {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }} {{ end }} +{{ end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/service.yaml index 2764566..ba84ef9 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/service.yaml @@ -1,3 +1,4 @@ +{{ if .Values.service.enabled }} apiVersion: v1 kind: Service metadata: @@ -47,4 +48,4 @@ spec: {{- end }} selector: {{- include "grafana.selectorLabels" . | nindent 4 }} - +{{ end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml index d826343..40f6a26 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml @@ -125,6 +125,7 @@ podPortName: grafana ## ref: http://kubernetes.io/docs/user-guide/services/ ## service: + enabled: true type: ClusterIP port: 80 targetPort: 3000 @@ -688,6 +689,8 @@ imageRenderer: # image-renderer deployment priority class priorityClassName: '' service: + # Enable the image-renderer service + enabled: true # image-renderer service port name portName: 'http' # image-renderer service port used by both service and deployment diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml index b2ed161..e8c2828 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml index 724d488..41128a6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml index d474a0c..4529496 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -161,7 +161,7 @@ spec: description: ProxyURL eg http://proxyserver:2195 Directs scrapes to proxy through this endpoint. type: string relabelings: - description: 'RelabelConfigs to apply to samples before ingestion. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config' + description: 'RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds relabelings for a few standard Kubernetes fields and replaces original scrape job name with __tmp_prometheus_job_name. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config' items: description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines ``-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' properties: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml index 7fd658e..6e5af51 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -35,6 +35,55 @@ spec: spec: description: Specification of desired Ingress selection for target discovery by Prometheus. properties: + basicAuth: + description: 'BasicAuth allow an endpoint to authenticate over basic authentication. More info: https://prometheus.io/docs/operating/configuration/#endpoint' + properties: + password: + description: The secret in the service monitor namespace that contains the password for authentication. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + username: + description: The secret in the service monitor namespace that contains the username for authentication. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + type: object + bearerTokenSecret: + description: Secret to mount to read bearer token for scraping targets. The secret needs to be in the same namespace as the probe and accessible by the Prometheus Operator. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object interval: description: Interval at which targets are probed using the configured prober. If not specified Prometheus' global scrape interval is used. type: string @@ -188,6 +237,99 @@ spec: type: array type: object type: object + tlsConfig: + description: TLS configuration to use when scraping the endpoint. + properties: + ca: + description: Struct containing the CA cert to use for the targets. + properties: + configMap: + description: ConfigMap containing data to use for the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + type: object + cert: + description: Struct containing the client cert file for the targets. + properties: + configMap: + description: ConfigMap containing data to use for the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + type: object + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keySecret: + description: Secret containing the client key file for the targets. + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object type: object required: - spec diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml index c3f13d9..be23c9f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -1467,6 +1467,11 @@ spec: enableAdminAPI: description: 'Enable access to prometheus web admin API. Defaults to the value of `false`. WARNING: Enabling the admin APIs enables mutating endpoints, to delete data, shutdown Prometheus, and more. Enabling this should be done with care and the user is advised to add additional authentication authorization via a proxy to ensure only clients authorized to perform these actions can do so. For more information see https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis' type: boolean + enableFeatures: + description: Enable access to Prometheus disabled features. By default, no features are enabled. Enabling disabled features is entirely outside the scope of what the maintainers will support and by doing so, you accept that this behaviour may break at any time without notice. For more information see https://prometheus.io/docs/prometheus/latest/disabled_features/ + items: + type: string + type: array enforcedNamespaceLabel: description: EnforcedNamespaceLabel enforces adding a namespace label of origin for each alert and metric that is user created. The label value will always be the namespace of the object that is being created. type: string @@ -2465,7 +2470,7 @@ spec: type: object type: object bearerToken: - description: bearer token for remote read. + description: Bearer token for remote read. type: string bearerTokenFile: description: File to read bearer token for remote read. @@ -2636,7 +2641,7 @@ spec: type: object type: object bearerToken: - description: File to read bearer token for remote write. + description: Bearer token for remote write. type: string bearerTokenFile: description: File to read bearer token for remote write. @@ -2646,6 +2651,16 @@ spec: type: string description: Custom HTTP headers to be sent along with each remote write request. Be aware that headers that are set by Prometheus itself can't be overwritten. Only valid in Prometheus versions 2.25.0 and newer. type: object + metadataConfig: + description: MetadataConfig configures the sending of series metadata to remote storage. + properties: + send: + description: Whether metric metadata is sent to remote storage or not. + type: boolean + sendInterval: + description: How frequently metric metadata is sent to remote storage. + type: string + type: object name: description: The name of the remote write queue, must be unique if specified. The name is used in metrics and logging in order to differentiate queues. Only valid in Prometheus versions 2.15.0 and newer. type: string diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml index 07a24df..cd8705a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml index 9dee64f..543546a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -149,7 +149,7 @@ spec: description: ProxyURL eg http://proxyserver:2195 Directs scrapes to proxy through this endpoint. type: string relabelings: - description: 'RelabelConfigs to apply to samples before scraping. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config' + description: 'RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds relabelings for a few standard Kubernetes fields and replaces original scrape job name with __tmp_prometheus_job_name. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config' items: description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines ``-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' properties: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml index a470d4b..aae4b9e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.46.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -3339,4 +3339,4 @@ status: kind: "" plural: "" conditions: [] - storedVersions: [] \ No newline at end of file + storedVersions: [] diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl index 8a3d1b7..66299eb 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl @@ -46,6 +46,10 @@ The longest name that gets created adds and extra 37 characters, so truncation s {{/* Generate basic labels */}} {{- define "kube-prometheus-stack.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }} chart: {{ template "kube-prometheus-stack.chartref" . }} release: {{ $.Release.Name | quote }} heritage: {{ $.Release.Service | quote }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml index cd3b10b..c550bee 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml @@ -91,7 +91,7 @@ spec: labelSelector: matchExpressions: - {key: app, operator: In, values: [alertmanager]} - - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-alertmanager]} + - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-alertmanager]} {{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml index 50fab14..0085e73 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml @@ -4,11 +4,13 @@ {{- $servicePort := .Values.alertmanager.service.port -}} {{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} {{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} -{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" -}} +apiVersion: networking.k8s.io/v1 + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} apiVersion: networking.k8s.io/v1beta1 -{{ else }} + {{- else -}} apiVersion: extensions/v1beta1 -{{ end -}} + {{- end }} kind: Ingress metadata: name: {{ $serviceName }} @@ -41,9 +43,16 @@ spec: pathType: {{ $pathType }} {{- end }} backend: + {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} serviceName: {{ $serviceName }} servicePort: {{ $servicePort }} - {{- end -}} + {{- end }} + {{- end -}} {{- end -}} {{- else }} - http: @@ -54,8 +63,15 @@ spec: pathType: {{ $pathType }} {{- end }} backend: + {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} serviceName: {{ $serviceName }} servicePort: {{ $servicePort }} + {{- end }} {{- end -}} {{- end -}} {{- if .Values.alertmanager.ingress.tls }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml index 3d673b2..51aabaa 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml @@ -11,17 +11,19 @@ metadata: items: {{ range $i, $e := until $count }} - kind: Ingress - {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} + {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + apiVersion: networking.k8s.io/v1 + {{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} apiVersion: networking.k8s.io/v1beta1 - {{ else }} + {{- else }} apiVersion: extensions/v1beta1 - {{ end -}} + {{- end }} metadata: name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} namespace: {{ template "kube-prometheus-stack.namespace" $ }} labels: app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager -{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} {{- if $ingressValues.labels }} {{ toYaml $ingressValues.labels | indent 8 }} {{- end }} @@ -45,8 +47,15 @@ items: pathType: {{ $pathType }} {{- end }} backend: + {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} servicePort: {{ $servicePort }} + {{- end }} {{- end -}} {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} tls: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceaccount.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceaccount.yaml index c5e6e92..c5f1230 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceaccount.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceaccount.yaml @@ -6,6 +6,8 @@ metadata: namespace: {{ template "kube-prometheus-stack.namespace" . }} labels: app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager {{ include "kube-prometheus-stack.labels" . | indent 4 }} {{- if .Values.alertmanager.serviceAccount.annotations }} annotations: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/serviceaccount.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/serviceaccount.yaml index ab41797..f0292e9 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/serviceaccount.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/serviceaccount.yaml @@ -6,6 +6,8 @@ metadata: namespace: {{ template "kube-prometheus-stack.namespace" . }} labels: app: {{ template "kube-prometheus-stack.name" . }}-operator + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator + app.kubernetes.io/component: prometheus-operator {{ include "kube-prometheus-stack.labels" . | indent 4 }} imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | indent 2 }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml index 4d45873..9e881f8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml @@ -1,14 +1,16 @@ -{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled }} -{{- $pathType := .Values.prometheus.ingress.pathType | default "" }} -{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} -{{- $servicePort := .Values.prometheus.service.port -}} -{{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} -{{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} -{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled -}} + {{- $pathType := .Values.prometheus.ingress.pathType | default "" -}} + {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" -}} + {{- $servicePort := .Values.prometheus.service.port -}} + {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} + {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} + {{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" -}} +apiVersion: networking.k8s.io/v1 + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} apiVersion: networking.k8s.io/v1beta1 -{{ else }} + {{- else -}} apiVersion: extensions/v1beta1 -{{ end -}} + {{- end }} kind: Ingress metadata: {{- if .Values.prometheus.ingress.annotations }} @@ -41,9 +43,16 @@ spec: pathType: {{ $pathType }} {{- end }} backend: + {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} serviceName: {{ $serviceName }} servicePort: {{ $servicePort }} - {{- end -}} + {{- end }} + {{- end -}} {{- end -}} {{- else }} - http: @@ -54,8 +63,15 @@ spec: pathType: {{ $pathType }} {{- end }} backend: + {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} serviceName: {{ $serviceName }} servicePort: {{ $servicePort }} + {{- end }} {{- end -}} {{- end -}} {{- if .Values.prometheus.ingress.tls }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml index 69de0f6..b09819e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml @@ -4,11 +4,13 @@ {{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}} {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} {{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} -{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" -}} +apiVersion: networking.k8s.io/v1 + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} apiVersion: networking.k8s.io/v1beta1 -{{ else }} + {{- else -}} apiVersion: extensions/v1beta1 -{{ end -}} + {{- end }} kind: Ingress metadata: {{- if .Values.prometheus.thanosIngress.annotations }} @@ -40,9 +42,16 @@ spec: pathType: {{ $pathType }} {{- end }} backend: + {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} serviceName: {{ $serviceName }} servicePort: {{ $thanosPort }} - {{- end -}} + {{- end }} + {{- end -}} {{- end -}} {{- else }} - http: @@ -53,8 +62,15 @@ spec: pathType: {{ $pathType }} {{- end }} backend: + {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + service: + name: {{ $serviceName }} + port: + number: {{ $thanosPort }} + {{- else }} serviceName: {{ $serviceName }} servicePort: {{ $thanosPort }} + {{- end }} {{- end -}} {{- end -}} {{- if .Values.prometheus.thanosIngress.tls }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml index 3314377..fe74f29 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml @@ -11,17 +11,19 @@ metadata: items: {{ range $i, $e := until $count }} - kind: Ingress - {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} + {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + apiVersion: networking.k8s.io/v1 + {{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} apiVersion: networking.k8s.io/v1beta1 - {{ else }} + {{- else }} apiVersion: extensions/v1beta1 - {{ end -}} + {{- end }} metadata: name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} namespace: {{ template "kube-prometheus-stack.namespace" $ }} labels: app: {{ include "kube-prometheus-stack.name" $ }}-prometheus -{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{ include "kube-prometheus-stack.labels" $ | indent 8 }} {{- if $ingressValues.labels }} {{ toYaml $ingressValues.labels | indent 8 }} {{- end }} @@ -45,8 +47,15 @@ items: pathType: {{ $pathType }} {{- end }} backend: + {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + service: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + port: + number: {{ $servicePort }} + {{- else }} serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} servicePort: {{ $servicePort }} + {{- end }} {{- end -}} {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} tls: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml index fdab5f5..124399e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml @@ -70,6 +70,12 @@ spec: logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }} +{{- if .Values.prometheus.prometheusSpec.enableFeatures }} + enableFeatures: +{{- range $enableFeatures := .Values.prometheus.prometheusSpec.enableFeatures }} + - {{ tpl $enableFeatures $ }} +{{- end }} +{{- end }} {{- if .Values.prometheus.prometheusSpec.scrapeInterval }} scrapeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceaccount.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceaccount.yaml index 862d5f8..7657831 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceaccount.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceaccount.yaml @@ -6,6 +6,8 @@ metadata: namespace: {{ template "kube-prometheus-stack.namespace" . }} labels: app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus {{ include "kube-prometheus-stack.labels" . | indent 4 }} {{- if .Values.prometheus.serviceAccount.annotations }} annotations: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml index c7abeef..6b210fa 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml @@ -1536,7 +1536,7 @@ prometheusOperator: ## image: repository: quay.io/prometheus-operator/prometheus-operator - tag: v0.46.0 + tag: v0.47.0 sha: "" pullPolicy: IfNotPresent @@ -1552,7 +1552,7 @@ prometheusOperator: ## prometheusConfigReloaderImage: repository: quay.io/prometheus-operator/prometheus-config-reloader - tag: v0.46.0 + tag: v0.47.0 sha: "" ## Set the prometheus config reloader side-car CPU limit @@ -1868,6 +1868,11 @@ prometheus: ## enableAdminAPI: false + # EnableFeatures API enables access to Prometheus disabled features. + # ref: https://prometheus.io/docs/prometheus/latest/disabled_features/ + enableFeatures: [] + # - exemplar-storage + ## Image of Prometheus. ## image: diff --git a/charts/kubezero-metrics/adjust_alarms.patch b/charts/kubezero-metrics/metrics-zdt.path similarity index 77% rename from charts/kubezero-metrics/adjust_alarms.patch rename to charts/kubezero-metrics/metrics-zdt.path index aee2b07..4b64f1a 100644 --- a/charts/kubezero-metrics/adjust_alarms.patch +++ b/charts/kubezero-metrics/metrics-zdt.path @@ -1,6 +1,26 @@ +diff -turN charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl charts/kube-prometheus-stack.zdt/charts/grafana/templates/_pod.tpl +--- charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl 2021-04-22 00:36:35.000000000 +0200 ++++ charts/kube-prometheus-stack.zdt/charts/grafana/templates/_pod.tpl 2021-04-25 11:16:06.807876896 +0200 +@@ -197,11 +197,16 @@ + - name: FOLDER_ANNOTATION + value: "{{ .Values.sidecar.dashboards.folderAnnotation }}" + {{- end }} ++ - name: SCRIPT ++ value: /opt/script.sh + resources: + {{ toYaml .Values.sidecar.resources | indent 6 }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} ++ - name: script-volume ++ mountPath: /opt/script.sh ++ subPath: script.sh + {{- end}} + - name: {{ .Chart.Name }} + {{- if .Values.image.sha }} diff -turN charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/etcd.yaml ---- charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml 2021-04-14 22:13:29.000000000 +0200 -+++ charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/etcd.yaml 2021-04-15 14:43:03.074281889 +0200 +--- charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml 2021-04-22 00:36:35.000000000 +0200 ++++ charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/etcd.yaml 2021-04-25 11:10:33.021207900 +0200 @@ -54,34 +54,6 @@ {{- if .Values.defaultRules.additionalRuleLabels }} {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} @@ -37,8 +57,8 @@ diff -turN charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yam annotations: message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": gRPC requests to {{`{{`}} $labels.grpc_method {{`}}`}} are taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' diff -turN charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/node-exporter.yaml ---- charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml 2021-04-14 22:13:29.000000000 +0200 -+++ charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/node-exporter.yaml 2021-04-15 14:49:41.614282790 +0200 +--- charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml 2021-04-22 00:36:35.000000000 +0200 ++++ charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/node-exporter.yaml 2021-04-25 11:10:33.021207900 +0200 @@ -30,7 +30,7 @@ summary: Filesystem is predicted to run out of space within the next 24 hours. expr: |- diff --git a/charts/kubezero-metrics/sync_grafana_dashboards.py b/charts/kubezero-metrics/sync_grafana_dashboards.py new file mode 100755 index 0000000..d376877 --- /dev/null +++ b/charts/kubezero-metrics/sync_grafana_dashboards.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 + +import sys +import json +import yaml +import requests +import textwrap +import io +import gzip +import base64 + +config_file = sys.argv[1] +configmap_file = sys.argv[2] + + +# read config file +with open(config_file, 'r') as yaml_contents: + config = yaml.safe_load(yaml_contents.read()) + + +if 'condition' in config: + configmap = '''{{- if %(condition)s }} +''' % config +else: + configmap = '' + +# Base configmap for KubeZero +configmap += '''apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%%s-%%s" (include "kubezero-lib.fullname" $) "%(configmap)s" | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} + labels: + grafana_dashboard: "1" +{{ include "kubezero-lib.labels" . | indent 4 }} +''' % config + +# Put all dashboards into a folder ? +if 'folder' in config: + configmap += ''' annotations: + k8s-sidecar-target-directory: %(folder)s +''' % config + +# compress ? +if 'gzip' in config and config['gzip']: + configmap += '''binaryData: +''' +else: + configmap += '''data: +''' + +for b in config['dashboards']: + response = requests.get(b['url']) + if response.status_code != 200: + print('Skipping the file, response code %s not equals 200' % response.status_code) + continue + raw_text = response.text + + obj = json.loads(raw_text) + + # Customize each dashboard + + # Set default tim in all charts to 1h + obj['time'] = { "from": "now-1h", "to": "now" } + obj['refresh'] = "30s" + + if 'tags' in b: + obj['tags'] = b['tags'] + + text = json.dumps(obj, indent=2) + text = text.replace("{{", "{{`{{").replace("}}", "}}`}}").replace("{{`{{", "{{`{{`}}").replace("}}`}}", "{{`}}`}}") + + if 'gzip' in config and config['gzip']: + # compress and base64 encode + buf = io.BytesIO() + f = gzip.GzipFile(mode='w', fileobj=buf, mtime=0) + f.write(text.encode()) + f.close() + + textb64 = base64.b64encode(buf.getvalue()).decode('utf-8') + textb64 = textwrap.indent(textb64, ' '*4) + configmap += ''' %(name)s.json.gz: +''' % b + configmap += textb64+'\n' + else: + text = textwrap.indent(text, ' '*4) + configmap += ''' %(name)s.json: +''' % b + configmap += text+'\n' + + +if 'condition' in config: + configmap += '{{- end }}'+'\n' + +# Write Configmap +with open(configmap_file, 'w') as f: + f.write(configmap) diff --git a/charts/kubezero-metrics/templates/dashboard-gunzip-cm.yaml b/charts/kubezero-metrics/templates/dashboard-gunzip-cm.yaml new file mode 100644 index 0000000..5adb1c8 --- /dev/null +++ b/charts/kubezero-metrics/templates/dashboard-gunzip-cm.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: script-configmap + namespace: {{ $.Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" $ | indent 4 }} +data: + script.sh: |- + #!/bin/sh + /bin/gunzip -k /tmp/dashboards/*/*.gz diff --git a/charts/kubezero-metrics/update.sh b/charts/kubezero-metrics/update.sh index 4d36445..10d76d4 100755 --- a/charts/kubezero-metrics/update.sh +++ b/charts/kubezero-metrics/update.sh @@ -1,10 +1,10 @@ #!/bin/bash -VERSION=14.9.0 +VERSION=15.2.0 rm -rf charts/kube-prometheus-stack curl -L -s -o - https://github.com/prometheus-community/helm-charts/releases/download/kube-prometheus-stack-${VERSION}/kube-prometheus-stack-${VERSION}.tgz | tar xfz - -C charts # The grpc alerts could be re-enabled with etcd 3.5 # https://github.com/etcd-io/etcd/pull/12196 -patch -p0 -i adjust_alarms.patch --no-backup-if-mismatch +patch -p0 -i metrics-zdt.path --no-backup-if-mismatch diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 2e4f52a..2c1c359 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -1,4 +1,8 @@ kube-prometheus-stack: + global: + rbac: + pspEnabled: false + defaultRules: create: true @@ -61,14 +65,6 @@ kube-prometheus-stack: limits: memory: 64Mi - namespaces: - releaseNamespace: true - additional: - - kube-system - - istio-system - - istio-ingress - - logging - admissionWebhooks: patch: tolerations: @@ -89,6 +85,8 @@ kube-prometheus-stack: action: replace prometheus-node-exporter: + rbac: + pspEnabled: false resources: requests: cpu: 20m @@ -103,6 +101,10 @@ kube-prometheus-stack: logFormat: json # externalUrl: + # gather all monitors + podMonitorSelectorNilUsesHelmValues: false + serviceMonitorSelectorNilUsesHelmValues: false + resources: requests: memory: 512Mi @@ -111,25 +113,66 @@ kube-prometheus-stack: memory: 3Gi # cpu: "1000m" + walCompression: true storageSpec: volumeClaimTemplate: spec: - storageClassName: ebs-sc-gp2-xfs + storageClassName: ebs-sc-gp3-xfs accessModes: ["ReadWriteOnce"] resources: requests: storage: 16Gi # Custom Grafana tweaks - # - persistence, plugins, auth grafana: enabled: true - persistence: - enabled: true - size: 4Gi - storageClassName: ebs-sc-gp2-xfs - deploymentStrategy: - type: Recreate + rbac: + pspEnabled: false + + grafana.ini: + server: + enable_gzip: true + analytics: + check_for_updates: false + security: + disable_gravatar: true + cookie_secure: true + strict_transport_security: true + #auth: + # disable_login_form: true + # disable_signout_menu: true + auth.anonymous: + enabled: true + dashboards: + min_refresh_interval: "30s" + # default_home_dashboard_path: 'dashboards/kubezero.json' + alerting: + enabled: false + date_formats: + default_timezone: UTC + + sidecar: + # We require at least 1.12.0 for the SCRIPT and relative folder names + image: + tag: 1.12.0 + dashboards: + searchNamespace: ALL + provider: + foldersFromFilesStructure: true + # For the gunzip script + extraContainerVolumes: + - name: script-volume + configMap: + name: script-configmap + defaultMode: 0777 + + #persistence: + # enabled: true + # size: 4Gi + # storageClassName: ebs-sc-gp2-xfs + #deploymentStrategy: + # type: Recreate + plugins: - grafana-piechart-panel service: @@ -141,6 +184,8 @@ kube-prometheus-stack: # Assign state metrics to control plane kube-state-metrics: + podSecurityPolicy: + enabled: false tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule