From abfffe55ad2e6970902029221f5b0df4711a9d10 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 25 Aug 2021 15:59:17 +0200 Subject: [PATCH] feat: metrics version bump --- charts/kubezero-metrics/Chart.yaml | 4 +- .../charts/kube-prometheus-stack/.helmignore | 2 + .../charts/kube-prometheus-stack/Chart.yaml | 6 +- .../charts/kube-prometheus-stack/README.md | 14 + .../charts/grafana/Chart.yaml | 4 +- .../charts/grafana/templates/_helpers.tpl | 13 +- .../charts/grafana/values.yaml | 2 +- .../charts/kube-state-metrics/Chart.yaml | 4 +- .../charts/kube-state-metrics/values.yaml | 2 +- .../prometheus-node-exporter/Chart.yaml | 4 +- .../templates/daemonset.yaml | 2 +- .../prometheus-node-exporter/values.yaml | 5 +- .../crds/crd-alertmanagerconfigs.yaml | 2 +- .../crds/crd-alertmanagers.yaml | 2 +- .../crds/crd-podmonitors.yaml | 130 +++- .../crds/crd-probes.yaml | 137 +++- .../crds/crd-prometheuses.yaml | 331 ++++++++- .../crds/crd-prometheusrules.yaml | 4 +- .../crds/crd-servicemonitors.yaml | 130 +++- .../crds/crd-thanosrulers.yaml | 2 +- .../templates/_helpers.tpl | 22 +- .../templates/alertmanager/extrasecret.yaml | 20 + .../templates/alertmanager/ingress.yaml | 2 +- .../alertmanager/podDisruptionBudget.yaml | 2 +- .../grafana/dashboards-1.14/apiserver.yaml | 2 +- .../dashboards-1.14/cluster-total.yaml | 2 +- .../dashboards-1.14/controller-manager.yaml | 2 +- .../grafana/dashboards-1.14/etcd.yaml | 2 +- .../k8s-resources-cluster.yaml | 2 +- .../k8s-resources-namespace.yaml | 2 +- .../dashboards-1.14/k8s-resources-node.yaml | 2 +- .../dashboards-1.14/k8s-resources-pod.yaml | 2 +- .../k8s-resources-workload.yaml | 2 +- .../k8s-resources-workloads-namespace.yaml | 2 +- .../grafana/dashboards-1.14/kubelet.yaml | 2 +- .../dashboards-1.14/namespace-by-pod.yaml | 2 +- .../namespace-by-workload.yaml | 2 +- .../node-cluster-rsrc-use.yaml | 399 ++++++---- .../dashboards-1.14/node-rsrc-use.yaml | 414 +++++++---- .../grafana/dashboards-1.14/nodes.yaml | 10 +- .../persistentvolumesusage.yaml | 2 +- .../grafana/dashboards-1.14/pod-total.yaml | 2 +- .../prometheus-remote-write.yaml | 2 +- .../grafana/dashboards-1.14/proxy.yaml | 2 +- .../grafana/dashboards-1.14/scheduler.yaml | 2 +- .../dashboards-1.14/workload-total.yaml | 2 +- .../prometheus-operator/deployment.yaml | 6 +- .../templates/prometheus/extrasecret.yaml | 20 + .../templates/prometheus/ingress.yaml | 2 +- .../prometheus/podDisruptionBudget.yaml | 2 +- .../templates/prometheus/prometheus.yaml | 4 + .../charts/kube-prometheus-stack/values.yaml | 106 ++- charts/kubezero-metrics/dashboards/build.sh | 2 + .../dashboards/jsonnetfile.lock.json | 6 +- .../dashboards/kube-mixin/alerts.yml | 685 ++++++++++++++++++ .../dashboards/kube-mixin/apiserver.json | 14 +- .../dashboards/kube-mixin/cluster-total.json | 2 +- .../kube-mixin/controller-manager.json | 4 +- .../kube-mixin/k8s-resources-cluster.json | 2 +- .../dashboards/kube-mixin/kubelet.json | 2 +- .../kube-mixin/namespace-by-pod.json | 2 +- .../kube-mixin/namespace-by-workload.json | 2 +- .../dashboards/kube-mixin/pod-total.json | 2 +- .../dashboards/kube-mixin/rules.yml | 685 ++++++++++++++++++ .../dashboards/kube-mixin/scheduler.json | 2 +- .../templates/grafana-dashboards-k8s.yaml | 16 +- charts/kubezero-metrics/update.sh | 2 +- charts/kubezero-metrics/values.yaml | 106 +-- 68 files changed, 2879 insertions(+), 502 deletions(-) create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/extrasecret.yaml create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/extrasecret.yaml create mode 100644 charts/kubezero-metrics/dashboards/kube-mixin/alerts.yml create mode 100644 charts/kubezero-metrics/dashboards/kube-mixin/rules.yml diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index b0fe43f..0221a23 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-metrics description: KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all Kubernetes integrations. type: application -version: 0.4.4 +version: 0.4.6 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -17,7 +17,7 @@ dependencies: version: ">= 0.1.4" repository: https://zero-down-time.github.io/kubezero/ - name: kube-prometheus-stack - version: 17.0.3 + version: 18.0.1 # Switch back to upstream once all alerts are fixed eg. etcd gpcr # repository: https://prometheus-community.github.io/helm-charts - name: prometheus-adapter diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/.helmignore b/charts/kubezero-metrics/charts/kube-prometheus-stack/.helmignore index 93bf1ec..1937f42 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/.helmignore +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/.helmignore @@ -24,3 +24,5 @@ OWNERS hack/ ci/ kube-prometheus-*.tgz + +unittests/ diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml index f40e049..decd533 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml @@ -6,7 +6,7 @@ annotations: url: https://github.com/prometheus-operator/kube-prometheus artifacthub.io/operator: "true" apiVersion: v2 -appVersion: 0.49.0 +appVersion: 0.50.0 dependencies: - condition: kubeStateMetrics.enabled name: kube-state-metrics @@ -19,7 +19,7 @@ dependencies: - condition: grafana.enabled name: grafana repository: https://grafana.github.io/helm-charts - version: 6.14.* + version: 6.15.* description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus @@ -47,4 +47,4 @@ sources: - https://github.com/prometheus-community/helm-charts - https://github.com/prometheus-operator/kube-prometheus type: application -version: 17.0.3 +version: 18.0.1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md index 4692551..33753f5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md @@ -83,6 +83,20 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. +### From 17.x to 18.x +Version 18 upgrades prometheus-operator from 0.49.x to 0.50.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + ### From 16.x to 17.x Version 17 upgrades prometheus-operator from 0.48.x to 0.49.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml index cb7214a..e2e2ba7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 8.0.5 +appVersion: 8.1.0 description: The leading tool for querying and visualizing time series and metrics. home: https://grafana.net icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png @@ -19,4 +19,4 @@ name: grafana sources: - https://github.com/grafana/grafana type: application -version: 6.14.1 +version: 6.15.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl index 868aa07..f0c06aa 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl @@ -117,13 +117,6 @@ new password and use it. {{- end -}} {{- end -}} -{{/* -Get KubeVersion removing pre-release information. -*/}} -{{- define "grafana.kubeVersion" -}} - {{- default .Capabilities.KubeVersion.Version (regexFind "v[0-9]+\\.[0-9]+\\.[0-9]+" .Capabilities.KubeVersion.Version) -}} -{{- end -}} - {{/* Return the appropriate apiVersion for rbac. */}} @@ -139,7 +132,7 @@ Return the appropriate apiVersion for rbac. Return the appropriate apiVersion for ingress. */}} {{- define "grafana.ingress.apiVersion" -}} - {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19.x" (include "grafana.kubeVersion" .)) -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) -}} {{- print "networking.k8s.io/v1" -}} {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} {{- print "networking.k8s.io/v1beta1" -}} @@ -159,12 +152,12 @@ Return if ingress is stable. Return if ingress supports ingressClassName. */}} {{- define "grafana.ingress.supportsIngressClassName" -}} - {{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18.x" (include "grafana.kubeVersion" .))) -}} + {{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}} {{- end -}} {{/* Return if ingress supports pathType. */}} {{- define "grafana.ingress.supportsPathType" -}} - {{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18.x" (include "grafana.kubeVersion" .))) -}} + {{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}} {{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml index 02f27ab..c09ddd9 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml @@ -70,7 +70,7 @@ livenessProbe: image: repository: grafana/grafana - tag: 8.0.5 + tag: 8.1.0 sha: "" pullPolicy: IfNotPresent diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml index 1371550..e9cd620 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 2.1.0 +appVersion: 2.1.1 description: Install kube-state-metrics to generate and expose cluster-level metrics home: https://github.com/kubernetes/kube-state-metrics/ keywords: @@ -16,4 +16,4 @@ name: kube-state-metrics sources: - https://github.com/kubernetes/kube-state-metrics/ type: application -version: 3.4.1 +version: 3.4.2 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml index 3c34e41..a09e47e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml @@ -2,7 +2,7 @@ prometheusScrape: true image: repository: k8s.gcr.io/kube-state-metrics/kube-state-metrics - tag: v2.1.0 + tag: v2.1.1 pullPolicy: IfNotPresent imagePullSecrets: [] diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml index 8e97c4f..052017f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.2.0 +appVersion: 1.2.2 description: A Helm chart for prometheus node-exporter home: https://github.com/prometheus/node_exporter/ keywords: @@ -15,4 +15,4 @@ name: prometheus-node-exporter sources: - https://github.com/prometheus/node_exporter/ type: application -version: 2.0.1 +version: 2.0.4 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml index 9076276..6c0322f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -129,7 +129,7 @@ spec: {{- end }} {{- end }} hostNetwork: {{ .Values.hostNetwork }} - hostPID: true + hostPID: {{ .Values.hostPID }} {{- if .Values.affinity }} affinity: {{ toYaml .Values.affinity | indent 8 }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml index 8c88bb9..27b8783 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. image: repository: quay.io/prometheus/node-exporter - tag: v1.2.0 + tag: v1.2.2 pullPolicy: IfNotPresent service: @@ -85,6 +85,9 @@ endpoints: [] # Expose the service to the host network hostNetwork: true +# Share the host process ID namespace +hostPID: true + ## If true, node-exporter pods mounts host / at /host/root ## hostRootFsMount: true diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml index 6033768..25e599e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml index 91b788b..f2ee201 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml index 9f4314a..6363bfe 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -43,6 +43,23 @@ spec: jobLabel: description: The label to use to retrieve the job name from. type: string + labelLimit: + description: Per-scrape limit on number of labels that will be accepted + for a sample. Only valid in Prometheus versions 2.27.0 and newer. + format: int64 + type: integer + labelNameLengthLimit: + description: Per-scrape limit on length of labels name that will be + accepted for a sample. Only valid in Prometheus versions 2.27.0 + and newer. + format: int64 + type: integer + labelValueLengthLimit: + description: Per-scrape limit on length of labels value that will + be accepted for a sample. Only valid in Prometheus versions 2.27.0 + and newer. + format: int64 + type: integer namespaceSelector: description: Selector to select which namespaces the Endpoints objects are discovered from. @@ -63,6 +80,33 @@ spec: description: PodMetricsEndpoint defines a scrapeable endpoint of a Kubernetes Pod serving Prometheus metrics. properties: + authorization: + description: Authorization section for this endpoint + properties: + credentials: + description: The secret's key that contains the credentials + of the request + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: + description: Set the authentication type. Defaults to Bearer, + Basic will cause an error + type: string + type: object basicAuth: description: 'BasicAuth allow an endpoint to authenticate over basic authentication. More info: https://prometheus.io/docs/operating/configuration/#endpoint' @@ -183,6 +227,90 @@ spec: type: string type: object type: array + oauth2: + description: OAuth2 for the URL. Only valid in Prometheus versions + 2.27.0 and newer. + properties: + clientId: + description: The secret or configmap containing the OAuth2 + client id + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + clientSecret: + description: The secret containing the OAuth2 client secret + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + endpointParams: + additionalProperties: + type: string + description: Parameters to append to the token URL + type: object + scopes: + description: OAuth2 scopes used for the token request + items: + type: string + type: array + tokenUrl: + description: The URL to fetch the token from + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object params: additionalProperties: items: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml index fe12f6b..a4ad2e2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -40,6 +40,33 @@ spec: description: Specification of desired Ingress selection for target discovery by Prometheus. properties: + authorization: + description: Authorization section for this endpoint + properties: + credentials: + description: The secret's key that contains the credentials of + the request + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: + description: Set the authentication type. Defaults to Bearer, + Basic will cause an error + type: string + type: object basicAuth: description: 'BasicAuth allow an endpoint to authenticate over basic authentication. More info: https://prometheus.io/docs/operating/configuration/#endpoint' @@ -109,11 +136,109 @@ spec: jobName: description: The job name assigned to scraped metrics by default. type: string + labelLimit: + description: Per-scrape limit on number of labels that will be accepted + for a sample. Only valid in Prometheus versions 2.27.0 and newer. + format: int64 + type: integer + labelNameLengthLimit: + description: Per-scrape limit on length of labels name that will be + accepted for a sample. Only valid in Prometheus versions 2.27.0 + and newer. + format: int64 + type: integer + labelValueLengthLimit: + description: Per-scrape limit on length of labels value that will + be accepted for a sample. Only valid in Prometheus versions 2.27.0 + and newer. + format: int64 + type: integer module: description: 'The module to use for probing specifying how to probe the target. Example module configuring in the blackbox exporter: https://github.com/prometheus/blackbox_exporter/blob/master/example.yml' type: string + oauth2: + description: OAuth2 for the URL. Only valid in Prometheus versions + 2.27.0 and newer. + properties: + clientId: + description: The secret or configmap containing the OAuth2 client + id + properties: + configMap: + description: ConfigMap containing data to use for the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: object + clientSecret: + description: The secret containing the OAuth2 client secret + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + endpointParams: + additionalProperties: + type: string + description: Parameters to append to the token URL + type: object + scopes: + description: OAuth2 scopes used for the token request + items: + type: string + type: array + tokenUrl: + description: The URL to fetch the token from + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object prober: description: Specification for the prober to use for probing targets. The prober.URL parameter is required. Targets cannot be probed if @@ -134,9 +259,19 @@ spec: required: - url type: object + sampleLimit: + description: SampleLimit defines per-scrape limit on number of scraped + samples that will be accepted. + format: int64 + type: integer scrapeTimeout: description: Timeout for scraping metrics from the Prometheus exporter. type: string + targetLimit: + description: TargetLimit defines a limit on the number of scraped + targets that will be accepted. + format: int64 + type: integer targets: description: Targets defines a set of static and/or dynamically discovered targets to be probed using the prober. diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml index c1566e6..753174f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -739,6 +739,35 @@ spec: description: Version of the Alertmanager API that Prometheus uses to send alerts. It can be "v1" or "v2". type: string + authorization: + description: Authorization section for this alertmanager + endpoint + properties: + credentials: + description: The secret's key that contains the credentials + of the request + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: + description: Set the authentication type. Defaults to + Bearer, Basic will cause an error + type: string + type: object bearerTokenFile: description: BearerTokenFile to read from filesystem to use when authenticating to Alertmanager. @@ -920,6 +949,37 @@ spec: inside of the cluster and will discover API servers automatically and use the pod's CA certificate and bearer token file at /var/run/secrets/kubernetes.io/serviceaccount/. properties: + authorization: + description: Authorization section for accessing apiserver + properties: + credentials: + description: The secret's key that contains the credentials + of the request + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + credentialsFile: + description: File to read a secret from, mutually exclusive + with Credentials (from SafeAuthorization) + type: string + type: + description: Set the authentication type. Defaults to Bearer, + Basic will cause an error + type: string + type: object basicAuth: description: BasicAuth allow an endpoint to authenticate over basic authentication @@ -2213,6 +2273,27 @@ spec: items: type: string type: array + enforcedLabelLimit: + description: Per-scrape limit on number of labels that will be accepted + for a sample. If more than this number of labels are present post + metric-relabeling, the entire scrape will be treated as failed. + 0 means no limit. Only valid in Prometheus versions 2.27.0 and newer. + format: int64 + type: integer + enforcedLabelNameLengthLimit: + description: Per-scrape limit on length of labels name that will be + accepted for a sample. If a label name is longer than this number + post metric-relabeling, the entire scrape will be treated as failed. + 0 means no limit. Only valid in Prometheus versions 2.27.0 and newer. + format: int64 + type: integer + enforcedLabelValueLengthLimit: + description: Per-scrape limit on length of labels value that will + be accepted for a sample. If a label value is longer than this number + post metric-relabeling, the entire scrape will be treated as failed. + 0 means no limit. Only valid in Prometheus versions 2.27.0 and newer. + format: int64 + type: integer enforcedNamespaceLabel: description: "EnforcedNamespaceLabel If set, a label will be added to \n 1. all user-metrics (created by `ServiceMonitor`, `PodMonitor` @@ -2233,11 +2314,13 @@ spec: type: integer enforcedTargetLimit: description: EnforcedTargetLimit defines a global limit on the number - of scraped targets. This overrides any TargetLimit set per ServiceMonitor - or/and PodMonitor. It is meant to be used by admins to enforce the - TargetLimit to keep overall number of targets under the desired - limit. Note that if TargetLimit is higher that value will be taken - instead. + of scraped targets. This overrides any TargetLimit set per ServiceMonitor + or/and PodMonitor. It is meant to be used by admins to enforce + the TargetLimit to keep the overall number of targets under the + desired limit. Note that if TargetLimit is lower, that value will + be taken instead, except if either value is zero, in which case + the non-zero value will be used. If both values are zero, no limit + is enforced. format: int64 type: integer evaluationInterval: @@ -3659,6 +3742,37 @@ spec: description: RemoteReadSpec defines the remote_read configuration for prometheus. properties: + authorization: + description: Authorization section for remote read + properties: + credentials: + description: The secret's key that contains the credentials + of the request + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + credentialsFile: + description: File to read a secret from, mutually exclusive + with Credentials (from SafeAuthorization) + type: string + type: + description: Set the authentication type. Defaults to Bearer, + Basic will cause an error + type: string + type: object basicAuth: description: BasicAuth for the URL. properties: @@ -3713,6 +3827,90 @@ spec: to differentiate read configurations. Only valid in Prometheus versions 2.15.0 and newer. type: string + oauth2: + description: OAuth2 for the URL. Only valid in Prometheus versions + 2.27.0 and newer. + properties: + clientId: + description: The secret or configmap containing the OAuth2 + client id + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + clientSecret: + description: The secret containing the OAuth2 client secret + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + endpointParams: + additionalProperties: + type: string + description: Parameters to append to the token URL + type: object + scopes: + description: OAuth2 scopes used for the token request + items: + type: string + type: array + tokenUrl: + description: The URL to fetch the token from + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object proxyUrl: description: Optional ProxyURL type: string @@ -3870,6 +4068,37 @@ spec: description: RemoteWriteSpec defines the remote_write configuration for prometheus. properties: + authorization: + description: Authorization section for remote write + properties: + credentials: + description: The secret's key that contains the credentials + of the request + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + credentialsFile: + description: File to read a secret from, mutually exclusive + with Credentials (from SafeAuthorization) + type: string + type: + description: Set the authentication type. Defaults to Bearer, + Basic will cause an error + type: string + type: object basicAuth: description: BasicAuth for the URL. properties: @@ -3945,6 +4174,90 @@ spec: to differentiate queues. Only valid in Prometheus versions 2.15.0 and newer. type: string + oauth2: + description: OAuth2 for the URL. Only valid in Prometheus versions + 2.27.0 and newer. + properties: + clientId: + description: The secret or configmap containing the OAuth2 + client id + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + clientSecret: + description: The secret containing the OAuth2 client secret + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + endpointParams: + additionalProperties: + type: string + description: Parameters to append to the token URL + type: object + scopes: + description: OAuth2 scopes used for the token request + items: + type: string + type: array + tokenUrl: + description: The URL to fetch the token from + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object proxyUrl: description: Optional ProxyURL type: string @@ -3987,6 +4300,12 @@ spec: remoteTimeout: description: Timeout for requests to the remote write endpoint. type: string + sendExemplars: + description: Enables sending of exemplars over remote write. + Note that exemplar-storage itself must be enabled using the + enableFeature option for exemplars to be scraped in the first + place. Only valid in Prometheus versions 2.27.0 and newer. + type: boolean tlsConfig: description: TLS Config to use for remote write. properties: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml index b78a7f7..b51f3ed 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -11,6 +11,8 @@ metadata: spec: group: monitoring.coreos.com names: + categories: + - prometheus-operator kind: PrometheusRule listKind: PrometheusRuleList plural: prometheusrules diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml index 56f0b85..ae3acce 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -46,6 +46,33 @@ spec: description: Endpoint defines a scrapeable endpoint serving Prometheus metrics. properties: + authorization: + description: Authorization section for this endpoint + properties: + credentials: + description: The secret's key that contains the credentials + of the request + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: + description: Set the authentication type. Defaults to Bearer, + Basic will cause an error + type: string + type: object basicAuth: description: 'BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints' @@ -169,6 +196,90 @@ spec: type: string type: object type: array + oauth2: + description: OAuth2 for the URL. Only valid in Prometheus versions + 2.27.0 and newer. + properties: + clientId: + description: The secret or configmap containing the OAuth2 + client id + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + clientSecret: + description: The secret containing the OAuth2 client secret + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + endpointParams: + additionalProperties: + type: string + description: Parameters to append to the token URL + type: object + scopes: + description: OAuth2 scopes used for the token request + items: + type: string + type: array + tokenUrl: + description: The URL to fetch the token from + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object params: additionalProperties: items: @@ -384,6 +495,23 @@ spec: \n Default & fallback value: the name of the respective Kubernetes `Endpoint`." type: string + labelLimit: + description: Per-scrape limit on number of labels that will be accepted + for a sample. Only valid in Prometheus versions 2.27.0 and newer. + format: int64 + type: integer + labelNameLengthLimit: + description: Per-scrape limit on length of labels name that will be + accepted for a sample. Only valid in Prometheus versions 2.27.0 + and newer. + format: int64 + type: integer + labelValueLengthLimit: + description: Per-scrape limit on length of labels value that will + be accepted for a sample. Only valid in Prometheus versions 2.27.0 + and newer. + format: int64 + type: integer namespaceSelector: description: Selector to select which namespaces the Kubernetes Endpoints objects are discovered from. diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml index bf68c45..ced22c3 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl index 61989b5..b762f22 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl @@ -97,18 +97,13 @@ Allow the release namespace to be overridden for multi-namespace deployments in {{- end -}} {{/* Allow KubeVersion to be overridden. */}} -{{- define "kube-prometheus-stack.ingress.kubeVersion" -}} - {{- $kubeVersion := default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} - {{/* Special use case for Amazon EKS, Google GKE */}} - {{- if and (regexMatch "\\d+\\.\\d+\\.\\d+-(?:eks|gke).+" $kubeVersion) (not .Values.kubeVersionOverride) -}} - {{- $kubeVersion = regexFind "\\d+\\.\\d+\\.\\d+" $kubeVersion -}} - {{- end -}} - {{- $kubeVersion -}} +{{- define "kube-prometheus-stack.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} {{- end -}} {{/* Get Ingress API Version */}} {{- define "kube-prometheus-stack.ingress.apiVersion" -}} - {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19.x" (include "kube-prometheus-stack.ingress.kubeVersion" .)) -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} {{- print "networking.k8s.io/v1" -}} {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} {{- print "networking.k8s.io/v1beta1" -}} @@ -125,5 +120,14 @@ Allow the release namespace to be overridden for multi-namespace deployments in {{/* Check Ingress supports pathType */}} {{/* pathType was added to networking.k8s.io/v1beta1 in Kubernetes 1.18 */}} {{- define "kube-prometheus-stack.ingress.supportsPathType" -}} - {{- or (eq (include "kube-prometheus-stack.ingress.isStable" .) "true") (and (eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18.x" (include "kube-prometheus-stack.ingress.kubeVersion" .))) -}} + {{- or (eq (include "kube-prometheus-stack.ingress.isStable" .) "true") (and (eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" (include "kube-prometheus-stack.kubeVersion" .))) -}} +{{- end -}} + +{{/* Get Policy API Version */}} +{{- define "kube-prometheus-stack.pdb.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" (include "kube-prometheus-stack.kubeVersion" .)) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} {{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/extrasecret.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/extrasecret.yaml new file mode 100644 index 0000000..ecd8f47 --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.alertmanager.extraSecret.data -}} +{{- $secretName := printf "alertmanager-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.alertmanager.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.extraSecret.annotations }} + annotations: +{{ toYaml .Values.alertmanager.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + app.kubernetes.io/component: alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.alertmanager.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml index f337502..b40cd62 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml @@ -1,5 +1,5 @@ {{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled }} -{{- $pathType := .Values.alertmanager.ingress.pathType | default "" }} +{{- $pathType := .Values.alertmanager.ingress.pathType | default "ImplementationSpecific" }} {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} {{- $servicePort := .Values.alertmanager.service.port -}} {{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/podDisruptionBudget.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/podDisruptionBudget.yaml index 1dbe809..c8d977e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/podDisruptionBudget.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/podDisruptionBudget.yaml @@ -1,5 +1,5 @@ {{- if and .Values.alertmanager.enabled .Values.alertmanager.podDisruptionBudget.enabled }} -apiVersion: policy/v1beta1 +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} kind: PodDisruptionBudget metadata: name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/apiserver.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/apiserver.yaml index 5ad5529..0e59a15 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/apiserver.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/apiserver.yaml @@ -1662,7 +1662,7 @@ data: }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": "cluster", "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml index a10f752..0b8a871 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -1823,7 +1823,7 @@ data: }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/controller-manager.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/controller-manager.yaml index 3717c8e..e239897 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/controller-manager.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/controller-manager.yaml @@ -1092,7 +1092,7 @@ data: }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": "cluster", "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml index 2185b7d..e57d358 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml @@ -1061,7 +1061,7 @@ data: "value": "prod" }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if (or .Values.grafana.sidecar.dashboards.multicluster.global.enabled .Values.grafana.sidecar.dashboards.multicluster.etcd.enabled) }}0{{ else }}2{{ end }}, "includeAll": false, "label": "cluster", "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml index 37d1729..51f7522 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -2965,7 +2965,7 @@ data: "value": "" }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml index 8d2afbe..47555a2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -2658,7 +2658,7 @@ data: "value": "" }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-node.yaml index 9c13032..9de94d7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-node.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -892,7 +892,7 @@ data: "value": "" }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml index 5109291..f7b7d2d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -2314,7 +2314,7 @@ data: "value": "" }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml index ba54c66..08748c5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -1846,7 +1846,7 @@ data: "value": "" }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml index bbf9c07..5e838ee 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -2033,7 +2033,7 @@ data: "value": "" }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/kubelet.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/kubelet.yaml index 613a6c2..9da2bcc 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/kubelet.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/kubelet.yaml @@ -2163,7 +2163,7 @@ data: }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": "cluster", "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml index 288aeed..b1fa693 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -1293,7 +1293,7 @@ data: }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml index f533068..c8ddf09 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -1533,7 +1533,7 @@ data: }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml index 22f5aea..68de5ea 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -21,23 +21,30 @@ metadata: data: node-cluster-rsrc-use.json: |- { + "__inputs": [ + + ], + "__requires": [ + + ], "annotations": { "list": [ ] }, - "editable": true, + "editable": false, "gnetId": null, - "graphTooltip": 0, + "graphTooltip": 1, "hideControls": false, + "id": null, "links": [ ], - "refresh": "10s", + "refresh": "30s", "rows": [ { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -48,26 +55,34 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -77,12 +92,11 @@ data: "steppedLine": false, "targets": [ { - "expr": "(\n instance:node_cpu_utilisation:rate5m{job=\"node-exporter\"}\n*\n instance:node_num_cpu:sum{job=\"node-exporter\"}\n)\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\"}))\n", + "expr": "((\n instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n *\n instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}))\n", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}}", - "legendLink": "/dashboard/file/node-rsrc-use.json", - "step": 10 + "legendFormat": "{{`{{`}} instance {{`}}`}}", + "refId": "A" } ], "thresholds": [ @@ -92,8 +106,8 @@ data: "timeShift": null, "title": "CPU Utilisation", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -111,17 +125,17 @@ data: "format": "percentunit", "label": null, "logBase": 1, - "max": 1, - "min": 0, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -134,26 +148,34 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 2, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -163,12 +185,11 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\"}\n/ scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\"}))\n", + "expr": "(\n instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", - "legendLink": "/dashboard/file/node-rsrc-use.json", - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -176,10 +197,10 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "CPU Saturation (load1 per CPU)", + "title": "CPU Saturation (Load1 per CPU)", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -197,17 +218,17 @@ data: "format": "percentunit", "label": null, "logBase": 1, - "max": 1, - "min": 0, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -217,11 +238,12 @@ data: "repeatRowId": null, "showTitle": true, "title": "CPU", - "titleSize": "h6" + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -232,26 +254,34 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 3, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -261,12 +291,11 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\"}\n/ scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\"}))\n", + "expr": "(\n instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", - "legendLink": "/dashboard/file/node-rsrc-use.json", - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -276,8 +305,8 @@ data: "timeShift": null, "title": "Memory Utilisation", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -295,17 +324,17 @@ data: "format": "percentunit", "label": null, "logBase": 1, - "max": 1, - "min": 0, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -318,26 +347,34 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 4, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -347,12 +384,11 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\"}", + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", - "legendLink": "/dashboard/file/node-rsrc-use.json", - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -362,8 +398,8 @@ data: "timeShift": null, "title": "Memory Saturation (Major Page Faults)", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -378,20 +414,20 @@ data: }, "yaxes": [ { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", + "format": "rds", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ] } @@ -401,11 +437,12 @@ data: "repeatRowId": null, "showTitle": true, "title": "Memory", - "titleSize": "h6" + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -416,33 +453,41 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 5, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ { - "alias": "/ Receive/", + "alias": "/Receive/", "stack": "A" }, { - "alias": "/ Transmit/", + "alias": "/Transmit/", "stack": "B", "transform": "negative-Y" } @@ -453,20 +498,18 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\"}", + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", - "legendLink": "/dashboard/file/node-rsrc-use.json", - "step": 10 + "refId": "A" }, { - "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\"}", + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", - "legendLink": "/dashboard/file/node-rsrc-use.json", - "step": 10 + "refId": "B" } ], "thresholds": [ @@ -474,10 +517,10 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Net Utilisation (Bytes Receive/Transmit)", + "title": "Network Utilisation (Bytes Receive/Transmit)", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -500,12 +543,12 @@ data: "show": true }, { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -518,26 +561,34 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 6, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ { "alias": "/ Receive/", @@ -555,20 +606,18 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\"}", + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", - "legendLink": "/dashboard/file/node-rsrc-use.json", - "step": 10 + "refId": "A" }, { - "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\"}", + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", - "legendLink": "/dashboard/file/node-rsrc-use.json", - "step": 10 + "refId": "B" } ], "thresholds": [ @@ -576,10 +625,10 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Net Saturation (Drops Receive/Transmit)", + "title": "Network Saturation (Drops Receive/Transmit)", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -594,7 +643,7 @@ data: }, "yaxes": [ { - "format": "rps", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -602,12 +651,12 @@ data: "show": true }, { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -617,11 +666,12 @@ data: "repeatRowId": null, "showTitle": true, "title": "Network", - "titleSize": "h6" + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -632,26 +682,34 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 7, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -661,12 +719,11 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\"}\n/ scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\"}))\n", + "expr": "(\n instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", - "legendLink": "/dashboard/file/node-rsrc-use.json", - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -676,8 +733,8 @@ data: "timeShift": null, "title": "Disk IO Utilisation", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -695,17 +752,17 @@ data: "format": "percentunit", "label": null, "logBase": 1, - "max": 1, - "min": 0, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -718,26 +775,34 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 8, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -747,12 +812,11 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\"}\n/ scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\"}))\n", + "expr": "(\n instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", - "legendLink": "/dashboard/file/node-rsrc-use.json", - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -762,8 +826,8 @@ data: "timeShift": null, "title": "Disk IO Saturation", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -781,17 +845,17 @@ data: "format": "percentunit", "label": null, "logBase": 1, - "max": 1, - "min": 0, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -801,11 +865,12 @@ data: "repeatRowId": null, "showTitle": true, "title": "Disk IO", - "titleSize": "h6" + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -816,26 +881,34 @@ data: "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 9, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, "lines": true, - "linewidth": 0, + "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], @@ -845,12 +918,11 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum without (device) (\n max without (fstype, mountpoint) (\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\"} - node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\"}\n )\n) \n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\"})))\n", + "expr": "sum without (device) (\n max without (fstype, mountpoint) ((\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", cluster=\"$cluster\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", cluster=\"$cluster\"}\n ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", cluster=\"$cluster\"})))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", - "legendLink": "/dashboard/file/node-rsrc-use.json", - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -860,8 +932,8 @@ data: "timeShift": null, "title": "Disk Space Utilisation", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -879,17 +951,17 @@ data: "format": "percentunit", "label": null, "logBase": 1, - "max": 1, - "min": 0, + "max": null, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -899,20 +971,21 @@ data: "repeatRowId": null, "showTitle": true, "title": "Disk Space", - "titleSize": "h6" + "titleSize": "h6", + "type": "row" } ], "schemaVersion": 14, "style": "dark", "tags": [ - + "node-exporter-mixin" ], "templating": { "list": [ { "current": { - "text": "default", - "value": "default" + "text": "Prometheus", + "value": "Prometheus" }, "hide": 0, "label": null, @@ -924,6 +997,33 @@ data: "refresh": 1, "regex": "", "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false } ] }, @@ -957,8 +1057,7 @@ data: ] }, "timezone": "utc", - "title": "USE Method / Cluster", - "uid": "", + "title": "Node Exporter / USE Method / Cluster", "version": 0 } {{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml index 52d24bf..212fbd2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -21,23 +21,30 @@ metadata: data: node-rsrc-use.json: |- { + "__inputs": [ + + ], + "__requires": [ + + ], "annotations": { "list": [ ] }, - "editable": true, + "editable": false, "gnetId": null, - "graphTooltip": 0, + "graphTooltip": 1, "hideControls": false, + "id": null, "links": [ ], - "refresh": "10s", + "refresh": "30s", "rows": [ { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -47,14 +54,21 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 1, + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": false, + "sideWidth": null, "total": false, "values": false }, @@ -63,26 +77,26 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Utilisation", - "legendLink": null, - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -92,8 +106,8 @@ data: "timeShift": null, "title": "CPU Utilisation", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -112,16 +126,16 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -133,14 +147,21 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": false, + "sideWidth": null, "total": false, "values": false }, @@ -149,26 +170,26 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Saturation", - "legendLink": null, - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -178,8 +199,8 @@ data: "timeShift": null, "title": "CPU Saturation (Load1 per CPU)", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -198,16 +219,16 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -217,11 +238,12 @@ data: "repeatRowId": null, "showTitle": true, "title": "CPU", - "titleSize": "h6" + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -231,14 +253,21 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, @@ -247,26 +276,26 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\", job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Memory", - "legendLink": null, - "step": 10 + "legendFormat": "Utilisation", + "refId": "A" } ], "thresholds": [ @@ -276,8 +305,8 @@ data: "timeShift": null, "title": "Memory Utilisation", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -296,16 +325,16 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -317,14 +346,21 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 4, + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": false, + "sideWidth": null, "total": false, "values": false }, @@ -333,26 +369,26 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Major page faults", - "legendLink": null, - "step": 10 + "legendFormat": "Major page Faults", + "refId": "A" } ], "thresholds": [ @@ -362,8 +398,8 @@ data: "timeShift": null, "title": "Memory Saturation (Major Page Faults)", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -378,20 +414,20 @@ data: }, "yaxes": [ { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", + "format": "rds", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true + }, + { + "format": "rds", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } ] } @@ -401,11 +437,12 @@ data: "repeatRowId": null, "showTitle": true, "title": "Memory", - "titleSize": "h6" + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -415,14 +452,21 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 6, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, @@ -431,11 +475,12 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ { "alias": "/Receive/", @@ -449,24 +494,22 @@ data: ], "spaceLength": 10, "span": 6, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Receive", - "legendLink": null, - "step": 10 + "refId": "A" }, { - "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "Transmit", - "legendLink": null, - "step": 10 + "refId": "B" } ], "thresholds": [ @@ -474,10 +517,10 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Net Utilisation (Bytes Receive/Transmit)", + "title": "Network Utilisation (Bytes Receive/Transmit)", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -500,12 +543,12 @@ data: "show": true }, { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -517,14 +560,21 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 7, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, @@ -533,42 +583,41 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ { - "alias": "/Receive/", + "alias": "/ Receive/", "stack": "A" }, { - "alias": "/Transmit/", + "alias": "/ Transmit/", "stack": "B", "transform": "negative-Y" } ], "spaceLength": 10, "span": 6, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Receive drops", - "legendLink": null, - "step": 10 + "legendFormat": "Receive", + "refId": "A" }, { - "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Transmit drops", - "legendLink": null, - "step": 10 + "legendFormat": "Transmit", + "refId": "B" } ], "thresholds": [ @@ -576,10 +625,10 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Net Saturation (Drops Receive/Transmit)", + "title": "Network Saturation (Drops Receive/Transmit)", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -594,7 +643,7 @@ data: }, "yaxes": [ { - "format": "rps", + "format": "Bps", "label": null, "logBase": 1, "max": null, @@ -602,12 +651,12 @@ data: "show": true }, { - "format": "short", + "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -616,12 +665,13 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Net", - "titleSize": "h6" + "title": "Network", + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -631,14 +681,21 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 8, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, @@ -647,26 +704,26 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}}", - "legendLink": null, - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -676,8 +733,8 @@ data: "timeShift": null, "title": "Disk IO Utilisation", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -696,16 +753,16 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] }, @@ -717,14 +774,21 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 9, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "show": true, + "rightSide": false, + "show": false, + "sideWidth": null, "total": false, "values": false }, @@ -733,26 +797,26 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 6, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}}", - "legendLink": null, - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -762,8 +826,8 @@ data: "timeShift": null, "title": "Disk IO Saturation", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -782,16 +846,16 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -801,11 +865,12 @@ data: "repeatRowId": null, "showTitle": true, "title": "Disk IO", - "titleSize": "h6" + "titleSize": "h6", + "type": "row" }, { "collapse": false, - "height": "250px", + "collapsed": false, "panels": [ { "aliasColors": { @@ -815,14 +880,21 @@ data: "dashLength": 10, "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 9, + "fill": 10, + "fillGradient": 0, + "gridPos": { + + }, + "id": 10, "legend": { + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, + "rightSide": false, "show": false, + "sideWidth": null, "total": false, "values": false }, @@ -831,26 +903,26 @@ data: "links": [ ], - "nullPointMode": "null as zero", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", + "repeat": null, "seriesOverrides": [ ], "spaceLength": 10, "span": 12, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { - "expr": "1 -\n(\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\"})\n/\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\"})\n)\n", + "expr": "sort_desc(1 -\n (\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n /\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n ) != 0\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}}", - "legendLink": null, - "step": 10 + "refId": "A" } ], "thresholds": [ @@ -860,8 +932,8 @@ data: "timeShift": null, "title": "Disk Space Utilisation", "tooltip": { - "shared": false, - "sort": 0, + "shared": true, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -880,16 +952,16 @@ data: "label": null, "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { - "format": "short", + "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, - "show": false + "show": true } ] } @@ -899,20 +971,21 @@ data: "repeatRowId": null, "showTitle": true, "title": "Disk Space", - "titleSize": "h6" + "titleSize": "h6", + "type": "row" } ], "schemaVersion": 14, "style": "dark", "tags": [ - + "node-exporter-mixin" ], "templating": { "list": [ { "current": { - "text": "default", - "value": "default" + "text": "Prometheus", + "value": "Prometheus" }, "hide": 0, "label": null, @@ -928,22 +1001,48 @@ data: { "allValue": null, "current": { - "text": "prod", - "value": "prod" + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_time_seconds, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + }, "datasource": "$datasource", "hide": 0, "includeAll": false, - "label": "instance", + "label": null, "multi": false, "name": "instance", "options": [ ], - "query": "label_values(up{job=\"node-exporter\"}, instance)", - "refresh": 1, + "query": "label_values(node_exporter_build_info{job=\"node-exporter\", cluster=\"$cluster\"}, instance)", + "refresh": 2, "regex": "", - "sort": 2, + "sort": 1, "tagValuesQuery": "", "tags": [ @@ -984,8 +1083,7 @@ data: ] }, "timezone": "utc", - "title": "USE Method / Node", - "uid": "", + "title": "Node Exporter / USE Method / Node", "version": 0 } {{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml index 9da7444..d311794 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml @@ -34,13 +34,13 @@ data: }, "editable": false, "gnetId": null, - "graphTooltip": 0, + "graphTooltip": 1, "hideControls": false, "id": null, "links": [ ], - "refresh": "", + "refresh": "30s", "rows": [ { "collapse": false, @@ -907,7 +907,7 @@ data: "schemaVersion": 14, "style": "dark", "tags": [ - + "node-exporter-mixin" ], "templating": { "list": [ @@ -984,8 +984,8 @@ data: "30d" ] }, - "timezone": "browser", - "title": "Nodes", + "timezone": "utc", + "title": "Node Exporter / Nodes", "version": 0 } {{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml index 93065ff..58483aa 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -466,7 +466,7 @@ data: }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": "cluster", "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml index d2842c3..a1aa780 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml @@ -1025,7 +1025,7 @@ data: }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml index a98581d..4fe7c55 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -1586,7 +1586,7 @@ data: } }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": true, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/proxy.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/proxy.yaml index e6b9b98..7490c12 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/proxy.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/proxy.yaml @@ -1172,7 +1172,7 @@ data: }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": "cluster", "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/scheduler.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/scheduler.yaml index 82edd36..5d0c6c3 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/scheduler.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/scheduler.yaml @@ -1015,7 +1015,7 @@ data: }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": "cluster", "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml index dc1e4ed..6ddae52 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml @@ -1203,7 +1203,7 @@ data: }, "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster.global.enabled }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml index 00144ab..e5cb317 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml @@ -70,8 +70,10 @@ spec: {{- else }} - --prometheus-config-reloader={{ .Values.prometheusOperator.prometheusConfigReloaderImage.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloaderImage.tag }} {{- end }} - - --config-reloader-cpu={{ .Values.prometheusOperator.configReloaderCpu }} - - --config-reloader-memory={{ .Values.prometheusOperator.configReloaderMemory }} + - --config-reloader-cpu-request={{ .Values.prometheusOperator.configReloaderCpu }} + - --config-reloader-cpu-limit={{ .Values.prometheusOperator.configReloaderCpu }} + - --config-reloader-memory-request={{ .Values.prometheusOperator.configReloaderMemory }} + - --config-reloader-memory-limit={{ .Values.prometheusOperator.configReloaderMemory }} {{- if .Values.prometheusOperator.alertmanagerInstanceNamespaces }} - --alertmanager-instance-namespaces={{ .Values.prometheusOperator.alertmanagerInstanceNamespaces | join "," }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/extrasecret.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/extrasecret.yaml new file mode 100644 index 0000000..17f3478 --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/extrasecret.yaml @@ -0,0 +1,20 @@ +{{- if .Values.prometheus.extraSecret.data -}} +{{- $secretName := printf "prometheus-%s-extra" (include "kube-prometheus-stack.fullname" . ) -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ default $secretName .Values.prometheus.extraSecret.name }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.extraSecret.annotations }} + annotations: +{{ toYaml .Values.prometheus.extraSecret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + app.kubernetes.io/component: prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- range $key, $val := .Values.prometheus.extraSecret.data }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml index 3992789..4930189 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml @@ -1,5 +1,5 @@ {{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled -}} - {{- $pathType := .Values.prometheus.ingress.pathType | default "" -}} + {{- $pathType := .Values.prometheus.ingress.pathType | default "ImplementationSpecific" -}} {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" -}} {{- $servicePort := .Values.prometheus.service.port -}} {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/podDisruptionBudget.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/podDisruptionBudget.yaml index cce4a85..93a30e7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/podDisruptionBudget.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/podDisruptionBudget.yaml @@ -1,5 +1,5 @@ {{- if and .Values.prometheus.enabled .Values.prometheus.podDisruptionBudget.enabled }} -apiVersion: policy/v1beta1 +apiVersion: {{ include "kube-prometheus-stack.pdb.apiVersion" . }} kind: PodDisruptionBudget metadata: name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml index b2b6890..76692b4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml @@ -70,6 +70,10 @@ spec: logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }} +{{- if .Values.prometheus.prometheusSpec.web }} + web: +{{ toYaml .Values.prometheus.prometheusSpec.web | indent 4 }} +{{- end }} {{- if .Values.prometheus.prometheusSpec.enableFeatures }} enableFeatures: {{- range $enableFeatures := .Values.prometheus.prometheusSpec.enableFeatures }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml index 0b2a607..fdceaf6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml @@ -197,7 +197,7 @@ alertmanager: # {{- $root := . -}} # {{ range .Alerts }} # *Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}` - # *Cluster:* {{ template "cluster" $root }} + # *Cluster:* {{ template "cluster" $root }} # *Description:* {{ .Annotations.description }} # *Graph:* <{{ .GeneratorURL }}|:chart_with_upwards_trend:> # *Runbook:* <{{ .Annotations.runbook }}|:spiral_note_pad:> @@ -366,14 +366,14 @@ alertmanager: bearerTokenFile: - ## metric relabel configs to apply to samples before ingestion. + ## Metric relabel configs to apply to samples before ingestion. ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -486,11 +486,11 @@ alertmanager: # selector: {} - ## The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. string false + ## The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. string false ## externalUrl: - ## The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, + ## The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, ## but the server serves requests under a different route prefix. For example for use with kubectl proxy. ## routePrefix: / @@ -558,7 +558,7 @@ alertmanager: # app: alertmanager ## SecurityContext holds pod-level security attributes and common container settings. - ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false + ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ ## securityContext: @@ -606,6 +606,16 @@ alertmanager: ## Use case is e.g. spanning an Alertmanager cluster across Kubernetes clusters with a single replica in each. forceEnableClusterMode: false + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. ## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml ## @@ -668,7 +678,11 @@ grafana: ## Annotations for Grafana dashboard configmaps ## annotations: {} - multicluster: false + multicluster: + global: + enabled: false + etcd: + enabled: false datasources: enabled: true defaultDatasourceEnabled: true @@ -730,14 +744,14 @@ grafana: # in grafana.ini path: "/metrics" - ## metric relabel configs to apply to samples before ingestion. + ## Metric relabel configs to apply to samples before ingestion. ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -768,7 +782,7 @@ kubeApiServer: component: apiserver provider: kubernetes - ## metric relabel configs to apply to samples before ingestion. + ## Metric relabel configs to apply to samples before ingestion. ## metricRelabelings: [] # - action: keep @@ -846,7 +860,7 @@ kubelet: # replacement: $1 # action: drop - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. # metrics_path is required to match upstream rules and charts ## cAdvisorRelabelings: @@ -891,7 +905,7 @@ kubelet: # replacement: $1 # action: drop - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. # metrics_path is required to match upstream rules and charts ## relabelings: @@ -946,14 +960,14 @@ kubeControllerManager: # Name of the server to use when validating TLS certificate serverName: null - ## metric relabel configs to apply to samples before ingestion. + ## Metric relabel configs to apply to samples before ingestion. ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -981,14 +995,14 @@ coreDns: ## proxyUrl: "" - ## metric relabel configs to apply to samples before ingestion. + ## Metric relabel configs to apply to samples before ingestion. ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1020,14 +1034,14 @@ kubeDns: ## proxyUrl: "" - ## metric relabel configs to apply to samples before ingestion. + ## Metric relabel configs to apply to samples before ingestion. ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1041,7 +1055,7 @@ kubeDns: # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. ## dnsmasqRelabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1098,14 +1112,14 @@ kubeEtcd: certFile: "" keyFile: "" - ## metric relabel configs to apply to samples before ingestion. + ## Metric relabel configs to apply to samples before ingestion. ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1156,14 +1170,14 @@ kubeScheduler: ## Name of the server to use when validating TLS certificate serverName: null - ## metric relabel configs to apply to samples before ingestion. + ## Metric relabel configs to apply to samples before ingestion. ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1208,14 +1222,14 @@ kubeProxy: ## https: false - ## metric relabel configs to apply to samples before ingestion. + ## Metric relabel configs to apply to samples before ingestion. ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. ## relabelings: [] # - action: keep @@ -1241,14 +1255,14 @@ kubeStateMetrics: ## namespaceOverride: "" - ## metric relabel configs to apply to samples before ingestion. + ## Metric relabel configs to apply to samples before ingestion. ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1293,7 +1307,7 @@ nodeExporter: ## scrapeTimeout: "" - ## metric relabel configs to apply to samples before ingestion. + ## Metric relabel configs to apply to samples before ingestion. ## metricRelabelings: [] # - sourceLabels: [__name__] @@ -1302,7 +1316,7 @@ nodeExporter: # replacement: $1 # action: drop - ## relabel configs to apply to samples before ingestion. + ## relabel configs to apply to samples before ingestion. ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1367,7 +1381,7 @@ prometheusOperator: tolerations: [] ## SecurityContext holds pod-level security attributes and common container settings. - ## This defaults to non root user with uid 2000 and gid 2000. *v1.PodSecurityContext false + ## This defaults to non root user with uid 2000 and gid 2000. *v1.PodSecurityContext false ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ ## securityContext: @@ -1483,14 +1497,14 @@ prometheusOperator: scrapeTimeout: "" selfMonitor: true - ## metric relabel configs to apply to samples before ingestion. + ## Metric relabel configs to apply to samples before ingestion. ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1562,7 +1576,7 @@ prometheusOperator: ## image: repository: quay.io/prometheus-operator/prometheus-operator - tag: v0.49.0 + tag: v0.50.0 sha: "" pullPolicy: IfNotPresent @@ -1578,7 +1592,7 @@ prometheusOperator: ## prometheusConfigReloaderImage: repository: quay.io/prometheus-operator/prometheus-config-reloader - tag: v0.49.0 + tag: v0.50.0 sha: "" ## Set the prometheus config reloader side-car CPU limit @@ -1767,6 +1781,18 @@ prometheus: # - secretName: thanos-gateway-tls # hosts: # - thanos-gateway.domain.com + # + + ## ExtraSecret can be used to store various data in an extra secret + ## (use it for example to store hashed basic auth credentials) + extraSecret: + ## if not set, name will be auto generated + # name: "" + annotations: {} + data: {} + # auth: | + # foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0 + # someoneelse:$apr1$DMZX2Z4q$6SbQIfyuLQd.xmo/P0m2c. ingress: enabled: false @@ -1868,14 +1894,14 @@ prometheus: bearerTokenFile: - ## metric relabel configs to apply to samples before ingestion. + ## Metric relabel configs to apply to samples before ingestion. ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + # relabel configs to apply to samples before ingestion. ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1921,6 +1947,10 @@ prometheus: ## enableAdminAPI: false + ## WebTLSConfig defines the TLS parameters for HTTPS + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#webtlsconfig + web: {} + # EnableFeatures API enables access to Prometheus disabled features. # ref: https://prometheus.io/docs/prometheus/latest/disabled_features/ enableFeatures: [] @@ -2332,7 +2362,7 @@ prometheus: runAsUser: 1000 fsGroup: 2000 - ## Priority class assigned to the Pods + ## Priority class assigned to the Pods ## priorityClassName: "" @@ -2344,7 +2374,7 @@ prometheus: thanos: {} ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. - ## if using proxy extraContainer update targetPort with proxy container port + ## if using proxy extraContainer update targetPort with proxy container port containers: [] ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes diff --git a/charts/kubezero-metrics/dashboards/build.sh b/charts/kubezero-metrics/dashboards/build.sh index 73fc256..49fb336 100755 --- a/charts/kubezero-metrics/dashboards/build.sh +++ b/charts/kubezero-metrics/dashboards/build.sh @@ -5,4 +5,6 @@ ./jb-linux-amd64 update mkdir -p kube-mixin +#jsonnet -J vendor -S -e 'std.manifestYamlDoc((import "mixin.libsonnet").prometheusAlerts)' > kube-mixin/alerts.yml +#jsonnet -J vendor -S -e 'std.manifestYamlDoc((import "mixin.libsonnet").prometheusRules)' > kube-mixin/rules.yml jsonnet -J vendor -m kube-mixin -e '(import "mixin.libsonnet").grafanaDashboards' diff --git a/charts/kubezero-metrics/dashboards/jsonnetfile.lock.json b/charts/kubezero-metrics/dashboards/jsonnetfile.lock.json index fe0c057..0b8ff00 100644 --- a/charts/kubezero-metrics/dashboards/jsonnetfile.lock.json +++ b/charts/kubezero-metrics/dashboards/jsonnetfile.lock.json @@ -18,7 +18,7 @@ "subdir": "grafana-builder" } }, - "version": "8df6a4ff1623781260c3236761c2ebe837958b3f", + "version": "ac31371db5008f92b054751dfc4f7ece6526250f", "sum": "GRf2GvwEU4jhXV+JOonXSZ4wdDv8mnHBPCQ6TUVd+g8=" }, { @@ -28,8 +28,8 @@ "subdir": "" } }, - "version": "25b5047a57352345e478b215370dfcd1fecee27d", - "sum": "z+ksn3PAtR/fYtxgsrrG4euIriyiefZsTUcJaVXdG1Q=" + "version": "2b27a09a667091cef74776b690ccceaf55995e29", + "sum": "j2jPdrcM3iuaUK+6V9jWn2M3Fapr0KtI8FZ1KQoHIGA=" } ], "legacyImports": false diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/alerts.yml b/charts/kubezero-metrics/dashboards/kube-mixin/alerts.yml new file mode 100644 index 0000000..1e5ad0d --- /dev/null +++ b/charts/kubezero-metrics/dashboards/kube-mixin/alerts.yml @@ -0,0 +1,685 @@ +"groups": +- "name": "kubernetes-apps" + "rules": + - "alert": "KubePodCrashLooping" + "annotations": + "description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf \"%.2f\" $value }} times / 10 minutes." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping" + "summary": "Pod is crash looping." + "expr": | + increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[10m]) > 0 + and + kube_pod_container_status_waiting{job="kube-state-metrics"} == 1 + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubePodNotReady" + "annotations": + "description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready" + "summary": "Pod has been in a non-ready state for more than 15 minutes." + "expr": | + sum by (namespace, pod) ( + max by(namespace, pod) ( + kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown"} + ) * on(namespace, pod) group_left(owner_kind) topk by(namespace, pod) ( + 1, max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeDeploymentGenerationMismatch" + "annotations": + "description": "Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch" + "summary": "Deployment generation mismatch due to possible roll-back" + "expr": | + kube_deployment_status_observed_generation{job="kube-state-metrics"} + != + kube_deployment_metadata_generation{job="kube-state-metrics"} + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeDeploymentReplicasMismatch" + "annotations": + "description": "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch" + "summary": "Deployment has not matched the expected number of replicas." + "expr": | + ( + kube_deployment_spec_replicas{job="kube-state-metrics"} + > + kube_deployment_status_replicas_available{job="kube-state-metrics"} + ) and ( + changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m]) + == + 0 + ) + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeStatefulSetReplicasMismatch" + "annotations": + "description": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch" + "summary": "Deployment has not matched the expected number of replicas." + "expr": | + ( + kube_statefulset_status_replicas_ready{job="kube-state-metrics"} + != + kube_statefulset_status_replicas{job="kube-state-metrics"} + ) and ( + changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[10m]) + == + 0 + ) + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeStatefulSetGenerationMismatch" + "annotations": + "description": "StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch" + "summary": "StatefulSet generation mismatch due to possible roll-back" + "expr": | + kube_statefulset_status_observed_generation{job="kube-state-metrics"} + != + kube_statefulset_metadata_generation{job="kube-state-metrics"} + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeStatefulSetUpdateNotRolledOut" + "annotations": + "description": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout" + "summary": "StatefulSet update has not been rolled out." + "expr": | + ( + max without (revision) ( + kube_statefulset_status_current_revision{job="kube-state-metrics"} + unless + kube_statefulset_status_update_revision{job="kube-state-metrics"} + ) + * + ( + kube_statefulset_replicas{job="kube-state-metrics"} + != + kube_statefulset_status_replicas_updated{job="kube-state-metrics"} + ) + ) and ( + changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m]) + == + 0 + ) + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeDaemonSetRolloutStuck" + "annotations": + "description": "DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15 minutes." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck" + "summary": "DaemonSet rollout is stuck." + "expr": | + ( + ( + kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} + != + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} + ) or ( + kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} + != + 0 + ) or ( + kube_daemonset_updated_number_scheduled{job="kube-state-metrics"} + != + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} + ) or ( + kube_daemonset_status_number_available{job="kube-state-metrics"} + != + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} + ) + ) and ( + changes(kube_daemonset_updated_number_scheduled{job="kube-state-metrics"}[5m]) + == + 0 + ) + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeContainerWaiting" + "annotations": + "description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} has been in waiting state for longer than 1 hour." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting" + "summary": "Pod container waiting longer than 1 hour" + "expr": | + sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0 + "for": "1h" + "labels": + "severity": "warning" + - "alert": "KubeDaemonSetNotScheduled" + "annotations": + "description": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled" + "summary": "DaemonSet pods are not scheduled." + "expr": | + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} + - + kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0 + "for": "10m" + "labels": + "severity": "warning" + - "alert": "KubeDaemonSetMisScheduled" + "annotations": + "description": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled" + "summary": "DaemonSet pods are misscheduled." + "expr": | + kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0 + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeJobCompletion" + "annotations": + "description": "Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than 12 hours to complete." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion" + "summary": "Job did not complete in time" + "expr": | + kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"} > 0 + "for": "12h" + "labels": + "severity": "warning" + - "alert": "KubeJobFailed" + "annotations": + "description": "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed" + "summary": "Job failed to complete." + "expr": | + kube_job_failed{job="kube-state-metrics"} > 0 + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeHpaReplicasMismatch" + "annotations": + "description": "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has not matched the desired number of replicas for longer than 15 minutes." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch" + "summary": "HPA has not matched descired number of replicas." + "expr": | + (kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics"} + != + kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"} + > + kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics"}) + and + (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"} + < + kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"}) + and + changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}[15m]) == 0 + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeHpaMaxedOut" + "annotations": + "description": "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 15 minutes." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout" + "summary": "HPA is running at max replicas" + "expr": | + kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"} + == + kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"} + "for": "15m" + "labels": + "severity": "warning" +- "name": "kubernetes-resources" + "rules": + - "alert": "KubeCPUOvercommit" + "annotations": + "description": "Cluster has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit" + "summary": "Cluster has overcommitted CPU resource requests." + "expr": | + sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="cpu"}) - max(kube_node_status_allocatable{resource="cpu"})) > 0 + and + (sum(kube_node_status_allocatable{resource="cpu"}) - max(kube_node_status_allocatable{resource="cpu"})) > 0 + "for": "10m" + "labels": + "severity": "warning" + - "alert": "KubeMemoryOvercommit" + "annotations": + "description": "Cluster has overcommitted memory resource requests for Pods by {{ $value }} bytes and cannot tolerate node failure." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryovercommit" + "summary": "Cluster has overcommitted memory resource requests." + "expr": | + sum(namespace_memory:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0 + and + (sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0 + "for": "10m" + "labels": + "severity": "warning" + - "alert": "KubeCPUQuotaOvercommit" + "annotations": + "description": "Cluster has overcommitted CPU resource requests for Namespaces." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuquotaovercommit" + "summary": "Cluster has overcommitted CPU resource requests." + "expr": | + sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"}) + / + sum(kube_node_status_allocatable{resource="cpu"}) + > 1.5 + "for": "5m" + "labels": + "severity": "warning" + - "alert": "KubeMemoryQuotaOvercommit" + "annotations": + "description": "Cluster has overcommitted memory resource requests for Namespaces." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryquotaovercommit" + "summary": "Cluster has overcommitted memory resource requests." + "expr": | + sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"}) + / + sum(kube_node_status_allocatable{resource="memory",job="kube-state-metrics"}) + > 1.5 + "for": "5m" + "labels": + "severity": "warning" + - "alert": "KubeQuotaAlmostFull" + "annotations": + "description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaalmostfull" + "summary": "Namespace quota is going to be full." + "expr": | + kube_resourcequota{job="kube-state-metrics", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) + > 0.9 < 1 + "for": "15m" + "labels": + "severity": "info" + - "alert": "KubeQuotaFullyUsed" + "annotations": + "description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotafullyused" + "summary": "Namespace quota is fully used." + "expr": | + kube_resourcequota{job="kube-state-metrics", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) + == 1 + "for": "15m" + "labels": + "severity": "info" + - "alert": "KubeQuotaExceeded" + "annotations": + "description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded" + "summary": "Namespace quota has exceeded the limits." + "expr": | + kube_resourcequota{job="kube-state-metrics", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) + > 1 + "for": "15m" + "labels": + "severity": "warning" + - "alert": "CPUThrottlingHigh" + "annotations": + "description": "{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh" + "summary": "Processes experience elevated CPU throttling." + "expr": | + sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace) + / + sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace) + > ( 25 / 100 ) + "for": "15m" + "labels": + "severity": "info" +- "name": "kubernetes-storage" + "rules": + - "alert": "KubePersistentVolumeFillingUp" + "annotations": + "description": "The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup" + "summary": "PersistentVolume is filling up." + "expr": | + ( + kubelet_volume_stats_available_bytes{job="kubelet"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet"} + ) < 0.03 + and + kubelet_volume_stats_used_bytes{job="kubelet"} > 0 + "for": "1m" + "labels": + "severity": "critical" + - "alert": "KubePersistentVolumeFillingUp" + "annotations": + "description": "Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup" + "summary": "PersistentVolume is filling up." + "expr": | + ( + kubelet_volume_stats_available_bytes{job="kubelet"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{job="kubelet"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{job="kubelet"}[6h], 4 * 24 * 3600) < 0 + "for": "1h" + "labels": + "severity": "warning" + - "alert": "KubePersistentVolumeErrors" + "annotations": + "description": "The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors" + "summary": "PersistentVolume is having issues with provisioning." + "expr": | + kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0 + "for": "5m" + "labels": + "severity": "critical" +- "name": "kubernetes-system" + "rules": + - "alert": "KubeVersionMismatch" + "annotations": + "description": "There are {{ $value }} different semantic versions of Kubernetes components running." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch" + "summary": "Different semantic versions of Kubernetes components running." + "expr": | + count(count by (git_version) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1 + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeClientErrors" + "annotations": + "description": "Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'" + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors" + "summary": "Kubernetes API server client is experiencing errors." + "expr": | + (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job, namespace) + / + sum(rate(rest_client_requests_total[5m])) by (instance, job, namespace)) + > 0.01 + "for": "15m" + "labels": + "severity": "warning" +- "name": "kube-apiserver-slos" + "rules": + - "alert": "KubeAPIErrorBudgetBurn" + "annotations": + "description": "The API server is burning too much error budget." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn" + "summary": "The API server is burning too much error budget." + "expr": | + sum(apiserver_request:burnrate1h) > (14.40 * 0.01000) + and + sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) + "for": "2m" + "labels": + "long": "1h" + "severity": "critical" + "short": "5m" + - "alert": "KubeAPIErrorBudgetBurn" + "annotations": + "description": "The API server is burning too much error budget." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn" + "summary": "The API server is burning too much error budget." + "expr": | + sum(apiserver_request:burnrate6h) > (6.00 * 0.01000) + and + sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) + "for": "15m" + "labels": + "long": "6h" + "severity": "critical" + "short": "30m" + - "alert": "KubeAPIErrorBudgetBurn" + "annotations": + "description": "The API server is burning too much error budget." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn" + "summary": "The API server is burning too much error budget." + "expr": | + sum(apiserver_request:burnrate1d) > (3.00 * 0.01000) + and + sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) + "for": "1h" + "labels": + "long": "1d" + "severity": "warning" + "short": "2h" + - "alert": "KubeAPIErrorBudgetBurn" + "annotations": + "description": "The API server is burning too much error budget." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn" + "summary": "The API server is burning too much error budget." + "expr": | + sum(apiserver_request:burnrate3d) > (1.00 * 0.01000) + and + sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) + "for": "3h" + "labels": + "long": "3d" + "severity": "warning" + "short": "6h" +- "name": "kubernetes-system-apiserver" + "rules": + - "alert": "KubeClientCertificateExpiration" + "annotations": + "description": "A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration" + "summary": "Client certificate is about to expire." + "expr": | + apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 604800 + "labels": + "severity": "warning" + - "alert": "KubeClientCertificateExpiration" + "annotations": + "description": "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration" + "summary": "Client certificate is about to expire." + "expr": | + apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 86400 + "labels": + "severity": "critical" + - "alert": "AggregatedAPIErrors" + "annotations": + "description": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapierrors" + "summary": "An aggregated API has reported errors." + "expr": | + sum by(name, namespace)(increase(aggregator_unavailable_apiservice_total[10m])) > 4 + "labels": + "severity": "warning" + - "alert": "AggregatedAPIDown" + "annotations": + "description": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapidown" + "summary": "An aggregated API is down." + "expr": | + (1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85 + "for": "5m" + "labels": + "severity": "warning" + - "alert": "KubeAPIDown" + "annotations": + "description": "KubeAPI has disappeared from Prometheus target discovery." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown" + "summary": "Target disappeared from Prometheus target discovery." + "expr": | + absent(up{job="kube-apiserver"} == 1) + "for": "15m" + "labels": + "severity": "critical" + - "alert": "KubeAPITerminatedRequests" + "annotations": + "description": "The apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapiterminatedrequests" + "summary": "The apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests." + "expr": | + sum(rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="kube-apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) ) > 0.20 + "for": "5m" + "labels": + "severity": "warning" +- "name": "kubernetes-system-kubelet" + "rules": + - "alert": "KubeNodeNotReady" + "annotations": + "description": "{{ $labels.node }} has been unready for more than 15 minutes." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready" + "summary": "Node is not ready." + "expr": | + kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0 + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeNodeUnreachable" + "annotations": + "description": "{{ $labels.node }} is unreachable and some workloads may be rescheduled." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodeunreachable" + "summary": "Node is unreachable." + "expr": | + (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1 + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeletTooManyPods" + "annotations": + "description": "Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods" + "summary": "Kubelet is running at capacity." + "expr": | + count by(node) ( + (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"}) + ) + / + max by(node) ( + kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1 + ) > 0.95 + "for": "15m" + "labels": + "severity": "info" + - "alert": "KubeNodeReadinessFlapping" + "annotations": + "description": "The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodereadinessflapping" + "summary": "Node readiness status is flapping." + "expr": | + sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2 + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeletPlegDurationHigh" + "annotations": + "description": "The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletplegdurationhigh" + "summary": "Kubelet Pod Lifecycle Event Generator is taking too long to relist." + "expr": | + node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10 + "for": "5m" + "labels": + "severity": "warning" + - "alert": "KubeletPodStartUpLatencyHigh" + "annotations": + "description": "Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletpodstartuplatencyhigh" + "summary": "Kubelet Pod startup latency is too high." + "expr": | + histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet"} > 60 + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeletClientCertificateExpiration" + "annotations": + "description": "Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletclientcertificateexpiration" + "summary": "Kubelet client certificate is about to expire." + "expr": | + kubelet_certificate_manager_client_ttl_seconds < 604800 + "labels": + "severity": "warning" + - "alert": "KubeletClientCertificateExpiration" + "annotations": + "description": "Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletclientcertificateexpiration" + "summary": "Kubelet client certificate is about to expire." + "expr": | + kubelet_certificate_manager_client_ttl_seconds < 86400 + "labels": + "severity": "critical" + - "alert": "KubeletServerCertificateExpiration" + "annotations": + "description": "Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletservercertificateexpiration" + "summary": "Kubelet server certificate is about to expire." + "expr": | + kubelet_certificate_manager_server_ttl_seconds < 604800 + "labels": + "severity": "warning" + - "alert": "KubeletServerCertificateExpiration" + "annotations": + "description": "Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletservercertificateexpiration" + "summary": "Kubelet server certificate is about to expire." + "expr": | + kubelet_certificate_manager_server_ttl_seconds < 86400 + "labels": + "severity": "critical" + - "alert": "KubeletClientCertificateRenewalErrors" + "annotations": + "description": "Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes)." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletclientcertificaterenewalerrors" + "summary": "Kubelet has failed to renew its client certificate." + "expr": | + increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0 + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeletServerCertificateRenewalErrors" + "annotations": + "description": "Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes)." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletservercertificaterenewalerrors" + "summary": "Kubelet has failed to renew its server certificate." + "expr": | + increase(kubelet_server_expiration_renew_errors[5m]) > 0 + "for": "15m" + "labels": + "severity": "warning" + - "alert": "KubeletDown" + "annotations": + "description": "Kubelet has disappeared from Prometheus target discovery." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown" + "summary": "Target disappeared from Prometheus target discovery." + "expr": | + absent(up{job="kubelet"} == 1) + "for": "15m" + "labels": + "severity": "critical" +- "name": "kubernetes-system-scheduler" + "rules": + - "alert": "KubeSchedulerDown" + "annotations": + "description": "KubeScheduler has disappeared from Prometheus target discovery." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown" + "summary": "Target disappeared from Prometheus target discovery." + "expr": | + absent(up{job="kube-scheduler"} == 1) + "for": "15m" + "labels": + "severity": "critical" +- "name": "kubernetes-system-controller-manager" + "rules": + - "alert": "KubeControllerManagerDown" + "annotations": + "description": "KubeControllerManager has disappeared from Prometheus target discovery." + "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown" + "summary": "Target disappeared from Prometheus target discovery." + "expr": | + absent(up{job="kube-controller-manager"} == 1) + "for": "15m" + "labels": + "severity": "critical" diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/apiserver.json b/charts/kubezero-metrics/dashboards/kube-mixin/apiserver.json index 1c940dc..9830c36 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/apiserver.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/apiserver.json @@ -968,7 +968,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_adds_total{job=\"kube-apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1049,7 +1049,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_depth{job=\"kube-apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1130,7 +1130,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name, le))", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"kube-apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1224,7 +1224,7 @@ "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "expr": "process_resident_memory_bytes{job=\"kube-apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1305,7 +1305,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])", + "expr": "rate(process_cpu_seconds_total{job=\"kube-apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1386,7 +1386,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "expr": "go_goroutines{job=\"kube-apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1490,7 +1490,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(apiserver_request_total{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "query": "label_values(apiserver_request_total{job=\"kube-apiserver\", cluster=\"$cluster\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/cluster-total.json b/charts/kubezero-metrics/dashboards/kube-mixin/cluster-total.json index 22d92c3..86e147d 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/cluster-total.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/cluster-total.json @@ -1637,7 +1637,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"cadvisor\"}, cluster)", "refresh": 2, "regex": "", "sort": 0, diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/controller-manager.json b/charts/kubezero-metrics/dashboards/kube-mixin/controller-manager.json index 7d27b58..373d6ab 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/controller-manager.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/controller-manager.json @@ -952,7 +952,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-controller-manager\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -972,7 +972,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-controller-manager\"}, instance)", + "query": "label_values(up{cluster=\"$cluster\", job=\"kube-controller-manager\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-cluster.json b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-cluster.json index c598fbd..1d0c584 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-cluster.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-cluster.json @@ -2599,7 +2599,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(node_cpu_seconds_total, cluster)", + "query": "label_values(up{job=\"cadvisor\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/kubelet.json b/charts/kubezero-metrics/dashboards/kube-mixin/kubelet.json index dac28b9..4207218 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/kubelet.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/kubelet.json @@ -1905,7 +1905,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kubelet\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/namespace-by-pod.json b/charts/kubezero-metrics/dashboards/kube-mixin/namespace-by-pod.json index 51df866..cbe880c 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/namespace-by-pod.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/namespace-by-pod.json @@ -1155,7 +1155,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"cadvisor\"}, cluster)", "refresh": 2, "regex": "", "sort": 0, diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/namespace-by-workload.json b/charts/kubezero-metrics/dashboards/kube-mixin/namespace-by-workload.json index ab91864..c34f988 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/namespace-by-workload.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/namespace-by-workload.json @@ -1367,7 +1367,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"cadvisor\"}, cluster)", "refresh": 2, "regex": "", "sort": 0, diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/pod-total.json b/charts/kubezero-metrics/dashboards/kube-mixin/pod-total.json index c5b4352..e854c41 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/pod-total.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/pod-total.json @@ -921,7 +921,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"cadvisor\"}, cluster)", "refresh": 2, "regex": "", "sort": 0, diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/rules.yml b/charts/kubezero-metrics/dashboards/kube-mixin/rules.yml new file mode 100644 index 0000000..1c2f4f3 --- /dev/null +++ b/charts/kubezero-metrics/dashboards/kube-mixin/rules.yml @@ -0,0 +1,685 @@ +"groups": +- "name": "kube-apiserver-burnrate.rules" + "rules": + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET"}[1d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope=~"resource|",le="1"}[1d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="namespace",le="5"}[1d])) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="cluster",le="40"}[1d])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[1d])) + "labels": + "verb": "read" + "record": "apiserver_request:burnrate1d" + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET"}[1h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope=~"resource|",le="1"}[1h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="namespace",le="5"}[1h])) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="cluster",le="40"}[1h])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[1h])) + "labels": + "verb": "read" + "record": "apiserver_request:burnrate1h" + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET"}[2h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope=~"resource|",le="1"}[2h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="namespace",le="5"}[2h])) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="cluster",le="40"}[2h])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[2h])) + "labels": + "verb": "read" + "record": "apiserver_request:burnrate2h" + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET"}[30m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope=~"resource|",le="1"}[30m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="namespace",le="5"}[30m])) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="cluster",le="40"}[30m])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[30m])) + "labels": + "verb": "read" + "record": "apiserver_request:burnrate30m" + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET"}[3d])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope=~"resource|",le="1"}[3d])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="namespace",le="5"}[3d])) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="cluster",le="40"}[3d])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[3d])) + "labels": + "verb": "read" + "record": "apiserver_request:burnrate3d" + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET"}[5m])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope=~"resource|",le="1"}[5m])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="namespace",le="5"}[5m])) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="cluster",le="40"}[5m])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[5m])) + "labels": + "verb": "read" + "record": "apiserver_request:burnrate5m" + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"LIST|GET"}[6h])) + - + ( + ( + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope=~"resource|",le="1"}[6h])) + or + vector(0) + ) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="namespace",le="5"}[6h])) + + + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET",scope="cluster",le="40"}[6h])) + ) + ) + + + # errors + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[6h])) + "labels": + "verb": "read" + "record": "apiserver_request:burnrate6h" + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + - + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[1d])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + "labels": + "verb": "write" + "record": "apiserver_request:burnrate1d" + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + - + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[1h])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + "labels": + "verb": "write" + "record": "apiserver_request:burnrate1h" + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + - + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[2h])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + "labels": + "verb": "write" + "record": "apiserver_request:burnrate2h" + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + - + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[30m])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + "labels": + "verb": "write" + "record": "apiserver_request:burnrate30m" + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + - + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[3d])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + "labels": + "verb": "write" + "record": "apiserver_request:burnrate3d" + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + - + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[5m])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + "labels": + "verb": "write" + "record": "apiserver_request:burnrate5m" + - "expr": | + ( + ( + # too slow + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + - + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[6h])) + ) + + + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + ) + / + sum by (cluster) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + "labels": + "verb": "write" + "record": "apiserver_request:burnrate6h" +- "name": "kube-apiserver-histogram.rules" + "rules": + - "expr": | + histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"LIST|GET"}[5m]))) > 0 + "labels": + "quantile": "0.99" + "verb": "read" + "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))) > 0 + "labels": + "quantile": "0.99" + "verb": "write" + "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod)) + "labels": + "quantile": "0.99" + "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod)) + "labels": + "quantile": "0.9" + "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job="kube-apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod)) + "labels": + "quantile": "0.5" + "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" +- "interval": "3m" + "name": "kube-apiserver-availability.rules" + "rules": + - "expr": | + avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30 + "record": "code_verb:apiserver_request_total:increase30d" + - "expr": | + sum by (cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + "labels": + "verb": "read" + "record": "code:apiserver_request_total:increase30d" + - "expr": | + sum by (cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + "labels": + "verb": "write" + "record": "code:apiserver_request_total:increase30d" + - "expr": | + sum by (cluster, verb, scope) (increase(apiserver_request_duration_seconds_count[1h])) + "record": "cluster_verb_scope:apiserver_request_duration_seconds_count:increase1h" + - "expr": | + sum by (cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_duration_seconds_count:increase1h[30d]) * 24 * 30) + "record": "cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d" + - "expr": | + sum by (cluster, verb, scope, le) (increase(apiserver_request_duration_seconds_bucket[1h])) + "record": "cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase1h" + - "expr": | + sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase1h[30d]) * 24 * 30) + "record": "cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d" + - "expr": | + 1 - ( + ( + # write too slow + sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + ( + # read too slow + sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="40"}) + ) + ) + + # errors + sum by (cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + ) + / + sum by (cluster) (code:apiserver_request_total:increase30d) + "labels": + "verb": "all" + "record": "apiserver_request:availability30d" + - "expr": | + 1 - ( + sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~"LIST|GET"}) + - + ( + # too slow + ( + sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"}) + or + vector(0) + ) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"}) + + + sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="40"}) + ) + + + # errors + sum by (cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + ) + / + sum by (cluster) (code:apiserver_request_total:increase30d{verb="read"}) + "labels": + "verb": "read" + "record": "apiserver_request:availability30d" + - "expr": | + 1 - ( + ( + # too slow + sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - + sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"}) + ) + + + # errors + sum by (cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + ) + / + sum by (cluster) (code:apiserver_request_total:increase30d{verb="write"}) + "labels": + "verb": "write" + "record": "apiserver_request:availability30d" + - "expr": | + sum by (cluster,code,resource) (rate(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET"}[5m])) + "labels": + "verb": "read" + "record": "code_resource:apiserver_request_total:rate5m" + - "expr": | + sum by (cluster,code,resource) (rate(apiserver_request_total{job="kube-apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + "labels": + "verb": "write" + "record": "code_resource:apiserver_request_total:rate5m" + - "expr": | + sum by (cluster, code, verb) (increase(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h])) + "record": "code_verb:apiserver_request_total:increase1h" + - "expr": | + sum by (cluster, code, verb) (increase(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h])) + "record": "code_verb:apiserver_request_total:increase1h" + - "expr": | + sum by (cluster, code, verb) (increase(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h])) + "record": "code_verb:apiserver_request_total:increase1h" + - "expr": | + sum by (cluster, code, verb) (increase(apiserver_request_total{job="kube-apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + "record": "code_verb:apiserver_request_total:increase1h" +- "name": "k8s.rules" + "rules": + - "expr": | + sum by (cluster, namespace, pod, container) ( + irate(container_cpu_usage_seconds_total{job="cadvisor", image!=""}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + "record": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate" + - "expr": | + container_memory_working_set_bytes{job="cadvisor", image!=""} + * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, + max by(namespace, pod, node) (kube_pod_info{node!=""}) + ) + "record": "node_namespace_pod_container:container_memory_working_set_bytes" + - "expr": | + container_memory_rss{job="cadvisor", image!=""} + * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, + max by(namespace, pod, node) (kube_pod_info{node!=""}) + ) + "record": "node_namespace_pod_container:container_memory_rss" + - "expr": | + container_memory_cache{job="cadvisor", image!=""} + * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, + max by(namespace, pod, node) (kube_pod_info{node!=""}) + ) + "record": "node_namespace_pod_container:container_memory_cache" + - "expr": | + container_memory_swap{job="cadvisor", image!=""} + * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, + max by(namespace, pod, node) (kube_pod_info{node!=""}) + ) + "record": "node_namespace_pod_container:container_memory_swap" + - "expr": | + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + "record": "cluster:namespace:pod_memory:active:kube_pod_container_resource_requests" + - "expr": | + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + "record": "namespace_memory:kube_pod_container_resource_requests:sum" + - "expr": | + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + "record": "cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests" + - "expr": | + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + "record": "namespace_cpu:kube_pod_container_resource_requests:sum" + - "expr": | + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + "record": "cluster:namespace:pod_memory:active:kube_pod_container_resource_limits" + - "expr": | + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + "record": "namespace_memory:kube_pod_container_resource_limits:sum" + - "expr": | + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} * on (namespace, pod, cluster) + group_left() max by (namespace, pod) ( + (kube_pod_status_phase{phase=~"Pending|Running"} == 1) + ) + "record": "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" + - "expr": | + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + "record": "namespace_cpu:kube_pod_container_resource_limits:sum" + - "expr": | + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) ( + 1, max by (replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + "labels": + "workload_type": "deployment" + "record": "namespace_workload_pod:kube_pod_owner:relabel" + - "expr": | + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + "labels": + "workload_type": "daemonset" + "record": "namespace_workload_pod:kube_pod_owner:relabel" + - "expr": | + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + "labels": + "workload_type": "statefulset" + "record": "namespace_workload_pod:kube_pod_owner:relabel" +- "name": "kube-scheduler.rules" + "rules": + - "expr": | + histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + "labels": + "quantile": "0.99" + "record": "cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + "labels": + "quantile": "0.99" + "record": "cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + "labels": + "quantile": "0.99" + "record": "cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + "labels": + "quantile": "0.9" + "record": "cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + "labels": + "quantile": "0.9" + "record": "cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + "labels": + "quantile": "0.9" + "record": "cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + "labels": + "quantile": "0.5" + "record": "cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + "labels": + "quantile": "0.5" + "record": "cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) + "labels": + "quantile": "0.5" + "record": "cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile" +- "name": "node.rules" + "rules": + - "expr": | + topk by(namespace, pod) (1, + max by (node, namespace, pod) ( + label_replace(kube_pod_info{job="kube-state-metrics",node!=""}, "pod", "$1", "pod", "(.*)") + )) + "record": "node_namespace_pod:kube_pod_info:" + - "expr": | + count by (cluster, node) (sum by (node, cpu) ( + node_cpu_seconds_total{job="node-exporter"} + * on (namespace, pod) group_left(node) + topk by(namespace, pod) (1, node_namespace_pod:kube_pod_info:) + )) + "record": "node:node_num_cpu:sum" + - "expr": | + sum( + node_memory_MemAvailable_bytes{job="node-exporter"} or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + node_memory_Cached_bytes{job="node-exporter"} + + node_memory_MemFree_bytes{job="node-exporter"} + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) by (cluster) + "record": ":node_memory_MemAvailable_bytes:sum" +- "name": "kubelet.rules" + "rules": + - "expr": | + histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job="kubelet"}) + "labels": + "quantile": "0.99" + "record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job="kubelet"}) + "labels": + "quantile": "0.9" + "record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile" + - "expr": | + histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job="kubelet"}) + "labels": + "quantile": "0.5" + "record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile" diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/scheduler.json b/charts/kubezero-metrics/dashboards/kube-mixin/scheduler.json index d72172f..de71562 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/scheduler.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/scheduler.json @@ -887,7 +887,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-scheduler\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml index 9a2e659..ad750c7 100644 --- a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml +++ b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml @@ -18,11 +18,11 @@ binaryData: apiserver.json.gz: H4sIAAAAAAAC/+2d62/bthbAv+evINQNSAa39StpXKD3Is2jK5C2WetuH9rCoCXa5iJLKkk58bLcv/0eUpJNSVTitE7sJBywNuJLFHkePx2eqBcbCDm9Hg2iWHDnJfryrZaUMPI9poxoZTgIQoEFDQNZeAFFUOhTLlQTuLxUzYhHBe77BEoH2OdEFQ4DIt56UBTEvp+UMByNumHoCxpBeV0VjqhH9sNAsNDnuf4019enwak2sQgHRLX/ouaUzAzKXRiJBHJ6TndE0KfjD2iTEzahLkE+mRAfhf2/iSvohGwhHHgoFCPC0JgIRl2OPMojH08JlAdIjCiUYD7qh5h5CDOCBiFDNIA/x2pRsI+imEUhJxw6+NNnTi2biIcF5mHMXKI9RVJDuMtoJPuvfppDRr2TcL65qnAEl83a/PpMXre1gvNs99LrqbxOLy9nY6sNnI3jjENProUzxuzUC8+C+SQ47CbUNOZtBRVKmpz3oYA1mTcV00iVC3IuHFV4CX8mMsHIAIRXTt5p1bmTlIVnRinxfRzxnLzmKrxSTUHg9OGSntgdkS4dkzAWhR3PRg7Za+yeDlkYB6Xx523+xH5MqqvzU1DlT5qdjtvecWr5Yjbs481m60UNFrZTQ+16DdWf7Xa2iu2eeO02bmFHK/2Wu29OlJ0TFoIYjkjMnXwr4tIxVkvUKlTk5P338AyNcTBFEYERA4HCAZKGh3DB0WYfBB0usaek/oxRAQpAA9Sqg4hPORrhCUF9QgKo52eEgQrw2HUJ5wNY8anqNcBcIBKE8XD03/wcE42Qs0hvHgdU5JsMcTwkOXVIhBefZxvTqNcLKzimQVZZrOKj8My0mVKUR1JaQ987xn2St32mVu9AbYjafsFiojW6zM9/rtD5CqWN+Z2hYCvZBPsmec3bW20doogGw26iho2qurKU5tcTWgd4rORpIlcOiRApla4Vm02yVc9VXNYWG5zhYLjA4M384JWaAFJwAMpwEsLK8bIoOHIVVeW71NaBOwrAjIM9KbXryhkZFh7MtBhQaWEdx1RxBB7uE/1HDb9d/7XQBmygsa8qv7KrWqp3YPyu37oBmAA5hpx7eVFF8mDO++d7hspw1nHBNU+dQ7tYyE5BRA2aOqC+vy8tpdp/aQNbDTCBjV34Y7cjbWBjt2QDpfGoUEB5F328ZLgmWNNGp1UaSNf3Sh1VrARjxuOgvFUCsyERC+wCOY/UnHBEJToQ1kvN6Es8wdTHfepTMW3VvQuo67/66mDf/+rUkOvHHPQeCn5Jf/zqXJZ3am4rBXi1HtyCEl5ultmQI+wKtUbNUhOfDEngHc3GKw8C0qlI0dlbVCpmZpEbVjCDhz1tHdAmrMQW+g/qdJ7V60XhFzMovSjuJ/CUd/2OpljCwf75sAW44FWUodHVb7c4A9ViMfUL5TydVzfUvMzW3Uj5VKf3qTHFk+G8n/b8OQ7CPsV8PyOVgg/qY2b0c47E12MSDMVIWdVSHanqtnQyid0RIowBQPdjDxQRAVz7ZCCQH4ansLsIA7HEDMxIp1P/FemqhoYxBisqCOFF7qDKvBQeTJa+YdijyQtL/QZ+PG8NEwUrCy/sxTDY493iq5neBPbUXOPGjCUTM9WCJ6yqoUFFDaPDESiAVzWX1HZKvCnWQKe/qKeko+gwlfYKXGW+lQTzaxVYWnluuLmqOEtv3ViEkEoMUPKSGXzioZnzI9lbykUsh98u15mVAXbLAyBWPmHghwULxEhEsBE4Esv+AZwEiBwxPBE4W5dUaWfqnnfzhQJedIyTBHcTRcQ7Tlx3uf7G3g8QDP2GNpfiBNHTRK3hv627dIjK3iTmZum+sbCXcupHCb6VBEHWfRrRgTBXpk71UE72dWIbl+FTi6oeMmF4j1Jq3MvcLA08OqFejHOOzOySVdApP7NzfE55eV792D1NZM9gY7LohVykIvSlbxuGXtUmbWaXVBjN/AhTfL7Im1SVf1vwlTcRSfkOanwGWRkOX2NeeuHTHIGxW+IJjFXautzove4hPaquvxvF0pnWmg13WvoWTA1O8cVQ/TE8ywVgZ/PppsqcD3HNVPxgFkyEEZx8fQawo51yUI5B6w1tG228bW3jbdjTgm7Hbz91a28OuzbStrxI27aNtNlI272MtLVspO1nI23Svj6uUNtH6VHK8TYbZXtEUTYzdlSyBoAB4gTstoc8cBsjgmb6xZF8u+xPkQvW/bZjaTs2lmZjabcXS7vayihFlnd53nz27DktGxQ3c4hPtnf2dtqvnRvR2Xz41nXDHzX39+v7Pzp8+7rhW80X7YPOjw6/fd3wh/XmUbt9A9K5NqLZMkY0SzK99IAmj8fS8m1K07eV/AW4kdjflyUC6Sl9fcmwINvjhfDjTiObFxfKhKPLy/UPbCqE+XT8Fj1FH1N3ZQOaKwlozuURpDzi9zZaeW+ewxyKvP/suUjIS8NQmTrJiIhZQDx0RkV6EgzNt8/Ptyo4dFkY+sJiqMXQdTrSba3qSDcjoIx6fo6CZNdX//vqAEJWMxF6jpZ71/Vgr2zu942/1Bmzpa9V09eKjlHrtwBg6/8oD5XBErbiMo9Oxvg6HcCqdDeon/yqirSZMrVOUdm/isowGtIJCWYW7Jbha9fCl4UvC1/yHDWBld73GCsFNWCPFydJJ71UtV+OKBchOMbxrNfaHYTdYxg6SJfb4tCKceghBKK4zYez+XA2H64QHFS/Z6pFB08+AIeefIb/97r7v/97cHh82D202XHLy47r2Ow4mx1ns+MeZ3acsraPKz3uL+VgbH6czY8rYciC7LEu2XKFkWyozIbKbLqcTZe7z+lyV/OIzZfzr2IamzBnE+ZswtwDTJi7OZeuNH2u0bBcarnUHuEuL38uw6I7TqBbLxq7V4fGOpHZFDqbQmdT6B5NCp20miqFrgLT7jifrtG0NGZpzNLY0hPq1ufs7D7DkU2psyl1NqXOptRdnVK3hoS4dE5rPWBOM1ZZUFtzUNtZYdhsU4aiNs9CdgpgBu4bex5PglQXf4fq68EZu0kEowHMLnBVcOyX7KKSzb5sj79tbakIWda2hqQjvuOQVnbzy0t0cSHvfy/gDbYE/SH3BO154DNgmyy7rZjdwrs9oLuNkNb6PsJ9D2UtHVTaFlQsqFhQqQAVj0RiZBllXRjlQG6HBZRVB5dGcvXuOaKs80NYSClAyvaPQIpBHRZhFEO3n0UU08HWrR55XfkbhJZPfo5PCmewdwgo5eOrTfnvKtWQgVxSfimcgPUS73OLRFNDPtmyWHMjrDmGrQvcqQUbe2pmT83sqdnj5Tz7TwbYWNRasV57VagXsVB+9EQmecPsA9Ebk3HIpr3+VBBuwLeb0duq8Gz9qeydWmZLYismMSXmD4DG1vw5bKCpACD2Y/EWQCyAJF/WEmQzoxA3imcRpKq0nR+IH1kMqcKQ/ZPPKOZSkC2J2MMue9j1mBjEfjPdMohlEOVnwt4wZGEs5AbaoMet0cab2SJb3HhsuHFbx1Dr/RwP+ShqI30Ch7sjMsZ/EsaTWSa5vmCmp8mtPMxO1UBgmYdzKXVO4z5hAZGRq9lYgowjH8tfhZ9pHThJLjTZnkuDxhc5esu+YOqRAY5900dU9VoTDDqjBDE041IhU7NPRGsUp9WG6msARWvqfI8Jm6bx9zLtScMt7XBeQqF0SIrfZJ7ti3b7jcLjaAuGfT/7wnjhMbSl1NdhATbN1krzVmB2wR96ZM/0NeTZQma/Wq2PNYYdoYY+2Sobuly3xOp2vcSKblZ8w2bmyrfM29C8bhtSZ6RvFwi7Wmv+RzYTJ19bmrIsMzdONzl5Jq0i5qSbDKSj6V3vfL1q5/PerEqDrtvzDHGWvemm7CQj0c2Tlh6GeGxk/ugysbl0PP/++fyj7OHZ00aGQ9n31hPjP+8WUQAhNu+cLkkvQ1mdSZztuew4jbp20dIvGuP5z9vazw39olXXa+bQ5jS1nxteYgq/Zc8g4VsTnGvvog+8ow+s36XZ1i/mH+h3Xnj6fLO55Jbvn1C9wjifu/upg8zc897JW5RIZVIRq3d2p94h7i7GDdLp7Hg7g4Hr7ew2XvT7eDBoe/1GP2k8mbni+sbl/wFvRsHeLqIAAA== cluster-total.json.gz: - H4sIAAAAAAAC/+1d6W/bOBb/nr9C0HSLFhuPLR85ChSDnE2BaSfbpDOYbQODlmhbiCxpSCpHi+zfviR1URIp24kTOzHnQyd+lJ/Id/5IPtI/NwzD7PddP4wINt8Z3y42YwqC/0QuggIN+H5AAHEDnxF/UhIlei4m7BH+yUiovGUQuR756NNGazOnOoAAHETIhrTBPEXBBJIxjLApPAN9MPBYO0ERFOhj15FQXTvwDwIvQIwhGg3Am9am0bYs+k+vt2lYb0XWPpjwF+/lYzFeG3seRKTQBXIb8uccgMeDACDHTNru+P8v6L93XCjQcUmpt+bIh+SjQyl+5HkxBYFwfB4EHnFDSm9xIhvOQeATFHhMoEPg4fj7buG7nutfCloIgQ/587HIU4GbduB5IMRQ5FRocCotI+Q6p0GuzFjIJYVd08/trkC4SQeQfL5ln1PpZLz5GNrZx7zXFxkNwRACIgxVoH4kEHH1yJu/BNcFAfMmPA6uz11SsRyTJETzIEII+sTYB75z7TpkbBafOXN/8OfGW0JDYgkouI5tIBljJnjguQBzA+SSzEUwAJxS6Aqzp9+hPyJczK0CHeKKhqZ6izl0Pa8gaUb4gIDj0oGKmlJre7ekbas9RduWXNud7KMHR9B3im+iYhr5e/hc7trgalQePDPdWF/ySHA0CcmtvOm/EAXVlgm4kbxj4voSKnJHY3ImjTfMzCRU+uxf3KSKVsmaAsSGkI2m1HQIsV3lR2h48iQduwJeBFOzquiBxgqJFTHqddI3S6ReljySyuKMemrB4002mtPA9cmngIuDE0zBs20os+gQUqv1CRhVI1LIuDETjdj7e0V6tftUaA5EkMf3oRfkAlREEAyRC/EfVxBRi4elIWJVf3Fl4JgA+7LSG0xgGELndyrTShsBaAQJFvKhmBFZtrgJ+TCY3vu0b/YbHE3euDTUwTc0jRFAuaI+TR/XAbqkKdiG7hXsD24JxH1uET9tL6I9QO+/m6+SP7+bmyyr8YG9/99389d/fzfvvr2iooSIWsu7VzSJB17EgunF27fG4NZ4kz3/VkyOLHgEaMIFSuPhBPZjSRYfSfkeA5vwnGsVmmPXP874/PyZvezursgJwSEP4uZekc4kzHWTEe+Sv3I1kjEd1TjwnJJ6WbePaZgs2wSjn43dIak2lHLDF6oLIxga+0zoxpdYBY6QD7Is/lOMCQDxBFuOCrHrt8sO3E+zius77pXrRNTXK86cPsPBQ96BG3DjlqL4ILIvY7srhZ5J4rIVRaZAqByq4timDDrfRD2l8SyjXVSGcAtuYI075Oa2H5bMzAMD6FU6yBqC0T7gMKdoeHF0rzweh/dW0cDyCJ6b2Obqd3FDFPP6wZACoQaHdDUO0ThE45DF4RAqIx9PXKKByPKAyHmiA6KxiMYiq45FbOrmE79G2ATe8HGcU28oDoSrN2uaKpKU0Z/sa8YvewpmWfO8DPfrGe7PzfCgnuHB3AwP6xkezs3wqJ7h0dwMj+sZHs/N8EM9ww9zMzypZ3gyO8Msxyg45u01OWVmuJ2DqCFN2unq5W7rX+b8cHvqGq+lWOTtVVBfYarxCKDPANj4waD1DNAM2yjgsip0ioW0Ewgc/o1iS5wlBWHRYFaShhMjZZ6ZKjKZC9jdenUpiU/p5GHSZhO9RCaVIG6ns8BcuqldwRwQ/U3/a3z61Dg8NE5O3k0m73AppzjQdieAr9q3izmFKq+amZOGfJfDPETUTA0nuPbNymNfEZOqWWwIAcU3yJcPWYG0CvCETrocWHpb5Lt8vPS7iEz35Ezolc2C6mLQeqhCnltnUYcfTQbUxaTqYBDmIcqQAeK10sf+MvWxR2e1dEqtnUMO65atjLV3jsNl6iOduacuYZyCeE68lqo4WqwqwnupQnCI9dbG8Spoo+wYxiEK2GLmeqrkw4o6yHpr5WSZWvksX0ZYBdkXV5XvJXqnuTPYBjuDTnvL2QZbQ2vY6ra73dbWVmdrx97aBsPmZUQl6UMCcSPZhXD9USNbPWmEgYN/C9Doo/Peeo3gkCnnfaeFX18BlD/2/lW/b0Nhl6isa8VyzUOVXZ5qVncNXlChCN9ILe3MYAIk26WyTZt27abNvfdpNp//vtcS5Lq/OLmCq5E210SsB08j1rUz18OnCQOpuYYxLFoDwR49cXxdH8keL8dknRjJr4GAPyzJdNdHwif3KJJ5cDHMGQFE3ABNUXgsAFURQnogprDDJ56HKTQs7jiMaqt0S3IepmYmKKnmVFV01lV11lR2znokS1LhWVvlqZSnbPtZWvEpk20uX0ugFWXD5bwtsetqN2orQdXVoLUVoVOqQusrQ9XVoeoK0SlVoupK0SnVolMqRuurRusqR5XVo1VdSqtI1UUFisICZXHBDFWldZWlU6pL6ypMa6pM68sZ1NWmUytOa4sT5AUK6urTaRWoykWOcqB7DrPHGQtSZyhKnbMwtW7RQ5F7xfxbyMHT17XkObo2TxdzdboNOeUEjbJyta56VVHBOlMVazWwyKtZFRWttVWt9ZWt6urWmgpXdZWrvNJVrHatDrVa9Sp1QXVpaW15aV2JaU2ZqarUVFluWhnXMx3ChkxltbsBGgPKiLOAwB0NAjUI1CDwZYLAp1/r1ijwIShQWZCmgaAGghoIqhZOn/6KnEot6SKuyFnq3USW4pj47opdTpRX7564mKaN26e4nKh8zn/1rwWYru+2XN/CeO57L0CJ+BjXApSJPLCUiIu8FOBh5/uVJ73aDz/pRUGfD+3CeeeXf8a/eBZNXzX0Ik/4J0ui0hz7HE/0k+JZvfrz/KUx5PhWn9h/3C4+6PagF4gT2gpcaFkaJ2icoHGCvgpo6UAhXTXTSEEjhRW622dZZXUdRVmd1V5oXZ1sd/R5b6oWd3NqCus6U/ZUrc5CN1UlDY+9pyprkGCnR91RXdymqOwwXhVbLWZPtIqz9MaoFI899r7oip1V0Rujs22MznYrxIvcDyXly5xm2Q2VjLMKARewzxk+/33OcH0K3tYYnHVbU8BZV4MzDc40OFtZcLaM864anc2HzqZeFKUBmgZoLx6grUAhWtn5FlN+9uRLlqpNxp5eslwMKu5NQ8VbGhVrVKxR8bNZsnzCmzQ0OH7Y0qX8hkiNkDVC1kuYGqzJwNruFLC2rcGaBmsarD2fJUyN1p7TUqYGbBqwacCmAdviANuOBmzPHbDVe3wMJPY9IL39v5h5/hoDYrjYOD84Nb7AFCv8Vo0TUfxDAGNCQvyu2QS2DR0X+L/awaQJWeoOkYshbg5ozGgm2KMRQ46GF2DcQCl3jClywA3gOw0nCj3XpsmvQR/0g2sPOiM4oexw05wxMWu0+izRKgerPtUMw6m0E6R/bof9xALP4AhLgakShhpNQ87xj4jMzy0GtfG1vE+PadP3vhBIWwgs2AgiwsjA8wzMrgnGcMQdXuPbJePbONTx30N57jh35Yei8e4T4t1djXc13s3x7q0xCWhkCJBx9vdnA+WpaTroHblkHA045A0zz2hyzEFRDs0+EDUpvKXCaFqtducX/jd9miW4Rre1Y7V3Ols9jWzXD9ke3dD/HZye3foJFloQvL03YNYQd7EQl8USBcwVQowGuRrkrjnIXYE61SOEGJa+T5nqRtJzM/kBS9baacW82GN5nDGxPYYT8CdEOO5/vO5KQ9ht3AsHoMv4ezQL51Zs5r+jaWZvI3ASelQQ/ijzSgqOMBFsv3Dfjcd/EbV6TQOISCC5EYaR+3YQcQzbaZVbYs2bVkuMXwLqLcxWCLzh9tmbFGPKVdIj1iCb8swy8+BouVSy6/o02Tlwz5NddSP3A3MS0bgreTyJU2aeKMWXByGjlONNOXRDL0ZicqicSqfTqtw1mwmItalCnvptspu85apQKWPmV9UNzBor30WbNmThQUjL5j8RRLeJEDZ7k80CO8Hp2gXqCN6Ufv3KxJdu+BV5FHDZEkUnSVSsDadOyJ0G/yftgllsLUMITpM/nGXhGBqJbRGG5zEv3qdyqfk6eXH7kb1YJv55fbjGq7pqS+/OY+nd9TXxKcbnwCGgOlZYYNoqNUNJolBYUWotguHK7UWqvFBq4LkSrWlKTAUpvP5eMUEQ5Yq6YzInnUe6/E39eCbwhuGifhg4fdcfBptGwu7tfZ0ndo/W47pHPJLpvrGRguS7GO65XGQJ0BvGk1TTD64bWT40eQpgNLPwtdClczSUfzkRST91VDH0mT0BBIt5oQBPTCvPAWI+MC3xQ6cltgh3jrWFvy0nNu6LdAxsYaEaktVvERlviYzFt7S74gfhjsFtR+xv2peC+H4EfFHI/Hp+kGDzdNLwOftNe6NpHAi2bEZ8mdkcDrc6PdBq9wb2EILtgd1xnO6w19rZ3W2BDtyNH77K5gOtjbv/A4FidTvKpwAA + H4sIAAAAAAAC/+1d62/bthb/nr9C0HqHFDeeLT/yKFAMeTYF1q63STfstoVBS7StRZY0ksqjQe7ffknqRUmkbCdO7MTchy4+lI/I8/yRPKRvNwzD7PddP4wINt8YX79vxRQE/4lcBAUa8P2AAOIGPiPeUhIlei4m7BH+yUiovGUQuR5579NGayunOoAAHETIhrTB/ISCCSRjGGFTeAb6YOCxdoIiKNDHriOhunbgHwZegBhDNBqAzdaW0bYs+k+vt2VYr0XWPpjwF+/nYzF+NvY9iEihC+Qm5M85AI8HAUCOmbTd8f9/p//ecaFAxyWl3pojH5L3DqX4kefFFATC8XkQeMQNKb3FiWw4h4FPUOAxgQ6Bh+Pvu4Xveq5/IWghBD7kz8ciTwVu2oHngRBDkVOhwam0jJDrfApyZcZCLinsin5udwXCdTqA5PMN+5xKJ+PNx9DOPua9/p7REAwhIMJQBep7AhFXj7z5c3BVEDBvwuPg6twlFcsxSUI0DyOEoE+MA+A7V65DxmbxmTP3B39uvC00JJaAgqvYBpIxZoIHngswN0AuyVwEA8Apha4we/oN+iPCxdwq0CGuaGiqt5hD1/MKkmaEdwg4Lh2oqCm1tvdK2rbaU7RtybXdyT56cAR9p/gmKqaRv4/P5a4NLkflwTPTjfUljwTHk5DcyJv+C1FQbZmAa8k7Jq4voSJ3NCZn0njDzExCpc/+yU2qaJWsKUBsCNloSk1HENtVfoSGJ0/SsUvgRTA1q4oeaKyQWBGjXiV9s0TqRckjqSzOqKcWPN5ko/kUuD75EHBxcIIpeLYNZRYdQmq1PgGjakQKGTdmohF7f69Ir3afCs2BCPL4PvSCXICKCIIhciH+/RIiavGwNESs6i+uDBwTYF9UeoMJDEPo/EZlWmkjAI0gwUI+FDMiyxbXIR8G03uf9s3exNFk06WhDm7SNEYA5Yr6NH1cBeiCpmAbupewP7ghEPe5RdzaXkR7gN5+M18lf34zt1hW4wN7+79v5i///mbefX1FRQkRtZY3r2gSD7yIBdPvr18bgxtjM3v+tZgcWfAI0IQLlMbDCezHkiw+kvI9ATbhOdcqNMeuf5Lxub3NXnZ3V+SE4JAHcXO/SGcS5rrJiHfJX7kayZiOahx4Tkm9rNsnNEyWbYLRz8bukFQbSrnhM9WFEQyNAyZ043OsAkfIB1kWvxVjAkA8wZajQuz67bID99Os4vqOe+k6EfX1ijOnz3DwkHfgGly7pSg+iOyL2O5KoWeSuGxFkSkQKoeqOLYpg85XUU9pPMto3ytDuAHXsMYdcnM7CEtm5oEB9CodZA3B6ABwmFM0vDi6Vx6Pw3uraGB5BM9NbGv1u7ghinn9YEiBUINDuhqHaByiccjicAiVkY8nLtFAZHlA5DzRAdFYRGORVcciNnXziV8jbAKv+TjOqTcUB8LVmzVNFUnK6A/2NeOnfQWzrHlehgf1DA/mZnhYz/BwboZH9QyP5mZ4XM/weG6GJ/UMT+Zm+K6e4bu5GZ7WMzydnWGWYxQc8/aanDIz3M5B1JAm7XT1cq/1L3N+uD11jddSLPL2KqivMNV4BNBnAGz8YNB6BmiGbRRwWRU6xULaKQQO/0axJc6SgrBoMCtJw4mRMs9MFZnMBexuvLqUxKd08jBps4leIpNKELfTWWAu3dSuYA6I/qL/NT58aBwdGaenbyaTN7iUUxxouxPAV+3bxZxClVfNzElDvsthHiFqpoYTXPlm5bEviEnVLDaEgOIb5MuHrEBaBXhCJ10OLL0t8l0+XvpdRKZ7cib0ymZBdTFoPVQhz62zqMOPJgPqYlJ1MAjzEGXIAPFa6eNgmfrYp7NaOqXWziGHdctWxto7x9Ey9ZHO3FOXMD6BeE68lqo4XqwqwnupQnCI9dbGySpoo+wYxhEK2GLmeqrk3Yo6yHpr5XSZWvkoX0ZYBdkXV5XvJXqnuTvYAbuDTnvb2QHbQ2vY6ra73db2dmd7197eAcPmRUQl6UMCcSPZhXD9USNbPWmEgYN/DdDovfPW+hnBIVPO204L/3wJUP7Y21f9vg2FXaKyrhXLNQ9VdnmqWd01eEGFInwjtbQzgwmQbJfKNm3atZs2996n2Xr++15LkOvB4uQKLkfaXBOxHj6NWNfOXI+eJgyk5hrGsGgNBHv8xPF1fSR7shyTdWIkvwYCfrck010fCZ/eo0jmwcUwZwQQcQM0ReGxAFRFCOmBmMIOn3geptCwuOMwqq3Sbcl5mJqZoKSaU1XRWVfVWVPZOeuRLEmFZ22Vp1Kesu1nacWnTLa5fC2BVpQNl/OOxK6r3aitBFVXg9ZWhE6pCq2vDFVXh6orRKdUiaorRadUi06pGK2vGq2rHFVWj1Z1Ka0iVRcVKAoLlMUFM1SV1lWWTqkuraswrakyrS9nUFebTq04rS1OkBcoqKtPp1WgKhc5yoHuOcweZyxInaEodc7C1LpFD0XuFfNvIQdPX9eS5+jaPF3M1ek25JQTNMrK1brqVUUF60xVrNXAIq9mVVS01la11le2qqtbaypc1VWu8kpXsdq1OtRq1avUBdWlpbXlpXUlpjVlpqpSU2W5aWVcz3QIGzKV1e4GaAwoI84CAnc1CNQgUIPAlwkCn36tW6PAh6BAZUGaBoIaCGogqFo4fforciq1pIu4ImepdxNZimPieyt2OVFevXvqYpo2bp7icqLyOf/VvxZgur7bcn0L47nvvQAl4mNcC1Am8sBSIi7yUoCHne9XnvRqP/ykFwV9PrQL551f/hn/4lk0fdXQizzhnyyJSnPsczzRT4pn9erP85fGkONbfWL/cbv4oNuDXiBOaCtwoWVpnKBxgsYJ+iqgpQOFdNVMIwWNFFbobp9lldV1FGV1VnuhdXWy3dHnvala3M2pKazrTNlTtToL3VSVNDz2nqqsQYKdHnVHdXGborLDeFVstZg90SrO0hujUjz22PuiK3ZWRW+MzrYxOtutEC9yP5SUL3OaZTdUMs4qBFzAPmf4/Pc5w/UpeFtjcNZtTQFnXQ3ONDjT4GxlwdkyzrtqdDYfOpt6UZQGaBqgvXiAtgKFaGXnW0z52ZMvWao2GXt6yXIxqLg3DRVva1SsUbFGxc9myfIJb9LQ4PhhS5fyGyI1QtYIWS9harAmA2t7U8DajgZrGqxpsPZ8ljA1WntOS5kasGnApgGbBmyLA2y7GrA9d8BW7/ExkDjwgPT2/2Lm+XMMiOFi4/zwk/EZpljh12qciOIfAhgTEuI3zSawbei4wP/FDiZNyFJ3iFwMcXNAY0YzwR6NGHI0vADjBkq5Y0yRA24A32k4Uei5Nk1+DfqgH1x50BnBCWWHm+aMiVmj1WeJVjlY9almGE6lnSD9czvsJxZ4BkdYCkyVMNRoGnKOv0dkfm4xqI2v5X16TJu+94VA2kJgwUYQEUYGnmdgdk0whiPu8BrfLhnfxqGO/x7Kc8e5Kz8UjXefEO/uabyr8W6Od2+MSUAjQ4CMs78+GihPTdNB78gl42jAIW+YeUaTYw6Kcmj2gahJ4S0VRtNqtTs/8b/p0yzBNbqtXau929nuaWS7fsj2+Jr+7/DT2Y2fYKEFwdt7A2YNcRcLcVksUcBcIcRokKtB7pqD3BWoUz1GiGHp+5SpbiQ9N5MfsGStnVbMiz2WxxkT22M4AX9AhOP+x+uuNITdxL1wALqIv0ezcG7FZv47mmb2NgInoUcF4Y8yr6TgCBPB9gv33Xj8F1Gr1zSAiASSG2EYuW8HEcewnVa5Jda8abXE+CWg3sJshcBrbp+9STGmXCY9Yg2yKc8sMw+Olkslu65Pk50D9z3ZVTdyPzAnEY27kseTOGXmiVJ8eRAySjnelEM39GIkJofKqXQ6rcpds5mAWJsq5KnfJrvJW64KlTJmflXdwKyx8l20aUMWHoS0bP4TQXSTCGGrN9kqsBOcrl2gjuB16devTHzhhl+QRwGXLVF0kkTF2nDqhNxp8H/SLpjF1jKE4DT5w1kWjqGR2BZheB7z4n0ql5qvkxe3H9mLZeKf14drvKqrtvTuPJbeXV8Tn2J8DhwCqmOFBaatUjOUJAqFFaXWIhiu3F6kygulBp4r0ZqmxFSQwuvvFRMEUa6oOyZz0nmky9/Uj2cCm1F4+3cwoLNbGziXLjVvOrvdMhKur+/rQ7GXtB7XS+IBTXeRjRQr38Woz+WSS/DeMJ6rmn5w1cjSoskzAaOZha+FLp2qofzLiUj6qb+KEdDsCVhYTA8FlGJaeSoQ04JpiR86LbFFuHqsLfxtObGNf0/HwNYXqpFZ/RaR8bbIWHxLuyt+EK4a3HHE/qZ9KYjvR8DXhswv54cJRE/nDh+zn7Y3msahYNJmxFebzeFwu9MDrXZvYA8h2BnYHcfpDnut3b29FujAvfjhy2xa0Nq4+z91+P/u0acAAA== controller-manager.json.gz: - H4sIAAAAAAAC/+1d3W/bNhB/z18haBuQAG4nf7XxgD6k6VoUa7a09baHrTBoiZa1SKRKUomzwPvbd6Q+TEl04rpt4jR8Maw7fhzvjnc/nWnpas9x3MkkImkmuPuT89eHTk5h+GMWMazRECFUIBFRIolXQAJiHHGhmsDlUjXDQSTQNMZAnaGYY0UMCRavAyCRLI5zCkPpfExpLKIU6J4izqMAH1MiGI15rX9U6xtH5EwTjOEZCDoHgtv3uJvT6IVqoaTMZQWqT+MYpbwmW40RtDgpIjheDaUPl/dE/hyPowTTTGgyrvg0puw58s9CRjPSGn/V5g8UZ3g9uy6Con/XG438wRO3UyezcIr2e/2nHafbG3WcgddxvMeHo4Nmu++CwQD1katRP9TmDZBAnGbMl2K5p4wmWMxxxmsDuTPKEiSX7hJKcJ0XoizElbNU5AQtytV2Pa8hVhKRktlk8Tm9MGkIWGIuXYDGwRs0xXXnMbU6QewMK50KlmGt0bIuP4uCU6rcvc5Q/tirk4jA7BzFJieoO6ymhzSNSDi+TJUq1vHapq/rE1oTlCgjnUvNOYI6Ai9Ew+BOwZZT1RjLzmaDM0TCDQbv1Qdf617gBS/Aw04paI63XcGVWlTMExqo+X1KCPYFbNJWu7GUyKD4lHIxixayt2tivIRg8z76Vw0/9H5otIHAYuyr6Nd2Vao6gYhys+lmsK/U7gHZ20oV+cLcX388MjBp1XFDnXMIZy3PBSI7Axc17NRZFMfHMvwo+8vA0u9CXOkewsfhSAaW7mErsMykQOYNKGfRx8uH60GI6o76rYH0/b52j6pkA2NmCWmbSiAWYrGBFfAiVTLxLNnP0is/zjhs52d/u98XX/92O84/dAqUs2yKH/l5looxe5QggkLZYHnQNtEqOgrIEROOWYR5u1kZPF4iXyjl9FpNYhxiErysxmsPAm6pcqx7tKk7VPGQG1QXCZXE3d/TBqNK2ldNcyG2yqDrDZaHO5dDeIsxB0hRH1/FEX13HTZ3l2qx2e6iUk732SdurDKUfdLeUp1+LWIlZLFVP239NeyA4gjx4zK7N1LMFDFjGoO0zOdvMAnFXAXNFg+v67ZRNo/U3u22iK8YCiJMRDMt35Aj+/U8qHy47TmgiJAc8XGBG2VOboQCdB6uCSl+xlgul6Eb5Jg13aSBzBwWhXPwvWCNJEVQMnGgz59RoOzSzERq3wi0Li4q37kRjsjoyQ1zK8ZFMXN3E+TRyq2t7OOmGFyFCBSaQWkqe0ufyOTwwzbP7IVgqgAzrGLtLKaNrc9wipExkeeB87dzzMDdsGFFkMR8vG5bFGmvSRUAy41SQsBPUxy8yXNim79VWmFI4P0Lys4+ZjjDExQEfKJ8Yrts03EiAisgPn72H3QsLyAN/TVMPhwcONNLZ78YbtW240g0d6uJ6qpc33LpXF2VgqgLKcty+cUzWcM15IJe5iir5VeS934ezYSZWaTAP8FqzltpNucoCJx3YMmtcmIzYFAmDLc5KhhMyjQZkSA6j4IM1RKROaWqm+q6ZAu0iHhbrmnmn+UebIhUSREUpHKamKy4GTD0Wh8Yq+imygTmJVyixSY3Ois3panBPWN592cUTzJp+Bzx1q2WliiM3fJMYWRpS/6kO6p7sgp9t+01qdUeM0ftgvoa9r2qGZnZ7+hFrSxUyTMutl69GFNtyBeAc6YUMdiLIHudX8LG+RONUWwQBq33NAt9zcrQA0B3A4vuLLrbGXTX2yF0F+BUzC2wu0fA7oW0mEV1d4zqYDombhMReV8B1O3yIiyms5juWkw3tJjOYjqL6eYRFxQybzL5mCEiohjve49Ho45jAHsF5MvyqAgQC2BdwCd5Nr59ENhxYnxgkeDWSPANWJf4lxYL3jUW/Abqe7a6Z5HgfUWCT7ZBgqYouAkUNCK+z8SCxoj8VcHg9ScfLBr8PDQ4uPMCHyAQMfFjuZnUCVm4LH/E3RbQdXwwoaT1Hj+u4N1tgrfeYvEJ8KyzQxrr35HG+tdq7PkOa2xwRxobXKux4x3W2PCONDa8VmMvduOm6RfQq3N0+tp5l2vdnoywJyMe8MmIh3vM86m9VbC3Crt0q3C4u3VjA1q5pdpxxwGlyq6nv70f10vJktFxMhbfRfFYTq6KxTD/fagVQxwUFeQpqsXOaCTmztvC5hYDPaTasXffC8f2/ICtGn8pKHhozw/Y8wP2/MC9wIGvfrYwcHsY+ApbFGhRoEWBFgVaFNhEgSNbELQFQXt24Cf5tBjqY84B4kknAayX4ISyy8n0UmC+JbBbdzb0dvHa6pzn7mO1E6V0C8juGJApp/8Gfpzc8XXYnyfraKQhn4UjFo48VDiiSk8lJvHTrKo0fc6jaK79r4oFJetAyfHp707GpVtbXGL/dvwA/3b8gBFJ1yISi0gsIpF5hk5CymgmpAFtQeS2fryqVG7Bx0MDH1/rv667vY5v+TervWIFLvfnOEF/YMZzKbuDnCwu86kCxM7yNyRA8lp5qQqtjGBZ1arGEjhJY1guCdvve9hreoOGNmpYrnzOcoBnKItNj3rWuSZoqN4KUQ8ua3yqelK9huk0Lk3Ll1fo0dT9mGF2WVTq29hPe7tEt0YNcfPR8JVdtOn3GsvRFIbiuHzRQWMZmip1PWyAVEtdadkKwi4k0gAfmR7KXinSLZKtPlYCFokMfUotG7rcpGI13SSPovvS5yYpDSYRmdGOUwx3YFZ+7yblFylINxK4uNIwf1vO79a5LUElzdy4MG2+Eo2RcTzOB9Lh6W3b21tn73oOW7dvbrJ0CWy2NvUXLjgtV0ebvg132SuzUv4OHQVXqphbvSGCXjzqlqCofPlDngJW3dII4BBbdS5UMikBrY5M3OHKl9yup1309Ytusvo+1L539Yu+p3NW0M3tad+7QR4QP5RrkBBcc6QbZ9EHfqIPrM/SG+gXq7eFuE8DXd5Slpr6/qUkf+nB+LhIk1WxrvJD5yT3w7xBpu7n3ac97GFvOMWzoTca4eHMG4ym3nAW+IPeCAeDvPF5lZi9veX/fzSxfHxpAAA= + H4sIAAAAAAAC/+1d3W/bNhB/z18haBuQAG4nf7XxgD6k6VoUa7q09baHrTBoiZa1SKJKUomzwPvbd6S+KIlKXHdNnIYvhnXHj+Pd8e6nMy1d7VmWPZsFcZJyZv9k/fmxl1Eo/pQGFCs0FMeEIx6QWBCvgATEMGBcNoHLtWyGvYCjeYiBukAhw5Lox5i/9oAUp2GYUShKllNCQh4kQHckcRl4+JjEnJKQ1foHtb5hEJ8pglG8AEGXQLCHDrMzGrmQLaSUmaxAdUkYooTVZKsxvBYnQTEOq6HU4bKeyF3iaRBhknJFxopPQkKfI/fMpySNW+NXbX5HYYq72XURJP27wWTijp7YvTqZ+nO0Pxg+7Vn9waRnjZye5Tw+nBw0233njUZoiGyF+rE2r4c4YiSlrhDLPqUkwnyJU1YbyF4QGiGxdDsmMa7zfJT6uHSWkhyhVbHavuM0xIqCuGA2WWxJLnQaAhZfChcgofcGzXHdeXStThA9w1KnnKZYabSuy08D75RId68zpD8O6qSYY3qOQp0T1B1W0UOSBLE/vUykKrp4bdPX9QmtYxRJI50LzVmcWByveMPgVs4WU9UY695mg1MU+xsMPqgP3ule4AUvwMNOCWiOtV3BFlqUzBPiyfldEsfY5bBJW+2mQiKN4hPC+CJYid62jvESgs2H4B85/Nj5odEGAou2r6Rf21Wq6gQiys2mW8C+krsHZG8rlWcLs9/+eKRhkrLjhjpnEM5angtEegYuqtmpiyAMj0X4kfYXgWXYh7jSP4SPw4kILP3DVmBZCIH0G1DMoo6XDTeAENWfDFsDqfu9c4/KZANjplHcNhVH1Md8AyvgVSJlYmm0nyZXbpgy2M7P/rK/z7/+Zfesv8kcKGfpHD9ysywVYvooQjHyRYP1QdtEVXTkkCNmDNMAs3azIni8RC6Xyhm0moTYx7H3shyvPQi4pcyx9tGm7lDGQ6ZRXcBlErd/SxqMMmlfNc2FaJVBuw2WhTubQXgLMQNIUR9fxhF1dx02d5dssdnuIkJO+9lnbqwilH3W3pKd3uaxErJY1U9Zfw07oDBA7LjI7o0UM0dUm8YgLbPlGxz7fCmDZouHu7ptlM0DuXf7LeIrirwAx7yZlm/IkcN6HpQ+3PYcUIQfH7FpjhtFTm6EAnTud4QUN6U0k0vTDXJMRzdhID2HBv4SfM/rkCQPSjoO9Pkj8KRdmplI7huOuuKi9J0b4YiInkwzt2Rc5DP3N0Eerdzayj52gsFVYo58PShNRG/hE6kYftzm6b0QTOVhimWsXYSksfUpTjDSJvIscP56jim4G9asCJKYi7u2RZ72mlQOsFwrJQT8JMHemywntvlbpRWKON6/IPTsU4pTPEOex2bSJ7bLNj0riGEFsYuf/QsdiwtIQ3+Oo48HB9b80trPh6va9iyB5m41UV0V61uvraurQhB5IWRZr//3TNZwDbGglxnKavmV4H1YBguuZ+Yp8A+wmvVOmM068jzrPVhyq5zYDBiEcs1tjgwGsyJNBrEXnAdeimqJSJ9S5U11XbIVWgWsLdc8dc8yD9ZEqigPCkI5TUyW3wxoenUHxjK6yTKBfgmXaLXJjU7lpiTRuGco7v604gkm8Z8j1rrVUhKFtluWKbQsZcmfdUd1T1ah7ra9JrXcY/qonVNfw76XNSM9+z25qJWFSnmm+darF2PKDfkCcM6cIAp7EWSv8wvYuHyiMPINQqH1nmKhr1kZegDobmTQnUF3O4PuBjuE7jyc8KUBdvcI2L0QFjOo7o5RHUxH+W0iIucrgLpdXoTBdAbTXYvpxgbTGUxnMN0yYJxA5o1mn1IU8yDE+87jyaRnacBeDvnSLCoCxAJY57FZlo1vHwT2rBAfGCS4NRJ8A9aN3UuDBe8aC34D9T1T3TNI8L4iwSfbIEFdFNwECmoR3xdiQW1E/qpg8PqTDwYNfhkaHN15gQ8QCJ+5odhM8oQsXBY/4m4L6HoumFDQBo8fl/DuNsHbYLX6DHjW2yGNDe9IY8NrNfZ8hzU2uiONja7V2PEOa2x8RxobX6uxF7tx0/QL6NU6On1tvc+0bk5GmJMRD/hkxMM95vnU3CqYW4VdulU43N26sQat3FLtuGeBUkXX018/TOulZMHoWSkN76J4LCaXxWKY/z7UiiEO8hLy5NViazLhS+tdbnODgR5S7di574Vjc37AVI3/Lyh4aM4PmPMD5vzAvcCBr342MHB7GPgKGxRoUKBBgQYFGhTYRIETUxA0BUFzduAn8bQY4mLGAOIJJwGsF+GI0MvZ/JJjtiWw6zobert4rTrnuftY7UQq3QCyOwZk0um/gR8nd3wd5ufJOhppyGfgiIEjDxWOyNJTgUncJC0rTV/yKJpr/6tiQEkXKDk+/c1KmXBrg0vM344f4N+OHzAi6RtEYhCJQSQiz5CZTyhJuTCgKYjc1o9XpcoN+Hho4ONr/dd1t9fxLf9mtZevwGbuEkfod0xZJmV/lJH5ZTaVh+hZ9oYESF6Vl8rQSmMsqlrlWBxHSQjLjf32+x72mt6goI0aliues+zhBUpD3aOeVa4OGsq3QtSDS4dPlU+qVzCdwiVJ8fIKNZran1JML/NKfRv7KW+X6NeoPm4+Gr60izL9XmM5isJQGBYvOmgsQ1GlqocNkGqhKyVbQdiFROrhI91D2UtF2nmyVceKwCKBpk+hZU2Xm1Qsp5tlUVQ8Z/2mJ6r3rHySA71JBjeZJE9MqunA8aXe2btCKrvObYkvaPrGucGz9SmMlOFpNpAKWm/bC5wuL6hntq7ddJP9C7jzJQ6w3YP2q5NN34Zf7BVJKXuFjkQrZcgtXxBBLh71C0xUvPshywBVtyQANESrzrlKZgWeVYGJPa6cxu47ysVQvehH1fex8r2vXgwdlVMhN3ugfO97WTz8WKxBIHDFY26cRR34iTqwOstgpF5ULwuxn3qqvIUsNfX9Q+LsnQfT4zxLlrW60g+tk8wPswapvJ23nw6wg53xHC/GzmSCxwtnNJk744XnjgYT7I2yxudlXnb21v8BquF9oXtpAAA= k8s-resources-cluster.json.gz: - H4sIAAAAAAAC/+1daW/bOBr+nl+h1XYHydZpLMd2DqBYNE4zLdBOM23awaItBFqibSG6hqJyNMj+9iWpizrsOLYTy/abTzEpkdR78dHLR+TdlqKoyHU9iqjluYF6rNyxIlZoWwFlv77/ZD/vG/wybFoU9W3MSikJsSgbupi+N1mJG9p2VEKQP7rwPJtaPitvisKRZeKe51Li2byLAbKD6H7bci8D0Y34SfCA4GDECtT9ZqBGZd61uEIMKxocKzU820Z+gOXWRMUIW8MRH7mqNZv+jZpW+MjFdtaS3JqoR7aFgp5ne0RI4b4hV/YRCYpdiQoTBaMP2B1SPmitWarD426jKPBCYvAHUM+J52A6wmGg5q4aWLbNm80XesRB4gF9zO53aehaNH+fZZbuslyKyRWyhWCc/OU2HmLXTFWfSeRqWDV4Lv2QENbzmFoH3YyrsdwxNcHIu5YMS6qhzDbtMXexBwozCUt1ee0xKxNXFVsXFdeWGSmvWCMZZlrMzfzcY8L86JlCdbxAQYHyCxMvL9VYO2iIKy3A560QZFoh76ZTrqs2HCZ2ExNMeN8D2ysoPsDEwsGnK0wI87iK8Qc+MvA4e2WVXD2tfCFFxmXlUAKKfR+bH5gMK+spIkNM8x5X9DpxIb7xxfNoyq7CbG6bIIq3XSZg3fBDPcCG55qBLuzgzmHFr38wC7fxD7WhGHbIhkFYyYv43x/q/fcXus7b0BOj/7mzozaKvWZ+RC0H65HoypdZLpOAMPWybUp+dYYM6pGi9GKVDUSIVN+ouap76VdeS3TEo6Bnm1x46kGzcdjM65mP+IwFDSnw5uq+jKwBra6kIoCrvfOvyldqsSgvAn+h+TR83xW9FBFsjnNhj9Ak4hd9VKe3vujXck3ryjJD5tJj3TW5NrDcoY2Z+AtWfoNurKA8uH5oXEb2VnxsHnlif+XSKShZdZGDq+8aH5XSyCMmyOrnuEU3eArzzwyRdUdo2QRt1Md25QB5pTc8QWIe1Ep1USCuvC2KxM1SufTEeWNt1PYhKqvi5yjOCwWv26p4vLUGBS2Y+GHir93EH4TONg/CYpB80j++DPtY9z1TZ3M/RawjorMpUfgG++dvpnsaHLPb7ioBwI6yp/A2RSsCSPBJJAx0ZNuegcRbzF3SHruV9cjARHVTABzKwOFzrAGl5zmORR0eEQBAAIAAALH2AGIfAAQAiBUHELbFJi2AD0uDDx+E/AE8AHgA8LBR4KEN4AHAQy2XHfh0fyymegc7HrnVP2LnzRWybD7X6/1biqfAC4X7L7gFRffeATiYBhx8FLKDdQlABoAMNgsZdAAZADKoeVohmtmfeWki6hTSC49DELBAAUgCkMRGIokuIAlAEquPJBa9RgE4YhYcASsVgCIARcgoYqvgkMx9fYyKzhSXvmfOH6XxKqs/e9e5b0fSAV3Ebpj/siN1zncYmdG8l6/7Yv0S9aNu5DOxLh7zzUirsy7fjDTL0OgAoFEJGjUBGkXQSKvGRiXhPQ004gAmw0c5THScoSP+OUgYMNnmPwrhKEm3+Bcf47BS/1bJ4Nc8YOdBNBN51Vna3t1d2u/9vTrmcibMy7EBmslbKG0WcFSwgwV9NsI1UEsgJL49BAwEGGiNMJBsTzk/BPAzZeZIYJ9DwD6QFnok9nl8Xiigt/Y0YVy4Dn+gi/IUExk7ziDEf9nf7sePu6enyrt3x45zHFRAEx9RFmjc8W0mc+TIMk3sqo8K2Olwzz2zom+Dx4DYPCoDrJEEibxuZn1WExuWg0RoKk9KdoSlKrNRvO5C4NMTG7mX1S6WXJdCGvWUsLCimN61q1BP8StlwG/5SsTn9a/2zL3DDup0W82DQ8McHLC/I4yQhkyzvW8Y2NDw3uVhsJuk54LdFCP+5wqR3SzWvX6R/f8br0rwbYJuRWF69+sXum5g29a1iQbyjccQ5Z9vKoxkPHjMGZEbOn3mkaV6kZNP5/XZbOwvj1zaHgJDux4viIK1ocODQb9pHmlH2Ghi1Bl0O+19rdXGWtdot1CnYG1pw8uxu5N62l31q82y7a41zu4mGNQ8hjfR1qbRbq++2k1YAaDgeRR8Wn8FK/8CFc+j4rdPp2J58X52RUfrcqDkeZR8Vl8/jpddwYvnUvDvdfbiPxLsuBo63ujXyYnG5o5XZB3CSc3NSxLk3qt/780uw4ASyx0+Tobj15BmWcNL2UzMsDFZwEqc2GvyCQhHxSU6dRIladr1uMZ0gjK80KXbfJO7bLUzeR3nwjvOS/GYYLEWMkmaye0NJZPrqsn5ZNFyrtmicg1E3HsSES9006bVEujp6tts8t3qBmvx7XLd4mGa72qJ82wDnWLddPi7ugIkpD9Dptt6kZAIcgOuyWo9pki9ogpISkBSWk+SUsFPgao0DU/7CLhKwNOuJU87Qz/xNjckqN7bpqGkl/6DVQAJe5H4J9kmh2NdZft6z1MMZIwKIgVO9vPDHbHXE8CdDYU7kVsC1nkMLbvQE2Ad4GUDLxt42cDLBl428LKBl1182wFCWE2p2dGbz1wKBnZ2zdnZC9QxUDvXmKCd2zsJ9FxTjvYCvBlo2kDTBpo20LSBpg00baBpA017Nnbfc60prz8He0Eb1AMNe5E2WaKTbqSW3i7d8oFpXX+7Byb1MzOJ0nwck2v1qx6wqoFVDTSj+tGMysslwDeahm9UKIvDvTgG0lGBiwRcJOAizclF6kVewAKUga0rrJwg14ysCZYJ5lgmeEJW0IkfzKnrC47SGHYHZS9G2Sd1VPZnpgPFGySObSrnKELSoOp6knL8uVWd+DUFbdeenuMv3LGVU+JxKAMqrytX5ykcHLRec+bO7FoHNgewOYDNsU5sDrGvjbRs42LKiQY6iaZyXbD8on1wxqzjZIb3vx/qq5c/1PvvzAJ5s3qSHfu5s/GEkEmypvEcuqnCPnlOYSeG7UdgZfOk3VuKaW+suE+XadxmhMQ3T+pvl2rkGyv2s9rv/Bbnm/+IVFfHoyiBrwB8BeArTO2zwFoosRYqdoQrLD4DNQG2hJOoCd0l7ghXxwQE7B43mfM5iZcB28Y9Pz7iS/KAjpaAjhprgQ32ARsANlglbLDUhDmAg0ng4AEiH6ADQAeQO3m+3EnZCyFfMhUmagMmAkxUN0zE95OAfMmqQaI3zNj4Rvu9RGcZNsp9Nnuc8mcBMgFkgoTK6oKHDoAHAA+rBB4gobIG6EH6FAMABAAIyLksKecyrcdCSuZxqKoLqApQ1SpSWJbDfQdYNZHEMs1eBICcnh85+YCcIPUyO0g4AJAAIGEluSyAEuqLEh7axgaAAgAFSLE8X4ol8cuiL0ImZSqQdAggCUDSKmdSlvOBNWClWTIqlduBAWACwASZlZUCDUcAGgA0rHRmBVDDSmVYADgAcIBMS70yLSWfhIxLdlW2ZeiuNi2sajUBVgGsqhusYm/w9jbDVgznyCfybReQ1iDQCUZmumaVP9GvoVSewfe94/zcUV4q5bauiSWxj6dvbAcA2OwA7P2n8y/bn7kWX/4l5L8DeAu2u4NUzQqnaloaYArAFDVM1UwLJ3JfIS0GVMzYJCCLObaCGREvHI7OQyrwRQQvAF3Ap0mQzVlWNucLC0X806T3nyCB85hTf1uwfW4ZUW3Gyb7RVFNQNHMHVtguCJc1YJQD2z2cFPxUJwVnuYudWp0YtavB4cALOQdMKLgqLQUartGJwHNrWLgwe3sDTdf8QOB5NB2/Dvrx6+AOHA86j5JP63jAt6Tiipd90HF9Dv5diI65GydRG1Q9l6rP6qhqOO0XTvuF037X6LTfJ6NzbOR5vg9Lc2ZCyyae2LsiXKNNPN73iddtN/IM36mjx7oK9e3qGeqzEwzgSN/pj/StWNusBcUAzvMFgiOQECpJCMqucmrxt6J+KHp5gJOwFY9bDYwRdtA3TIJobNFpLdFyJ7/DRORSNMa8aphZpnoZsndYF7NAraZtUez4NntId5h6GnuzD6hkz5kRSOv4OTIDxTfCJkw8QKFdWGMWLiTXVtFN+UIozlvqGFNK3FgmNUi1ns9FWYy06t8hJrfiY64q8gMP9zxG5w2TlQ7xTWF+SAOW1P1W4XEkgSHb/hY/f+ExHhDlGBlWC28KfkciYGkyZPGbzcYmfmNX0CXGSd9hGrQqLk+0Es/vj1GJ6EmPIu22ywK4bvihHmAGI5L3nRRC7FRrrfWQ1uL5TNYu8w2hmuDPZCBqvrY0Yl5WfXFsE9EjSRVhgC+ihuQIFUUhHpXuIx+0hOxi7xtEcEB1vetdLWHlsWk8LlNzt/kWmwxJdnMskvRjXXleUjvSTkFaU/qxL//QnOz/jvS/Jv/Yb8o10vFaLel/zYzM9WfyDJwwKlnDg73IDXflhuVeWm35h/Rp3oEpjzcZS058vzxBHVG/XvTigJnCKs/xQ4qVz0nGUtlTepJtq6FgRqksph12B6ZmNg2tpaFWt91u97v7XbQ/6DSP0GF08VUaq5tb9/8Hdtob7zQsAQA= + H4sIAAAAAAAC/+1daW/bOhb9nl+h0XQGydRpLMd2FqAYNE7zWqB9zWvTPgzaQqAl2tZE26OoLA0yv31IaqMWO47txLJ98ykmJZK6G48uj8i7LUVRket6FFHLcwP1WLljRazQtgLKfn3/yX7eN/hl2LQo6tuYlVISYlE2dDF9b7ISN7TtqIQgf3TheTa1fFbeFIUjy8Q9z6XEs3kXA2QH0f225V4Gohvxk+ABwcGIFaj7zUCNyrxrcYUYVjQ4Vmp4to38AMutiYoRtoYjPnJVazb9GzWt8JGL7awluTVRj2wLBT3P9oiQwn1DruwjEhS7EhUmCkYfsDukfNBas1SHx91GUeCFxOAPoJ4Tz8F0hMNAzV01sGybN5sv9IiDxAP6mN3v0tC1aP4+yyzdZbkUkytkC8E4+cttPMSumao+k8jVsGrwXPohIaznMbUOuhlXY7ljaoKRdy0ZllRDmW3aY+5iDxRmEpbq8tpjViauKrYuKq4tM1JesUYyzLSYm/m5x4T50TOF6niBggLlFyZeXqqxdtAQV1qAz1shyLRC3k2nXFdtOEzsJiaY8L4HtldQfICJhYNPV5gQ5nEV4w98ZOBx9soquXpa+UKKjMvKoQQU+z42PzAZVtZTRIaY5j2u6HXiQnzji+fRlF2F2dw2QRRvu0zAuuGHeoANzzUDXdjBncOKX/9gFm7jH2pDMeyQDYOwkhfxvz/U++8vdJ23oSdG/3NnR20Ue838iFoO1iPRlS+zXCYBYepl25T86gwZ1CNF6cUqG4gQqb5Rc1X30q+8luiIR0HPNrnw1INm47CZ1zMf8RkLGlLgzdV9GVkDWl1JRQBXe+dfla/UYlFeBP5C82n4vit6KSLYHOfCHqFJxC/6qE5vfdGv5ZrWlWWGzKXHumtybWC5Qxsz8Res/AbdWEF5cP3QuIzsrfjYPPLE/sqlU1Cy6iIHV981PiqlkUdMkNXPcYtu8BTmnxki647QsgnaqI/tygHySm94gsQ8qJXqokBceVsUiZulcumJ88baqO1DVFbFz1GcFwpet1XxeGsNClow8cPEX7uJPwidbR6ExSD5pH98Gfax7numzuZ+ilhHRGdTovAN9s9fTPc0OGa33VUCgB1lT+FtilYEkOCTSBjoyLY9A4m3mLukPXYr65GBieqmADiUgcPnWANKz3Mcizo8IgCAAAABAGLtAcQ+AAgAECsOIGyLTVoAH5YGHz4I+QN4APAA4GGjwEMbwAOAh1ouO/Dp/lhM9Q52PHKrf8TOmytk2Xyu1/u3FE+BFwr3X3ALiu69A3AwDTj4KGQH6xKADAAZbBYy6AAyAGRQ87RCNLM/89JE1CmkFx6HIGCBApAEIImNRBJdQBKAJFYfSSx6jQJwxCw4AlYqAEUAipBRxFbBIZn7+hgVnSkufc+cP0rjVVZ/9q5z346kA7qI3TD/ZUfqnO8wMqN5L1/3xfol6kfdyGdiXTzmm5FWZ12+GWmWodEBQKMSNGoCNIqgkVaNjUrCexpoxAFMho9ymOg4Q0f8c5AwYLLNfxTCUZJu8S8+xmGl/q2Swa95wM6DaCbyqrO0vbu7tN/7e3XM5UyYl2MDNJO3UNos4KhgBwv6bIRroJZASHx7CBgIMNAaYSDZnnJ+COBnysyRwD6HgH0gLfRI7PP4vFBAb+1pwrhwHf5AF+UpJjJ2nEGI/7C/3Y8fd09PlXfvjh3nOKiAJj6iLNC449tM5siRZZrYVR8VsNPhnntmRd8GjwGxeVQGWCMJEnndzPqsJjYsB4nQVJ6U7AhLVWajeN2FwKcnNnIvq10suS6FNOopYWFFMb1rV6Ge4lfKgN/ylYjP61/tmXuHHdTptpoHh4Y5OGB/RxghDZlme98wsKHhvcvDYDdJzwW7KUb89xUiu1mse/0i+/+fvCrBtwm6FYXp3a9f6LqBbVvXJhrINx5DlL+/qTCS8eAxZ0Ru6PSZR5bqRU4+nddns7E/PXJpewgM7Xq8IArWhg4PBv2meaQdYaOJUWfQ7bT3tVYba12j3UKdgrWlDS/H7k7qaXfVrzbLtrvWOLubYFDzGN5EW5tGu736ajdhBYCC51Hwaf0VrPwDVDyPit8+nYrlxfvZFR2ty4GS51HyWX39OF52BS+eS8G/1dmLf0+w42roeKNfJycamztekXUIJzU3L0mQe6/+tTe7DANKLHf4OBmOX0OaZQ0vZTMxw8ZkAStxYq/JJyAcFZfo1EmUpGnX4xrTCcrwQpdu803ustXO5HWcC+84L8VjgsVayCRpJrc3lEyuqybnk0XLuWaLyjUQce9JRLzQTZtWS6Cnq2+zyXerG6zFt8t1i4dpvqslzrMNdIp10+Fv6gqQkP4ImW7rRUIiyA24Jqv1mCL1iiogKQFJaT1JSgU/BarSNDztI+AqAU+7ljztDP3E29yQoHpvm4aSXvo3VgEk7EXin2SbHI51le3rPU8xkDEqiBQ42c8Pd8ReTwB3NhTuRG4JWOcxtOxCT4B1gJcNvGzgZQMvG3jZwMsGXnbxbQcIYTWlZkdvPnMpGNjZNWdnL1DHQO1cY4J2bu8k0HNNOdoL8GagaQNNG2jaQNMGmjbQtIGmDTTt2dh9z7WmvP4c7AVtUA807EXaZIlOupFaert0ywemdf3tHpjUz8wkSvNxTK7Vr3rAqgZWNdCM6kczKi+XAN9oGr5RoSwO9+IYSEcFLhJwkYCLNCcXqRd5AQtQBrausHKCXDOyJlgmmGOZ4AlZQSd+MKeuLzhKY9gdlL0YZZ/UUdmfmQ4Ub5A4tqmcowhJg6rrScrx51Z14tcUtF17eo6/cMdWTonHoQyovK5cnadwcNB6zZk7s2sd2BzA5gA2xzqxOcS+NtKyjYspJxroJJrKdcHyi/bBGbOOkxne/36or17+UO+/MwvkzepJduznzsYTQibJmsZz6KYK++Q5hZ0Yth+Blc2Tdm8ppr2x4j5dpnGbERLfPKm/XaqRb6zYz2q/81ucb/49Ul0dj6IEvgLwFYCvMLXPAmuhxFqo2BGusPgM1ATYEk6iJnSXuCNcHRMQsHvcZM7nJF4GbBv3/PiIL8kDOloCOmqsBTbYB2wA2GCVsMFSE+YADiaBgweIfIAOAB1A7uT5cidlL4R8yVSYqA2YCDBR3TAR308C8iWrBoneMGPjG+33Ep1l2Cj32exxyp8FyASQCRIqqwseOgAeADysEniAhMoaoAfpUwwAEAAgIOeypJzLtB4LKZnHoaouoCpAVatIYVkO9x1g1UQSyzR7EQByen7k5ANygtTL7CDhAEACgISV5LIASqgvSnhoGxsACgAUIMXyfCmWxC+LvgiZlKlA0iGAJABJq5xJWc4H1oCVZsmoVG4HBoAJABNkVlYKNBwBaADQsNKZFUANK5VhAeAAwAEyLfXKtJR8EjIu2VXZlqG72rSwqtUEWAWwqm6wir3B29sMWzGcI5/It11AWoNAJxiZ6ZpV/kS/hlJ5Bt/3jvNzR3mplNu6JpbEPp6+sR0AYLMDsPefzr9sf+ZafPmnkP8O4C3Y7g5SNSucqmlpgCkAU9QwVTMtnMh9hbQYUDFjk4As5tgKZkS8cDg6D6nAFxG8AHQBnyZBNmdZ2ZwvLBTxT5Pef4IEzmNO/W3B9rllRLUZJ/tGU01B0cwdWGG7IFzWgFEObPdwUvBTnRSc5S52anVi1K4GhwMv5BwwoeCqtBRouEYnAs+tYeHC7O0NNF3zA4Hn0XT8OujHr4M7cDzoPEo+reMB35KKK172Qcf1Ofh3ITrmbpxEbVD1XKo+q6Oq4bRfOO0XTvtdo9N+n4zOsZHn+T4szZkJLZt4Yu+KcI028XjfJ1633cgzfKeOHusq1LerZ6jPTjCAI32nP9K3Ym2zFhQDOM8XCI5AQqgkISi7yqnF34r6oejlAU7CVjxuNTBG2EHfMAmisUWntUTLnfwOE5FL0RjzqmFmmeplyN5hXcwCtZq2RbHj2+wh3WHqaezNPqCSPWdGIK3j58gMFN8ImzDxAIV2YY1ZuJBcW0U35QuhOG+pY0wpcWOZ1CDVej4XZTHSqn+FmNyKj7mqyA883PMYnTdMVjrEN4X5IQ1YUvdbhceRBIZs+1v8/IXHeECUY2RYLbwp+B2JgKXJkMVvNhub+I1dQZcYJ32HadCquDzRSjy/P0Yloic9irTboX/3X6/PkIKBzCuLTUMMKaQAYqdaZ62HdBbPZrJumWcIxQR/JMNQ87Wl8fKy6otji4geSKoIA3wRNSTHpygG8Zh0H3mgJSQX+94gAgOq613vagknj03icZmau8232FRIsptjkaSf6sqzktqR9gnSmtKPffmH5mT/d6T/NfnHflOukQ7Xakn/a2ZkrD+TZ+B0UckWHuxFbrgrNyz30mrLP6QP8w5MebzJWHLi++UJ4oj69aIXh8sUVHmOH1KsfE7ylcqe0pMsWw0FL0plEe2wOzA1s2loLQ21uu12u9/d76L9Qad5hA6ji6/SSN3cuv8/Uzjq1DIsAQA= k8s-resources-namespace.json.gz: H4sIAAAAAAAC/+1da1PbuBr+zq/w8fbskLOwJIEAZaZzpkDZdqbdskB75sy2k1FsJfHg2K5sEyjD+e1HF9uRbTn3i5O8fCKSLcvvTY9ePbKedzRNR47jBiiwXMfXz7RnWkQLbcsP6K+/v9OfL3vsMmxaAWrZmJYGJMS8rOPg4INJS5zQtkUJQV73znXtwPJoeZUXdi0TX7hOQFybPaKNbF/cb1vOvc8fw38S3CbY79IC/bDq66LM7fMreLdE52ip4do28nwst8YrutjqdFnP9Vq16j3qSYWHHGwPWpJb4/XItpB/4dou4VJ42ZMrW4j42UfxChP53Y/Y6QSs07Vqrg4X3RYg3w2JwV5AvyZuDwddHPp66qq2Zdus2XShS3qIv6CH6f1OEDpWkL7PMnN32biDHTNR7+CtHzqqDjIJh4TQ1gtqe+ixqMZyCmr8rtuXjEeqCaj92QV3PSA7HEhRqktriFoSvyrbOq/oW6ZQULZGMr6kmJnytWs5wSfX5OphBRrytZ+YuGlBRxpAHazUssdaIci0QvaYRr5ObRxU7CYmmLBnt203o1wfEwv7nx8wIdSrFP33PWTgIpuklUw9h+nCABn3yq74AfY8bH6kMlTWB4h0cJD2qqxn8Qvxo8ffxw97uw6Va9NBPcx72vRcs2nQ4IDoQ8hZ8l/T8MJm6FPZNn1MS02/yc3kjDbRtAgK8LNhh7SD5M03/VX07zd9T0taZuXJj2/6S0U70Njz78NW5qlNGnW4N9J/flBrC/wJ297T4hZoBe04e5q+lxXCwHUDq8dei2kyf5nlUIVwz8u7Cq+mnaFecYWMwGVCrecuoXGUR2X9rZ6qepF+pY0m6LLA69om06V+Ut07rabNjvX4isYpKdan6m67VjtQVwZ8zNAvrr9oXwKLDix8rNF227Q5LRZ4JfO4ZAR5zgYRRLBZFGFcEsSDTjaENIMnj/fDckzrwTJDGnEKo0l8rW85HZt2D2Wc8BE9Wn6+c63QuBfukBUDC4xROGHSyihdZ6akvqs4aCaBkY/R6vd4Qo94DO8cGCZ9HAnyJmmjFraVHWSVbucc8aG4lqsT44TyNjFQVHPl0hunjXevtC+hrIreIztsZbxwR/F6G41L6oBLAJcALpkQl9hWzwJUsjRUIsQNmAQwCWCSLcAkh4BJAJOUEpMMoEAP91zy1Oy75J4OAHSYDpqtpwBPjAmSFv9Ba9iVVo/qRPxYWopEvAzgkTwe+cQlA4kSACUASrYblBwBKAFQAqBkWfkRgCQTQxLIkgAgAUCSlCaOSd3YwyjrVFHpBxoEuBOpq2/cfopQk3ToLnLHNN0lcdL3GJliCE3X3Vo/eX33WPhMpItJiDT1xqYQaap5lNUAlJVDWdXNRFnDgxA3W9bWj5CKWdtPJtz5cGQw42aX/nJVP3rduMhfkVisEghE5pgPoF3LNLFzK+BFwc2MSzfg1ykvSaxYXStpOo9AUlByoiieE6CAB0PFd/X69LC6JeIrhHDj4P1aXQn4c268YQujrSdtlz5uFkg+EnMLdV8l7T0/0ye+vOgFF1LR3hcCByp9rsKJHCcRtYFsRMQ0J56YcF+aeFrDQDAtpIDbpD+lSU4c036ffTV4YrmOjqsbIGAR81Yp3qKoO7Vwx5t6ZtDBnBbmWTQp5fSS09xhZgkzyw2aWcr2NMwPYWI5PMHP55XHMK+E7P2EaH7y9L0fPNkTTW7v8gONMHY8QBL/pX/7nz7tX15q79+f9XpnvgJOeCig4cYpbjMeKcWkTJ9uKqkGAYNZZGQjylhrxJEiraBpX9jEhtVDPD7VVZPCe7UDRZV3fN51bqNR1yXwRr8kNLhoptt3dOWlXwjzWX2oar4y79V+eatQTzF4S6nPCXst6gu5er5omYyr02v3ZniWBRQ8loLPy69g7Z+g4llUfLE4Fcs8hOkV/XFYug+UPJaSL8vrx0K94MWzKfhdmb342jXXQ7tFifkFKvf3A/PguHFaw0fH+Ag3jJMWOqqi6snx0fHh6wZun7Tqhwf3p/5+nAz09z3X/PcDIvuDqeSbV4P/f2VVcd4xzjrywkHacZB05BW0wTevmk0D00nnMFvzVHosQxwpuXVJIjz4/V8H08vQD4jldCaTYXGadQOXbPh3RBbAn8omxfVhDKt5ry4wZpwQ0VkikjMueS88o322HvDZAoj1ayTz80XIfPWmHvETQflDlX9ROoebijS6RhK/BHfbVtW/09dgefcvtmRdruVdghyfaVKtxwTgKapg+ReWfzd0+Tftp7D8Ow6v+ATWf4FXDLxi4BUDr7h48jb3fYTaiI2EwCdeEJ94LrsngVI8hFK8WglvCqs43svKMhPabv/A1QxkdHEFSMYrnmXycA+zzC2dZcpuCRPNSXjGpzDPBJ4x8Ixn5Bkr4w8QnMpFNRYQYSYFA9u45GzjOeoYqIobTDiOdA2c41JzjufgzUA73njacTonc3N7WwFlz6Lsq1I7dKTmi3zaDRQ9oaL/WAdF3/aRB3qeSc/vy6ln2C5SrFbYLgLbRdZou8jSv+gLu0RyzGUhedgrsLCNIisz8qGEdVD7SraIjC922CWyhb4G+0PmoXTiz6rmNRLp1VJEyulK2yPUP5YiVL+PvO2R6Xt9PUh6sD0MtocBcW8tiHuwQ2xi4t7rdFEU4VlbtZ4OpD4g9QGpb9aPhwov0G6wgel8TztHjimsCVajysnwO/f8GXV9xxAanbqDsktP9Zte2TdUB5rbjh3b1K6RQNGg6pLy/WZWdezXAWi79Kw/b+6OrV0Sl0EZUHlZSYCLcHDQesnZgNNrfcupQ1rgap5r+kAhAgrRplCI+Ef9pKUVBwdszbdJxFguFn3FRwBHLrX8L7Ou+je1QdZ8M06Tfa9sOYWoSNpBNIxur7jPlynu2Lg9gVi2Ud4XKzHvLRb45SoN3BSQfBvl/m6lhr7Fgr8q/bdso+Tzn0J5ZTy2FIgLQFwA4sLYPgsMhhyDQfGN20wZ8BQ2+CO3k/MUjlf4ZdFyJiNK/2HRFeGn4QQN+Brj8rERW5sHZLQCZLS3EbigBrgAcME64YIVp80BGKiBwQg2HyADQAaQM1leziTvhZAnGQsP1QEPAR5axzzJqpZZARAVZErGYb4DLFo+LPIAFkHCZHqAcAgAAQDCWiZMACGUEyGM2jAFIAFAAuROlpc7if0y64uQQRkLIB0BQAKAtM4ZlFXxdwEnTZZJUW46BbAEYAkyKmsFGBoAGAAwrHVGBRDD2mRWADQAaIAMS7kyLDmfhEzL4KrBNyn2a2NDqmOAVACpygap6Ozd3qW4imIbcf7DbgZftf0mwchMVqiyx0gogdUQXNXofa9ov2n5p/SJJXGH5/GYCgC1aYDah8/Xt7s3TOe//YfrpAK4DPZaQzpnndM5J4A9AHuUMJ0zGnakdhQtGnzM/WGAQKbat9QlbtjpXocBxyEChgAKgb1LkB1aVXbolgYhdh75h8+QEJrkmJraKSCvHPLajrNoxFCTUTR1B1p4lBEubcDIB7YXONtmUWfbDHIclVJ94ni/BsfZzOXD1VzBqvQVaLhEZ9jMrGHuwnRGB5ou+RE2s2g6mg560XSwAgdazKLkyzIeSSWpWDHZBx2X56iaueiYuXEctUHVM6n6qoyqhvNp4HwaOJ9mk86nWSo/ZCvPpBkm4QVwY7bxHJqNJDlt4wE3K1g23spzbcaISdsn6HebYdAl4kHAITnjH5KjWIgtBR8CTsgB1iYwJpSMCW1fu7TYDK4V8qeMIFDsRP3WfaOLe+grJr7om/iKh1ibZXeYiNzzxqhXdQaWqd+HdL7tYBrC9aStAPc8m76k00k8TbdppyR7HhiBRDpIMS8C/MhtwsRtFNqZBXHuQnKtikPLVm1x2lILTCl2Y5mBIdW6HhNlNtLqP0JMnnjGQsXUYOGexei0YdLSDn7MjA9JwJIev5N5HUlgyLa/Ru+feY0RoiyQoVp4Y5BRYgFLgyGN33ScNvFbW8HtKJJ+j2rQUlweayUa+SdRCX9SU0TaXWajTYp4mpbTdhNMUVErqz5KWdEwJiuVugTXiP9X/Hw9XZvrKCtTXxyZgngTqSL08Z1oSA5Ma2Ef1QXbR4IA52Mhyv3bL9IG7s2wnJ14GHsRQdviwozCdVvgR91x+/u1mHNKcV9Upqdu8yyKnsjg5kgkyXZ2GcjoDel7WrWq9ONQ/lHrDf5vSP/X5B+HVblG+tJ5Xfq/Zgr7/R6/AyNCS8Yx8ilyw8dyw/JT6kfyD2kj64kp9zfuS0p8P11OjNK/3F1EI2yCw92eFwZYu4mz8tqB9mdshdrutRtTivSQEwD10wZqHNerJ6eG2T6hf68xQjVkmkeHhoGNmnAQ/SEZ5as7L/8HGrDTw1cWAQA= k8s-resources-node.json.gz: @@ -34,18 +34,18 @@ binaryData: k8s-resources-workloads-namespace.json.gz: H4sIAAAAAAAC/+1d6W/bOBb/nr9Cq+kOktmksZ2rCVAsGqfBDNDOdNK0i8WkMGiJtoXIkkpRORp4/vbhoYOSqPhIYkvJ65dGpERS7+LvPb4n360Zhok8z6eIOr4XmkfGHWtija4TUnb11zd2Odnkt2HboajvYtZKSYRF29DD9DebtXiR68oWgoLRue+71AlYe0s0jhwbd32PEt/lUwyQG8rnXce7DMU04pLgAcHhiDWYO63QlG3+tbhDLEsujrVavuuiIMTqaKJjhJ3hiK/c7Oy1ghsz7QiQh91sJHU00Y9cB4Vd3/WJoMJkU+3sIxIWpxIdNgpHH7A3pHzR7VapD1c9RlHoR8TiL2B+Iv4Y0xGOQjN318Bx3fKwDqd3O9fk4iH27JR52TtdDXXTc/pFhGCPVvSO0U1Vj+NV9IQj/1oRDaWHMulyK566Qm6U0Ujpy9OfyYm4qzi66Lh2bEH+VrFHEa20mQvqJ9/x6EffFsTnDQYKjR+Y+HnyB5jxx6NoiLU8DPgoBNlOxKfZK/fpWc/IbmOCCZ974Po0P2eIiYPDP64wIUxn8uJaFNlMbPlY3yNGZmPLIPg7oyjNi1KiM76Y9qfTzu7hXrd8RyqxZS5m4tgqdTD1trH3WSy96mFuAjKzoL0llWJ9r8LpTqk3pMi61IhRQZBmIaDrjJ0p5Ds9fLPTeiHkU67ymhQGyMJVto91ckPR7uRb42lKahxSHATY/sAWqVUaisgQ0xnUAd8EgkdhNF6/8AzDY1re89AYi9X2At/uWWwjQmwicpT+1bOCqBeFTNN7IWatdtgTRuuIDdNzCKL4znIjtkjy9sJ8Ff95YW4a6ci8Pb24MCcX3i+G762nTZts5g2+oCHxo6Dn4gFdv/bJpesje9NI/urR2wBvZKP20g72+NFl1Jdv4F/z1RPsoj5251xZYTLey/8XS94w+rdG5bIuvLLED3wyRmK3pc6YE0/IcOk2ZgwxYab+FFlU6FCnQnpP0/Hu7pLZJxOmlNmlWMxkYlaMwEToUsEjBSnHgRDVuQxEKlIWchFZF1xgKEXs3sJmzM2BmPAjRGx2mYzFWhLb/ZrJI+PIxjIJPn3/eAYElrZ9leSt2l0WJm6lfaYjjqV919agIP52pwx1auYSfZ9HzoDqO6nwAMzupy/GF24x8/iFpnvUXREeIoLtKuzoE1reGiU4FPrOZ3Q827ly7IhhyUqcmNwrvJD8ym7QjROW19WPrEu5s5SJbo5jjMhJUmCZyQVO/1Q1Ek7RrnCr9K9wi25mwX2ZpLLpCNUIFN8cqmTJ9YfHSHhP7VKfBP/axyT6LyMY5Y3n0vwVvoRexeR7TEFBa8XWVLeYAQ8wKipO3PobsxnCy9Z3n/nXOU86XdB5rHKqPN2nh7Lvs/ND9I/2pbbErHiZDnTZf+6A/1xyC9rP039+NK+lvJZpbktIb925nPjz8kYjhR1nSOL/7N/Wx49bJyfGr78ejcdHoQZOBIgyc+NVj5nslNL5NBdzmc8iz3O8ofHJt6sc5lhMtObWSoxFnkeLvrONLWeMhIlq6fzfS70OxZ3nwsU8dtG0+1KEY54QZl8Mm7ljpvbWL4SrrXkvd75yBTZ+eqfhUDV+y3HQi8Z9pg6l/shzlK11MQbrUd6qudtpGneP68vds/vDhcDgmRjcrT+DjX8Dix/C4pOnY3EMlMTl4oz+cF/cGpg8E5Pf11ePJXtBix/G4NM6a/H/4hB3M1hcdcz0hBx+vW1vo/b+LjoYtHZ2DgeHh/jNoYXRntXfPcCHfbzfPti+fBNuJSHfcCs5NvjvFSJbWdTg7avs7595VxJiTgLMojGLMGfxZdGRjPr2Va9nYdcVjSLqHDf0OvdK43Ulp+tgbhI5NM75ZGBvFrE3ueOqerK55pxViLn9+pftxWkYUuJ4w/loWH2qMu9JtOVHHl1fzZHu/Qe69x5/iUwvzcFXSJEIe2qt/wLnYuU5CB6IQLipiYk88IARkgKWnBRQAyk6fiopSimfSUwCO3rJIfrcfFDOquUhNcjMCmSmC5anAVK0DZr47DXxZBXWW+bogMQ0UmLeg+1+LrYb9LDBenhqNiCD8U+elVmvDEaCvJBzUs/HNKih6YIMR8hwfKYZjnk9hQzHUoajpkRwB1IcSymOUCIIJYJQIgglgqpfZxiZ6zHGY5/cCqTveEPmwdFe/5bi+X2QdMh/sS7e4IyZkZAXEz5rhUcCBYJQIDhLgaAUVKgRfLoawdVS+LmUCX4UVIRKwVr40WIrAz/6hfrRWlUEV7rkSpc96V3wpEsOAhQLQrHgfM4zFAs+82LBSqwHWcP1qReUGPBBDIaSwZqXDD4ij6He6GFcrnXVYMxrKBysdeHgI2gz1A5C7SDUDkLtINQONsLeQO0g1A5C7WA9awchWwAqB+teOZgeXUPi+wrEpgumB0oHQRWhehCEBgoIwX43uXwQFHGVilj7CsI4ug5FhFBECMmPjUh+hDrCuZMf87l2iYXnY7XHJiRGQmIkJEY+MDGyK7XAOMMWdq6wcYw8W0oTHJ/VM0vyOAgfyOtzjtCYewLMrn3G5OLMPmM8MPxBoti28QlJFA2srmfiZPBgVid6TYHb9U+gfHTFNk6Iz6EMsLymuZRPouDA9bpnVy7MdciqvPYM6huBtmarYdmV+mw6yKyEzErIrHxpmZXr/ABRfI51PTs59DDlAtIjEtbJE1n5+dapx3J/F77a+hezP3z4XhIx/bYhDxGN0imi7gzxSY8Q/14sB3SjAWeGj573ea+g0BgMgqQ0T1KOlyopiUkJpMsAotIkUemuxqiArDRQVk5WalZsGYkAkWmSyLxfrXkBmWmgzNT/6/rxQefvUuzq+AVASJKDJDlIkptZZyFbrpQtp/nq/j6kxJVS4lqQEidT4vZX9KlziHbqUCDvef2fR8SFT/iF8+Z/Cfr+7Eb4HPTywR5PbAOotwKot/kcgM4BAB0AOo0COi/3tA6QzjKRzpTcfoA6AHUgqrW8qFZZCyGSNQvAewMADwBe7QAekzeIZAG+WyW+e8dUh2ms0U0EMAN6nMpJnvZRWh4E8A/gH0S6GguEDgEIARBqFBCCSBcgoVohIaVqFsAQgCGIha0oFjajwkKkbC6AWGgDhAgIsSlJXy+wxggg4lLTvmb5BhbAwOXDwABgIMTEFoc8bYA8AHmamf4FmAcwz1Iwz7SPQQLsAdgD0a/lRb8SvSzqIkS5ZoJ8HYB8APkaHeV6gZ+6AORXh2iX9hPBAP8A/kHUq1EQaAcgEECgZke9AAMBBlpV9AtgEMAgiILVKwpW0smKaNhavGoztEZ4jL5iEsqVtXdlM/+VPv6EjcilGIztZMNMKk1u1QnblrD4MQ45FsXjwGWv6A1TLWMIIqSKLGcioKCnHIak+EZIhI0HKHILO7tQH7VXB0n57/fhvJxWCFKiwiqWVHr9gLOraFbN7xEmt0LvdJiTf9WRG+S8WLLWIb4pfAYyNVbK9GuF11EIhlz3a/z+hdeYQsoKGuqJNwOsTgis7IXMNrNd3MbvXA1IraL+mHHQ0dyecCXGBfOwRMzUk1Z2PUUejjfwN414uA09szrTmBVvUSpTmUoIjoR/JvOb+d7SQnmb/uZYFOSbKB1RiM/lQKpZmk8+UER9DZ15c8/yIyE5O61ijzSUZruV4/1UtQ1c/3bM76jS3PSGheWPab/jObEtzfN8ySB0UvwhdvN+O/QUalL49ZP5dKRO9JpHGS+d4AtxP996loYyZTTZGFVdkSl/ahlNheJxjLlWECeKJD4PI7+W4M2JxFeOIGaMrAbSrzM9/3qrnbg6pjD0vM3MPRY4DB6S7OGYJKnjr/ob5p5ymKpaf3NHvVB+MNzcU/5uqxc7LbVH+WhHR/m7bUv5/Za8A/fIFeGYOos68L46sDpLZ1e9UBDyga2uN1lLjnw/fBHeMb+cd2MwnH5l1x8HEYPgZ8kvoRnbxu+JFBrrSQ1GKAXSjETk0URvDgb9ln3YPsRWC6O9wf7e7k67s4vb+9ZuB+3Jm69SVN5am/wDnjkd7N3vAAA= kubelet.json.gz: - H4sIAAAAAAAC/+1dW2/bOBZ+z68QhH1oAW8R23ESLzAPbXrBYjrTTNvpAtsWBi3RNieyqFJUEm+Q+e17SOpC2VIc144tJydP0SHFy+E533d4EX1z4DjuYMDCKJGx+y/n6/eWkQj6I2GCWjIShlwSyXiohDcgAmHAYqmzwOOtzkZ9JskwoCAdkSCmWjgOqfy3D6IwCQIjESSafOY8kCwC+aEWTphPz3goBQ/i0vus9G7AwgurYREJqc7/VbfJtAzkPpEk5onwVFvcc8GnVE5oErutLMeI0cCHCkdsnHfJvEpHJAlkXJIu1JyLpySKWDiuSJET0OGEB/58Ueot7uuWkWHMg0RSt1VOjyWN4ky52d9tqfgkZEr9bshD6uYJtwdzeUHbzD/n5Ta4E3g8KYpzr+D5yHq+zsYlfZ6p54Wi9dB08sdFDbk8KhuNFno84OK3VAOXJEjs7hvryFKJoMRO/CuJJRvN8uREcjuZC0ZDY6hVyYL6iUc/VLRJtYoEXmFJxbCTWP7O5e9ggK6VUh5qbUzqZbc0kKZzuTnfMUhRkIxZ+IWKOG36SWGpkogxleWmlVpOryOh3omT6bOLZEgDKgchaGgQkim98QLQGRW/fHP/kf77zW05f/EhSNLc39zb5+WGj7iYEm1ekk3pIKag2LichYVQFHTwLfEkF7YhGL3RMQ39t3k55ZcFHWlMcF8umm5hPZJJDSbuxyQMwcmcX017LTeWgoRxBGYSShs1TNos0m/HYBGmmlThCBMbhImj+8FEF2GigTAhjGMNIu7H90OKlsNCcKjQo7/8DRmzBwUhzoePTk3hA48noVy7hocGqZubrLbb240A1jnoFcGqOWB1ej+wOkKwajBYeTBNICyEgh8MsvIqnipwnWUKQPRqDnq1O/eDrx7CV4Pg6xLsCbx8SkIyBkCRXJJgYITrAljLUQ5HIS/ARALF6scBHw2uuAj8vUKel7oHzhetGMAfgF3EngZhz/H9sOcYseeJYE8GPT6NGahyn7HntekCgk9Dwadzz7XoEwSfBoGPADwoL0h72sgHVAgu1p1Tfe1Nvz/fH4gx/u28UV3fLMCQgJH4TNmxtqNiIIdExAvl+SSevKfhWCq/ax+W5LQq+3L0AmuEokqCd4L4zHTncHX3L81zqraiTqrd/7Rwfz2M5ZpAT+PwZfw53aGUIqFWseRyPN955YiJSEdlLvuUXFdkn7KwQirYeCI/Mb+iVgDlqwop5P0P8/UQZTufmTkoBq+oI/duVdSCdgAN88SS8CqtpX0XcKomnHNwnAzqlKAwgIiCdQDSjemC9UTqLWUKSVyalmr5orGBqn0qqAaQUcAtDxE0otohbXW4xr8/XAKegMbmWg0O5dEqSwdP8i4W6lY0F1H/PShl0Qd/GvUE+LlCIg5KMvvnJgarBr957Lsb+pzhzHmWFzxQQFGg5VaB0bm5KbfjJ7HSjlK+2hg6pW8BeeaHX8k/TdhILiakoPsha5XzkViRDXhRdvjgxvZFQGC/whu5kHMYpL1tkEEzC312yXyYvboLnpfl0ZFE0YBrcs3mgHCYeBfGyOZcPgvQVHftCELt7i7mrkaUHB7yKK5o4oxc0ztsuzAdHs2ZTEDAUheaoBL4+BWJqQ0sFmouZDewuSC2unJQEXg2uZWZbT9Vwl5Yqaxh7D4yNjJ20xlbT1c2TdyZtOWUqROJu0zcZsKE9I30jfT9UPTdWXb0s31UTd9Wf5C/kb93wN8TFksO6Dwd/EgIMHdAnx2+6Pdbzn2I3U9S6oupx0M/HhgQf3CObzkBfY5EXyb6bDCcfl9OnGw0kfTXI/19oHwk/MYtsHfaNYTfRsJHwm/UhF2dLof6hFyk8zvObq7B5tslbujePZi69RNKu+LigorHqTXTtxrFvdpuiHPOfeeTMlBcxcBVDAxqdrgJURvVdDCqwahmP5Yxth7u7GCxYiMxz2oqrQuGHmxFaOtKbWZI9DpVOIZFuM6DIdH213lOayKi7vYiInXzy5tpJGfVSf+lgmMYhWHUSotDQP0CFFJs9mwvWiqqVCTQctJPcMyp+t1t+qjGaFGpPU3ZBvpkxsvBA5u4VoKBQSPWSuoigyOMDDAyeEyRwdaOd2I8sEY8gOdAMSrAqGDH50C7vZqgoIdBAQYFj3DX5R7rCHftEKz6NfsqocOOD5HuW/zwGo+X4rYDxhHN2HY46tTEEcd4EANDgkYdL/XGgidRfoPWKtsIG2P/LX0X2swvRM70ADjpABRacQSuBeBaAHL4znYIakn8BEkcSXw/TlMuY/etTO7xk9C7CB/n6zhfR64v7A4Q1hMsyu4VVUdpAzai3swLqEMvgQIccHrl2FzseJbfrwkQ8J5GDBAa9hEpGOJAUPWThtub4jf/8trNH/x//+adY/SM83ecvyOn73L+XkfPeCkj0vO+fA1p8XZGjo/v273mcXfWGeRvnI8jd2/9HF7vuOa3uPE+RqTuPaTux/vZffOo28cv7pG6kbp3Rd3H3RrqXulmxXnyW8bdCzT9k+Q9n7Z59rZ/nQzpe2cL48BAcuAFyhGApH/A2MiVPpGr/dVID4ZAyTovXmztR9w619cbvEPxwVXT3aJqurWqedVE1RxtUTVHtao5a6JqeltUTa9WNa+3G9h+PD/Db0JxHwkD2t0FtCc1v8XbwUs1cS2q4WtRFaS83lrUakdAQVXDlpOIYCenP1Xt+h9oQFMWqj6aQcAfA8ElK2T4LTH86TKCr/ntzk4XV6xwxWrXJB8J7tE4BvpWYwg8PqVTLmaD4UzSeL0NpKe3T/Sb1h3S7Ho0q01vD6i2Ie184nR7el+6PUK6RbrdNd3qaXPGuV6U5LPkTV2f+PRI9+z8TyeJlXEh7643vZ0oTezBFLcZ7XzivFvcNbSMeHtIvEi8uybeMR+MueCJVPrGee1qFPsu1xxyLHLszjj2ILVy5QfKxPUBnENjk67gV4W9u7E3oVPyhYrYfOdvrt0HKJkZi/aJuDDvAToWI6e9X4RUTezz2iSdRgEB8x/n1uaqo8DWeN9UUVNJtZJea8X6dESSYE612rTs1KqR0tfzzjlR9ZhkZm0HClYq15cflMEBxD8SKmbpkuB8QFHSebskHVM1yq6dNXNYq/p5jykFAMGXtP9z3bBUaethafhT6Mr+ZpOFAPU+fRlUsXSmSDelA7usKYwIq3gn03LFK8tUrKsbGETRJ/r1j+ixcMRbTlrc82rld5YpP4Vae5DAxLWG4z+y+t1y6kJDlaw6czq0pidWQhLTz6agyjhnS+N9WDfec1he4zfLRjrj5bWGWn28IZJQRwv5DUJ3LgDM35JwW+yTPw4rOchA/tYALtPqTqF2ZEIWN+RX/2xn/A/hRypzS69FzLvQUWr6cqqS/ONWm6TdnhXOtA+th6790J4W//es/9v2Q/fQTrGilI71f9s3OPg964Mxgdx+ltZiF3xsF2zX0jmyH4pfAnVPfLu9WVtK6vsf1zG2++fns5QdsxDwV2N8RproiZ3bbXdPR6Td6/m9frt3ctwfDX3qn/ZPiXfYH/l9k/kyJ+HDg9v/A9gbKcQ1tQAA + H4sIAAAAAAAC/+1dW2/bOBZ+z68QhH1oAW8R23ESLzAPbXrBYjrTTNrpAtsWBiPRNieyqFJUEq+R+e17SOpCWVKcqy0nJ0/RIcXL4Tnfd3gRvdhxHHc0YmGUyNj9l/PtR8dIBP2ZMEEtGQlDLolkPFTCBYhAGLBY6izweKWzUZ9JchpQkI5JEFMtnIRU/tsHUZgEgZEIEk2/cB5IFoF8VwunzKdHPJSCB3HpfVZ6N2DhmdWwiIRU5/+m22RaBnKfSBLzRHiqLe6x4DMqpzSJ3U6WY8xo4EOFYzbJu2RepWOSBDIuSSs15+IZiSIWTmpS5BR0OOWBv1yUeov7umXkNOZBIqnbKafHkkZxptzs76pUfBIypX435CF184SrnaW8oG3mH/NyG9wpPB4UxbkX8LxnPV9m45I+z9VzpWg9NL38saohl0dlo9FCjwdc/JZq4JwEid19Yx1ZKhGU2Il/JbFk43menEhuJ3PBaGgMtS5ZUD/x6KeaNqlWkcArLKkYdhLL37n8HQzQtVLKQ62NSb3slgbSdC4352sGKQqSCQu/UhGnTT8oLFUSMaGy3LRSy+llJNQ7cTJ7cZac0oDKUQgaGoVkRhdeADqj4pfv7j/Sf7+7HecvfgqSNPd39+plueFjLmZEm5dkMzqKKSg2LmdhIRQFHXxPPMmFbQhGb3RCQ/99Xk75ZUHHGhPc11XTLaxHMqnBxD1JwhCczPnVtNdyYylIGEdgJqG0UcOkzSP9dgwWYapJFY4w8YAwsXczmOgjTLQQJoRxrFHE/fhmSNFxWAgOFXr0l78hY/agIMT5dOI0FD7yeBLKe9fw2CC1WGS1XV09CGAdg14RrNoDVoc3A6s9BKsWg5UH0wTCQij40SArr+K5AtdRpgBEr/agV7d3M/gaIHy1CL7OwZ7Ay2ckJBMAFMklCUZGeF8A6zjK4SjkBZhIoFj9OOLj0QUXgb9VyPNa98D5qhUD+AOwi9jTIuzZvxn27CP2PBPsyaDHpzEDVW4z9rw1XUDwaSn49G64Fn2A4NMi8BGAB+UFaU8b+YgKwcV951TfBrMfL7cHYox/O+9U1x8WYEjASHyk7FjbUTGQp0TElfJ8Ek8/0nAild91d0tyWpd9NXqBNUJRJcEHQXxmurN7e/cvzXPqtqIO6t3/sHB/PYzlmkBPk/B1/CXdoZQioVax5Hyy3HnliIlIR2Up+4xc1mSfsbBGKthkKj8zv6ZWAOWLGink/Q/z9RBlO5+ZOSgGr6kj925VVEU7gIZ5Ykl4kdbSvQ44VROOOThOBnVKUBhARME6AOkmtGI9kXpLmUISl6alWl41NlC1TwXVADIOuOUhgkZUO6StDtf496dzwBPQ2FKrwaE8Wmfp4EneWaVuRXMR9T+CUqo+eGfUE+DnCok4KMnsn5sYrB78lrHveuhzTufOi7zgkQKKAi3XCozOYlFuxx2x0o5SvtkYOqPvAXmWh1/JP0/ZWFYTUtD9lLXKOSFWZANelB0+WNi+CAjs13gjF3IJg7S3jTJoZqHPzpkPs1e34nlZHh1JFA24JJdsCQhPE+/MGNmSy2cBmuquHUGo3d1q7npEyeEhj+KKJs7JJb3GtgvT4dGSyQQELLXSBJXAJ29ITG1gsVCzkt3AZkVsdWWnJvBscysz236uhF1ZqWxg7CEyNjJ22xlbT1cemrgzaccpUycSd5m4zYQJ6RvpG+n7sei7t+roZ3evnr6t/iB/I39vgL+nLJYc0Hk2+pkQYO6Avth9NRx2nJsQu5+k1BdTj4d+PDIg/ugc33EC+hKJvkz02WA4w6GcOtloIunfj/S3gfKR8Fu3wN7rNhB+FwkfCb9VE3Z1uhzqE7JK59ec3bwHm6+XuKF7N2Dqzh2UdsHFGRVPU2umbw2Ke7PeEOeY+85nZaC4ioGrGBjUbHATojGq6WFUg1HNdixjrD3c2cBixYPEPLdTaVMw9GgrQmtXajtDorepwjEswnUeDInWv85z2BAR9dcXEambX97NIjmvT/ovFRzDKAyjbrU4BNQvQCHFZs/6oqWiSkUCHSf9BMecqt/cpo9qjBaV2tOWbaDPZrwcPLCJayUYGLRiraQpMtjDyAAjg6cUGazteCfGA/eIB/AcKEYFGBVs+Bxof9AQFAwwKMCg4AnuutxgHeG6HYLbfs1+m9Bhw4dIty1+eIvHS3HbAeOIdmw77PUa4oh9PIiBIUGrjpd6E8GTKL9B6zbbCA/G/mv6LrSdX4gc6QFw0gEotOIIXAvAtQDk8I3tEDSS+AGSOJL4dpymXMXua5nc4yeh1xE+ztdxvo5cX9gdIKwnWJTdK6qO0gZsTL25F1CHngMFOOD0yrG52PAsf9gQIOA9jRggtOwjUjDEkaDqJw3XN8Vv/+W1D3/w/+O7D47RM87fcf6OnL7J+XsTPeOljEjP2/I1pMXbGTk+vW/32sfdWWeQv3E+jty99nN4g/2G3+LG+xiRureQup/uZ/fto24fv7hH6kbq3hR17/cbqPtWNysuk98q7q7Q9B3Jeznt4dnb/nUypO+NLYwDA8mRFyhHAJL+CWMjb/WJXOOvRnowBErWe/VqbT/i1ru8fMA7FB9dNf01qqbfqJo3bVTN3hpVs9eomqM2qmawRtUMGlXzdr2B7cnxEX4TivtIGNBuLqA9aPgt3h5eqolrUS1fi6oh5futRd3uCCio6rTjJCLYyOlPVbv+BxrQloWqEzMI+GMguGSFDL8mhj9cRfANv93Z6+OKFa5YbZrkI8E9GsdA32oMgcdndMbFfHQ6lzS+3wbS89sn+k3rDmn2fjSrTW8LqLYl7XzmdHt4U7rdQ7pFut003eppc8a5XpTks+SHuj7x+ZHu0fGfThIr40Levd/0dqo0sQVT3Ha085nzbnHX0CriHSDxIvFumngnfDThgidS6Rvntbej2A+55pBjkWM3xrE7qZUrP1Amrg/g7BqbdAW/KOzdjb0pnZGvVMTmO39z7T5AydxYtE/EmXkP0LEYOe39IqRqYp/XJuksCgiY/yS3NlcdBbbGe1FHTSXVSnqpFevTMUmCJdVq07JT60ZKX8+75ET1Y5KZtR0oWKlcX35QBgcQ/0yomKdLgssBRUnn3ZJ0QtUou3bWzGGt6pc9phQABF/T/i91w1KlrYeV4U+hK/ubTRYC1Pv0dVDH0pki3ZQO7LJmMCKs5p1MyzWvrFKxrm5kEOVFEi2WqOeq46SFvqwfgt6qIUgB1x4qMHSt5/iPrBVuObXSXCWrz5wOsOmPlZDE9IspqDbaWdOo7zaN+hKiN3jPqvHO2PnOA559wiGSUMcM+T1C1y4DOFUjqbsJYXutZCeD+isDu0yrOwXcsQlc3JBf/LObRQEQhKQyt/RaxLwzHaumL6cqyT9xtanaHVhBTXfXeujbD91Z8f/A+r9rP/R37RQrVulZ/3d9g4Y/sj4YE8jtZ2UtdsH7dsF2Lb09+6H4PVD3wLfbm7WlpL7/cR1pu39+OUo5MgsEfzXGZ6SJnt65/W7/cEy6g4E/GHYHB/vD8alP/cPhIfF2h2N/aDKf51S8u3P1f+y0cu87tQAA namespace-by-pod.json.gz: - H4sIAAAAAAAC/+1daW/bOBr+3l8hqN0ixTpTSz6SFAgWObcF2k6mSbvYbQODlmhbiCxpKCpHg+xvH5K6SIqU7CTNMVY/tDVJvSTf8+FLUrp+YRjmaOQFUYJj853x/bSTliD4Z+IhyJWBIAgxwF4Y0MJrUkQKfS/GtAn7ZWSlrGaceD7+EJBKq1OWugCDOEyQA0mFeYTCOcQzmMQm1wYGYOzTeowSyJXPPFdR6jlhsBf6IaIE0XQM1rodw7Ys8tdg0DGsNzzpAMxZxzvlXIzXxo4PERaGgK8i1s4F8WwcAuSaWd0N+/eU/H3DmAJdD0ujNacBxB9cUhIkvp+WIBDNTsLQx15EyruskE5nLwwwCn3K0Anw4/R5T3jW94IzTgoRCCBrn7I8Z7jphL4PohjylIQKt1IzRZ57FJbCTJksCeyC/Lb7XMFlPoHs9xX9nXOnoM3mYBc/y1GfFmUIRhBgbqpc6QcMEROPuvpLeCEwmFXFs/DixMMVzTFxVmjuJQjBABu7IHAvPBfPTLHNsfeTtZsNuYpME1B4kepANseS8cCZwRNvDsOkMhmH6uUucM6mKEwCVyWbEH0DfgLVVTFnWqTspb215fSHvKIyhbd7G0TR7a2O0Se63/1tc0tQ+pduvw96IFfhUgKNxmi60PHmgEmukLk5CdGcCY5wbQ5HMUQe5J6ZgmQKRaWag8t8llaXV565F+QVfDEVpcwRKosZcUiz0Hc/gjEUjUZu8QmgM8j4R1Whop5a1d+SVN+yG1TfqtCeQW86wzyt1Bp65c+AqPc58GV1EU0941wUecH0JNVCS1UuKkk5H87dnVMeGzg0MLzEnGoYWRU3D24mGmIIBNMGYnZJrKJ0RBf2id4dhYQLsagQVB2OiasQ+G5SFrHWn0Lm/4lpBAF0MPFoQpsTOhyJo2EkxqtUfz3ou78rapg1+47I0FQygIQ5ruhUmLYLJyDxsUwsnWw6xfxPR6r3AlGnRIf1ivI8jgAxUKlJEnjMAnejmB/XjTCuTElElUr5cg4RSsPptfhMYUFVLohzy31UGibR2foUQRhI42TK7sLL6iQ5baESE+puOsv0ewV9n7gLbceWtuNBIZY7dI84Paz0bWv73tD0rVUy9mDh8yoGVvqfKIzxxKO9m6ZceEjwRh7lBt1/cPUIVp9hZdpHmB/4RKCF3v9MSEihz1EJi54Cp8Zqfn67I1WExQM1TiSueAlSgs6IA4Wyrfu+iA57BBla1ib5a3OLBktrUwiWE9pzNbBQyjydlIxNwq211RMI8LGrIhuGFQmdZB6IvMYATSGuYSW8jFjfcTJf8wg2gmvEDWJAhoVGBG9ehOiMYHYHeudwNL7CMB5hgnD9a8dPYhJttn+Yr7L//jA7hVvZ/v8Pzsn8MG++v8qD07tXxBGQkVI3efrmjSilegiQmUCMQSB7ZCH+HQIHM44KJmr6cAoD97DoQqRLlJJhP3OnTkEEN8bzmQz3MNVKIVDQ8uOZN8HVCgk8fiG8N8KJsUuZbHxJWe5WAWMKg4piZr68LW3ytsRq620ppCsHc3tBM8rdTKMlsYafs9DupHNsgW4LdMWC5ZBuv0W6LdJtkW6LdFuk2yLdX4J0MRFJPPdwC3UfDuqeZDzHq4F2merWjDAfDEXEmtGwqka0kRNi+NB4qZtaUb0swd16grtLE9yrJ7i3NMH9eoL7SxM8qCd4sDTBw3qCh4sTjEJXQ4vW1Khu8xqGRgABOk84AyTIjLPAhZcFjVs/lmbvZ8BjfRjLezK0MN2C4QesWhYUQLkcShUo0wIDxMZPiEIubhPfChFk0WTihyWEN2MHhYxZwqBoQHsPgcueEGtChEVmEf8gccOFsaMNhyzofITBNJ1xV47v3ORifOXXLXWA74FY7XkYXst4UglF5UJbwvTEy5cR6L/kz/qnT+v7+8b79+/m83exFOy4lbItxjIivCqeyCrKzU9zHxE9NdzwQoTQrNlX5FfDYARIzEGBesoiihdnlgenmee6EmAvlhXkWYSbzbdgerF3WE39rIYI1GFqETEEyXxMTEspBmF1t4QQVKBkpeSw+5hyyAFibgrGEXDOII5XUxR79yuK6Fai4AxitaWx/xSkIRuGsY/CKFpVX3XwRA1ktaVy+JhSOQqfINfFE4a3Yrr7dgNYm93hhgO3+j3Q7wJoD/qD/nBj4Fhbk8nAeXuWEB4GEMN4PUtrecF0nawC/xWi6Qd323qN4ITKYrvXjV+fA7Re5rLKRBarIA9tvxqNHCgnHzlpVxaedxW0jJyriamnvqdtjK+MNcIXfcaPHe3U5PoqWlLJ9dm3y/UJ5WR2Eb/M1tvUo+dNnyQ7d38lO3PljNI4sgr83HsQ9Vwhhu4/pIK6KdBZBb4ePKiirhBjDxsZW0UCd96KOsYA87nuHKCkDNDt4DzejQhrS50WHz6xKxH3ehWCofq9HKOXxy3MMUCVQ0zsbo0qI07Loar5gvsftlDwbwRcDzJ76P6C4/+2Zvtjo9zTYFYldkUYNQ124pPs5pC8BQ/Op4rSfLuyWkNvEh3MI3ylvjP1P7opUqlJD+zIpNJjOnIpoue8jtNTNHJddgpAoh6Txv/J9nfEZZzJXKSCkPLYx83i20j2YttIS5630u3bRJBoIgkL06p/iShJqnZJLGyERflBMLG5fpdK7Q7SEwi/Z+eapHk27TPx50gwiV2VjS7izUkc+5geLxEH+jdYz+lPcCx1TOP6mvRyc3PrFVz9GY7v9xE4s8yj8q5bkcm45s0YIBYRZUNOdz5t2VBHeTTwAtc799wEFAcvbqqnQ+j9x3IAl+DSk5zvOElT1RVvMc9ME4s7f/kZTam12hkVnuX7aWWIV+CybrO11B26MyIqBT2bq8pS+eF0F8SwokWpx600V5yM5OfRiFef0hBzxV5VeFA5NK3DB5stPmjxQYsPnlyGcnUAQr4P1iKEFiE8FYTApWwEr8NnbISK+0vY2JqEzZYiYVOzp6mAN1qIUwdzaqDOYnBHCXlqYY+WoSVT5es7MgZScbfgcK+rvb7DOC1NXoGLFsBGenxUj5EacFI9VtLjJT1masJNemfVgJ/qMFTd9RpZJio8pcdUGlylxVaL4qs6jNWAs+qwVg3eqsdcetzViL1q8Zcag2lxWBMW0+Ix5TWvp7rVuSA4WwCgLQzS6oCaBqzVX2arPWWhBm+1AE7K8ix0AFMF5+ognQbWLQTtqo5EDfE0MK8W6tXBPT3ka/CkVehXnUIVAirtqFTVKFKoqA5r1eGtGsylv1arwV6L3Ph8DlPgbW2xk2YtKqtkpxaHZVYLy1pY1sKypwnLHuXEVIvLFsVljbcxWmjWQrO/PTSrZLMf/pyTbHx3eeHr4yUpe5rdRMtus5T3lKXsN8HhXguHWzjcwuHnkaV8yGPZLSq+bbZSff+yhcYtNG6zli1KU2YtG2Fav4VpLUxrYdozyVq2OO2ZZC9bqNZCtTaL+VhZzAOEKCK8TRLzRTZyM3upCK3tdVNatBn3ibHYmcE5+AZRnI7f2kyL6XsJ85dBp8+RSFlqrVm+0MQsesNwHvmEEcF0kW/GlYhK+e7M7O3jmvdn5rUqcJ1/O45PqKpVrXj5O4eKudryNeu81zX/TCC6Ykqt/KJdyXNLKJ3CS+miNffluaJ7OecrXBTxv3HvFlfeBuFXDwt9fy/jFX+g2wtIIHbhjvqN0WpGzokwPEXznMFZaF+Gu6ynUepQ16i6jUisHXnBJOwYGbk3ar7bTXzPohGvIES7GXPjP/L+TbG2MlBapm6cSTWdCVeRxPAkJaS81aIWtfnbP3kaIGGvEq/cEiLFIydMmBL0unJN6hVNqyvIvsECKc/X4yvC6LnGCvkWSktcRAOJKXuBl7lPUep3OZl40zEK3PjGrPcOgsZLtwVuqfCqTxwsp/K/ZPLLGMmZF31F/vFV4KhueqUmZD0HE5JE94AmNNBZzuAOBtOkwPfkssvVlUaFX+iQH1nM+uliXJ1HyLnT68qgsGQQrdMhaH1vCkSuEYVOGAt3VTcxa6bti1S9UKFPpT8gTOgM5h2B3HMz4eKrQK0VPwbwUrF/WRuusaq+XtP7y2h6f0VUXPqiNEs7FGul4rsy4cV6YfP5p2PSxV35WOQREIDKhzOejfLB8OI1B9w6ktd9wQWbVqnnvM6bFv+j1+VruGucNvd/K/u6wWk+B5p3q6qdvhee8JAnzPdi9/kf3D3tDZcfbz4WgX0/Q5b7NL+e7GXL23zd/bl4P6fx1vicoyhj7Sh04xRLmQnbdjA3xxtgc9yzh+4GGE6sSbdv9/vd4bA33HSGG2CSNj4v1tbdFzd/ATu1gJODfQAA + H4sIAAAAAAAC/+1dW2/bOBZ+768Q1G6RYpOpJV+SFAgWuW4LtJ1sk3ax2wYGLdG2JrKkoahcGmR/+5LUjaRIyU7SxBmrD21NUofkuX48JKWbF4ZhDodeECU4Nt8Z38/W0xIE/0w8BLkyEAQhBtgLA1p4Q4pIoe/FmDZhv4yslNWMEs/HHwJSaa2XpS7AIA4T5EBSYR6jcAbxFCaxybWBARj5tB6jBHLlU89VlHpOGOyHfogoQTQZgbXOumFbFvmr3183rDc86QDMWMe75VyM18auDxEWhoCvI9bOBfF0FALkmlndLfv3jPx9y5gCXQ9LozUnAcQfXFISJL6fliAQTU/D0MdeRMo7rJBOZz8MMAp9ytAx8OP0eU941veCc04KEQgga5+yPGe46YS+D6IY8pSECrdSM0GeexyWwkyZLAnskvy2e1zBVT6B7Pc1/Z1zp6DN5mAXP8tRnxVlCEYQYG6qXOkHDBETj7r6S3gpMJhVxdPw8tTDFc0xcVZo7icIwQAbeyBwLz0XT02xzYn3k7WbDriKTBNQeJnqQDbHkvHAmcJTbwbDpDIZh+rlHnDOJyhMAlclmxB9A34C1VUxZ1qk7KW9ve30BryiMoW3u5tE0e3tdaNHdL/z29a2oPQv3V4PdEGuwqUEGo3RdKHjzQCTXCFzcxyiGRMc4doMDmOIPMg9MwHJBIpKNQNX+SytDq88My/IK/hiKkqZI1QWU+KQpqHvfgQjKBqN3OITQOeQ8Y+qQkU9taq/Lam+ZTeovlWhPYXeZIp5Wqk1dMufAVHvC+DL6iKaesa5KPKCyWmqhZaqXFSScj6cu7ugPDZwaGB4hTnVMLIqbh7cTDTEEAgmDcTsklhF6YguHBC9Ow4JF2JRIag6nBBXIfDdpCxirT+FzP8T0wgC6GDi0YQ2p3Q4EkfDSIxXqf560Hd/V9Qwa/YdkaGpZAAJc1zRmTBtF45B4mOZWDrZdIr5n3Wp3gtEnRId1ivK8zgCxEClJkngMQvci2J+XLfCuDIlEVUq5csFRCgNpzfiM4UFVbkgzi33UWmYROcbEwRhII2TKbsLr6qT5LSFSkyou11fpN9r6PvEXWg7trQd9wux3KN7xOlhpW9b2/empm+tkrEHC59XMbDS/0RhjMce7d005cIjgjfyKNfv/I2rR7D6DCvTPsL8wCcCLfT+Z0xCCn2OSlj0FDg1VvPz212pIiweqHEiccVLkBJ0ThwolG3d90V02CXI0LK2yF9b2zRYWltCsBzTnquBhVLm6aRkbBJure2uQICPXRXZMKxI6CSzQOQ1BmgCcQ0r4VXE+o6T2ZpHsBFcI24QAzIsNCR48zJE5wSzO9C7gMPRNYbxEBOE6984fhKTaLPzw3yV/feHuV64lZ3//eCczA/z9vurPDi9e0UcARkpdZNnb96IUqqHAJkJxBgEskcW4t8RcDDjqGCipg8nMHCPii5EukQpGfYzd+sURHBjPJ/JcI9SrRQCBS0/mXpjXK2QwOMXwnsjHBt7lMnGl5TlbhUwpjCoKGbmy9vSFm9LrLbelkK6cjB35jSj3M00WhJr+DkL7U46xxbotkBXLFgM6fZapNsi3Rbptki3Rbot0v0lSBcTkcQzD7dQ9/Gg7mnGc7waaJepbs0I88FQRKwZDatqRBs5IYYPjZe6qRXVixLcqye4tzDB/XqC+wsTPKgneLAwwcN6gocLEzyqJ3g0P8EodDW0aE2N6javYWgEEKDzmDNAgsw4C5x7WdC49WNp9n76PNaHsbwnQwvTLRh+wKplQQGUy6FUgTItMEBs/IQo5OI28a0QQRZNxn5YQngzdlDImCUMiga09xC47AmxJkRYZBbxDxI3XBg72nDIgs5HGEzSGXfk+M5NLsbXft1SB/geiNWeh+G1jCeVUFQutCVMT7x8GYH+Q/5sfPq0cXBgvH//bjZ7F0vBjlsp22IsI8Kr4omsotz8NA8Q0VPDDS9FCM2afUV+NQxGgMQcFKinLKJ4cWZ5cJp6risB9mJZQZ5FuNl8C6YXe4fV1M9qiEAdpuYRQ5DMRsS0lGIQVncLCEEFSlZKDntPKYccIOamYBwD5xzieDVFsf+woojuJArOIFZbGgfLIA3ZMIwDFEbRqvqqwyU1kNWWytFTSuU4XEKuiycM78R09+0msLY6g00Hbve6oNcB0O73+r3BZt+xtsfjvvP2PCE8DCCG8UaW1vKCyQZZBf4jRJMP7o71GsExlcVOtxO/vgBoo8xllYksVkEe2nk1HDpQTj5y0q4sPO8raBk5VxNTy76nbYyujTXCF33Gjx3t1OT6KlpSyfXZd8v1CeVkdhG/zNbb1JPnTZeSnXu/kp25ckZpHFkFfu4/inquEEMPHlNB3RTorAJfDx9VUVeIsUeNjK0igXtvRZ1ggPlcdw5QUgbodnCe7kaEta1Oiw+W7ErEg16FYKh+P8fo5XELcwRQ5RATu1ujyojTcqhqPuf+hy0U/BMB14PMHjq/4Pi/rdn+2Cz3NJhViV0RRk2C3fg0uzkkb8GDi4miNN+urNbQm0SHswhfq+9M/ZduilRq0gM7Mqn0mI5ciug5r5P0FI1cl50CkKjHpPG/s/0dcRlnMhepIKQ89nE7/zaSPd820oLnrXT7NhEkmkjCwqTqXyJKkqpdEgsbYVF+EExsrt+lUruD9ATC79m5JmmeTftM/DkSTGJXZaOLeHMSxz6mx0vEgf4F1nP6ExwLHdO4uSG93N7eeQVXf4bj+0MEzizzqLzrVmQybngzBohFRNmQ051PWzbUYR4NvMD1Ljw3AcXBi9vq6RB6/7EcwBW48iTnO0rSVHXFW8wy08Tizl9+RlNqrXZGhWf5flYZ4jW4qttsLXWH7oyISkHP5qqyVH442QMxrGhR6nErzRUnI/l5NOLVZRpirtirCg8qh6Z1+GCrxQctPmjxwdJlKFcHIOT7YC1CaBHCsiAELmUjeB0+YyNUPFzCxtYkbLYVCZuaPU0FvNFCnDqYUwN15oM7SshTC3u0DC2ZKl/fkTGQirsFh7sd7fUdxmlp8gpcNAc20uOjeozUgJPqsZIeL+kxUxNu0jurBvxUh6HqrtfIMlHhKT2m0uAqLbaaF1/VYawGnFWHtWrwVj3m0uOuRuxVi7/UGEyLw5qwmBaPKa95LetW55zgbA6ANjdIqwNqGrBWf5mt9pSFGrzVAjgpyzPXAUwVnKuDdBpYNxe0qzoSNcTTwLxaqFcH9/SQr8GTVqFfdQpVCKi0o1JVo0ihojqsVYe3ajCX/lqtBnvNc+PzOUyBt7X5Tpq1qKySnZofllktLGthWQvLlhOWPcmJqRaXzYvLGm9jtNCshWZ/eWhWyWY//jkn2fju88LXp0tSdjW7iZbdZikfKEvZa4LD3RYOt3C4hcPPI0v5mMeyW1R812yl+v5lC41baNxmLVuUpsxaNsK0XgvTWpjWwrRnkrVscdozyV62UK2Fam0W86mymIcIUUR4lyTmi2zkZvZSEVrb7aS0aDPuE2OxM4Uz8A2iOB2/tZUW0/cS5i+DTp8jkbLUWrN8oYlZ9IbhLPIJI4LJPN+MKxGV8t2Z2dvHNe/PzGtV4Dr/dhyfUFWrWvHydw4Vc7Xla9Z5r2v+mUB0zZRa+UW7kueWUDqBV9JFa+7Lc0X3cs5XuCjif+PeLa68DcKvHub6/l7GK/5AtxeQQOzCXfUbo9WMnBFheIrmOYOz0L4Id1lPw9ShriXRzR/hiIAEB7gXHokmBAqsGxnVN2r2203sz4ISrydEyRmP43/lwzDF2sp4aZm6cSbcdEJcRRLD05SQ8nKLWuLmb3/naYCEvVG8clmIFA+dMGG60O3INalzNK2OoAINhkgtfSO+JoyeaYyRb6E0yHkUkVi0F3iZFxWFf58DikRLCvj4xqx3EoLiS5cG7qj3qi8dLKb5v2TyixjJuRd9Rf7JdeCoLnylJmQ9BxOSRPeIJtTXWU7/HgbTpMAP5LnLRZZGhV/oACBZ0/rpmlydTsi50+3I2LBkEK3TAWl9bwpgrhGFThhzd1U3MWuq7YtUvVCBUKU/IExY78/WBXLPzYSLjwO1VvwU+EvF/kVtuMaqenpN7y2i6b0VUXHpw9Is+1AsmYrPy4SXG4XN51+QSdd45WORR0AAKh/OeDbMB8OL1+xzy0le9wUXbFqlnvM6b1r8j26Hr+Fuc9rc/63sIwdn+Rxo+q2qdvpeeMIDnjDfi93jf3DXtTddfrz5WAT2/QxZCtT8erqfrXLz5ffn4jWdxlvjc46ijLXj0I1TLGUmbPfB3Bptgq1R1x64m2Awtsadnt3rdQaD7mDLGWyCcdr4olhid17c/h8c4vp8in0AAA== namespace-by-workload.json.gz: - H4sIAAAAAAAC/+1dbW/bOBL+nl8h6HpFcudcLdtx0gDFIa9Nge1er0m7uGsDg5ZoW4gsaSkqiRvkfvuR1BslkZKdOH6JuR+68YxMkTPD4cMZDv2wpWl6r2e7fogD/VD7cd2IKAj+GdoIcjTguh4G2PZcSnwgJEJ07ADTR9gnLaYyTj+0HfzJJUyjkVEtgEHghciEhKF/Qd4Y4hEMA517Brqg71A+RiHk6CPbElBt03NPPMdDtEE07IPtZkNrGQb5Z2+voRk7fNMuGLMXH2Vj0d5qRw5EONcFPPHZcxYIRn0PIEuPeY/s/9fk30cmFGjZuNBbfehC/MkiFDd0nIiCgD+68jwH2z6hNxmRDufEczHyHCrQAXCC6Pt27ruO7d5wWvCBC9nzkcgTgeum5zjADyDfUo5hlThDZFtfvEyZkZALCrsjn1sdjnCfDCD+PKGfE+mkbbMxtNKPWa+vUxqCPgSYGypH/YQhYuoRs796dzkBM1Yw8u6ubFyyHB3HRP0kRAi6WDsGrnVnW3ik55+5tH+x50ZdjhFbAvLuIhuIx5gKHjg2CJgBMklmIugDRsl1hdrTb9AdYibmZo4Og5KGameLPrAdJydpSviIgGWTgfKakmv7fUHbRqtG24ZY2+30owOH0LXybyJiGrpHwZV4aoPbYXHw1HQjfYk9wdnYxxMx678QeWXOGNwL3jG2XQEV2cMRvhT6G2pmAip59g9mUnmrpCwP0SGkoymwTmFgltvDxD05go7dAieEiVmV9EB8hcCKKPUu7pvBU28KM5LI4pLM1NyM1+lovni2iz97TByMoHMz24Qii/YhsVoXg2HZI/m0NWqiIX3/Xp5e7j4RmgURZP594HiZACUeJIDIhsG/biEiFg8LQwxk/Q1KAw8wMG9KvQkw9H1o/UZkWuJhgIYQB9x6yK+IdLW499kwqN57pG/mdhCOt23i6uA2WcYwIK2iHlk+7jx0Q5ZgE9q3sNefYBj0mEU8mE5IeoA+/NTfxH/+1Bt0VWMD+/A/Qk8//dQff7whIoWIWM3hG7KYe05Iner1zk/3b5rnatvpsw3fswh1iLzQ7zlwgLdpFxwPWI3kjx51hTta+pVeyiDfPbwJ+5D+0fPuyBgOEXRAH87a34aWtEk5//g7T2Gvp+3Q/5Oh7Wj9iZb2cod0nptZ1BF6aMyMg/j2MexFVpF/JJHNOTAxww9Gjh25sfO0nYeHtDPa42O+KQQHbEXSj/J0ai7M0FLiY/xXZpN4RFQz8hyrYKu03+fE5xcNnNIvR/YAlxmFhe4rMSzNG2jH1IK0r5E9WdzilkKSB97BAcTQQtHFRX6sVfRGvWSJtF3LvrWtkDiukmdKnmFIKOvAPbi3C0tSPzRvoklU8KPj2P+UNJmguqLfjRy11IP+4PWUOOeUdl0awgTcw4q5ndnbsV+wMzYVSh2kDG94DBhmy1tetFSVHo/WqmbewLLlKDOxxup3cYsX8+ZhqhyhAlR1FKhSoEqBqvmBKiIjNxjbWKEqharmgqquYoPCClgpYLXqwMokvmrsVggbw3s2jisyG/IDYepNWbUiSRr6Tr+m/eVI0ljKnrXB4+oGj2du8KS6wZOZGzytbvB05gbPqhs8m7nB8+oGz2du8GN1gx9nbvCiusGL6RtM1hhJgym7YkWZeueQ4cEBwR9JVPl986/67DuH2ti7IQm+75UAbG7X9AL4VQOB9ovuEqZAmYGJPCarXKeoQ7uAwGLfyHOiNZITFnFlBWlYEehn61JJJjNh1IlTtSCx3anYSZp0zxrLpOTCzWRDm0k3sSuY4aH/kP92P3/ePT3VLi4Ox+PDoLCiWNC0x4BlU1r5FYUor7wux4ws+6SfImKmmkUgpV567BuiUtXzDB8QdINc8ZAlOCsHTsj+0YKFt4WuzcZLvotw/TxOhV5K4pTjWpuhCvHKOo063HDcJ1NMqA4KYJ6jDBEc3ih9HC9TH0dkgw6GUE0OMahbtjI2fnKcLlMfyb49mRLaFxDtiDdSFWfzVYX/JFVwE2KztXG+CtooTgztFHk0LruZKvm4ohNks7VysUyt/CEMIqyC6PMh5SdJ3nq33zroD/b3TdMwul2rNWgb7XZ/0No76HQP9rum+Y6mI5ALMQx243yK7Q53k9jJPz00/GR9MN4iOKBa+dBuBm9vAdrNMhNZWoIxWPKBpR7YxzRN8abXMyGXECvagjiY81xbKO5EyykFdb5nJTNRLP1dyEEFGAiS3KL0VKsyPfXkhFRDZSs33EaO52cj4Hao3MgrNJGTxZiIciNrbCOni1lqEjfiR7sMZSRrZSRnC8YjykrW0UrOl+NKrChgoYxlrYzl45JcirKWdbSWiyec0Hz2ScxLDDB//iaJ8kQCkJ2AS+pkcwdM+DLZHGN+VbLGe/FJna6gTLYiECmoi5DVRlTVR1TUSExbqS2olaisl5DKU3T6SVg7IZJtKt9Wk6PlZcPkvC+w63I3Kmsq5HUVlbUVNfUV1TUW8joLea1FTb2FvOaipu6ipvaiuv6iqgZDWodR1qWwHkN+pk1yrk16tm2K+oyqGo2aOo2qWo2Keo3q03Tyuo3a2o3Ks3Hi83HyOo66Wg5pEL3o6FQUbLHYYcrSjinKO2Yt8aiKqkuABA8mcoCiPgkkBhyVoCMPPJIjPTWFtdIakKo6EEktyFT1IGUvKa4LkdSGVNaHVNeIyOtEKmpF5PUi4poRvm6kPNRy/YjQn8iLNCoLNaqKNSoKNmRFG9LCjdK41nQIWyKVVWbWFaAVEadBtAcK0SpEqxDt60S0KmmnIO3SIK30pLpCtQrVKlQrC2kv/k7DUpHJPO40XOplki1JmPz9it0mmZX1XHwKyLoxWcRtksW7jF7BdZLtA7G+ufFMcfdR8TaKusuPipyF3n5U5M18/VHtTUbiIujaSvDWdJXgmU7L+woCZF1o5m5DeWXXGXGjTzYN+Vp1dUXk5l5mlLxtta8yiiPWQtSwjlcX4fy1BNUXFxXGkCF2dTXRy3bxWXc+vgLkU7r0UQp9DAV9FPRR0EeVRinsM3fsk4Q2FfhR4GeF7mVc1qnUtiTcZrTmeixVlI/fkHOpnZosvtF+ShpflHyYJo8v4i41kS/iPzmTP1UynkeI8my86EqFMlqcLhmft5VpkOMmZuQLUhIhzJdOyKsKyfXNyEvg55rl46e7pexVpuFx8XLRaZLwgnGWQe0c0uv++qfX/c05NLohcFN4arQWb3YU3lR4U+HNVceb6rIFBTgXCDhr72JVmFNhzlePOVfgSGdx8s3nIOei48odye+SGHsqrjynuLJRg/O7CucrnK9w/rrFldUFSAruLz6+LL5WXmF+hflVnFnFmVtPwZ/7Cn8q/Knw59rFmRUAVQB0KfFmhUEVBlVx52XFnc8QolD3KWHnrbjnevzTT5TbbkZt0ccyR6IH5giOwXeIgqj/xkFEpj+6zH6GCqCb6HsEA2RWq2e/PaWnb8Nw7DtEEO4wnYUEswSYs/UHUQWU8BfCLTgAoYMlPxCecEW7BgYkCyFwsaklU5yH+xzX86lSi15X/zOEaMKMWrQt4GRu5KhDeF+4sDnVHPf6YpQ+V2fmfI/HXxgGJ0peDlNsYhJZ8VUTtksWagseOaK6Lokgx0QZtuDxRMDx0j+LdNmbepFD3U7Bhe0OvIYWN7cjlnurTu7xasQbCLFuJtzg38n79Ty31FFKEz8cazUaCccIA3gVNSQsiptK1SDEnqjIkJB7phcyI2g3i5zIK+pGM6f7mhlIZb4bTIigx5JZyD8hnInTWCCZyrZrx+4zr/XnHPp9bGRAdUev9g4vYfHpy59s8y8y+llmyY3tf0PO5cQ1RZWi0Rwy1ByqXsV8x5uMi7cY8gtZ+sBLzKDF7tUeC5u1Zcw7ZjlPnXIrJK55T1S12FVO1D3ZGrf3jKVtQSafxXUkhr8l26PpAXSigKA4lplIp90sXceXCojyZHtd+dtEN7eKVSFTxtSvqhqYMZK+i7C2RPtEoRchQmjsjRu55tZtrU2CYGoWL2WLJBL/rHO4YlZ15JbemcXSOxti4ltJROgxim3YTEtxVGMQhVx117vbTee8zsyc0vTc13yboHWUfTmWWS/pDK9efY+L+PC2n3PBupHZOW/zusF/aDd5DnerQYv727Ai3V8nY6Bh8rLZyd/CN9zlG+bf0urwH7gLWfYtvr9JX3Li++Wx/Iv+7eokDkQlEbLf0x89195pvydIS9tOfhc+wlN6yNKfer/fb4H97h7ottoAtg+MdrPV7Jr71vsOMLrNQfTwbRoJa249/h/ptNL4j6cAAA== + H4sIAAAAAAAC/+1dbW/bOBL+nl8h6HpFcufcWrbjpAGKQ16bAtu9XpN2cdcGBi3RtjaypKWoJG6Q++1HUm+UREp24vgl5n7oxjMyRc4Mhw9nOPTDlqbpvZ7t+iEO9EPt+3UjoiD4Z2gjyNGA63oYYNtzKfGBkAjRsQNMH2GftJjKOP3QdvBHlzCNRka1AAaBFyITEob+GXljiEcwDHTuGeiCvkP5GIWQo49sS0C1Tc898RwP0QbRsA+2mw2tZRjkn729hmbs8E27YMxefJSNRXurHTkQ4VwX8MRnz1kgGPU9gCw95j2y/1+Tfx+ZUKBl40Jv9aEL8UeLUNzQcSIKAv7oyvMcbPuE3mREOpwTz8XIc6hAB8AJou/bue86tnvDacEHLmTPRyJPBK6bnuMAP4B8SzmGVeIMkW199jJlRkIuKOyOfG51OMJ9MoD484R+TqSTts3G0Eo/Zr2+TmkI+hBgbqgc9SOGiKlHzP7i3eUEzFjByLu7snHJcnQcE/WTECHoYu0YuNadbeGRnn/m0v7Jnht1OUZsCci7i2wgHmMqeODYIGAGyCSZiaAPGCXXFWpPv0J3iJmYmzk6DEoaqp0t+sB2nJykKeEDApZNBsprSq7tdwVtG60abRtibbfTjw4cQtfKv4mIaegeBVfiqQ1uh8XBU9ON9CX2BGdjH0/ErP9C5JU5Y3AveMfYdgVUZA9H+FLob6iZCajk2d+ZSeWtkrI8RIeQjqbAOoWBWW4PE/fkCDp2C5wQJmZV0gPxFQIrotS7uG8GT70pzEgii0syU3MzXqej+ezZLv7kMXEwgs7NbBOKLNqHxGpdDIZlj+TT1qiJhvT9e3l6uftEaBZEkPn3geNlApR4kAAiGwb/uoWIWDwsDDGQ9TcoDTzAwLwp9SbA0Peh9SuRaYmHARpCHHDrIb8i0tXi3mfDoHrvkb6Z20E43raJq4PbZBnDgLSKemT5uPPQDVmCTWjfwl5/gmHQYxbxYDoh6QF6/0N/E//5Q2/QVY0N7P3/CD399EN//P6GiBQiYjWHb8hi7jkhdarXOz/cv2meq22nzzZ8zyLUIfJCv+fAAd6mXXA8YDWSP3rUFe5o6Vd6KYN89/Am7EP6R8+7I2M4RNABfThrfxta0ibl/OPvPIW9nrZD/0+GtqP1J1rayx3SeW5mUUfooTEzDuLbx7AXWUX+kUQ258DEDD8YOXbkxs7Tdh4e0s5oj4/5phAcsBVJP8rTqbkwQ0uJj/FfmU3iEVHNyHOsgq3Sfp8Tn180cEq/HNkDXGYUFrovxLA0b6AdUwvSvkT2ZHGLWwpJHngHBxBDC0UXF/mxVtEb9ZIl0nYt+9a2QuK4Sp4peYYhoawD9+DeLixJ/dC8iSZRwY+OY/9T0mSC6op+N3LUUg/6nddT4pxT2nVpCBNwDyvmdmZvx37BzthUKHWQMrzhMWCYLW950VJVejxaq5p5A8uWo8zEGqvfxS1ezJuHqXKEClDVUaBKgSoFquYHqoiM3GBsY4WqFKqaC6q6ig0KK2ClgNWqAyuT+KqxWyFsDO/ZOK7IbMgPhKk3ZdWKJGnoG/2a9pcjSWMpe9YGj6sbPJ65wZPqBk9mbvC0usHTmRs8q27wbOYGz6sbPJ+5wQ/VDX6YucGL6gYvpm8wWWMkDabsihVl6p1DhgcHBH8kUeV3zb/qs+8camPvhiT4vlcCsLld0wvgVw0E2k+6S5gCZQYm8piscp2iDu0CAot9I8+J1khOWMSVFaRhRaCfrUslmcyEUSdO1YLEdqdiJ2nSPWssk5ILN5MNbSbdxK5ghof+Q/7b/fRp9/RUu7g4HI8Pg8KKYkHTHgOWTWnlVxSivPK6HDOy7JN+ioiZahaBlHrpsa+ISlXPM3xA0A1yxUOW4KwcOCH7RwsW3ha6Nhsv+S7C9fM4FXopiVOOa22GKsQr6zTqcMNxn0wxoToogHmOMkRweKP0cbxMfRyRDToYQjU5xKBu2crY+Mlxukx9JPv2ZEpon0G0I95IVZzNVxX+k1TBTYjN1sb5KmijODG0U+TRuOxmquTDik6QzdbKxTK18rswiLAKos+HlJ8keeuX/dZBf7C/b5qG0e1arUHbaLf7g9beQad7sN81zV9oOgK5EMNgN86n2O5wN4md/NNDw4/We+MtggOqlfftZvD2FqDdLDORpSUYgyUfWOqBfUzTFG96PRNyCbGiLYiDOc+1heJOtJxSUOd7VjITxdLfhRxUgIEgyS1KT7Uq01NPTkg1VLZyw23keH42Am6Hyo28QhM5WYyJKDeyxjZyupilJnEjfrTLUEayVkZytmA8oqxkHa3kfDmuxIoCFspY1spYPizJpShrWUdruXjCCc1nn8S8xADz52+SKE8kANkJuKRONnfAhC+TzTHmVyVrvBOf1OkKymQrApGCughZbURVfURFjcS0ldqCWonKegmpPEWnn4S1EyLZpvJtNTlaXjZMzvsCuy53o7KmQl5XUVlbUVNfUV1jIa+zkNda1NRbyGsuauouamovqusvqmowpHUYZV0K6zHkZ9ok59qkZ9umqM+oqtGoqdOoqtWoqNeoPk0nr9uord2oPBsnPh8nr+Ooq+WQBtGLjk5FwRaLHaYs7ZiivGPWEo+qqLoESPBgIgco6pNAYsBRCTrywCM50lNTWCutAamqA5HUgkxVD1L2kuK6EEltSGV9SHWNiLxOpKJWRF4vIq4Z4etGykMt148I/Ym8SKOyUKOqWKOiYENWtCEt3CiNa02HsCVSWWVmXQFaEXEaRHugEK1CtArRvk5Eq5J2CtIuDdJKT6orVKtQrUK1spD24u80LBWZzONOw6VeJtmShMnfrdhtkllZz8XHgKwbk0XcJlm8y+gVXCfZPhDrmxvPFHcfFW+jqLv8qMhZ6O1HRd7M1x/V3mQkLoKurQRvTVcJnum0vK8gQNaFZu42lFd2nRE3+mTTkK9VV1dEbu5lRsnbVvsqozhiLUQN63h1Ec5fS1B9cVFhDBliV1cTvWwXn3Xn4ytAPqVLH6XQx1DQR0EfBX1UaZTCPnPHPkloU4EfBX5W6F7GZZ1KbUvCbUZrrsdSRfn4DTmX2qnJ4hvtp6TxRcmHafL4Iu5SE/ki/pMz+VMl43mEKM/Gi65UKKPF6ZLxeVuZBjluYka+ICURwnzphLyqkFzfjLwEfq5ZPn66W8peZRoeFy8XnSYJLxhnGdTOIb3ur3963d+cQ6MbAjeFp0Zr8WZH4U2FNxXeXHW8qS5bUIBzgYCz9i5WhTkV5nz1mHMFjnQWJ998DnIuOq7ckfwuibGn4spziisbNTi/q3C+wvkK569bXFldgKTg/uLjy+Jr5RXmV5hfxZlVnLn1FPy5r/Cnwp8Kf65dnFkBUAVAlxJvVhhUYVAVd15W3PkMIQp1nxJ23op7rsc//US57WbUFn0scyR6YI7gGHyDKIj6bxxEZPqjy+xnqAC6ib5HMEBmtXr221N6+jYMx75DBOEO01lIMEuAOVt/EFVACX8h3IIDEDpY8gPhCVe0a2BAshACF5taMsV5uM9xPZ8qteh19T9DiCbMqEXbAk7mRo46hPeFC5tTzXGvL0bpc3Vmzrd4/IVhcKLk5TDFJiaRFV81YbtkobbgkSOq65IIckyUYQseTwQcL/2zSJe9qRc51O3Qf/jD65PF3wTWrU1WE7L+N7S41R2x+Ft14o8XJd5OiJEzGQf/Trqh57ml/lKa+OFYudGAOEYYwKuoIWFt3FQaByH2RLWGhNwzvZDZQrtZ5ETOUTeaOROomYh0pu8GEyLosWQy8k8IJ+Q0hkhmtO3asRfNK/85Z3+JlaTwckevdhIvYfjpy59s+i8y+llmyY3tf0XO5cQ1RQWj0Rwy1ByqXsx8x5uMi5cZ8utZ+sBLzKDFbtkeC3u2Zcw7ZjlPnXIrJK55T1S12FVO1D3ZGrf3jKVtQSafhXckhr8l26rpAXSiuKA4pJlIp90s3cqXCojyZFte+dtEF7iKVSFTxtSvqhqYMZK+i7C2RNtFoRchQmjsjRu55tZtrU1iYWoWL2WnJBL/rHO4YlZ15JbemcXSOxti4ltJYOgxCnHYTEtxcGMQRV5117vbTee8zsyc0vTc13yboHWUfTmWWS/pDK9efY8L/PC2n3PBupHZOW/zusF/aDd5Dne5QYv727Ai3V8nY6DR8rLZyd/CN9zlG+bf0urwH7h7WfYtvr9JX3Li++mxNIz+9eokjkclgbLf0t8+137RfkuQlrad/Dx8hKf0kGVB9X6/3wL73T3QbbUBbB8Y7War2TX3rXcdYHSbg+jh2zQg1tx6/D9hH5uqlqcAAA== persistentvolumesusage.json.gz: H4sIAAAAAAAC/+1bX2/bOBJ/96cQiF0gOXh3JSdu7AX2oc0ihwXaXm6b9h4uC4OWaJkXmtRSVP5skO9+Q1KSKYmqbaSbpq2BtohmhqPhzHDmx6lyPwgCNJtRnhUqRz8H//1jaCmS/FlQSRwa5lworKjgmngPJCAymisjAo8PRowkVOE5I0BdYJYTQ0w5Ub8lQOIFY5Yicba8EIIpmgE9NMQlTcip4EoKljfW08ZaRvmVY5gkCzB0CQR0FObI0sSNkTBWWluBGgvGcJY3bGswkg4nw5ywtSpXneFjRnF+KpiQxisPQ5c5xzJvazSMBOfL14SnSlsdhR0e6VumcC4KGesdoHMpVkQtSWH3XEstKGNabYf4T4kTSriq/F0zU0mTc+HZgPH7qEFiJCU8qTPAdUTKX+YXZeyVLBq2a4nr1M+ICymtWR7mCt/2MCj3MyRNl+odpJLPh8DPl+LGvzKHRf+hiYlKlW0OW0H+sx6l15gVJmparcNq+hMytxbqMG7KN0dtjpPsNVmbdy4oV2+E2aghNPMgI5AoXOHU6wiU6dU6Iwqtftzl+XMQIpUQSaR+54IJhVrcjGDlcR/KiaQk/9c1kZBsxLOjPMMx6TsUwNTRnjaJCsdXPm/mimQZSV6DU717UFimRDVPdftkG0Fym5mNHlzyIMiLVXBD1VIU6oByeDmPyTDg4P/D4ECJ7OogGgYHV8WcMKJm14IVKzIDMZXPYgybo+puNr9TJL+PWQEmyl8u0Xflj5doGPxPzIFSrtcEjlfEeEUL1g+aA6HNofJCdO1rYobpSkvZx0v0cHh4qG3+4ZGG42tMmT7ST2w5/EHDdjQWQq5MciFFtYUmo7pikLpEwnk8w7ESsn2gnCJ2Vut7D6U/eKdt7KqD9mJaF3qJGqyH4Xa58wV6/yldfyYJ2ej6Vy3XO0/NIqKWGgoIlngKjLb8DHqmrzpp3rslXSg/U5mehj4YL1lrg/e5rqxNwRrP3LdbDpZdcFExhey05KqjzNRdZl5NeUKvaVJAA+rtLpWswVZNy27xLc27ds2L+MoWQk+7W5WdRfumFRuks8q/qr+71i3SoEX/Fu7wLdmiLq/T0RyMbubAqSHMa6BmivQVNiiwm54WbniXWcARdujOjncqEF/GJtyjNvBsrYGGYxwvyQXkCxQ730mKNVB+BV07laLg3gNhZT7oXOlnd1MEyXSOD8bhMIhORsPgeDwMwh+nJ4ftxDVio6MTkBtNQS7UcpNpj5xWMz62f0HdIeotPFtC8zrkJThrslNcGLB234HAlT+iMOzC4IoZbn0U6zL5WqdY3lOYaqk3WF4RuRHdfvwacdQklZ3ClyU9kHeFs4zy9MJWuaiPt0X5KKuXrUiBEoEit6p7/q4rp+90rCvlEvN0C+WjLTsbJMGvkGPnFTJvZUL3QhALzkmsoO105C60RR7HZyJXC6prB0I+xhnczd/Rv4z6cfh9Swa6tXetoX90qXHVG7iAb1H5bQPvXndsxtqNobc/vfQwRb1wS5+Xt46jNlFeMXu1aB1Ufck2swATf11BjgDbRdEE/plMdQmJJp1KsyhY36VSv8XVZ9WNdJGbHnUUlcfdKOrHCBpOnmocw7uh2vleBEnZxbYltL3k5tZUIdznci+qb0UfNexzIG9z6/npkj8/j/1DV5tPei8YbbwX7HAR2+42gCZwbqYh+luh/UZsnkOXYkTHsqnftAO3SE7aRdJIbFckhbYT/bJjfaw60k4l0ix6W7a8apA3aK+r1/gHRCX1N0gSM2D2s38XN40Zcl30LsogNie3dWh/xflyLrBMAtCAmvzK28sXDqMMlQTpgRPJ/Rj5UWPk4/0YeT9G/rbHyI8YBVItlc8KKC5f5RjQTGDtJj/9CPax43tr1xc0tv8c2fLECROYwfHGjHlek2Nr7n50/CxGx1xw8qVPjp/xHvaD4/3g+JMMjsf7wfF+cLwfHH9Dg+OPTkCfGttunsk+4fXgW5rFPhos74exX+MwdlDuAOUAGVf4gz40xsro2JLVnX1VAuXWfg2scLoOlDmLkpP1Jxcozimq1SqyyhjsnKfdz5wH7Rg7Q8cGnq1im5AFLpgvvVyuDx6bj6Gb2KwH19fAwYGSDldk1TfbLixCfxZE3hlA6YOczkfVUYOaknanrkPkvH7Q2o7jMMxYBTtb23Bc6fphC4Bc+cqpZFDmoAgn5KWvR9aORGWhdnWtICLUs6bysmfJJheb183sJXabD0SHQfmSQ39IRptCUs4F3NDBGTB+z/9dWYWa3I75muYXLgNu9+cwoCdfWEVu/X3qLAh3zoK3VRfeJQ+4b9GnzoTtQMWDgyr2KfMkKXNeYzWLVk41VtslfWzQP3/u9APShx5E+nVk2KAaTNlfWzKTyrrf15dFcfNDVM1Dq3ugRSLrZRmNr8x/ZJWLS5fMKqDtYkQ0XqcfikLn4ch9iNaphMbOz5H7cBS6nPXUFo2cn6PENuM/qj3oq4GTaRvf4ip+4Sp23zI6dh/WgwN0krj2VrY03PeXMDdi9P7itERrFVZcn7HAHrLyl7sKMxZC02g6n47whEzC42iejF+c4MWUJHgejeLJcRhb4esaH4aDh/8D0iZvTe82AAA= pod-total.json.gz: - H4sIAAAAAAAC/+1d6W/bOBb/nr9C0GQXCdadsXzkKNAPSWc6KNB2sk2mC2wbGIxE20RkUUNRSTyB929fHjpIipSd9EiCqB/S+D2Kxzt/fLSY2y3P8ycTlKQ5zfyX3ufznqQQ+FeOCFRoIEkwBRThhBNvGYkRY5RR3kR88gqq4FzkKKZvE8YMejU1AhRkOCchZAz/hOAFpHOYZ77SBibgIuZ8SnKo0OcoslBRiJPXOMaEd0hmF2Cn3/MGQcB+jMc9L9hVu07AQgx8VK/F+6d3FENCtSnQZSraRSCbX2BAIr/grcT/5+znSggFRogas/VnCaRvI0ZJ8jiWFALS+RnGMUUpo/cFkS/nNU4owTEX6BTEmXweac/GKLlUtJCCBIr2UuSlwP0QxzFIM6j2pDGiBmdGUHSCa2VKIRsKu2afByOFcFMuoPi85J9L6VR9izUMqo/1rM8rGoEpBFRZqkJ9SyER6rGzP+JrTcCClc3x9RmiDcvxaUH0X+eEwIR6xyCJrlFE577e5hT9LdrN9xRGYQkEX0sbKNZYCx6Ec3iGFhDnjcWE3C6PQXg5IzhPIptuMPkE4hzaWZniWoz20+DwMBztqYYqDH4w3GeGPjjseSNm+/2fDw41o/8pGo3AEJQmXGtgrTP6EQzRAgjNVTr3p5gshOKY1BZwkkGCoPLMDOQzqBvVAtyUqwz6qvEsUFIyVDJXpSkRros5C0hzHEfvwAXUncZs8R6QSyjkx02hYZ5O0z80TD8YrDH9oNH3HKLZnKp9SW8Y1h8TZt5XIDbNRXf1QnJpipLZmbTCwEbXjaRejxLurriMPYo9Cm+oYhpewVLWoazE0RkByWxNZ4O6s4bRMVv4ldndCWZSyHSD4OZwykKFJnefi0i0fo9F/GeukSQwpCyiaW3O+HQMieJUz1fSfhGMoz8sHOHNcagLVGoGsDSnkM61ZUdwCvKYmp3Jxcollv96Bh8luk3pAWubyzxLQQhfetspjnyjYZ4g4YfHaabObqXNrjAV3bCkdK4gITKp3urPVH7UlIW+wjJSyWRJLl/MCISJMU9h8hG8aS5VsRmuN4236t1l3CWMYxY0nAMHzoHHlXK+YngCI/fYA+fY+46xnaYmHqwiX8PN6iiU4oxOER/d903iG4Y6ylw37v9D4RPYfEbQnI+IaPCeAQx3FJqyxMKf4xrW4wWVLut/+OXIYODqgZZQkjViBaOQSxZGoenxcaxjxCHDh0FwwH4cHPKUGRxoKXPKR26mF96z2o/sZsCSbnA41DpQM1hDNwIxsn7yRaLLmgIyg7RFlPAmFWNn+WIHMYQEd1gwpIBNi0wY6rzG5JIh9xCiKzi5WFKYTSjDufFtGOcZyzmvvvjbxa9f/F4VXF7974sSahjHY7FGENn/X/zV5+0yY73cZnGBTZzHzvPdXV1p7big8IiMgsQM01pSfANCKgSseawfwxlMojfVEHq/zEYFIPSP2uxFi2qq2Nl030gj1bIHp5/O0ZQ2GQai/MhU4eGpd8xl7n2UGoiaKFJio4osvFl1rQPVtQS33bUw3074rzb0qjLqrHUs0fBDke9DucYO/XboVyfcDf6OOvjbwd8O/nbwt4O/Hfz9jvCXMpVkC0Q7/Ptg+PesUAF9HhD4ISvvgaP0Pn5kpfdvWnIHMQLZ63KLUOdu/wKQBi4WZzjvYDKjQsB9jQ5tzddtC3g41U43OOF3AiIEhWv3v0eZObDrea9GzyJA6EMxQc2So+ysOKEy4zm4mlmopbE3OfzE6rdFSpf2s7n/QoKbHIkBza4k8jOphG8dTiUkM3lFSjF6z1jj/wi7MkOqLyK/pSMrhlipuxCYmXbMidfFMIPWDct9ITzPPDYrTSGzRJbjZs34kvIuudnlme7x5d5Cb86UGkECRQKdxpj6a8KBzF9/FCDZWGfmmq8FlFAQXpoCZSk4TWH0TmIVfaJPrQrmXSy9HdbmPnDgTjn/9paNslo5U79G5xJWE8RaQKBo9/6AoCh9WY9Yq/PvW9WrAREJ0vRrTKi+dZB+OymTA9tcoCsU5aDK4qsm1ODH7vUEbsANMmLxRR5eSlMzgsei8FS+YssXCIzW9thUBZrP540pLsFNW+With2+t9WNgld/bAAyxrNjkMGGFckA3Ghu2Xur61hbD3lMUywN+7mihWZZzgEX9ju40MGFDi48bNWgwws1XihLBR1g6ADDYwEMSkFHC0JqPUdjfLtyzsBRzjmwlHPsUnahHSfiaUM9Lchn02/zWhBQKwpyCrQWqnleZEIim3RrCQfOkyIh6UNLRG3Oox0pudFSO2Jag5rakZMbPbkR1DoU5Y5Va9BUG6JqO8gxVWJDV26E5UBZTqS1KdpqQ1xrUFcb8mpBX+0IzI3C1iKxVjRmR2ROVLYOmTnRmfVA8S5FHTZ/noUfAqZtCNU2gGsbQ7Y22OaAbu2nqA4Y1wblWuGcUQIqzoLKb0F5J1Jb5hFnE9y1ATwHyNsI6DXjih3wOUBfK/BrA39uALgmsDaBYHMJTUBodavaVNPUYqIu5NWGvloQmPtbHQ4ktslXDZ7CElRfW4uEO4xmL11tDNKM1XcorUNpHUp7NCitqqV1MO0JwDTlyzodUuuQ2jNFao1S94//ipTpfF/zTvIDVjAdJ49B0JUwv1EJczhYg44HHTru0HGHjp9GDTMimA/egeSnVMv0fpVa65Byh5S7mmYH2jaoaa5FbcMOtXWorUNtT6Sm2cG2p1nb7JBbh9y6GudD1Th/I4QDxPuUOLeKmfMwxGMH5w77si/eTLkjMwvncAE+QZLJ+QcHkkyXMSyvLJDPscRZW61/mV9AwiI+FNdJyL4oXKQxE0Qy2+TS0xpg3dpeRy5uynC8klxybVi7vPxULbfaTa26qEQByQq3vhJEjbr+XzkkS2HU1itZa5kHGnUGb4yX0ZWrU6vhzYqw9spJ/Em5AcP6Xom6mdjoAtlCVup3wVHC8nIEj+z3GtgFuWDKQJbmpYCLTH8X6YqRJjKg7nBzm7BcO0HJFPe8ortdu9wH6+ReZCPVQJh1C+Fm/y7H93VuY6KcZm9caFWuRGHkGTyTHVnfj7Gr2v/5X2ofIBcXXjTeN2LkSYhzYQTDvsmRUdEP+pru13ggl/mLbMkEvXB4odrC6ombWCBzZZSgInzqWv+abzWuel4FI3f99uigWbzxosE9Db4a+94m/10WfxcnuUTpnyQ+XSah7Z0x6UJB50JtLuTwm0fnLO7910pswO7kQN8oZeiXWP0437mTLJ6lPxmK/IH+NHZlovFXJKAfZM91tcJh1luunZSfwVjWuuxlulI6w765yaoFxHmuHal7NMsO16EKlzI2HqptYcHcORZjbdl2c9YYwYTQGy96WndPzYWrGyE7L36IjYxN/Hf14RavGrktfXQXSx89ExM3/sSIKONVtYfqNkF8/aLy+fLCQFksqR9LEQMGpH64kNmknIyqXn+s1GVU29dCsB/Udq7avB+oH4Z9laO8UT1Qfg+KP6pyXq6B17GbZuceRe14T+1YHWUwUj8oNyjsR+p8y7lo4vsbi6MF/8+z10W5qKxjfZAIDCUz7xfvpAB1fi4O8vx9EBz09/ZDeDgaglEfwMF4NB7t7Y/D4HA6HYey8VVVnupvrf4PBQMVW4dnAAA= + H4sIAAAAAAAC/+2daW/bOBqAv/dXCJrsIsG6M5aPHAX6IclMFwXaTrbJdIFtg4CWaJsbWdRQVBJP4P3ty0MHSZGykx5JEPVDGpMUj5fv8fCVpdy+8Dz/4gIlaU4z/5X3+bwnSwj8M0cEKmUgSTAFFOGEF96yIlYYo4zyJuKTV5SKmkmOYvo2YZVBry6NAAUZzkkIWYV/QvAC0jnMM19pAxMwiXk9JTlUyucospSiECfHOMaEd0hmE7Dd73mDIGA/xuOeF+yoXSdgIQY+rNfi/d07jCGh2hToMhXtIpDNJxiQyC/qVuL/c/ZzJYQCI0SN2fqzBNK3EStJ8jiWJQSk8zOMY4pSVt4XhXw5xzihBMdcoFMQZ/J6pF0bo+RS2YUUJFC0lyIvBe6HOI5BmkG1J60iatTMCIpOcL2ZUsjGhl2zz4ORUnBTLqD4vOSfS+lUfYs1DKqP9azPqzICUwioslSl9C2FRGyPvfojvtYELKqyOb4+Q7ShOT4tCv3jnBCYUO8IJNE1iujc19ucor9Eu/muUlFoAsHXUgeKNdaCB+EcnqEFxHljMSHXyyMQXs4IzpPItjeYfAJxDu1VmWJarOynwcFBONpVFVUo/GC4xxR9cNDzRkz3+z/vH2hK/1M0GoEhKFW43oG1xuhHMEQLIHau2nN/islCbByT2gJeZJAgqFwzA/kM6kq1ADflKoO+qjwLlJQVajHfSlMifC/mzCHNcRy9AxOoG43Z4j0gl1DIj6tCQz2dqn9gqH4wWKP6QaPvOUSzOVX7ktYwrD8mTL2vQGyqi27qheTSFCWzM6mFga1cV5J6PYq7u+Iy9ij2KLyhimp4RZWyDmUljs4ISGZrOhvUnTWUjunCr0zvTjCTQqYrBFeHU+YqNLn7XESi9Xss/D8zjSSBIWUeTWtzxqdjSBSneryS+otgHP1uqRHWHIe6QOXOABbmlKJzbdkRnII8pmZncrFyieW/nlGPEl2ndIe1xWWepSCEr7ytFEe+0TBPkLDDozRTZ7fSZleoiq5YUjpXkBAZVG/1ayo7aspCX2HpqWSwJJcvZwTCxJinUPkI3jSXqugM3zetbtW7y7hLGMfMaTgHDpwDj6vN+YrhCYzcYw+cY+85xnaqmriw8nwNM6u9UIozOkV8dN83C98w6ihj3bj/N6WewOY1osx5ifAG7xlguL3QlAUWfh3fYd1fUGmy/odfDo0KXF3Q4kqyhq9gJeSSuVFoWnwc64w4ZHwYBPvsx/4BD5nBvhYyp3zkZnjhPav9yG4GLOgGB0OtAzWCNfZGECPrJ18kuqwpIDNIW0QJb1IxdpYvthEjJLjNnCEFbFrkglHnNSaXjNxDiK7gxWRJYXZBGefGt2GcZyzmvP7ibxW/fvF7lXN5/b8viqthNR7zNaKQ/f/FX33eKiPWqy3mF9jEue8839nRN62dCwqLyChITDetBcU3IKRCwJrF+jGcwSR6Uw2h98t0VAChf9imL5pXU8XOpvtGKqkWPXj56RxNabPCIMqPbCs8PPWOuMy9j3IHoiZFSjaqioU1q6a1r5qWqG03LcyPE/7rDa2q9DprDUs0/FDE+1CusaPfjn71grvh76jD3w5/O/zt8LfD3w5/vyP+UrYl2QLRjn8fjH/Pii2gzwOBHzLzHjhS7+NHlnr/pil3ECOQHZdHhDp2+xNAGlws7uG8g8mMCgH3tXJoa77uWMDdqXZ3gxf8k4AIQWHa/e+RZg7s+7xb07NwEPpQTFCz5DA7K+5Qmf4cXM0spaWyN2v4HavfFild2u/N/QcS3KyRDGh2JcnPLCX86HAqkcysK0KK0XvGGv9b6JXpUn3h+S0dWRlipZ5CYGbqMS+8LoYZtB5Y7ovwPPLYtDSFTBNZjJs1/UvKu+Rql2e6xZdnC70529QIEigC6DTG1F/jDmT8+r2AZGOdmWu+FiihILw0BcpCcJrC6J1kFX2iTy0L5k2W3jZrcx8cuFPMv71lo6xWztCvlXMJqwFiLRAou3t/IChSX9ZbrNX971vVqgERAdK0a0yofnSQdntRBgd2uEBXKMpBFcVXTdTgt93rCdyAG2T44kkeXkpVM5zHorBUvmLLFwiM1nbfVDmaz+eNKS7BTVvmotYdfrbVlYJnf2wAGePZEchgQ4ukA240t5y91XWszYc8pimWiv1caaGZlnPgwl6HCx0udLjwsFmDjhdqXihTBR0wdMDwWIBBSehoTkjN52gV3y6dM3Ckc/Yt6Ry7lF204ySeNuppIZ9Nv81rIaBWCnIKtBaqeb/IRCKbdGsJB847RULSBxaP2pxHOym5aamdmNZQUzs5uenJTVDrKMrtq9bQVBtRtd3IMbfERlduwnJQlpO0NqWtNuJaQ11t5NVCX+0E5qawtSTWSmN2InNS2Toyc9KZ9YbiXZI6bP48Cj8Epm2Iahvg2sbI1oZtDnRrv4vqwLg2lGvFOSMFVNwLKr8F5Z3I3TJvcTbhrg3wHJC3Eeg1/Yod+BzQ1wp+bfDnBsA1jrUJgs0lNIHQala1qqapRUVd5NVGXy0E5v5Wh4PENvmqwVNYgmpra0m4YzR76mpjSDNW31FaR2kdpT0aSqtyaR2mPQFMU76s05FaR2rPlNQaqe4f/xUp0/i+5pnkB8xgOu48BkGXwvxGKczhYA0dDzo67ui4o+OnkcOMCOaDd5D8lHKZ3q9y1zpS7ki5y2l20LZBTnMttQ07auuoraO2J5LT7LDtaeY2O3LryK3LcT5UjvM3Qjgg3ifF+aKYOXdD3Hfw2mFf9sWbKe/IzMI5XIBPkGRy/sG+LKbLGJavLJDXscBZa61/mU8gYR4fitdJyL4oXKQxE0Qy2+SlpzVg3doeRy7elOF4JLmstbF2+fJTNd1qV7XqRSUKJCu19StBVK/r/5lDshRKbX0lay3zQCudwRvjYXTl1anV8GZGWHvkJP6kvAHD+lyJepjY6AWyhazU74KjhMXlCB7a32tgF+SCbQayNC8FXET6u0hXjHQhHep2nt7+F08YM4QgukIsmjAU6HlFrzt28Q/Wib8ISqqeMCUXMs7+VU7D12sb8+Vl9sbF5soFKRV5Bs9kR9bHZOw77v/8D7UPkIv3XjQeO2LFFyHOhS4M+2aNdI5+0NdUYI0hckt/mS2ZoBcOY1RbWA1yE0VkFo0SVHhRffO/5suNTEsqmtzx252EpvjG8wb31Ptq7Htr/ndZ/F2M5BKlf5D4dJmEtkfHpAkFnQm1mZDDbh6dsbiPYStxDruTAX2jyKG/y+rH2c6dZPEs7cnYyB9oT2NXJBp/RQD6QfpcJy0cav3CdaDyMxjLlJc9W1dKZ9g3z1q1gHid62DqHs1y0HVshWszNh6qbWHB3DkWq3phO9RZfQQTQm+86GndPTUTrl4M2VnxQ5xnbOK/qw23WNXIremju2j66JmouPGXRkQ2r0pBVC8VxNcvK5sv3xsocyb1ZSliYEDqiwuZXZSTUbfXHyvpGVX3NRfsB7WeqzrvB+qHYV+tUR6sHii/B8XfVjkv18DT2U21c4+idryrdqyOMhipH5QXKexF6nzLuWji+wuLOwz+H2fHRdaoTGd9kASGkpn3i3dSQJ2fi/t5/h4I9vu7eyE8GA3BqA/gYDwaj3b3xmFwMJ2OQ9n4qspS9V+s/g+n66MejmcAAA== proxy.json.gz: H4sIAAAAAAAC/+1dW3PbthJ+96/gsD0zzoyao3uszuTBcU4ymSapm6jtQ5PRQCRE8ZgCGAC0pXp0fvtZgBeBJGjLcR1JMV4y4i4ui93F7oclHVwfOY47mYQkTgR3f3b++txKKQx/SUKGNRoihAokQkok8RpIQIxCLlQTeFyrZtgPBZpGGKgzFHGsiAHB4o0PJJJEUUphKJ6PKY1EGAO9rYjz0MdnlAhGI17qH5b6RiG50ARjeAaCzoHg9trcTWn0SrVQUqayAtWjUYRiXpKtxPBrnBgRHG2G0odLeyJvjsfhAtNEaDJu+DSi7AXyLgJGE1Ibf9PmDxQluJldFkHRf+iORl5/6LbKZBZM0XG396zldLqjltNvt5z205PRk2q7H/x+H/WQq1E/l+b1kUCcJsyTYrnnjC6wmOOElwZyZ5QtkFy6SyjBZV6AkgAXzlKQF2iZr7bTblfEWoQkZ1ZZfE6vTBoClphLF6CR/xZNcdl5TK3eIXaBlU4FS7DWaF2Wn4X+OVXuXmYof+yWSURgdokikxOUHVbTQxyHJBivYqWKJl7d9GV9QmuCFspIl1JzjqCOwEtRMbiTseVUJca6td3gDJFgi8G75cEb3Qu84CV42DkFzfG6K7hSi4r5jvpqfo8Sgj0Bm7TWbiwlMig+plzMwqXs7ZoYryDYfAz/VsMP2v+qtIHAYuyr6Dd2Vap6BxHldtPNYF+p3QOy15Uq0oW57/99amDSouOWOucQzmqeC0R2AS5q2KmzMIrOZPhR9peBpdeBuNI5gX9ORjKwdE5qgWUmBTJvQDmLPl46XBdCVGfUqw2k7/fGPaqSDYyZLEjdVAKxAIstrICXsZKJJ4vjJL72ooTDdn7+yf0x+/nJbTn/pVOgXCRT/FPM6HL1yV0/qVtlExAFpIUJxyzEvN4sjxevkCeUPrq1JhEOMPFfFePVBwFPVGnVPd3WA4oQyA3aCoXK2+7vcYVR5OnrqoUQ2yTNZhulEc7lENEizAFFlMdXoUPfUCfVDaVabLehqJTTfX7HvZRHrzttJ9XpfRYeIXFt+mnrL8EFFIWIn+UJvZJVpogZMxdkYj5/i0kg5ipO1ni4qdtWCTxU27VTI75myA8xEdVMfEta7JVTn/LhuueAIgJyysdVqKg3uQwaOF7CWCqYiQuJpYkTkgYOC4M5eJ/fJEsWiiRaqHKg05+hr0xTzT9q6wjUFA2V+/Bbd48MmtwwuWJcZVN3tgEctZRaSzpujMFdiECBGYvGsrf0i0QOP6jzzJ4I1vIxwyrEziJa2f4MxxgZ83caPH+9xAxcDhtWBLnLw01bI8t2ZSkh+ngXRiEhzMcx9t+mmbDO/6pkwpDAxzJlqIwx4SviTdKfLIFQOPETps5UkCcA3/h84sExQWydgFpOSGBFxMPP/wdt8wfITH8NFp+ffNP0JJf6j6eoir2lzK9SxFRzFsn7OA9nwszMctsHqXbnI9jB+VAVeNtEVw0BlAnDcUVt70me+0Lih5ehn6BSdjHnSXU4Lku2RMuQ1+WaJt5F6pOG2LPIdrlUTBVbZaDe0Ks51BXxSh33zUtYoeU2B5aNF9LY4H2RPMUZxZNMGrxAvHZk0mK/sVsa/Ns1urbeOx2LDmEJ+h6z2ORnp/812MSwE7aBJoZu90UmJvjxoMDkxuqIxSUHikvmIRcUcsxi8iVBRIQRPm4/HY1ad8YqafY5SLByfZ1Pv14fBmhJMctbsBHxVs5oJObOb5n5LIjZMYg5eAhzWADmqLIDzZE6o76B2KDClpn9gV6V3kwV8oyzzVd+H1RsyZeAbaYUMd+BEdwyPy9jzYcaI9sWDFofacZ5yJdTjwDRDWy1yVab9gnVDfeo2kSwuKLsQoI4CffAIQNbcPrW2O19agTnfGMEW3mylSdbeXpUOGVoK0+28vToMUpT5cn5WvDyD1egnOnKOc6pLSfCtih1N2Bjy1O2PGXLU7Y8ZctTKex7ZstTtjy1T9Cvv/PyFEASMfEiuZnUX5nAI58oz7g/jGt5YEtJ6z59upOSVHe5vANwa+2j6no7Ul3vRtW9OATV9Xekuv6Nqjs7BNUNdqS6wY2qe7kfJ61fQJXO6fkb50Oqfls/foT1YyPr4ErId1uFrSKXjxMn9jhhjxP7dJw42d9KsgG23L9+3ARjQIuy7fmvH8flYrJktJyERbuoJsvJ12vn+hrmP4SKMgQ+UWAcW0q2pWRbSralZFtKTrHfyH5BYL8g2Bfc1+k+MuDXWMByMuj3+j8W+X098nuNLfCzwM8CPwv8LPCrAr+KfLbqZ6t+j/UjAkBjHuYcYJ10EsB3C7ygbDWZrgTm967ire23nk347J3SswVhOwZhys+/g5eOe74O+9qxgkA6FoFYBGIRSPY3p8c5DPHipCgo3fGDqBv/DsXikCYccnb+u5Nw6ckWiuy4HjSX2jv0mtAeL8KCkAoI6VoQYkGIBSEyz9BJQBlNhDSgLXs84GupQssWbzw2vPFApY89X8f3/DbqKFuBy705XqA/MOOplJ1+ShardCofsYv0HiHIVxsvVdGUESxrV8VYAi/iCJZLgvqtSEdVb9AARgm+5VcT+HiGksh0O4LONaFBdXdSObg0+FRxn4sG4zQujfMrnvRo6n5JMFtlJfg63NPuYOqUqAGuXqBS2EWb/qiyHE1hKIry64Aqy9BUqethC3Ca60rLVhB2IXf6+NR0dUmhSDfLr/pYC7BIaOiTa9nQ5TYVq+kmaRRV/yHGJKb+JCQz2nKy4Z6Yld+9TflZCtKNBC6uNMx/y+d3y9yaoJJmbpyZNl2Jxkg4HqcD6Yj0W9u73WTvcg5r2je3WToHNvcy9cP93yfrzTdM34f3HOVJKr14TqGXIgQX1yrRq586OUbKb0xKM8KmWxyCJtmmc6aSSY5vdaDiDjau5Xba2kNPf+gsNr8H2u+O/tBr65wNknO72u+On8bHz/kaJCLX/OrWWfSBh/rA+izdvv6wuWLLfebr8uaylNT3NyXptUHjsyxrFh/0S9dLaYk6xLvDXhd3hwMft7uj4Unf67en3Y43fTaceqg/G/XTxpdFam4frf8PZcHkVKRwAAA= scheduler.json.gz: - H4sIAAAAAAAC/+1dW3PbthJ+96/gsD0z9oyaI+riWJnJg+OcZDJNWjdR24c2o4FIiOIxRTAAaMv16Pz2swBvIAnakqL4EuHFI+7isthd7H4L0uTNgWXZk0kQxQln9gvrr8+dlELxlySgWKGhKCIc8YBEgngDJCCGAeOyCVyuZDPsBRxNQwzUGQoZlkQ/wvydB6QoCcOUQlE8HxMS8iAGelcS54GHz0jEKQlZpX9Q6RsG0YUiGMUzEHQOBLvfZXZKI1eyhZQylRWoLglDFLOKbBWG1+DEKMJhOZQ6XNoTuXM8DhaYJFyRseSTkNBXyL3wKUmixvhlmz9QmOB2dlUESf+hNxq5g2O7UyVTf4oOe/3nHcvpjTrWoNuxus9ORkf1dj94gwHqI1uhfq7M6yGOGEmoK8SyzylZYD7HCasMZM8IXSCxdDsiEa7yfJT4uHCWgrxAy3y1TrdbE2sRRDmzzmJzcqXTELD4XLgACb33aIqrzqNr9QHRCyx1ymmClUarqvw08M6JdPcqQ/pjr0qKOKaXKNQ5QdVhFT3EcRD54+tYqqKN1zR9VZ/QOkILaaRLoTmLE4vjJa8Z3MrYYqoKY9VZb3CKIn+NwXvVwVvdC7zgNXjYOQHNsaYr2EKLkvmBeHJ+l0QRdjls0ka7sZBIo/iYMD4LlqK3rWO8gWDzKfhHDj/s/qvWBgKLtq+k39pVquoDRJS7TTeDfSV3D8jeVCpPF2b/8u9TDZMUHdfUOYNw1vBcINILcFHNTp0FYXgmwo+0vwgsfQfiinMCf05GIrA4J43AMhMC6TegmEUdLx2uByHKGfUbA6n7vXWPymQDYyaLqGkqjqiP+RpWwMtYysSSxWES37hhwmA7v/zb/jH7+bfdsf5LpkC5SKb4JwZR30tCQV8dNS1TBkUOqWHCMA0wazbLY8Yb5HKpk16jSYh9HHlvivGag4A3ytRqn67rBUUYZBqNBVzmbvv3uMYocvVN3UqIlomz3U5plLMZRLUQM0AS1fFl+FA31Ul9U8kW620qIuS0X264n/IIttGWkp1+yUIkJK+yn7L+CmRAYYDYWZ7Ua5lliqg2e0E2ZvP3OPL5XMbKBg+3dVsriQdyyzoN4luKvABHvJ6N70iN/Wr6kz7c9BxQhB+dsnEGF0UqrkUAdOm3RBI3oTSVS9MNUktLN2EgPYcG/hx8z2uRJItFOg70+TPwpF3qCUjuG47awqH0nTtRiAiaTDO3ZFxlMzvrAI5GSm0kHTvG4CoRR74ei8ait/CJRAw/bPL0Xgim8jDFMsTOQlLb+hTHGGnzdxo4f73EFNwNa1YEucvFbdsiy3ZVKSHyuBdaISHMxzH23qeZsMnfKplQxPFhkS0muAfZIL2CODjxEiorKsgQgG48NnGhSOAbpZ+OFUSwpsjFL/8H7fMLyEt/DRefj46s6bV1mI1Str3XnHWTL2i1sm5uchHgArSxQT7rbK31aRB5Rt1WpoZbVP5qZypXnByFPqEBny/2Xf+lTqxCJ7cY42xnxrgU8NgEntISqUJu0f3rrbB0LTmJ5bxJy7tGZhO8T/NgxvXMDIR/Kj3mI5h0K0RehyuEcs3ZioQikxyki0BxGXgJqsBgPaCXJ3lVyZZoGbCmXNPEvUgTqAYnLTJIIhRTLwSzEwhNr3ZYVmAreTapX8I1Wq5zulI6KIk1jhmKIyeteIJJ/FeINc53FJiq7Zbi1G6Drqx3o+DwFJag7jFTRL2wBqaIMkXU3hdR84BxAjlmMfmSoIgHIT7sPhuNOtYW1VWagDZDOduAnI4V4qPvuMTaxCattdceGuObFWCbGGS9ymwPrXM/5dkmplqjbttDOz3B4i0Eu0butTUa8bn1W2Z3U809cDX35Gu5p1XJHdS2oB6yZtR3sPFlpNOzP5KryvNEhTzjbPdVn+Ip9uRrKPKmBFHPghHsKj+/8Tg/VhjZtqDQ+kAxzrd8pGgPStvhNqWtLv6tU9tqS9ivLG61sfibVre331035e3XlbeDB79HCEiET9xQbCb58CVcson0jN2cyndcsKeg9Z49K7DefeK53nL5DW/23Zv6+g+kvv6t6nv1VNQ3eCD1DW5V39lTUd/wgdQ3vFV9j6Tg+hnUaZ2ev7M+piYw98zMPTNzz2yvCotjU1iYwuIxFRYnj/e+mQa07OZ8uRXFWKBL0eP810/j6nmzYHSshIYPcc4sJpeHzDD/arXzJ+x3jnMg/PEC47w3R8vmaNkcLZujZXO0LBHgc/PUlHlq6rGgP6dn4F8T/r39j0F/26O/t9iAPwP+DPgz4M+Avzr4OzHHf+b4zzxX8EK8pIS4mDFAdsJJAOIt8ILQ68n0mmO2o//zu1+cVj4B+vgx2gepbAPEHhiISWe/TyyjZX09Hnvk6zA3IasoZGRQiEEhBoXIV8ZxfJhDETdOinOlHT4blR4kGSzShkXOzn+3Eibc2cCRPYMj3e8Bi5inobYGIjX5DBIxSGRfkYhPJj6hJOHCgLv5d1kDOVpvURWaNpjjoe9FzYX2nv4RyCNfx/d8Z+ogW4EtguIC/YEpS6V0BimZX6dTeYhepK/fh5xVeqmMqDTCAjcWY3G8iENYbuQ3PyZwUPcGBWRUIFz+Nl8Pz1AS6l4orHJ1iFB+cqAaXFp8qngNugLlFC6J8y8jqNHU/pJgep2dxzchn/LpAqdC9XH9veOFXZTpD2rLURSGwjB/i35tGYoqVT2sAVBzXSnZCsIu5E8Pn+re+F0o0s5yrDrWAiwSaPrkWtZ0uUvFcrpJGkUPhc9NYuJNgmhGOlY23JFe+b27lJ+lINVI4OJSw+y3fH67ym0IKmj6xplp05UojIThcTqQikrv297dNntXc1jbvrnL0jmw2drUuzlfWunfUfl0veQgT0bpd1kkSilCbfHVAXL1k5NjofyDAmnkL7vFAaAgWnbOVDLJcawKSOxh6UK201Uu+uqFU75Zxx4qvx31ot9VOSVis3vKb8dL4+DnfA0CeSv+c+cs6sDH6sDqLL2BelF+gcJ+7qny5rJU1PcPidI36o/PsuxYe0NMFubsRBbtdg8fT49RfzqYet7MGfSeT/toOETI6TuOezxMc7V9WaTh7sHq/10T1QPHZwAA + H4sIAAAAAAAC/+1dW3PbthJ+96/gsD0z9oyaI+riWJnJg+OcZDJNWjdR24c2o4FIiOIxRTAAaMv16Pz2swBvIAnakqL4EuHFI+4C4GJ3sfstSIM3B5ZlTyZBFCec2S+svz53UgrFX5KAYoWGoohwxAMSCeINkIAYBozLJnC5ks2wF3A0DTFQZyhkWBL9CPN3HpCiJAxTCkXxfExIyIMY6F1JnAcePiMRpyRklf5BpW8YRBeKYBTPQNA5EOx+l9kpjVzJFlLKVFaguiQMUcwqslUYXoMTowiH5VDqcGlP5M7xOFhgknBFxpJPQkJfIffCpySJGuOXbf5AYYLb2VURJP2H3mjkDo7tTpVM/Sk67PWfdyynN+pYg27H6j47GR3V2/3gDQaoj2yF+rlyXw9xxEhCXSGWfU7JAvM5TlhlIHtG6AKJqdsRiXCV56PEx4WzFOQFWuazdbrdmliLIMqZdRabkyudhoDF58IFSOi9R1NcdR5dqw+IXmCpU04TrDRaVeWngXdOpLtXGdIfe1VSxDG9RKHOCaoOq+ghjoPIH1/HUhVtvKbpq/qE1hFaSCNdCs1ZnFgcL3nN4FbGFreqMFad9QanKPLXGLxXHbzVvcALXoOHnRPQHGu6gi20KJkfiCfv75Iowi6HRdpoNxYSaRQfE8ZnwVL0tnWMNxBsPgX/yOGH3X/V2kBg0faV9Fu7SlV9gIhyt+lmsK7k6gHZm0rl6cTsX/59qmGSouOaOmcQzhqeC0R6AS6qWamzIAzPRPiR9heBpe9AXHFO4M/JSAQW56QRWGZCIP0CFHdRx0uH60GIckb9xkDqem9dozLZwJjJImqaiiPqY76GFfAyljKxZHGYxDdumDBYzi//tn/Mfv5td6z/kilQLpIp/olB1PeSUNBXR03LlEGRQ2qYMEwDzJrN8pjxBrlc6qTXaBJiH0fem2K85iDgjTK12qfrekERBplGYwGXudv+Pa4xilx9U7cSomXibLdTGuVsBlEtxAyQRHV8GT7URXVSX1SyxXqLigg57Zcbrqc8gm20pGSnX7IQCcmr7KfMvwIZUBggdpYn9VpmmSKqzV6Qjdn8PY58PpexssHDbd3WSuKBXLJOg/iWIi/AEa9n4ztSY7+a/qQPNz0HFOFHp2ycwUWRimsRAF36LZHETShN5dJ0g9TS0k0YSM+hgT8H3/NaJMlikY4Dff4MPGmXegKS64ajtnAofedOFCKCJtPcWzKusjs76wCORkptJB07xuAqEUe+HovGorfwiUQMP2zy9F4IpvIwxTLEzkJSW/oUxxhp83caOH+9xBTcDWtmBLnLxW3LIst2VSkh8rgXWiEhzMcx9t6nmbDJ3yqZUMTxYZEtJrgH2SC9gjg48RIqKyrIEIBuPDZxoUjgG6WfjhVEMKfIxS//B+3zC8hLfw0Xn4+OrOm1dZiNUra915x1k09otbJubnIR4AK0sUE+62yt9WkQeUbdVqaGW1T+amcqV5wchT6hAZ8v9l3/pU6sQie3GONsZ8a4FPDYBJ7SEqlCbtH9662wdC05iem8Scu7RmYTvE/zYMb1zAyEfyo95iOYdCtEXocrhHLN3oqEIpMcpItAcRl4CarAYD2glzt5VcmWaBmwplzTxL1IE6gGJy0ySCIUUy8Esx0ITa92WFZgK7k3qZ/CNVqus7tSOiiJNY4Zii0nrXiCSfxXiDX2dxSYqu2W4tRug67Md6Pg8BSmoK4xU0S9sAamiDJF1N4XUfOAcQI5ZjH5kqCIByE+7D4bjTrWFtVVmoA2QznbgJyOFeKj77jE2sQmrbXXHhrjmxVgmxhkvcpsD61zP+XZJqZao27bQzs9weItBLtG7rU1GvG59Vtmd1PNPXA19+RruadVyR3UlqAesmbUd7DwZaTTsz+Sq8r7RIU842z1Vd/iKdbkayjypgRRz4IR7Co/f/A4P1YY2bKg0PpAMc63fKVoD0rb4TalrS7+rVPbakvYryxutbH4m1a3tz9dN+Xt15W3gwd/RghIhE/cUCwm+fIlXLKJ9Izd7Mp3XLCnoPWePSuw3n3iud5y+Q0f9t2b+voPpL7+rep79VTUN3gg9Q1uVd/ZU1Hf8IHUN7xVfY+k4PoZ1Gmdnr+zPqYmMM/MzDMz88xsrwqLY1NYmMLiMRUWJ4/3uZkGtOxmf7kVxVigS9Hj/NdP4+p+s2B0rISGD7HPLG4uN5nh/qvVzt+w3znOgfDHC4zz3mwtm61ls7VstpbN1rJEgM/NW1PmranHgv6cnoF/Tfj39j8G/W2P/t5iA/4M+DPgz4A/A/7q4O/EbP+Z7T/zXsELcUgJcTFjgOyEkwDEW+AFodeT6TXHbEf/53e/OK18A/TxY7QPUtkGiD0wEJPOfp9YRsv6ejz2yOdhHkJWUcjIoBCDQgwKkUfGcXyYQxE3Top9pR2+G5VuJBks0oZFzs5/txIm3NnAkT2DI93vAYuYt6G2BiI1+QwSMUhkX5GITyY+oSThwoC7+XdZAzlaH1EVmjaY46GfRc2F9p7+Fsgjn8f3/GTqIJuBLYLiAv2BKUuldAYpmV+nt/IQvUiP34ecVXqpjKg0wgI3FmNxvIhDmG7kNz8mcFD3BgVkVCBcfpqvh2coCXUHCqtcHSKUnxyoBpcWnyqOQVegnMIlcf5lBDWa2l8STK+z/fgm5FM+XeBUqD6unzte2EW5/UFtOorCUBjmp+jXpqGoUtXDGgA115WSrSDsQv708KnuxO9CkXaWY9WxFmCRQNMn17Kmy10qlrebpFFUHOLdclx3x8rGPtJboneXJbJ8pFoM/F2qm/2WC2NXuQ2pBU3fOLNzOi2FkTA8TgdSIep9G7/bZvxqQmtbRHeZPUc5W9t9N5tNK/2BlU/XSw7yzJR+pEVCliLuFp8gIFc/OTkwyr8ukKaBslscACSiZedMJZMc1KroxB6WLmQ7XeWir1445TE79lD57agX/a7KKeGb3VN+O14aFD/ncxAwXPGfO++iDnysDqzepTdQL8rPUdjPPVXeXJaK+v4hUXq8/vgsS5W142KymGcnsoK3e/h4eoz608HU82bOoPd82kfDIUJO33Hc42GauO3LIid3D1b/B+dQ2oPUZwAA workload-total.json.gz: H4sIAAAAAAAC/+2dX3PbNhLA3/0pOJxMx75TW1GybDczfXDS5i4zvbtc7LYzTTwaiIQkjCmCBUHLqkf32Q8A/wEkQFGOLNsxXhJzlwKBxe7ipwUk3R04jjseoyhOaeK+dj5d9TIJgX+miEBJBqIIU0ARjrjwjomYMEQJ5beIKyeXCs0kRSF9HzGl16ukAaAgwSnxIVO4HwheQDqHaeJK98AITEKupySFknyOAo0U+Th6i0NMeINkNgGH/Z4z8Dz2z2jUc7wjuekILMSDz6uxON845yEkVOkCXcXivgAk8wkGJHBz3Vr8f8X+XQujwADRWm/dWQTp+4BJojQMMwkB8fwS45CimMn7QsiH8xZHlOCQG3QKwiR7PVJeG6LoWpqFGERQ3J+ZvDC46+MwBHEC5ZYURdDQzAgKPuBqMjMj1yZsya4Hx5LgthhAfr3i14V1yrbFGAblZdXrq1JGYAwBlYYqSd9TSMT06NUf8VIxsFAlc7y8RLThOS7Nhe7blBAYUecNiIIlCujcVe+5QH+J++YnkiL3BIKXmQ/kYywND0IEEuGAwpKVCSZASJSucH/6BUYzKszcV+QwaczQxmhxpygMFUtzwT8ICBAbqDxT5tn+oTbb3mDDbHv62R6WlyGcwShQn8TMNIvOk0t9aIObWX3w3HWz+dJngp8XMV3pVX9AgpuaBbjVPGOBIo2UoNmcXmjzDXczjZTd+7twKdUruQoTPoRyNDXVTzDxm+1Rlp5CTcduQJjCwq0a88ByhcaLuHSZ982Tpde1iGS2uGCRqkS8y0fzAaOI/gsLcwiBK0W2D3UeHUPmtREFs2ZGinlr3EVT/vyRKm92nxktgASK/D4NcWVAQwZJIEEw+c8NJMzjYW2Iiam/SWPgCQX+daM3CYVxDINfmE0bOgrIDNJEWg/lFZGvFrexGAaf9zHrm3+YpItDxFIdPGTLGAWsVTJmy8cSk2u2BPsQ3cDxZEVhMhYeceeHKesB+fGz+yr/87Pb46uaGNiP/2Py8uqzu/70ipkUEuY1r1+xxRyHKU+qV0efo785OHIOy3t7MQ6YdEZwGo9DOKWHvAshBkGv+GPMU+GRU75kXCrYa19fpxPI/xjjJRvDawJDMIHb9rfnFG0KTXEhK0QveHP8fzbCI2eycg5571n3pdjiqRCThXAPlt0XcJz5hXpLYZ13wKeCIDxFnSWyd2U7d3cOe5KzXqutEDgVy5F7rsq5rwgvK4Xr/K/KIemczcsch0HNUXmX37GEX/duLr+YoyltKmqr3EfmVQ6eOm+4+zgfM2cKpJWt5JE7ObsBIlChnt+yJDaop6JxsT6iKEA3KEhZ1mqkpeIegUFVB27BLaqtR5PUv84iqJZEF3nyaUxigXT1pJtlaWP6/CTPU5GZS9lVYwgrcAtbArtytTdxzcVEHDQ6yBV49gYIYFOdLlunGrdnC1VfdbBqLapcrPf0u3ggm/nlAZUiaCGqY0tUlqgsUe2OqJiNomSBqEUqi1RfjlSXuTdRS1WWqp46VUn1QQWg5PKgothdddAzlAdHmvKg3somJDRhYRsatuBh1wq1BhNbUdFoTx0yarFRZ9vKvp4kU20j7HyiScDNbrTipBkpW7FyA1q246UZMc2YuQE1zbi5ATk3YGc7erbhpxFBm3OpRVEzjhqQ1IilHdC0DU83IGobpragajuumpF1I7a2oqseX80IuwljjShbT3QGpGWBZ4uED0O0Ham2A9luQbdthGugXHmBV2i3jXjbqLeVfFX6PWcRxEJ6U0HRiL9tCGzA4E4o3EyReiQ2YHErGrfjsRmRWzDZjMp6XJaRuTnUJjprk4mZT1sZtY1TW1jVxKtGZm2M65kO4UA3Zab3DJZmdYXQzjh7anHW4qzF2a8TZ22F1vLs4/CsrpprkdYirUVaGWkbmy37P8VZhO9OT3E+6vFZz7D/f/bEzs+WFnf++T5hS8ZqH+dn6wc4voYDtAP9fP+wzXmP+ibcpgMfdc1eT3zUdVsf+dh4ekMojMc3FH+W318Mup3eqKa0+Z6CQWwEfWUT+Cs7wiGNvnjDoBjUnol90Qc42IOe9vGNvFStJYbneFyDj7jzYY3aGCpat8cxnvAh16+AepqnXA3YIw3Ico/lHss9ti5qwWcn4FPUNC35WPKxB1GdgaHO5nk7PYmq24V/IUdRB4P2vXu1oa6b97pdhy679zrto27f6/T33r/vtAUv46F5D75x2kSLit224GtT3AEbX+I+fM1KOrx86G34LmU2NhK+IFvefHL78E32fGa78MXue3GO1PmQ+doL2HyvUWy3rXfNOJtEu4NN9fj5b6rHL+ec6AthTe1B0Y2wObSwaWHTwuZTh82ytmlp09LmA9OmdMrTAqcFzhcKnE/gFGc9+HZzdnPvFWXTlvaxrSjvqKI83AD5Iwv5FvIt5D+3inJAMO+GZX3L+vuqLDs/ZT5ngd8Cv60wW/bsUGHeBJ8nFj4tfFr4fHYVZkuflj73X2m2AGoB1FacH6vi/DMhnHPvU3A+yHvO0xDPHVw77Gdt8duk3+dK/DlcgN8gSbL+e2eZmK7C/BfFyHX2OgYAlde6PNUTtl4xTy6fRuEiDpkholmXH1yrQFFhfApvhccEcArSsIYFIrxkre4tQ/HDa3LxW+9q5e+rSawvaXFc/HqcnHXdP1NIVsKptT8HV9ncU6QzyJ3W1f9sW/n4en1e+WBZ+Fs+/towJFOut/zxutxW8iclUMRW6QCeh7oPchkMuWCTgTS3FwbO1/1trCueNM4S6mFJFiia4p6TN3ekt/tgk93z1Uh2EObdwrjJf4vnu6q20VEu09+cz2o2EkmRJvAya0j7KTj9VLvf/V1uA6QU6z5VyMRjH6fCCYb9uibLiq7XV+Z+QwRym3+brJihF4YolO/QRmIXD2ShjCKUp0911r/koO+6V1HqkdueHRSPr30M554OXz773i7/IIPfJkiuUfwrCS9Wka/7ZGgWQt5zCKHa1O0xhAxx8yDB8pDvydbVe6+tYmlHq0fx7HvH0mPYxobaXkMtgHGIV4v6N4PKyFje8NzCz1gSWddqIo8RnMKBnmJg3s9quw7bvg3btrAdmdhy9AVIuSfPr4qpBv8/MNVG3ASGWRVev4FQWGfYb3znZWkgrjPVmMxP0303sn4qTJPR+VFtA/Pmxmcx1YGuPqNNJswIvdGipzT33Fbeou5so/hRShM6828bwy1RdWz29ONtPP34hbj4QVGJXWc1RSRmKa8mTrOtDjfCy2/LmHeFm3OZq7wsRuxtMqlenNtsXHRGnl53JFVaZd9XUrDrVX4u+7zryRfDvqyRvkFkIP3tBdncXxVj4DtTTbczP0Vu+ERuWH7K4Fi+kL756DSQ+1v0RTHfX1hserq/Xr7NC8BFZfrfWT0CRTPne+d3+b2Zm4pzBu7p4GwyPT31fc87OQkG06E3HE6mg9HZ8cnZ6YnvZzfflFXn/sH6/+UubNfagwAA diff --git a/charts/kubezero-metrics/update.sh b/charts/kubezero-metrics/update.sh index 79d1f8d..5ecfc94 100755 --- a/charts/kubezero-metrics/update.sh +++ b/charts/kubezero-metrics/update.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION=17.0.3 +VERSION=18.0.1 PG_VER=1.10.1 rm -rf charts/kube-prometheus-stack diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index f553225..04272a2 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -5,6 +5,9 @@ kube-prometheus-stack: defaultRules: create: true + # Disable kubeapi error budget rules for now as they are buggy + rules: + kubeApiserverSlos: false #additionalRuleLabels: # clusterName: myTestCluster # awsRegion: eu-central-1 @@ -133,6 +136,7 @@ kube-prometheus-stack: rbac: pspEnabled: false + # Disabled as we use the upstream kube-mixin dashboards directly defaultDashboardsEnabled: false grafana.ini: @@ -208,57 +212,57 @@ kube-prometheus-stack: # Add sns-forwarder to AlertManager pod, see: https://github.com/DataReply/alertmanager-sns-forwarder # uses the alertmanager serviceaccount to assume IAM role, requires annotation: kubezero.com/sns_forwarder_arn_prefix to point to SNSAlertHub # eg: "arn:aws:sns:eu-central-1:123456789012:" - #containers: - #- name: alertmanager-sns-forwarder - # image: datareply/alertmanager-sns-forwarder:latest - # imagePullPolicy: Always - # env: - # - name: SNS_FORWARDER_ARN_PREFIX - # valueFrom: - # fieldRef: - # fieldPath: metadata.annotations['kubezero.com/sns_forwarder_ARN_PREFIX'] - # - name: AWS_ROLE_ARN - # valueFrom: - # fieldRef: - # fieldPath: metadata.annotations['kubezero.com/sns_forwarder_AWS_ROLE_ARN'] - # - name: AWS_WEB_IDENTITY_TOKEN_FILE - # value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" - # - name: AWS_STS_REGIONAL_ENDPOINTS - # value: regional - # volumeMounts: - # - name: aws-token - # mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" - # readOnly: true - # resources: - # limits: - # memory: 64Mi - # cpu: 100m - # requests: - # cpu: 25m - # memory: 32Mi - # ports: - # - containerPort: 9087 - # name: webhook-port - # livenessProbe: - # httpGet: - # path: /health - # port: webhook-port - # initialDelaySeconds: 30 - # timeoutSeconds: 10 - # readinessProbe: - # httpGet: - # path: /health - # port: webhook-port - # initialDelaySeconds: 10 - # timeoutSeconds: 10 - #volumes: - #- name: aws-token - # projected: - # sources: - # - serviceAccountToken: - # path: token - # expirationSeconds: 86400 - # audience: "sts.amazonaws.com" + containers: + - name: alertmanager-sns-forwarder + image: datareply/alertmanager-sns-forwarder:latest + imagePullPolicy: Always + env: + - name: SNS_FORWARDER_ARN_PREFIX + valueFrom: + fieldRef: + fieldPath: metadata.annotations['kubezero.com/sns_forwarder_ARN_PREFIX'] + - name: AWS_ROLE_ARN + valueFrom: + fieldRef: + fieldPath: metadata.annotations['kubezero.com/sns_forwarder_AWS_ROLE_ARN'] + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + - name: AWS_STS_REGIONAL_ENDPOINTS + value: regional + volumeMounts: + - name: aws-token + mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" + readOnly: true + resources: + limits: + memory: 64Mi + cpu: 100m + requests: + cpu: 25m + memory: 32Mi + ports: + - containerPort: 9087 + name: webhook-port + livenessProbe: + httpGet: + path: /health + port: webhook-port + initialDelaySeconds: 30 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: /health + port: webhook-port + initialDelaySeconds: 10 + timeoutSeconds: 10 + volumes: + - name: aws-token + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 86400 + audience: "sts.amazonaws.com" # Metrics adapter prometheus-adapter: