From cd56f0a7aa3658cca41fa8d4ba95f18eef84fc02 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 9 Dec 2021 19:03:55 +0100 Subject: [PATCH] feat: metrics version bump --- charts/kubezero-metrics/Chart.yaml | 13 +- .../charts/kube-prometheus-stack/Chart.yaml | 11 +- .../charts/kube-prometheus-stack/README.md | 55 +- .../charts/grafana/Chart.yaml | 4 +- .../charts/grafana/README.md | 43 +- .../charts/grafana/templates/_pod.tpl | 30 +- .../templates/dashboards-json-configmap.yaml | 3 + .../grafana/templates/headless-service.yaml | 6 +- .../templates/image-renderer-deployment.yaml | 4 + .../grafana/templates/networkpolicy.yaml | 37 + .../charts/grafana/values.yaml | 48 +- .../charts/kube-state-metrics/Chart.yaml | 4 +- .../kube-state-metrics/templates/_helpers.tpl | 32 + .../templates/clusterrolebinding.yaml | 5 +- .../templates/deployment.yaml | 22 +- .../templates/kubeconfig-secret.yaml | 5 +- .../kube-state-metrics/templates/pdb.yaml | 8 +- .../templates/podsecuritypolicy.yaml | 5 +- .../templates/psp-clusterrole.yaml | 5 +- .../templates/psp-clusterrolebinding.yaml | 5 +- .../kube-state-metrics/templates/role.yaml | 10 +- .../templates/rolebinding.yaml | 5 +- .../kube-state-metrics/templates/service.yaml | 11 +- .../templates/serviceaccount.yaml | 5 +- .../templates/servicemonitor.yaml | 64 +- .../templates/stsdiscovery-role.yaml | 5 +- .../templates/stsdiscovery-rolebinding.yaml | 5 +- .../charts/kube-state-metrics/values.yaml | 18 +- .../prometheus-node-exporter/Chart.yaml | 3 +- .../templates/_helpers.tpl | 15 + .../templates/daemonset.yaml | 4 +- .../templates/endpoints.yaml | 2 +- .../templates/monitor.yaml | 4 +- .../templates/service.yaml | 2 +- .../prometheus-node-exporter/values.yaml | 2 + .../crds/crd-alertmanagerconfigs.yaml | 259 ++- .../crds/crd-alertmanagers.yaml | 1288 +++++++++++++-- .../crds/crd-podmonitors.yaml | 4 +- .../crds/crd-probes.yaml | 48 +- .../crds/crd-prometheuses.yaml | 1398 +++++++++++++++-- .../crds/crd-prometheusrules.yaml | 4 +- .../crds/crd-servicemonitors.yaml | 4 +- .../crds/crd-thanosrulers.yaml | 1322 ++++++++++++++-- .../templates/_helpers.tpl | 33 + .../templates/alertmanager/alertmanager.yaml | 4 +- .../templates/alertmanager/service.yaml | 2 +- .../alertmanager/serviceperreplica.yaml | 2 +- .../exporters/core-dns/servicemonitor.yaml | 2 +- .../kube-api-server/servicemonitor.yaml | 2 +- .../servicemonitor.yaml | 2 +- .../exporters/kube-dns/servicemonitor.yaml | 2 +- .../exporters/kube-etcd/servicemonitor.yaml | 2 +- .../exporters/kube-proxy/servicemonitor.yaml | 4 +- .../kube-scheduler/servicemonitor.yaml | 2 +- .../kube-state-metrics/serviceMonitor.yaml | 10 +- .../exporters/kubelet/servicemonitor.yaml | 32 +- .../node-exporter/servicemonitor.yaml | 8 +- .../grafana/configmap-dashboards.yaml | 2 +- .../grafana/configmaps-datasources.yaml | 8 +- .../templates/grafana/servicemonitor.yaml | 4 +- .../admission-webhooks/job-patch/psp.yaml | 5 +- .../prometheus-operator/deployment.yaml | 17 +- .../templates/prometheus/csi-secret.yaml | 12 + .../templates/prometheus/prometheus.yaml | 2 +- .../charts/kube-prometheus-stack/values.yaml | 160 +- .../charts/prometheus-pushgateway/Chart.yaml | 4 +- .../templates/deployment.yaml | 10 +- .../templates/ingress.yaml | 21 +- .../charts/prometheus-pushgateway/values.yaml | 12 +- .../configs/dashboards/apiserver.json | 93 +- .../configs/dashboards/cluster-total.json | 2 +- .../dashboards/controller-manager.json | 57 +- .../dashboards/k8s-resources-cluster.json | 178 ++- .../dashboards/k8s-resources-namespace.json | 157 +- .../dashboards/k8s-resources-node.json | 70 +- .../configs/dashboards/k8s-resources-pod.json | 142 +- .../dashboards/k8s-resources-workload.json | 162 +- .../k8s-resources-workloads-namespace.json | 174 +- .../configs/dashboards/kubelet.json | 52 +- .../configs/dashboards/namespace-by-pod.json | 2 +- .../dashboards/namespace-by-workload.json | 42 +- .../dashboards/persistentvolumesusage.json | 22 +- .../configs/dashboards/pod-total.json | 2 +- .../configs/dashboards/proxy.json | 72 +- .../configs/dashboards/scheduler.json | 68 +- .../configs/dashboards/workload-total.json | 28 +- .../configs/jsonnetfile.lock.json | 52 +- .../configs/rules/alertmanager-prometheusRule | 1 + .../configs/rules/etcd-mixin-prometheusRule | 4 +- .../rules/kube-state-metrics-prometheusRule | 2 +- .../configs/rules/kubernetes-prometheusRule | 84 +- .../rules/node-exporter-prometheusRule | 6 +- .../rules/prometheus-operator-prometheusRule | 20 +- .../configs/rules/prometheus-prometheusRule | 3 +- .../templates/grafana-dashboards-k8s.yaml | 34 +- .../templates/rules/etcd-mixin.yaml | 4 +- .../templates/rules/kubernetes.yaml | 83 +- .../templates/rules/node-exporter.yaml | 4 +- .../templates/rules/prometheus-operator.yaml | 15 + charts/kubezero-metrics/update.sh | 1 + charts/kubezero-metrics/values.yaml | 5 + charts/kubezero-metrics/zdt.patch | 31 +- charts/kubezero/Chart.yaml | 2 +- charts/kubezero/values.yaml | 2 +- containers/admin/v1.21/kubezero_121.sh | 4 + 105 files changed, 5517 insertions(+), 1354 deletions(-) create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/networkpolicy.yaml create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/csi-secret.yaml diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index 8176aae8..2461b032 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-metrics description: KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all Kubernetes integrations. type: application -version: 0.5.4 +version: 0.6.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -11,22 +11,23 @@ keywords: - alertmanager - grafana maintainers: - - name: Quarky9 + - name: Stefan Reimer + email: stefan@zero-downtime.net dependencies: - name: kubezero-lib version: ">= 0.1.4" repository: https://cdn.zero-downtime.net/charts/ - name: kube-prometheus-stack - version: 18.1.0 + version: 23.1.5 # Switch back to upstream once all alerts are fixed eg. etcd gpcr # repository: https://prometheus-community.github.io/helm-charts - name: prometheus-adapter - version: 2.17 + version: 3.0.0 repository: https://prometheus-community.github.io/helm-charts condition: prometheus-adapter.enabled - name: prometheus-pushgateway - version: 1.10.1 + version: 1.13.0 # Switch back to upstream once namespaces are supported # repository: https://prometheus-community.github.io/helm-charts condition: prometheus-pushgateway.enabled -kubeVersion: ">= 1.18.0" +kubeVersion: ">= 1.20.0" diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml index d6df5123..eba1fa72 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml @@ -6,20 +6,20 @@ annotations: url: https://github.com/prometheus-operator/kube-prometheus artifacthub.io/operator: "true" apiVersion: v2 -appVersion: 0.50.0 +appVersion: 0.52.0 dependencies: - condition: kubeStateMetrics.enabled name: kube-state-metrics repository: https://prometheus-community.github.io/helm-charts - version: 3.4.* + version: 4.1.* - condition: nodeExporter.enabled name: prometheus-node-exporter repository: https://prometheus-community.github.io/helm-charts - version: 2.0.* + version: 2.2.* - condition: grafana.enabled name: grafana repository: https://grafana.github.io/helm-charts - version: 6.16.* + version: 6.18.* description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus @@ -32,7 +32,6 @@ keywords: - kube-prometheus kubeVersion: '>=1.16.0-0' maintainers: -- name: vsliouniaev - name: bismarck - email: gianrubio@gmail.com name: gianrubio @@ -47,4 +46,4 @@ sources: - https://github.com/prometheus-community/helm-charts - https://github.com/prometheus-operator/kube-prometheus type: application -version: 18.1.0 +version: 23.1.5 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md index 33753f54..d745a3d4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md @@ -83,7 +83,58 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. +### From 22.x to 23.x + +Port names have been renamed for Istio's +[explicit protocol selection](https://istio.io/latest/docs/ops/configuration/traffic-management/protocol-selection/#explicit-protocol-selection). + +| | old value | new value | +|-|-----------|-----------| +| `alertmanager.alertmanagerSpec.portName` | `web` | `http-web` | +| `grafana.service.portName` | `service` | `http-web` | +| `prometheus-node-exporter.service.portName` | `metrics` (hardcoded) | `http-metrics` | +| `prometheus.prometheusSpec.portName` | `web` | `http-web` | + +### From 21.x to 22.x + +Due to the upgrade of the `kube-state-metrics` chart, removal of its deployment/stateful needs to done manually prior to upgrading: + +```console +kubectl delete deployments.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +or if you use autosharding: + +```console +kubectl delete statefulsets.apps -l app.kubernetes.io/instance=prometheus-operator,app.kubernetes.io/name=kube-state-metrics --cascade=orphan +``` + +### From 20.x to 21.x + +The config reloader values have been refactored. All the values have been moved to the key `prometheusConfigReloader` and the limits and requests can now be set separately. + +### From 19.x to 20.x + +Version 20 upgrades prometheus-operator from 0.50.x to 0.52.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 18.x to 19.x + +`kubeStateMetrics.serviceMonitor.namespaceOverride` was removed. +Please use `kube-state-metrics.namespaceOverride` instead. + ### From 17.x to 18.x + Version 18 upgrades prometheus-operator from 0.49.x to 0.50.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: ```console @@ -98,6 +149,7 @@ kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheu ``` ### From 16.x to 17.x + Version 17 upgrades prometheus-operator from 0.48.x to 0.49.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: ```console @@ -111,11 +163,12 @@ kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheu kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.49.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml ``` - ### From 15.x to 16.x + Version 16 upgrades kube-state-metrics to v2.0.0. This includes changed command-line arguments and removed metrics, see this [blog post](https://kubernetes.io/blog/2021/04/13/kube-state-metrics-v-2-0/). This version also removes Grafana dashboards that supported Kubernetes 1.14 or earlier. ### From 14.x to 15.x + Version 15 upgrades prometheus-operator from 0.46.x to 0.47.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: ```console diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml index c2f1e9f0..4aeff42d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 8.1.5 +appVersion: 8.3.1 description: The leading tool for querying and visualizing time series and metrics. home: https://grafana.net icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png @@ -19,4 +19,4 @@ name: grafana sources: - https://github.com/grafana/grafana type: application -version: 6.16.9 +version: 6.18.2 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md index 20e8d091..96dc9029 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md @@ -59,8 +59,8 @@ This version requires Helm >= 3.1.0. | `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | | `priorityClassName` | Name of Priority Class to assign pods | `nil` | | `image.repository` | Image repository | `grafana/grafana` | -| `image.tag` | Image tag (`Must be >= 5.0.0`) | `8.0.3` | -| `image.sha` | Image sha (optional) | `80c6d6ac633ba5ab3f722976fb1d9a138f87ca6a9934fcd26a5fc28cbde7dbfa` | +| `image.tag` | Image tag (`Must be >= 5.0.0`) | `8.2.5` | +| `image.sha` | Image sha (optional) | `2acf04c016c77ca2e89af3536367ce847ee326effb933121881c7c89781051d3` | | `image.pullPolicy` | Image pull policy | `IfNotPresent` | | `image.pullSecrets` | Image pull secrets | `{}` | | `service.enabled` | Enable grafana service | `true` | @@ -75,6 +75,7 @@ This version requires Helm >= 3.1.0. | `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` | | `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` | | `service.externalIPs` | service external IP addresses | `[]` | +| `headlessService` | Create a headless service | `false` | | `extraExposePorts` | Additional service ports for sidecar containers| `[]` | | `hostAliases` | adds rules to the pod's /etc/hosts | `[]` | | `ingress.enabled` | Enables Ingress | `false` | @@ -141,7 +142,8 @@ This version requires Helm >= 3.1.0. | `sidecar.image.sha` | Sidecar image sha (optional) | `""` | | `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | | `sidecar.resources` | Sidecar resources | `{}` | -| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable | `false` | +| `sidecar.securityContext` | Sidecar securityContext | `{}` | +| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable. If set to `true` the sidecar will create unique filenames where duplicate data keys exist between ConfigMaps and/or Secrets within the same or multiple Namespaces. | `false` | | `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` | | `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` | | `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` | @@ -158,16 +160,18 @@ This version requires Helm >= 3.1.0. | `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | | `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | | `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | -| `sidecar.dashboards.searchNamespace` | If specified, the sidecar will search for dashboard config-maps inside this namespace. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces | `nil` | +| `sidecar.dashboards.searchNamespace` | Namespaces list. If specified, the sidecar will search for dashboards config-maps inside these namespaces.Otherwise the namespace in which the sidecar is running will be used.It's also possible to specify ALL to search in all namespaces. | `nil` | +| `sidecar.dashboards.script` | Absolute path to shell script to execute after a configmap got reloaded. | `nil` | | `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.dashboards.extraMounts` | Additional dashboard sidecar volume mounts. | `[]` | | `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | | `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | | `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `nil` | -| `sidecar.datasources.searchNamespace` | If specified, the sidecar will search for datasources config-maps inside this namespace. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces | `nil` | +| `sidecar.datasources.searchNamespace` | Namespaces list. If specified, the sidecar will search for datasources config-maps inside these namespaces.Otherwise the namespace in which the sidecar is running will be used.It's also possible to specify ALL to search in all namespaces. | `nil` | | `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | | `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | | `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | -| `sidecar.notifiers.searchNamespace` | If specified, the sidecar will search for notifiers config-maps (or secrets) inside this namespace. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces | `nil` | +| `sidecar.notifiers.searchNamespace` | Namespaces list. If specified, the sidecar will search for notifiers config-maps (or secrets) inside these namespaces.Otherwise the namespace in which the sidecar is running will be used.It's also possible to specify ALL to search in all namespaces. | `nil` | | `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | | `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | | `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | @@ -190,7 +194,7 @@ This version requires Helm >= 3.1.0. | `command` | Define command to be executed by grafana container at startup | `nil` | | `testFramework.enabled` | Whether to create test-related resources | `true` | | `testFramework.image` | `test-framework` image repository. | `bats/bats` | -| `testFramework.tag` | `test-framework` image tag. | `v1.1.0` | +| `testFramework.tag` | `test-framework` image tag. | `v1.4.1` | | `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | | `testFramework.securityContext` | `test-framework` securityContext | `{}` | | `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | @@ -230,6 +234,11 @@ This version requires Helm >= 3.1.0. | `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | | `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | | `imageRenderer.resources` | Set resource limits for image-renderer pdos | `{}` | +| `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` | +| `networkPolicy.allowExternal` | Don't require client label for connections | `true` | +| `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` | + + ### Example ingress with path @@ -526,3 +535,23 @@ imageRenderer: ### Image Renderer NetworkPolicy By default the image-renderer pods will have a network policy which only allows ingress traffic from the created grafana instance + +### High Availability for unified alerting + +If you want to run Grafana in a high availability cluster you need to enable +the headless service by setting `headlessService: true` in your `values.yaml` +file. + +As next step you have to setup the `grafana.ini` in your `values.yaml` in a way +that it will make use of the headless service to obtain all the IPs of the +cluster. You should replace ``{{ Name }}`` with the name of your helm deployment. + +```yaml +grafana.ini: + ... + unified_alerting: + enabled: true + ha_peers: {{ Name }}-headless:9094 + alerting: + enabled: false +``` diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl index f28b2946..f7cae314 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl @@ -87,7 +87,7 @@ initContainers: imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} env: - name: METHOD - value: LIST + value: {{ .Values.sidecar.datasources.watchMethod }} - name: LABEL value: "{{ .Values.sidecar.datasources.label }}" {{- if .Values.sidecar.datasources.labelValue }} @@ -104,7 +104,7 @@ initContainers: {{- end }} {{- if .Values.sidecar.datasources.searchNamespace }} - name: NAMESPACE - value: "{{ .Values.sidecar.datasources.searchNamespace }}" + value: "{{ .Values.sidecar.datasources.searchNamespace | join "," }}" {{- end }} {{- if .Values.sidecar.skipTlsVerify }} - name: SKIP_TLS_VERIFY @@ -112,6 +112,10 @@ initContainers: {{- end }} resources: {{ toYaml .Values.sidecar.resources | indent 6 }} +{{- if .Values.sidecar.securityContext }} + securityContext: +{{- toYaml .Values.sidecar.securityContext | nindent 6 }} +{{- end }} volumeMounts: - name: sc-datasources-volume mountPath: "/etc/grafana/provisioning/datasources" @@ -139,7 +143,7 @@ initContainers: {{- end }} {{- if .Values.sidecar.notifiers.searchNamespace }} - name: NAMESPACE - value: "{{ .Values.sidecar.notifiers.searchNamespace }}" + value: "{{ .Values.sidecar.notifiers.searchNamespace | join "," }}" {{- end }} {{- if .Values.sidecar.skipTlsVerify }} - name: SKIP_TLS_VERIFY @@ -147,6 +151,10 @@ initContainers: {{- end }} resources: {{ toYaml .Values.sidecar.resources | indent 6 }} +{{- if .Values.sidecar.securityContext }} + securityContext: +{{- toYaml .Values.sidecar.securityContext | nindent 6 }} +{{- end }} volumeMounts: - name: sc-notifiers-volume mountPath: "/etc/grafana/provisioning/notifiers" @@ -189,7 +197,7 @@ containers: {{- end }} {{- if .Values.sidecar.dashboards.searchNamespace }} - name: NAMESPACE - value: "{{ .Values.sidecar.dashboards.searchNamespace }}" + value: "{{ .Values.sidecar.dashboards.searchNamespace | join "," }}" {{- end }} {{- if .Values.sidecar.skipTlsVerify }} - name: SKIP_TLS_VERIFY @@ -199,16 +207,22 @@ containers: - name: FOLDER_ANNOTATION value: "{{ .Values.sidecar.dashboards.folderAnnotation }}" {{- end }} + {{- if .Values.sidecar.dashboards.script }} - name: SCRIPT - value: /opt/script.sh + value: "{{ .Values.sidecar.dashboards.script }}" + {{- end }} resources: {{ toYaml .Values.sidecar.resources | indent 6 }} +{{- if .Values.sidecar.securityContext }} + securityContext: +{{- toYaml .Values.sidecar.securityContext | nindent 6 }} +{{- end }} volumeMounts: - name: sc-dashboard-volume mountPath: {{ .Values.sidecar.dashboards.folder | quote }} - - name: script-volume - mountPath: /opt/script.sh - subPath: script.sh + {{- if .Values.sidecar.dashboards.extraMounts }} + {{- toYaml .Values.sidecar.dashboards.extraMounts | trim | nindent 6}} + {{- end }} {{- end}} - name: {{ .Chart.Name }} {{- if .Values.image.sha }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/dashboards-json-configmap.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/dashboards-json-configmap.yaml index 59e0be64..30190671 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/dashboards-json-configmap.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/dashboards-json-configmap.yaml @@ -9,6 +9,9 @@ metadata: labels: {{- include "grafana.labels" $ | nindent 4 }} dashboard-provider: {{ $provider }} + {{- if $.Values.sidecar.dashboards.label }} + {{ $.Values.sidecar.dashboards.label }}: "1" + {{- end }} {{- if $dashboards }} data: {{- $dashboardFound := false }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/headless-service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/headless-service.yaml index 2fa816e0..1df42e96 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/headless-service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/headless-service.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "statefulset")}} +{{- if or .Values.headlessService (and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "statefulset"))}} apiVersion: v1 kind: Service metadata: @@ -15,4 +15,8 @@ spec: selector: {{- include "grafana.selectorLabels" . | nindent 4 }} type: ClusterIP + ports: + - protocol: TCP + port: 3000 + targetPort: 3000 {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml index af35f88b..39769955 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml @@ -78,6 +78,10 @@ spec: - name: {{ .Values.imageRenderer.service.portName }} containerPort: {{ .Values.imageRenderer.service.port }} protocol: TCP + livenessProbe: + httpGet: + path: / + port: {{ .Values.imageRenderer.service.portName }} env: - name: HTTP_PORT value: {{ .Values.imageRenderer.service.port | quote }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/networkpolicy.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/networkpolicy.yaml new file mode 100644 index 00000000..fc243828 --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/networkpolicy.yaml @@ -0,0 +1,37 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.labels }} +{{ toYaml .Values.labels | indent 4 }} +{{- end }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + podSelector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + ingress: + - ports: + - port: {{ .Values.service.targetPort }} + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ template "grafana.fullname" . }}-client: "true" + {{- if .Values.networkPolicy.explicitNamespacesSelector }} + namespaceSelector: + {{ toYaml .Values.networkPolicy.explicitNamespacesSelector | indent 12 }} + {{- end }} + - podSelector: + matchLabels: + {{- include "grafana.labels" . | nindent 14 }} + role: read + {{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml index cc2a756f..ad0129ab 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml @@ -23,6 +23,9 @@ serviceAccount: replicas: 1 +## Create a headless service for the deployment +headlessService: false + ## Create HorizontalPodAutoscaler object for deployment type # autoscaling: @@ -70,7 +73,7 @@ livenessProbe: image: repository: grafana/grafana - tag: 8.1.5 + tag: 8.3.1 sha: "" pullPolicy: IfNotPresent @@ -84,7 +87,7 @@ image: testFramework: enabled: true image: "bats/bats" - tag: "v1.1.0" + tag: "v1.4.1" imagePullPolicy: IfNotPresent securityContext: {} @@ -615,7 +618,7 @@ smtp: sidecar: image: repository: quay.io/kiwigrid/k8s-sidecar - tag: 1.12.3 + tag: 1.14.2 sha: "" imagePullPolicy: IfNotPresent resources: {} @@ -625,6 +628,7 @@ sidecar: # requests: # cpu: 50m # memory: 50Mi + securityContext: {} # skipTlsVerify Set to true to skip tls verification for kube api calls # skipTlsVerify: true enableUniqueFilenames: false @@ -639,15 +643,19 @@ sidecar: folder: /tmp/dashboards # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead defaultFolderName: null - # If specified, the sidecar will search for dashboard config-maps inside this namespace. + # Namespaces list. If specified, the sidecar will search for config-maps/secrets inside these namespaces. # Otherwise the namespace in which the sidecar is running will be used. - # It's also possible to specify ALL to search in all namespaces + # It's also possible to specify ALL to search in all namespaces. searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: WATCH # search in configmap, secret or both resource: both # If specified, the sidecar will look for annotation with this name to create folder and put graph here. # You can use this parameter together with `provider.foldersFromFilesStructure`to annotate configmaps and create folder structure. folderAnnotation: null + # Absolute path to shell script to execute after a configmap got reloaded + script: null # provider configuration that lets grafana manage the dashboards provider: # name of the provider, should be unique @@ -664,6 +672,8 @@ sidecar: allowUiUpdates: false # allow Grafana to replicate dashboard structure from filesystem foldersFromFilesStructure: false + # Additional dashboard sidecar volume mounts + extraMounts: [] datasources: enabled: false # label that the configmaps with datasources are marked with @@ -674,6 +684,8 @@ sidecar: # Otherwise the namespace in which the sidecar is running will be used. # It's also possible to specify ALL to search in all namespaces searchNamespace: null + # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. + watchMethod: LIST # search in configmap, secret or both resource: both notifiers: @@ -748,3 +760,29 @@ imageRenderer: # requests: # cpu: 50m # memory: 50Mi + +networkPolicy: + ## @param networkPolicy.enabled Enable creation of NetworkPolicy resources. Only Ingress traffic is filtered for now. + ## + enabled: false + ## @param networkPolicy.allowExternal Don't require client label for connections + ## The Policy model to apply. When set to false, only pods with the correct + ## client label will have network access to grafana port defined. + ## When true, grafana will accept connections from any source + ## (with the correct destination port). + ## + allowExternal: true + ## @param networkPolicy.explicitNamespacesSelector A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed + ## If explicitNamespacesSelector is missing or set to {}, only client Pods that are in the networkPolicy's namespace + ## and that match other criteria, the ones that have the good label, can reach the grafana. + ## But sometimes, we want the grafana to be accessible to clients from other namespaces, in this case, we can use this + ## LabelSelector to select these namespaces, note that the networkPolicy's namespace should also be explicitly added. + ## + ## Example: + ## explicitNamespacesSelector: + ## matchLabels: + ## role: frontend + ## matchExpressions: + ## - {key: role, operator: In, values: [frontend]} + ## + explicitNamespacesSelector: {} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml index e9cd6209..113f9970 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 2.1.1 +appVersion: 2.2.4 description: Install kube-state-metrics to generate and expose cluster-level metrics home: https://github.com/kubernetes/kube-state-metrics/ keywords: @@ -16,4 +16,4 @@ name: kube-state-metrics sources: - https://github.com/kubernetes/kube-state-metrics/ type: application -version: 3.4.2 +version: 4.1.1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/_helpers.tpl index 6ae0e647..88f5c06a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/_helpers.tpl @@ -45,3 +45,35 @@ Allow the release namespace to be overridden for multi-namespace deployments in {{- .Release.Namespace -}} {{- end -}} {{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kube-state-metrics.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Generate basic labels +*/}} +{{- define "kube-state-metrics.labels" }} +helm.sh/chart: {{ include "kube-state-metrics.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "kube-state-metrics.name" . }} +{{- include "kube-state-metrics.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kube-state-metrics.selectorLabels" }} +app.kubernetes.io/name: {{ include "kube-state-metrics.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrolebinding.yaml index 223c285b..cf9f628d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrolebinding.yaml @@ -3,10 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "kube-state-metrics.labels" . | indent 4 }} name: {{ template "kube-state-metrics.fullname" . }} roleRef: apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml index a4c4c0b2..5e666c55 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml @@ -8,18 +8,11 @@ metadata: name: {{ template "kube-state-metrics.fullname" . }} namespace: {{ template "kube-state-metrics.namespace" . }} labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - app.kubernetes.io/instance: "{{ .Release.Name }}" - app.kubernetes.io/managed-by: "{{ .Release.Service }}" - app.kubernetes.io/version: "{{ .Chart.AppVersion }}" - {{- if .Values.customLabels }} -{{ toYaml .Values.customLabels | indent 4 }} - {{- end }} + {{- include "kube-state-metrics.labels" . | indent 4 }} spec: selector: matchLabels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} replicas: {{ .Values.replicas }} {{- if .Values.autosharding.enabled }} serviceName: {{ template "kube-state-metrics.fullname" . }} @@ -28,11 +21,7 @@ spec: template: metadata: labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - app.kubernetes.io/instance: "{{ .Release.Name }}" - {{- if .Values.customLabels }} -{{ toYaml .Values.customLabels | indent 8 }} - {{- end }} + {{- include "kube-state-metrics.labels" . | indent 8 }} {{- if .Values.podAnnotations }} annotations: {{ toYaml .Values.podAnnotations | indent 8 }} @@ -77,6 +66,9 @@ spec: {{- if .Values.metricLabelsAllowlist }} - --metric-labels-allowlist={{ .Values.metricLabelsAllowlist | join "," }} {{- end }} + {{- if .Values.metricAnnotationsAllowList }} + - --metric-annotations-allowlist={{ .Values.metricAnnotationsAllowList | join "," }} + {{- end }} {{- if .Values.metricAllowlist }} - --metric-allowlist={{ .Values.metricAllowlist | join "," }} {{- end }} @@ -107,8 +99,10 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" ports: - containerPort: {{ .Values.service.port | default 8080}} + name: "http" {{- if .Values.selfMonitor.enabled }} - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + name: "metrics" {{- end }} livenessProbe: httpGet: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/kubeconfig-secret.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/kubeconfig-secret.yaml index a7800d7a..6af00845 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/kubeconfig-secret.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/kubeconfig-secret.yaml @@ -5,10 +5,7 @@ metadata: name: {{ template "kube-state-metrics.fullname" . }}-kubeconfig namespace: {{ template "kube-state-metrics.namespace" . }} labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - app.kubernetes.io/instance: "{{ .Release.Name }}" - app.kubernetes.io/managed-by: "{{ .Release.Service }}" + {{- include "kube-state-metrics.labels" . | indent 4 }} type: Opaque data: config: '{{ .Values.kubeconfig.secret }}' diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/pdb.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/pdb.yaml index d3ef8104..cbcf3a37 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/pdb.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/pdb.yaml @@ -5,13 +5,7 @@ metadata: name: {{ template "kube-state-metrics.fullname" . }} namespace: {{ template "kube-state-metrics.namespace" . }} labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - app.kubernetes.io/instance: "{{ .Release.Name }}" - app.kubernetes.io/managed-by: "{{ .Release.Service }}" -{{- if .Values.customLabels }} -{{ toYaml .Values.customLabels | indent 4 }} -{{- end }} + {{- include "kube-state-metrics.labels" . | indent 4 }} spec: selector: matchLabels: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/podsecuritypolicy.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/podsecuritypolicy.yaml index e822ba0e..3299056a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/podsecuritypolicy.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/podsecuritypolicy.yaml @@ -4,10 +4,7 @@ kind: PodSecurityPolicy metadata: name: {{ template "kube-state-metrics.fullname" . }} labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "kube-state-metrics.labels" . | indent 4 }} {{- if .Values.podSecurityPolicy.annotations }} annotations: {{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/psp-clusterrole.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/psp-clusterrole.yaml index 217abc95..69047d4f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/psp-clusterrole.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/psp-clusterrole.yaml @@ -3,10 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "kube-state-metrics.labels" . | indent 4 }} name: psp-{{ template "kube-state-metrics.fullname" . }} rules: {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml index b96503f2..03c56d57 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml @@ -3,10 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "kube-state-metrics.labels" . | indent 4 }} name: psp-{{ template "kube-state-metrics.fullname" . }} roleRef: apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml index 86436527..9ad54ae2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml @@ -10,9 +10,15 @@ kind: ClusterRole metadata: labels: app.kubernetes.io/name: {{ template "kube-state-metrics.name" $ }} - helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version }} - app.kubernetes.io/managed-by: {{ $.Release.Service }} app.kubernetes.io/instance: {{ $.Release.Name }} + app.kubernetes.io/component: metrics + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/version: "{{ $.Chart.Version }}" + app.kubernetes.io/part-of: {{ template "kube-state-metrics.name" $ }} + helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version }} + {{- if $.Values.customLabels }} + {{ toYaml $.Values.customLabels | nindent 4 }} + {{- end }} name: {{ template "kube-state-metrics.fullname" $ }} {{- if eq $.Values.rbac.useClusterRole false }} namespace: {{ . }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml index 6809e2bb..606a79f4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml @@ -5,10 +5,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" $ }} - helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version }} - app.kubernetes.io/managed-by: {{ $.Release.Service }} - app.kubernetes.io/instance: {{ $.Release.Name }} + {{- include "kube-state-metrics.labels" . | indent 4 }} name: {{ template "kube-state-metrics.fullname" $ }} namespace: {{ . }} roleRef: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/service.yaml index f03e6d65..853cf469 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/service.yaml @@ -4,13 +4,7 @@ metadata: name: {{ template "kube-state-metrics.fullname" . }} namespace: {{ template "kube-state-metrics.namespace" . }} labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - app.kubernetes.io/instance: "{{ .Release.Name }}" - app.kubernetes.io/managed-by: "{{ .Release.Service }}" -{{- if .Values.customLabels }} -{{ toYaml .Values.customLabels | indent 4 }} -{{- end }} + {{- include "kube-state-metrics.labels" . | indent 4 }} annotations: {{- if .Values.prometheusScrape }} prometheus.io/scrape: '{{ .Values.prometheusScrape }}' @@ -38,5 +32,4 @@ spec: loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" {{- end }} selector: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "kube-state-metrics.selectorLabels" . | indent 4 }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/serviceaccount.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/serviceaccount.yaml index 365ccec5..e1229eb9 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/serviceaccount.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/serviceaccount.yaml @@ -3,10 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "kube-state-metrics.labels" . | indent 4 }} name: {{ template "kube-state-metrics.serviceAccountName" . }} namespace: {{ template "kube-state-metrics.namespace" . }} {{- if .Values.serviceAccount.annotations }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml index 7d1cd7aa..54d76319 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml @@ -5,30 +5,58 @@ metadata: name: {{ template "kube-state-metrics.fullname" . }} namespace: {{ template "kube-state-metrics.namespace" . }} labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - app.kubernetes.io/instance: "{{ .Release.Name }}" - app.kubernetes.io/managed-by: "{{ .Release.Service }}" - {{- if .Values.prometheus.monitor.additionalLabels }} -{{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }} - {{- end }} -{{- if .Values.customLabels }} -{{ toYaml .Values.customLabels | indent 4 }} -{{- end }} + {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- with .Values.prometheus.monitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: + jobLabel: app.kubernetes.io/name selector: matchLabels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} endpoints: - port: http - {{- if .Values.prometheus.monitor.honorLabels }} + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} honorLabels: true - {{- end }} - {{ if .Values.selfMonitor.enabled }} + {{- end }} + {{- if .Values.prometheus.monitor.metricRelabelings }} + metricRelabelings: + {{- toYaml .Values.prometheus.monitor.metricRelabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- if .Values.selfMonitor.enabled }} - port: metrics - {{- if .Values.prometheus.monitor.honorLabels }} + {{- if .Values.prometheus.monitor.interval }} + interval: {{ .Values.prometheus.monitor.interval }} + {{- end }} + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} + {{- if .Values.prometheus.monitor.honorLabels }} honorLabels: true - {{- end }} - {{ end }} + {{- end }} + {{- if .Values.prometheus.monitor.metricRelabelings }} + metricRelabelings: + {{- toYaml .Values.prometheus.monitor.metricRelabelings | nindent 8 }} + {{- end }} + {{- if .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml .Values.prometheus.monitor.relabelings | nindent 8 }} + {{- end }} + {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/stsdiscovery-role.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/stsdiscovery-role.yaml index 9770b049..489de147 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/stsdiscovery-role.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/stsdiscovery-role.yaml @@ -5,10 +5,7 @@ metadata: name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} namespace: {{ template "kube-state-metrics.namespace" . }} labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "kube-state-metrics.labels" . | indent 4 }} rules: - apiGroups: - "" diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml index 2e432779..73b37a4f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml @@ -5,10 +5,7 @@ metadata: name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} namespace: {{ template "kube-state-metrics.namespace" . }} labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "kube-state-metrics.labels" . | indent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml index a09e47e3..da09c892 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml @@ -2,7 +2,7 @@ prometheusScrape: true image: repository: k8s.gcr.io/kube-state-metrics/kube-state-metrics - tag: v2.1.1 + tag: v2.2.4 pullPolicy: IfNotPresent imagePullSecrets: [] @@ -30,7 +30,9 @@ service: loadBalancerIP: "" annotations: {} +## Additional labels to add to all resources customLabels: {} + # app: kube-state-metrics hostNetwork: false @@ -63,7 +65,12 @@ prometheus: enabled: false additionalLabels: {} namespace: "" + interval: "" + scrapeTimeout: "" + proxyUrl: "" honorLabels: false + metricRelabelings: [] + relabelings: [] ## Specify if a Pod Security Policy for kube-state-metrics must be created ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ @@ -133,6 +140,15 @@ metricDenylist: [] metricLabelsAllowlist: [] # - namespaces=[k8s-label-1,k8s-label-n] +# Comma-separated list of Kubernetes annotations keys that will be used in the resource' +# labels metric. By default the metric contains only name and namespace labels. +# To include additional annotations provide a list of resource names in their plural form and Kubernetes +# annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. +# A single '*' can be provided per resource instead to allow any annotations, but that has +# severe performance implications (Example: '=pods=[*]'). +metricAnnotationsAllowList: [] + # - pods=[k8s-annotation-1,k8s-annotation-n] + # Available collectors for kube-state-metrics. # By default, all available resources are enabled, comment out to disable. collectors: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml index 052017ff..1389d055 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml @@ -9,10 +9,9 @@ keywords: maintainers: - email: gianrubio@gmail.com name: gianrubio -- name: vsliouniaev - name: bismarck name: prometheus-node-exporter sources: - https://github.com/prometheus/node_exporter/ type: application -version: 2.0.4 +version: 2.2.2 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/_helpers.tpl index e8260688..2fa13132 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/_helpers.tpl @@ -64,3 +64,18 @@ Allow the release namespace to be overridden for multi-namespace deployments in {{- .Release.Namespace -}} {{- end -}} {{- end -}} + +{{/* +Create the namespace name of the service monitor +*/}} +{{- define "prometheus-node-exporter.monitor-namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- if .Values.prometheus.monitor.namespace -}} + {{- .Values.prometheus.monitor.namespace -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml index 6c0322f0..cf589441 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -21,7 +21,7 @@ spec: {{- toYaml .Values.podAnnotations | nindent 8 }} {{- end }} spec: - automountServiceAccountToken: false + automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }} serviceAccountName: {{ template "prometheus-node-exporter.serviceAccountName" . }} {{- if .Values.securityContext }} securityContext: @@ -62,7 +62,7 @@ spec: fieldPath: status.hostIP {{- end }} ports: - - name: metrics + - name: {{ .Values.service.portName }} containerPort: {{ .Values.service.port }} protocol: TCP livenessProbe: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/endpoints.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/endpoints.yaml index 8daaeaaf..d070c492 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/endpoints.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/endpoints.yaml @@ -12,7 +12,7 @@ subsets: - ip: {{ . }} {{- end }} ports: - - name: metrics + - name: {{ .Values.service.portName }} port: 9100 protocol: TCP {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml index 88f9015c..01c2b76c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml @@ -3,7 +3,7 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: {{ template "prometheus-node-exporter.fullname" . }} - namespace: {{ template "prometheus-node-exporter.namespace" . }} + namespace: {{ template "prometheus-node-exporter.monitor-namespace" . }} labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} {{- if .Values.prometheus.monitor.additionalLabels }} {{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }} @@ -14,7 +14,7 @@ spec: app: {{ template "prometheus-node-exporter.name" . }} release: {{ .Release.Name }} endpoints: - - port: metrics + - port: {{ .Values.service.portName }} scheme: {{ $.Values.prometheus.monitor.scheme }} {{- if $.Values.prometheus.monitor.bearerTokenFile }} bearerTokenFile: {{ $.Values.prometheus.monitor.bearerTokenFile }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/service.yaml index b0a447fe..cfccaa8e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/service.yaml @@ -17,7 +17,7 @@ spec: {{- end }} targetPort: {{ .Values.service.targetPort }} protocol: TCP - name: metrics + name: {{ .Values.service.portName }} selector: app: {{ template "prometheus-node-exporter.name" . }} release: {{ .Release.Name }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml index 27b87831..0d2c9aa8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml @@ -11,6 +11,7 @@ service: port: 9100 targetPort: 9100 nodePort: + portName: metrics listenOnAllInterfaces: true annotations: prometheus.io/scrape: "true" @@ -57,6 +58,7 @@ serviceAccount: name: annotations: {} imagePullSecrets: [] + automountServiceAccountToken: false securityContext: fsGroup: 65534 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml index 25e599e5..3c2ca571 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml @@ -1,11 +1,11 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.6.2 creationTimestamp: null name: alertmanagerconfigs.monitoring.coreos.com spec: @@ -198,7 +198,8 @@ spec: description: Whether or not to notify about resolved alerts. type: boolean smarthost: - description: The SMTP host through which emails are sent. + description: The SMTP host and port through which emails + are sent. E.g. example.com:25 type: string text: description: The text body of the email notification. @@ -391,8 +392,42 @@ spec: httpConfig: description: HTTP client configuration. properties: + authorization: + description: Authorization header configuration for + the client. This is mutually exclusive with BasicAuth + and is only available starting from Alertmanager + v0.22+. + properties: + credentials: + description: The secret's key that contains the + credentials of the request + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + type: + description: Set the authentication type. Defaults + to Bearer, Basic will cause an error + type: string + type: object basicAuth: - description: BasicAuth for the client. + description: BasicAuth for the client. This is mutually + exclusive with Authorization. If both are defined, + BasicAuth takes precedence. properties: password: description: The secret in the service monitor @@ -683,8 +718,42 @@ spec: httpConfig: description: HTTP client configuration. properties: + authorization: + description: Authorization header configuration for + the client. This is mutually exclusive with BasicAuth + and is only available starting from Alertmanager + v0.22+. + properties: + credentials: + description: The secret's key that contains the + credentials of the request + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + type: + description: Set the authentication type. Defaults + to Bearer, Basic will cause an error + type: string + type: object basicAuth: - description: BasicAuth for the client. + description: BasicAuth for the client. This is mutually + exclusive with Authorization. If both are defined, + BasicAuth takes precedence. properties: password: description: The secret in the service monitor @@ -959,8 +1028,42 @@ spec: httpConfig: description: HTTP client configuration. properties: + authorization: + description: Authorization header configuration for + the client. This is mutually exclusive with BasicAuth + and is only available starting from Alertmanager + v0.22+. + properties: + credentials: + description: The secret's key that contains the + credentials of the request + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + type: + description: Set the authentication type. Defaults + to Bearer, Basic will cause an error + type: string + type: object basicAuth: - description: BasicAuth for the client. + description: BasicAuth for the client. This is mutually + exclusive with Authorization. If both are defined, + BasicAuth takes precedence. properties: password: description: The secret in the service monitor @@ -1350,8 +1453,42 @@ spec: httpConfig: description: HTTP client configuration. properties: + authorization: + description: Authorization header configuration for + the client. This is mutually exclusive with BasicAuth + and is only available starting from Alertmanager + v0.22+. + properties: + credentials: + description: The secret's key that contains the + credentials of the request + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + type: + description: Set the authentication type. Defaults + to Bearer, Basic will cause an error + type: string + type: object basicAuth: - description: BasicAuth for the client. + description: BasicAuth for the client. This is mutually + exclusive with Authorization. If both are defined, + BasicAuth takes precedence. properties: password: description: The secret in the service monitor @@ -1635,8 +1772,42 @@ spec: httpConfig: description: The HTTP client's configuration. properties: + authorization: + description: Authorization header configuration for + the client. This is mutually exclusive with BasicAuth + and is only available starting from Alertmanager + v0.22+. + properties: + credentials: + description: The secret's key that contains the + credentials of the request + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + type: + description: Set the authentication type. Defaults + to Bearer, Basic will cause an error + type: string + type: object basicAuth: - description: BasicAuth for the client. + description: BasicAuth for the client. This is mutually + exclusive with Authorization. If both are defined, + BasicAuth takes precedence. properties: password: description: The secret in the service monitor @@ -1864,8 +2035,42 @@ spec: httpConfig: description: HTTP client configuration. properties: + authorization: + description: Authorization header configuration for + the client. This is mutually exclusive with BasicAuth + and is only available starting from Alertmanager + v0.22+. + properties: + credentials: + description: The secret's key that contains the + credentials of the request + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + type: + description: Set the authentication type. Defaults + to Bearer, Basic will cause an error + type: string + type: object basicAuth: - description: BasicAuth for the client. + description: BasicAuth for the client. This is mutually + exclusive with Authorization. If both are defined, + BasicAuth takes precedence. properties: password: description: The secret in the service monitor @@ -2143,8 +2348,42 @@ spec: httpConfig: description: HTTP client configuration. properties: + authorization: + description: Authorization header configuration for + the client. This is mutually exclusive with BasicAuth + and is only available starting from Alertmanager + v0.22+. + properties: + credentials: + description: The secret's key that contains the + credentials of the request + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + type: + description: Set the authentication type. Defaults + to Bearer, Basic will cause an error + type: string + type: object basicAuth: - description: BasicAuth for the client. + description: BasicAuth for the client. This is mutually + exclusive with Authorization. If both are defined, + BasicAuth takes precedence. properties: password: description: The secret in the service monitor diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml index f2ee201f..3df6ea42 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml @@ -1,11 +1,11 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.6.2 creationTimestamp: null name: alertmanagers.monitoring.coreos.com spec: @@ -340,10 +340,71 @@ spec: The requirements are ANDed. type: object type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. This field is beta-level + and is only honored when PodAffinityNamespaceSelector + feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace" items: type: string type: array @@ -435,10 +496,66 @@ spec: requirements are ANDed. type: object type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied to the + union of the namespaces selected by this field and + the ones listed in the namespaces field. null selector + and null or empty namespaces list means "this pod's + namespace". An empty selector ({}) matches all namespaces. + This field is beta-level and is only honored when + PodAffinityNamespaceSelector feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object namespaces: - description: namespaces specifies which namespaces the - labelSelector applies to (matches against); null or - empty list means "this pod's namespace" + description: namespaces specifies a static list of namespace + names that the term applies to. The term is applied + to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. null or + empty namespaces list and null namespaceSelector means + "this pod's namespace" items: type: string type: array @@ -532,10 +649,71 @@ spec: The requirements are ANDed. type: object type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. This field is beta-level + and is only honored when PodAffinityNamespaceSelector + feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace" items: type: string type: array @@ -627,10 +805,66 @@ spec: requirements are ANDed. type: object type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied to the + union of the namespaces selected by this field and + the ones listed in the namespaces field. null selector + and null or empty namespaces list means "this pod's + namespace". An empty selector ({}) matches all namespaces. + This field is beta-level and is only honored when + PodAffinityNamespaceSelector feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object namespaces: - description: namespaces specifies which namespaces the - labelSelector applies to (matches against); null or - empty list means "this pod's namespace" + description: namespaces specifies a static list of namespace + names that the term applies to. The term is applied + to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. null or + empty namespaces list and null namespaceSelector means + "this pod's namespace" items: type: string type: array @@ -788,10 +1022,11 @@ spec: CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will - be unchanged. The $(VAR_NAME) syntax can be escaped with a - double $$, ie: $$(VAR_NAME). Escaped references will never - be expanded, regardless of whether the variable exists or - not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + be unchanged. Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the variable + exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array @@ -800,10 +1035,12 @@ spec: The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference - in the input string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). Escaped - references will never be expanded, regardless of whether the - variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: + i.e. "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. More info: + https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array @@ -820,13 +1057,15 @@ spec: type: string value: description: 'Variable references $(VAR_NAME) are expanded - using the previous defined environment variables in + using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the - input string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). Escaped - references will never be expanded, regardless of whether - the variable exists or not. Defaults to "".' + input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. "$$(VAR_NAME)" will produce the string + literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists + or not. Defaults to "".' type: string valueFrom: description: Source for the environment variable's value. @@ -853,9 +1092,10 @@ spec: type: object fieldRef: description: 'Selects a field of the pod: supports - metadata.name, metadata.namespace, metadata.labels, - metadata.annotations, spec.nodeName, spec.serviceAccountName, - status.hostIP, status.podIP, status.podIPs.' + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' properties: apiVersion: description: Version of the schema the FieldPath @@ -1275,6 +1515,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -1443,6 +1700,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -1452,7 +1726,7 @@ spec: type: object resources: description: 'Compute Resources required by this container. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' properties: limits: additionalProperties: @@ -1462,7 +1736,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -1475,13 +1749,14 @@ spec: resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object securityContext: - description: 'Security options the pod should run with. More - info: https://kubernetes.io/docs/concepts/policy/security-context/ - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + description: 'SecurityContext defines the security options the + container should be run with. If set, the fields of SecurityContext + override the equivalent fields of PodSecurityContext. More + info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: description: 'AllowPrivilegeEscalation controls whether @@ -1578,6 +1853,30 @@ spec: to the container. type: string type: object + seccompProfile: + description: The seccomp options to use by this container. + If seccomp options are provided at both the pod & container + level, the container options override the pod options. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. The profile + must be preconfigured on the node to work. Must be + a descending path, relative to the kubelet's configured + seccomp profile location. Must only be set if type + is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile + will be applied. Valid options are: \n Localhost - + a profile defined in a file on the node should be + used. RuntimeDefault - the container runtime default + profile should be used. Unconfined - no profile should + be applied." + type: string + required: + - type + type: object windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext @@ -1594,6 +1893,19 @@ spec: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. This field is + alpha-level and will only be honored by components + that enable the WindowsHostProcessContainers feature + flag. Setting this field without the feature flag + will result in errors when validating the Pod. All + of a Pod's containers must have the same effective + HostProcess value (it is not allowed to have a mix + of HostProcess containers and non-HostProcess containers). In + addition, if HostProcess is true then HostNetwork + must also be set to true. + type: boolean runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified @@ -1612,8 +1924,7 @@ spec: can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. This is a beta feature enabled by - the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be @@ -1720,6 +2031,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -1888,10 +2216,11 @@ spec: CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will - be unchanged. The $(VAR_NAME) syntax can be escaped with a - double $$, ie: $$(VAR_NAME). Escaped references will never - be expanded, regardless of whether the variable exists or - not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + be unchanged. Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the variable + exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array @@ -1900,10 +2229,12 @@ spec: The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference - in the input string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). Escaped - references will never be expanded, regardless of whether the - variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: + i.e. "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. More info: + https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array @@ -1920,13 +2251,15 @@ spec: type: string value: description: 'Variable references $(VAR_NAME) are expanded - using the previous defined environment variables in + using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the - input string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). Escaped - references will never be expanded, regardless of whether - the variable exists or not. Defaults to "".' + input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. "$$(VAR_NAME)" will produce the string + literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists + or not. Defaults to "".' type: string valueFrom: description: Source for the environment variable's value. @@ -1953,9 +2286,10 @@ spec: type: object fieldRef: description: 'Selects a field of the pod: supports - metadata.name, metadata.namespace, metadata.labels, - metadata.annotations, spec.nodeName, spec.serviceAccountName, - status.hostIP, status.podIP, status.podIPs.' + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' properties: apiVersion: description: Version of the schema the FieldPath @@ -2375,6 +2709,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -2543,6 +2894,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -2552,7 +2920,7 @@ spec: type: object resources: description: 'Compute Resources required by this container. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' properties: limits: additionalProperties: @@ -2562,7 +2930,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -2575,13 +2943,14 @@ spec: resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object securityContext: - description: 'Security options the pod should run with. More - info: https://kubernetes.io/docs/concepts/policy/security-context/ - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + description: 'SecurityContext defines the security options the + container should be run with. If set, the fields of SecurityContext + override the equivalent fields of PodSecurityContext. More + info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: description: 'AllowPrivilegeEscalation controls whether @@ -2678,6 +3047,30 @@ spec: to the container. type: string type: object + seccompProfile: + description: The seccomp options to use by this container. + If seccomp options are provided at both the pod & container + level, the container options override the pod options. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. The profile + must be preconfigured on the node to work. Must be + a descending path, relative to the kubelet's configured + seccomp profile location. Must only be set if type + is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile + will be applied. Valid options are: \n Localhost - + a profile defined in a file on the node should be + used. RuntimeDefault - the container runtime default + profile should be used. Unconfined - no profile should + be applied." + type: string + required: + - type + type: object windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext @@ -2694,6 +3087,19 @@ spec: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. This field is + alpha-level and will only be honored by components + that enable the WindowsHostProcessContainers feature + flag. Setting this field without the feature flag + will result in errors when validating the Pod. All + of a Pod's containers must have the same effective + HostProcess value (it is not allowed to have a mix + of HostProcess containers and non-HostProcess containers). In + addition, if HostProcess is true then HostNetwork + must also be set to true. + type: boolean runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified @@ -2712,8 +3118,7 @@ spec: can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. This is a beta feature enabled by - the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be @@ -2820,6 +3225,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -2949,6 +3371,14 @@ spec: logLevel: description: Log level for Alertmanager to be configured with. type: string + minReadySeconds: + description: Minimum number of seconds for which a newly created pod + should be ready without any of its container crashing for it to + be considered available. Defaults to 0 (pod will be considered available + as soon as it is ready) This is an alpha field and requires enabling + StatefulSetMinReadySeconds feature gate. + format: int32 + type: integer nodeSelector: additionalProperties: type: string @@ -3010,7 +3440,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources - allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -3022,7 +3452,7 @@ spec: description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object retention: @@ -3066,7 +3496,7 @@ spec: support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". - If not specified defaults to "Always".' + If not specified, "Always" is used.' type: string runAsGroup: description: The GID to run the entrypoint of the container process. @@ -3116,6 +3546,27 @@ spec: the container. type: string type: object + seccompProfile: + description: The seccomp options to use by the containers in this + pod. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. The profile must be + preconfigured on the node to work. Must be a descending + path, relative to the kubelet's configured seccomp profile + location. Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile + will be applied. Valid options are: \n Localhost - a profile + defined in a file on the node should be used. RuntimeDefault + - the container runtime default profile should be used. + Unconfined - no profile should be applied." + type: string + required: + - type + type: object supplementalGroups: description: A list of groups applied to the first process run in each container, in addition to the container's primary GID. If @@ -3158,6 +3609,17 @@ spec: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. This field is alpha-level + and will only be honored by components that enable the WindowsHostProcessContainers + feature flag. Setting this field without the feature flag + will result in errors when validating the Pod. All of a + Pod's containers must have the same effective HostProcess + value (it is not allowed to have a mix of HostProcess containers + and non-HostProcess containers). In addition, if HostProcess + is true then HostNetwork must also be set to true. + type: boolean runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified @@ -3212,6 +3674,217 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + ephemeral: + description: 'EphemeralVolumeSource to be used by the Prometheus + StatefulSets. This is a beta field in k8s 1.21, for lower versions, + starting with k8s 1.19, it requires enabling the GenericEphemeralVolume + feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes' + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC to + provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC will + be deleted together with the pod. The name of the PVC will + be `-` where `` is the + name from the `PodSpec.Volumes` array entry. Pod validation + will reject the pod if the concatenated name is not valid + for a PVC (for example, too long). \n An existing PVC with + that name that is not owned by the pod will *not* be used + for the pod to avoid using an unrelated volume by mistake. + Starting the pod is then blocked until the unrelated PVC + is removed. If such a pre-created PVC is meant to be used + by the pod, the PVC has to updated with an owner reference + to the pod once the pod exists. Normally this should not + be necessary, but it may be useful when manually reconstructing + a broken cluster. \n This field is read-only and no changes + will be made by Kubernetes to the PVC after it has been + created. \n Required, must not be nil." + properties: + metadata: + description: May contain labels and annotations that will + be copied into the PVC when creating it. No other fields + are allowed and will be rejected during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the PVC + that gets created from this template. The same fields + as in a PersistentVolumeClaim are also valid here. + properties: + accessModes: + description: 'AccessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'This field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the + provisioner or an external controller can support + the specified data source, it will create a new + volume based on the contents of the specified data + source. If the AnyVolumeDataSource feature gate + is enabled, this field will always have the same + contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is + required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to populate + the volume with data, if a non-empty volume is desired. + This may be any local object from a non-empty API + group (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume binding + will only succeed if the type of the specified object + matches some installed volume populator or dynamic + provisioner. This field will replace the functionality + of the DataSource field and as such if both fields + are non-empty, they must have the same value. For + backwards compatibility, both fields (DataSource + and DataSourceRef) will be set to the same value + automatically if one of them is empty and the other + is non-empty. There are two important differences + between DataSource and DataSourceRef: * While DataSource + only allows two specific types of objects, DataSourceRef allows + any non-core object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed values + (dropping them), DataSourceRef preserves all values, + and generates an error if a disallowed value is specified. + (Alpha) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is + required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum resources + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: A label query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement is + a selector that contains values, a key, and + an operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. This array + is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required by + the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem is + implied when not included in claim spec. + type: string + volumeName: + description: VolumeName is the binding reference to + the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object volumeClaimTemplate: description: A PVC spec to be used by the Prometheus StatefulSets. properties: @@ -3269,20 +3942,52 @@ spec: type: array dataSource: description: 'This field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot - - Beta) * An existing PVC (PersistentVolumeClaim) * - An existing custom resource/object that implements data - population (Alpha) In order to use VolumeSnapshot object - types, the appropriate feature gate must be enabled - (VolumeSnapshotDataSource or AnyVolumeDataSource) If - the provisioner or an external controller can support - the specified data source, it will create a new volume - based on the contents of the specified data source. - If the specified data source is not supported, the volume - will not be created and the failure will be reported - as an event. In the future, we plan to support more - data source types and the behavior of the provisioner - may change.' + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified + data source, it will create a new volume based on the + contents of the specified data source. If the AnyVolumeDataSource + feature gate is enabled, this field will always have + the same contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to populate + the volume with data, if a non-empty volume is desired. + This may be any local object from a non-empty API group + (non core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only + succeed if the type of the specified object matches + some installed volume populator or dynamic provisioner. + This field will replace the functionality of the DataSource + field and as such if both fields are non-empty, they + must have the same value. For backwards compatibility, + both fields (DataSource and DataSourceRef) will be set + to the same value automatically if one of them is empty + and the other is non-empty. There are two important + differences between DataSource and DataSourceRef: * + While DataSource only allows two specific types of objects, + DataSourceRef allows any non-core object, as well + as PersistentVolumeClaim objects. * While DataSource + ignores disallowed values (dropping them), DataSourceRef preserves + all values, and generates an error if a disallowed value + is specified. (Alpha) Using this field requires the + AnyVolumeDataSource feature gate to be enabled.' properties: apiGroup: description: APIGroup is the group for the resource @@ -3312,7 +4017,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount - of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -3325,7 +4030,7 @@ spec: of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object selector: @@ -3553,16 +4258,19 @@ spec: type: object maxSkew: description: 'MaxSkew describes the degree to which pods may - be unevenly distributed. It''s the maximum permitted difference - between the number of matching pods in any two topology domains - of a given topology type. For example, in a 3-zone cluster, - MaxSkew is set to 1, and pods with the same labelSelector - spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 - to become 1/1/1; scheduling it onto zone1(zone2) would make - the ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - It''s a required field. Default value is 1 and 0 is not allowed.' + be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global minimum. + For example, in a 3-zone cluster, MaxSkew is set to 1, and + pods with the same labelSelector spread as 1/1/0: | zone1 + | zone2 | zone3 | | P | P | | - if MaxSkew is + 1, incoming pod can only be scheduled to zone3 to become 1/1/1; + scheduling it onto zone1(zone2) would make the ActualSkew(2-0) + on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming + pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that satisfy + it. It''s a required field. Default value is 1 and 0 is not + allowed.' format: int32 type: integer topologyKey: @@ -3575,17 +4283,20 @@ spec: whenUnsatisfiable: description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule - (default) tells the scheduler not to schedule it - ScheduleAnyway - tells the scheduler to still schedule it It''s considered - as "Unsatisfiable" if and only if placing incoming pod on - any topology violates "MaxSkew". For example, in a 3-zone - cluster, MaxSkew is set to 1, and pods with the same labelSelector - spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod - can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) - as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In - other words, the cluster can still be imbalanced, but scheduler - won''t make it *more* imbalanced. It''s a required field.' + (default) tells the scheduler not to schedule it. - ScheduleAnyway + tells the scheduler to schedule the pod in any location, but + giving higher precedence to topologies that would help reduce + the skew. A constraint is considered "Unsatisfiable" for + an incoming pod if and only if every possible node assigment + for that pod would violate "MaxSkew" on some topology. For + example, in a 3-zone cluster, MaxSkew is set to 1, and pods + with the same labelSelector spread as 3/1/1: | zone1 | zone2 + | zone3 | | P P P | P | P | If WhenUnsatisfiable is + set to DoNotSchedule, incoming pod can only be scheduled to + zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on + zone2(zone3) satisfies MaxSkew(1). In other words, the cluster + can still be imbalanced, but scheduler won''t make it *more* + imbalanced. It''s a required field.' type: string required: - maxSkew @@ -3807,12 +4518,15 @@ spec: this volume properties: defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a value between 0 and 0777. Defaults - to 0644. Directories within the path are not affected - by this setting. This might be in conflict with other - options that affect the file mode, like fsGroup, and the - result can be other mode bits set.' + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal value between + 0000 and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. Directories within + the path are not affected by this setting. This might + be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits + set.' format: int32 type: integer items: @@ -3832,8 +4546,11 @@ spec: description: The key to project. type: string mode: - description: 'Optional: mode bits to use on this file, - must be a value between 0 and 0777. If not specified, + description: 'Optional: mode bits used to set permissions + on this file. Must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires + decimal values for mode bits. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other @@ -3861,8 +4578,9 @@ spec: type: boolean type: object csi: - description: CSI (Container Storage Interface) represents storage - that is handled by an external CSI driver (Alpha feature). + description: CSI (Container Storage Interface) represents ephemeral + storage that is handled by certain external CSI drivers (Beta + feature). properties: driver: description: Driver is the name of the CSI driver that handles @@ -3909,11 +4627,15 @@ spec: properties: defaultMode: description: 'Optional: mode bits to use on created files - by default. Must be a value between 0 and 0777. Defaults - to 0644. Directories within the path are not affected - by this setting. This might be in conflict with other - options that affect the file mode, like fsGroup, and the - result can be other mode bits set.' + by default. Must be a Optional: mode bits used to set + permissions on created files by default. Must be an octal + value between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. Defaults to + 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' format: int32 type: integer items: @@ -3939,8 +4661,11 @@ spec: - fieldPath type: object mode: - description: 'Optional: mode bits to use on this file, - must be a value between 0 and 0777. If not specified, + description: 'Optional: mode bits used to set permissions + on this file, must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires + decimal values for mode bits. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other @@ -4007,6 +4732,239 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + ephemeral: + description: "Ephemeral represents a volume that is handled + by a cluster storage driver. The volume's lifecycle is tied + to the pod that defines it - it will be created before the + pod starts, and deleted when the pod is removed. \n Use this + if: a) the volume is only needed while the pod runs, b) features + of normal volumes like restoring from snapshot or capacity + \ tracking are needed, c) the storage driver is specified + through a storage class, and d) the storage driver supports + dynamic volume provisioning through a PersistentVolumeClaim + (see EphemeralVolumeSource for more information on the + connection between this volume type and PersistentVolumeClaim). + \n Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. \n Use CSI for light-weight local ephemeral + volumes if the CSI driver is meant to be used that way - see + the documentation of the driver for more information. \n A + pod can use both types of ephemeral volumes and persistent + volumes at the same time. \n This is a beta feature and only + available when the GenericEphemeralVolume feature gate is + enabled." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC to + provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC + will be deleted together with the pod. The name of the + PVC will be `-` where `` is the name from the `PodSpec.Volumes` array entry. + Pod validation will reject the pod if the concatenated + name is not valid for a PVC (for example, too long). \n + An existing PVC with that name that is not owned by the + pod will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC + is meant to be used by the pod, the PVC has to updated + with an owner reference to the pod once the pod exists. + Normally this should not be necessary, but it may be useful + when manually reconstructing a broken cluster. \n This + field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. \n Required, must + not be nil." + properties: + metadata: + description: May contain labels and annotations that + will be copied into the PVC when creating it. No other + fields are allowed and will be rejected during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the PVC + that gets created from this template. The same fields + as in a PersistentVolumeClaim are also valid here. + properties: + accessModes: + description: 'AccessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'This field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the + provisioner or an external controller can support + the specified data source, it will create a new + volume based on the contents of the specified + data source. If the AnyVolumeDataSource feature + gate is enabled, this field will always have the + same contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API + group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to + populate the volume with data, if a non-empty + volume is desired. This may be any local object + from a non-empty API group (non core object) or + a PersistentVolumeClaim object. When this field + is specified, volume binding will only succeed + if the type of the specified object matches some + installed volume populator or dynamic provisioner. + This field will replace the functionality of the + DataSource field and as such if both fields are + non-empty, they must have the same value. For + backwards compatibility, both fields (DataSource + and DataSourceRef) will be set to the same value + automatically if one of them is empty and the + other is non-empty. There are two important differences + between DataSource and DataSourceRef: * While + DataSource only allows two specific types of objects, + DataSourceRef allows any non-core object, as + well as PersistentVolumeClaim objects. * While + DataSource ignores disallowed values (dropping + them), DataSourceRef preserves all values, and + generates an error if a disallowed value is specified. + (Alpha) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API + group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum resources + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. If Requests + is omitted for a container, it defaults to + Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: A label query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required + by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem + is implied when not included in claim spec. + type: string + volumeName: + description: VolumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object fc: description: FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the @@ -4341,12 +5299,14 @@ spec: and downward API properties: defaultMode: - description: Mode bits to use on created files by default. - Must be a value between 0 and 0777. Directories within - the path are not affected by this setting. This might - be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits - set. + description: Mode bits used to set permissions on created + files by default. Must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML accepts + both octal and decimal values, JSON requires decimal values + for mode bits. Directories within the path are not affected + by this setting. This might be in conflict with other + options that affect the file mode, like fsGroup, and the + result can be other mode bits set. format: int32 type: integer sources: @@ -4379,13 +5339,17 @@ spec: description: The key to project. type: string mode: - description: 'Optional: mode bits to use - on this file, must be a value between - 0 and 0777. If not specified, the volume - defaultMode will be used. This might be - in conflict with other options that affect - the file mode, like fsGroup, and the result - can be other mode bits set.' + description: 'Optional: mode bits used to + set permissions on this file. Must be + an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML + accepts both octal and decimal values, + JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set.' format: int32 type: integer path: @@ -4440,13 +5404,17 @@ spec: - fieldPath type: object mode: - description: 'Optional: mode bits to use - on this file, must be a value between - 0 and 0777. If not specified, the volume - defaultMode will be used. This might be - in conflict with other options that affect - the file mode, like fsGroup, and the result - can be other mode bits set.' + description: 'Optional: mode bits used to + set permissions on this file, must be + an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML + accepts both octal and decimal values, + JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set.' format: int32 type: integer path: @@ -4512,13 +5480,17 @@ spec: description: The key to project. type: string mode: - description: 'Optional: mode bits to use - on this file, must be a value between - 0 and 0777. If not specified, the volume - defaultMode will be used. This might be - in conflict with other options that affect - the file mode, like fsGroup, and the result - can be other mode bits set.' + description: 'Optional: mode bits used to + set permissions on this file. Must be + an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML + accepts both octal and decimal values, + JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set.' format: int32 type: integer path: @@ -4577,8 +5549,6 @@ spec: type: object type: object type: array - required: - - sources type: object quobyte: description: Quobyte represents a Quobyte mount on the host @@ -4729,12 +5699,15 @@ spec: this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' properties: defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a value between 0 and 0777. Defaults - to 0644. Directories within the path are not affected - by this setting. This might be in conflict with other - options that affect the file mode, like fsGroup, and the - result can be other mode bits set.' + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal value between + 0000 and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. Directories within + the path are not affected by this setting. This might + be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits + set.' format: int32 type: integer items: @@ -4754,8 +5727,11 @@ spec: description: The key to project. type: string mode: - description: 'Optional: mode bits to use on this file, - must be a value between 0 and 0777. If not specified, + description: 'Optional: mode bits used to set permissions + on this file. Must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires + decimal values for mode bits. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml index 6363bfe7..55522bf2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml @@ -1,11 +1,11 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.6.2 creationTimestamp: null name: podmonitors.monitoring.coreos.com spec: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml index a4ad2e23..82853fac 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml @@ -1,11 +1,11 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.6.2 creationTimestamp: null name: probes.monitoring.coreos.com spec: @@ -153,6 +153,50 @@ spec: and newer. format: int64 type: integer + metricRelabelings: + description: MetricRelabelConfigs to apply to samples before ingestion. + items: + description: 'RelabelConfig allows dynamic rewriting of the label + set, being applied to samples before ingestion. It defines ``-section + of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' + properties: + action: + description: Action to perform based on regex matching. Default + is 'replace' + type: string + modulus: + description: Modulus to take of the hash of the source label + values. + format: int64 + type: integer + regex: + description: Regular expression against which the extracted + value is matched. Default is '(.*)' + type: string + replacement: + description: Replacement value against which a regex replace + is performed if the regular expression matches. Regex capture + groups are available. Default is '$1' + type: string + separator: + description: Separator placed between concatenated source label + values. default is ';'. + type: string + sourceLabels: + description: The source labels select values from existing labels. + Their content is concatenated using the configured separator + and matched against the configured regular expression for + the replace, keep, and drop actions. + items: + type: string + type: array + targetLabel: + description: Label to which the resulting value is written in + a replace action. It is mandatory for replace actions. Regex + capture groups are available. + type: string + type: object + type: array module: description: 'The module to use for probing specifying how to probe the target. Example module configuring in the blackbox exporter: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml index 753174f6..74702148 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml @@ -1,11 +1,12 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.6.2 + argocd.argoproj.io/sync-options: Replace=true creationTimestamp: null name: prometheuses.monitoring.coreos.com spec: @@ -415,10 +416,71 @@ spec: The requirements are ANDed. type: object type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. This field is beta-level + and is only honored when PodAffinityNamespaceSelector + feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace" items: type: string type: array @@ -510,10 +572,66 @@ spec: requirements are ANDed. type: object type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied to the + union of the namespaces selected by this field and + the ones listed in the namespaces field. null selector + and null or empty namespaces list means "this pod's + namespace". An empty selector ({}) matches all namespaces. + This field is beta-level and is only honored when + PodAffinityNamespaceSelector feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object namespaces: - description: namespaces specifies which namespaces the - labelSelector applies to (matches against); null or - empty list means "this pod's namespace" + description: namespaces specifies a static list of namespace + names that the term applies to. The term is applied + to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. null or + empty namespaces list and null namespaceSelector means + "this pod's namespace" items: type: string type: array @@ -607,10 +725,71 @@ spec: The requirements are ANDed. type: object type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. This field is beta-level + and is only honored when PodAffinityNamespaceSelector + feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace" items: type: string type: array @@ -702,10 +881,66 @@ spec: requirements are ANDed. type: object type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied to the + union of the namespaces selected by this field and + the ones listed in the namespaces field. null selector + and null or empty namespaces list means "this pod's + namespace". An empty selector ({}) matches all namespaces. + This field is beta-level and is only honored when + PodAffinityNamespaceSelector feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object namespaces: - description: namespaces specifies which namespaces the - labelSelector applies to (matches against); null or - empty list means "this pod's namespace" + description: namespaces specifies a static list of namespace + names that the term applies to. The term is applied + to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. null or + empty namespaces list and null namespaceSelector means + "this pod's namespace" items: type: string type: array @@ -1202,10 +1437,11 @@ spec: CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will - be unchanged. The $(VAR_NAME) syntax can be escaped with a - double $$, ie: $$(VAR_NAME). Escaped references will never - be expanded, regardless of whether the variable exists or - not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + be unchanged. Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the variable + exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array @@ -1214,10 +1450,12 @@ spec: The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference - in the input string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). Escaped - references will never be expanded, regardless of whether the - variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: + i.e. "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. More info: + https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array @@ -1234,13 +1472,15 @@ spec: type: string value: description: 'Variable references $(VAR_NAME) are expanded - using the previous defined environment variables in + using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the - input string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). Escaped - references will never be expanded, regardless of whether - the variable exists or not. Defaults to "".' + input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. "$$(VAR_NAME)" will produce the string + literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists + or not. Defaults to "".' type: string valueFrom: description: Source for the environment variable's value. @@ -1267,9 +1507,10 @@ spec: type: object fieldRef: description: 'Selects a field of the pod: supports - metadata.name, metadata.namespace, metadata.labels, - metadata.annotations, spec.nodeName, spec.serviceAccountName, - status.hostIP, status.podIP, status.podIPs.' + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' properties: apiVersion: description: Version of the schema the FieldPath @@ -1689,6 +1930,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -1857,6 +2115,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -1866,7 +2141,7 @@ spec: type: object resources: description: 'Compute Resources required by this container. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' properties: limits: additionalProperties: @@ -1876,7 +2151,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -1889,13 +2164,14 @@ spec: resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object securityContext: - description: 'Security options the pod should run with. More - info: https://kubernetes.io/docs/concepts/policy/security-context/ - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + description: 'SecurityContext defines the security options the + container should be run with. If set, the fields of SecurityContext + override the equivalent fields of PodSecurityContext. More + info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: description: 'AllowPrivilegeEscalation controls whether @@ -1992,6 +2268,30 @@ spec: to the container. type: string type: object + seccompProfile: + description: The seccomp options to use by this container. + If seccomp options are provided at both the pod & container + level, the container options override the pod options. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. The profile + must be preconfigured on the node to work. Must be + a descending path, relative to the kubelet's configured + seccomp profile location. Must only be set if type + is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile + will be applied. Valid options are: \n Localhost - + a profile defined in a file on the node should be + used. RuntimeDefault - the container runtime default + profile should be used. Unconfined - no profile should + be applied." + type: string + required: + - type + type: object windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext @@ -2008,6 +2308,19 @@ spec: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. This field is + alpha-level and will only be honored by components + that enable the WindowsHostProcessContainers feature + flag. Setting this field without the feature flag + will result in errors when validating the Pod. All + of a Pod's containers must have the same effective + HostProcess value (it is not allowed to have a mix + of HostProcess containers and non-HostProcess containers). In + addition, if HostProcess is true then HostNetwork + must also be set to true. + type: boolean runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified @@ -2026,8 +2339,7 @@ spec: can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. This is a beta feature enabled by - the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be @@ -2134,6 +2446,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -2273,6 +2602,15 @@ spec: items: type: string type: array + enforcedBodySizeLimit: + description: 'EnforcedBodySizeLimit defines the maximum size of uncompressed + response body that will be accepted by Prometheus. Targets responding + with a body larger than this many bytes will cause the scrape to + fail. Example: 100MB. If defined, the limit will apply to all service/pod + monitors and probes. This is an experimental feature, this behaviour + could change or be removed in the future. Only valid in Prometheus + versions 2.28.0 and newer.' + type: string enforcedLabelLimit: description: Per-scrape limit on number of labels that will be accepted for a sample. If more than this number of labels are present post @@ -2384,10 +2722,11 @@ spec: CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will - be unchanged. The $(VAR_NAME) syntax can be escaped with a - double $$, ie: $$(VAR_NAME). Escaped references will never - be expanded, regardless of whether the variable exists or - not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + be unchanged. Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the variable + exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array @@ -2396,10 +2735,12 @@ spec: The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference - in the input string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). Escaped - references will never be expanded, regardless of whether the - variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: + i.e. "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. More info: + https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array @@ -2416,13 +2757,15 @@ spec: type: string value: description: 'Variable references $(VAR_NAME) are expanded - using the previous defined environment variables in + using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the - input string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). Escaped - references will never be expanded, regardless of whether - the variable exists or not. Defaults to "".' + input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. "$$(VAR_NAME)" will produce the string + literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists + or not. Defaults to "".' type: string valueFrom: description: Source for the environment variable's value. @@ -2449,9 +2792,10 @@ spec: type: object fieldRef: description: 'Selects a field of the pod: supports - metadata.name, metadata.namespace, metadata.labels, - metadata.annotations, spec.nodeName, spec.serviceAccountName, - status.hostIP, status.podIP, status.podIPs.' + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' properties: apiVersion: description: Version of the schema the FieldPath @@ -2871,6 +3215,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -3039,6 +3400,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -3048,7 +3426,7 @@ spec: type: object resources: description: 'Compute Resources required by this container. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' properties: limits: additionalProperties: @@ -3058,7 +3436,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -3071,13 +3449,14 @@ spec: resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object securityContext: - description: 'Security options the pod should run with. More - info: https://kubernetes.io/docs/concepts/policy/security-context/ - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + description: 'SecurityContext defines the security options the + container should be run with. If set, the fields of SecurityContext + override the equivalent fields of PodSecurityContext. More + info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: description: 'AllowPrivilegeEscalation controls whether @@ -3174,6 +3553,30 @@ spec: to the container. type: string type: object + seccompProfile: + description: The seccomp options to use by this container. + If seccomp options are provided at both the pod & container + level, the container options override the pod options. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. The profile + must be preconfigured on the node to work. Must be + a descending path, relative to the kubelet's configured + seccomp profile location. Must only be set if type + is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile + will be applied. Valid options are: \n Localhost - + a profile defined in a file on the node should be + used. RuntimeDefault - the container runtime default + profile should be used. Unconfined - no profile should + be applied." + type: string + required: + - type + type: object windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext @@ -3190,6 +3593,19 @@ spec: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. This field is + alpha-level and will only be honored by components + that enable the WindowsHostProcessContainers feature + flag. Setting this field without the feature flag + will result in errors when validating the Pod. All + of a Pod's containers must have the same effective + HostProcess value (it is not allowed to have a mix + of HostProcess containers and non-HostProcess containers). In + addition, if HostProcess is true then HostNetwork + must also be set to true. + type: boolean runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified @@ -3208,8 +3624,7 @@ spec: can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. This is a beta feature enabled by - the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be @@ -3316,6 +3731,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -3444,6 +3876,14 @@ spec: logLevel: description: Log level for Prometheus to be configured with. type: string + minReadySeconds: + description: Minimum number of seconds for which a newly created pod + should be ready without any of its container crashing for it to + be considered available. Defaults to 0 (pod will be considered available + as soon as it is ready) This is an alpha field and requires enabling + StatefulSetMinReadySeconds feature gate. + format: int32 + type: integer nodeSelector: additionalProperties: type: string @@ -4306,6 +4746,59 @@ spec: enableFeature option for exemplars to be scraped in the first place. Only valid in Prometheus versions 2.27.0 and newer. type: boolean + sigv4: + description: Sigv4 allows to configures AWS's Signature Verification + 4 + properties: + accessKey: + description: AccessKey is the AWS API key. If blank, the + environment variable `AWS_ACCESS_KEY_ID` is used. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + profile: + description: Profile is the named AWS profile used to authenticate. + type: string + region: + description: Region is the AWS region. If blank, the region + from the default credentials chain used. + type: string + roleArn: + description: RoleArn is the named AWS profile used to authenticate. + type: string + secretKey: + description: SecretKey is the AWS API secret. If blank, + the environment variable `AWS_SECRET_ACCESS_KEY` is used. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: object tlsConfig: description: TLS Config to use for remote write. properties: @@ -4507,7 +5000,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources - allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -4519,7 +5012,7 @@ spec: description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object retention: @@ -4690,7 +5183,7 @@ spec: support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". - If not specified defaults to "Always".' + If not specified, "Always" is used.' type: string runAsGroup: description: The GID to run the entrypoint of the container process. @@ -4740,6 +5233,27 @@ spec: the container. type: string type: object + seccompProfile: + description: The seccomp options to use by the containers in this + pod. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. The profile must be + preconfigured on the node to work. Must be a descending + path, relative to the kubelet's configured seccomp profile + location. Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile + will be applied. Valid options are: \n Localhost - a profile + defined in a file on the node should be used. RuntimeDefault + - the container runtime default profile should be used. + Unconfined - no profile should be applied." + type: string + required: + - type + type: object supplementalGroups: description: A list of groups applied to the first process run in each container, in addition to the container's primary GID. If @@ -4782,6 +5296,17 @@ spec: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. This field is alpha-level + and will only be honored by components that enable the WindowsHostProcessContainers + feature flag. Setting this field without the feature flag + will result in errors when validating the Pod. All of a + Pod's containers must have the same effective HostProcess + value (it is not allowed to have a mix of HostProcess containers + and non-HostProcess containers). In addition, if HostProcess + is true then HostNetwork must also be set to true. + type: boolean runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified @@ -4937,6 +5462,217 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + ephemeral: + description: 'EphemeralVolumeSource to be used by the Prometheus + StatefulSets. This is a beta field in k8s 1.21, for lower versions, + starting with k8s 1.19, it requires enabling the GenericEphemeralVolume + feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes' + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC to + provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC will + be deleted together with the pod. The name of the PVC will + be `-` where `` is the + name from the `PodSpec.Volumes` array entry. Pod validation + will reject the pod if the concatenated name is not valid + for a PVC (for example, too long). \n An existing PVC with + that name that is not owned by the pod will *not* be used + for the pod to avoid using an unrelated volume by mistake. + Starting the pod is then blocked until the unrelated PVC + is removed. If such a pre-created PVC is meant to be used + by the pod, the PVC has to updated with an owner reference + to the pod once the pod exists. Normally this should not + be necessary, but it may be useful when manually reconstructing + a broken cluster. \n This field is read-only and no changes + will be made by Kubernetes to the PVC after it has been + created. \n Required, must not be nil." + properties: + metadata: + description: May contain labels and annotations that will + be copied into the PVC when creating it. No other fields + are allowed and will be rejected during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the PVC + that gets created from this template. The same fields + as in a PersistentVolumeClaim are also valid here. + properties: + accessModes: + description: 'AccessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'This field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the + provisioner or an external controller can support + the specified data source, it will create a new + volume based on the contents of the specified data + source. If the AnyVolumeDataSource feature gate + is enabled, this field will always have the same + contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is + required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to populate + the volume with data, if a non-empty volume is desired. + This may be any local object from a non-empty API + group (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume binding + will only succeed if the type of the specified object + matches some installed volume populator or dynamic + provisioner. This field will replace the functionality + of the DataSource field and as such if both fields + are non-empty, they must have the same value. For + backwards compatibility, both fields (DataSource + and DataSourceRef) will be set to the same value + automatically if one of them is empty and the other + is non-empty. There are two important differences + between DataSource and DataSourceRef: * While DataSource + only allows two specific types of objects, DataSourceRef allows + any non-core object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed values + (dropping them), DataSourceRef preserves all values, + and generates an error if a disallowed value is specified. + (Alpha) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is + required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum resources + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: A label query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement is + a selector that contains values, a key, and + an operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. This array + is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required by + the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem is + implied when not included in claim spec. + type: string + volumeName: + description: VolumeName is the binding reference to + the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object volumeClaimTemplate: description: A PVC spec to be used by the Prometheus StatefulSets. properties: @@ -4994,20 +5730,52 @@ spec: type: array dataSource: description: 'This field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot - - Beta) * An existing PVC (PersistentVolumeClaim) * - An existing custom resource/object that implements data - population (Alpha) In order to use VolumeSnapshot object - types, the appropriate feature gate must be enabled - (VolumeSnapshotDataSource or AnyVolumeDataSource) If - the provisioner or an external controller can support - the specified data source, it will create a new volume - based on the contents of the specified data source. - If the specified data source is not supported, the volume - will not be created and the failure will be reported - as an event. In the future, we plan to support more - data source types and the behavior of the provisioner - may change.' + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified + data source, it will create a new volume based on the + contents of the specified data source. If the AnyVolumeDataSource + feature gate is enabled, this field will always have + the same contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to populate + the volume with data, if a non-empty volume is desired. + This may be any local object from a non-empty API group + (non core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only + succeed if the type of the specified object matches + some installed volume populator or dynamic provisioner. + This field will replace the functionality of the DataSource + field and as such if both fields are non-empty, they + must have the same value. For backwards compatibility, + both fields (DataSource and DataSourceRef) will be set + to the same value automatically if one of them is empty + and the other is non-empty. There are two important + differences between DataSource and DataSourceRef: * + While DataSource only allows two specific types of objects, + DataSourceRef allows any non-core object, as well + as PersistentVolumeClaim objects. * While DataSource + ignores disallowed values (dropping them), DataSourceRef preserves + all values, and generates an error if a disallowed value + is specified. (Alpha) Using this field requires the + AnyVolumeDataSource feature gate to be enabled.' properties: apiGroup: description: APIGroup is the group for the resource @@ -5037,7 +5805,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount - of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -5050,7 +5818,7 @@ spec: of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object selector: @@ -5389,7 +6157,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -5401,7 +6169,7 @@ spec: description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object sha: @@ -5445,6 +6213,49 @@ spec: version: description: Version describes the version of Thanos to use. type: string + volumeMounts: + description: VolumeMounts allows configuration of additional VolumeMounts + on the output StatefulSet definition. VolumeMounts specified + will be appended to other VolumeMounts in the thanos-sidecar + container. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: Path within the container at which the volume + should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are + propagated from the host to container and the other way + around. When not set, MountPropagationNone is used. This + field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves similarly + to SubPath but environment variable references $(VAR_NAME) + are expanded using the container's environment. Defaults + to "" (volume's root). SubPathExpr and SubPath are mutually + exclusive. + type: string + required: + - mountPath + - name + type: object + type: array type: object tolerations: description: If specified, the pod's tolerations. @@ -5540,16 +6351,19 @@ spec: type: object maxSkew: description: 'MaxSkew describes the degree to which pods may - be unevenly distributed. It''s the maximum permitted difference - between the number of matching pods in any two topology domains - of a given topology type. For example, in a 3-zone cluster, - MaxSkew is set to 1, and pods with the same labelSelector - spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 - to become 1/1/1; scheduling it onto zone1(zone2) would make - the ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - It''s a required field. Default value is 1 and 0 is not allowed.' + be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global minimum. + For example, in a 3-zone cluster, MaxSkew is set to 1, and + pods with the same labelSelector spread as 1/1/0: | zone1 + | zone2 | zone3 | | P | P | | - if MaxSkew is + 1, incoming pod can only be scheduled to zone3 to become 1/1/1; + scheduling it onto zone1(zone2) would make the ActualSkew(2-0) + on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming + pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that satisfy + it. It''s a required field. Default value is 1 and 0 is not + allowed.' format: int32 type: integer topologyKey: @@ -5562,17 +6376,20 @@ spec: whenUnsatisfiable: description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule - (default) tells the scheduler not to schedule it - ScheduleAnyway - tells the scheduler to still schedule it It''s considered - as "Unsatisfiable" if and only if placing incoming pod on - any topology violates "MaxSkew". For example, in a 3-zone - cluster, MaxSkew is set to 1, and pods with the same labelSelector - spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod - can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) - as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In - other words, the cluster can still be imbalanced, but scheduler - won''t make it *more* imbalanced. It''s a required field.' + (default) tells the scheduler not to schedule it. - ScheduleAnyway + tells the scheduler to schedule the pod in any location, but + giving higher precedence to topologies that would help reduce + the skew. A constraint is considered "Unsatisfiable" for + an incoming pod if and only if every possible node assigment + for that pod would violate "MaxSkew" on some topology. For + example, in a 3-zone cluster, MaxSkew is set to 1, and pods + with the same labelSelector spread as 3/1/1: | zone1 | zone2 + | zone3 | | P P P | P | P | If WhenUnsatisfiable is + set to DoNotSchedule, incoming pod can only be scheduled to + zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on + zone2(zone3) satisfies MaxSkew(1). In other words, the cluster + can still be imbalanced, but scheduler won''t make it *more* + imbalanced. It''s a required field.' type: string required: - maxSkew @@ -5794,12 +6611,15 @@ spec: this volume properties: defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a value between 0 and 0777. Defaults - to 0644. Directories within the path are not affected - by this setting. This might be in conflict with other - options that affect the file mode, like fsGroup, and the - result can be other mode bits set.' + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal value between + 0000 and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. Directories within + the path are not affected by this setting. This might + be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits + set.' format: int32 type: integer items: @@ -5819,8 +6639,11 @@ spec: description: The key to project. type: string mode: - description: 'Optional: mode bits to use on this file, - must be a value between 0 and 0777. If not specified, + description: 'Optional: mode bits used to set permissions + on this file. Must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires + decimal values for mode bits. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other @@ -5848,8 +6671,9 @@ spec: type: boolean type: object csi: - description: CSI (Container Storage Interface) represents storage - that is handled by an external CSI driver (Alpha feature). + description: CSI (Container Storage Interface) represents ephemeral + storage that is handled by certain external CSI drivers (Beta + feature). properties: driver: description: Driver is the name of the CSI driver that handles @@ -5896,11 +6720,15 @@ spec: properties: defaultMode: description: 'Optional: mode bits to use on created files - by default. Must be a value between 0 and 0777. Defaults - to 0644. Directories within the path are not affected - by this setting. This might be in conflict with other - options that affect the file mode, like fsGroup, and the - result can be other mode bits set.' + by default. Must be a Optional: mode bits used to set + permissions on created files by default. Must be an octal + value between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. Defaults to + 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' format: int32 type: integer items: @@ -5926,8 +6754,11 @@ spec: - fieldPath type: object mode: - description: 'Optional: mode bits to use on this file, - must be a value between 0 and 0777. If not specified, + description: 'Optional: mode bits used to set permissions + on this file, must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires + decimal values for mode bits. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other @@ -5994,6 +6825,239 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + ephemeral: + description: "Ephemeral represents a volume that is handled + by a cluster storage driver. The volume's lifecycle is tied + to the pod that defines it - it will be created before the + pod starts, and deleted when the pod is removed. \n Use this + if: a) the volume is only needed while the pod runs, b) features + of normal volumes like restoring from snapshot or capacity + \ tracking are needed, c) the storage driver is specified + through a storage class, and d) the storage driver supports + dynamic volume provisioning through a PersistentVolumeClaim + (see EphemeralVolumeSource for more information on the + connection between this volume type and PersistentVolumeClaim). + \n Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. \n Use CSI for light-weight local ephemeral + volumes if the CSI driver is meant to be used that way - see + the documentation of the driver for more information. \n A + pod can use both types of ephemeral volumes and persistent + volumes at the same time. \n This is a beta feature and only + available when the GenericEphemeralVolume feature gate is + enabled." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC to + provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC + will be deleted together with the pod. The name of the + PVC will be `-` where `` is the name from the `PodSpec.Volumes` array entry. + Pod validation will reject the pod if the concatenated + name is not valid for a PVC (for example, too long). \n + An existing PVC with that name that is not owned by the + pod will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC + is meant to be used by the pod, the PVC has to updated + with an owner reference to the pod once the pod exists. + Normally this should not be necessary, but it may be useful + when manually reconstructing a broken cluster. \n This + field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. \n Required, must + not be nil." + properties: + metadata: + description: May contain labels and annotations that + will be copied into the PVC when creating it. No other + fields are allowed and will be rejected during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the PVC + that gets created from this template. The same fields + as in a PersistentVolumeClaim are also valid here. + properties: + accessModes: + description: 'AccessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'This field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the + provisioner or an external controller can support + the specified data source, it will create a new + volume based on the contents of the specified + data source. If the AnyVolumeDataSource feature + gate is enabled, this field will always have the + same contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API + group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to + populate the volume with data, if a non-empty + volume is desired. This may be any local object + from a non-empty API group (non core object) or + a PersistentVolumeClaim object. When this field + is specified, volume binding will only succeed + if the type of the specified object matches some + installed volume populator or dynamic provisioner. + This field will replace the functionality of the + DataSource field and as such if both fields are + non-empty, they must have the same value. For + backwards compatibility, both fields (DataSource + and DataSourceRef) will be set to the same value + automatically if one of them is empty and the + other is non-empty. There are two important differences + between DataSource and DataSourceRef: * While + DataSource only allows two specific types of objects, + DataSourceRef allows any non-core object, as + well as PersistentVolumeClaim objects. * While + DataSource ignores disallowed values (dropping + them), DataSourceRef preserves all values, and + generates an error if a disallowed value is specified. + (Alpha) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API + group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum resources + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. If Requests + is omitted for a container, it defaults to + Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: A label query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required + by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem + is implied when not included in claim spec. + type: string + volumeName: + description: VolumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object fc: description: FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the @@ -6328,12 +7392,14 @@ spec: and downward API properties: defaultMode: - description: Mode bits to use on created files by default. - Must be a value between 0 and 0777. Directories within - the path are not affected by this setting. This might - be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits - set. + description: Mode bits used to set permissions on created + files by default. Must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML accepts + both octal and decimal values, JSON requires decimal values + for mode bits. Directories within the path are not affected + by this setting. This might be in conflict with other + options that affect the file mode, like fsGroup, and the + result can be other mode bits set. format: int32 type: integer sources: @@ -6366,13 +7432,17 @@ spec: description: The key to project. type: string mode: - description: 'Optional: mode bits to use - on this file, must be a value between - 0 and 0777. If not specified, the volume - defaultMode will be used. This might be - in conflict with other options that affect - the file mode, like fsGroup, and the result - can be other mode bits set.' + description: 'Optional: mode bits used to + set permissions on this file. Must be + an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML + accepts both octal and decimal values, + JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set.' format: int32 type: integer path: @@ -6427,13 +7497,17 @@ spec: - fieldPath type: object mode: - description: 'Optional: mode bits to use - on this file, must be a value between - 0 and 0777. If not specified, the volume - defaultMode will be used. This might be - in conflict with other options that affect - the file mode, like fsGroup, and the result - can be other mode bits set.' + description: 'Optional: mode bits used to + set permissions on this file, must be + an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML + accepts both octal and decimal values, + JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set.' format: int32 type: integer path: @@ -6499,13 +7573,17 @@ spec: description: The key to project. type: string mode: - description: 'Optional: mode bits to use - on this file, must be a value between - 0 and 0777. If not specified, the volume - defaultMode will be used. This might be - in conflict with other options that affect - the file mode, like fsGroup, and the result - can be other mode bits set.' + description: 'Optional: mode bits used to + set permissions on this file. Must be + an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML + accepts both octal and decimal values, + JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set.' format: int32 type: integer path: @@ -6564,8 +7642,6 @@ spec: type: object type: object type: array - required: - - sources type: object quobyte: description: Quobyte represents a Quobyte mount on the host @@ -6716,12 +7792,15 @@ spec: this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' properties: defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a value between 0 and 0777. Defaults - to 0644. Directories within the path are not affected - by this setting. This might be in conflict with other - options that affect the file mode, like fsGroup, and the - result can be other mode bits set.' + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal value between + 0000 and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. Directories within + the path are not affected by this setting. This might + be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits + set.' format: int32 type: integer items: @@ -6741,8 +7820,11 @@ spec: description: The key to project. type: string mode: - description: 'Optional: mode bits to use on this file, - must be a value between 0 and 0777. If not specified, + description: 'Optional: mode bits used to set permissions + on this file. Must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires + decimal values for mode bits. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml index b51f3ed5..a110d48e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml @@ -1,11 +1,11 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.6.2 creationTimestamp: null name: prometheusrules.monitoring.coreos.com spec: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml index ae3acce5..02bf69a4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml @@ -1,11 +1,11 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.6.2 creationTimestamp: null name: servicemonitors.monitoring.coreos.com spec: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml index ced22c3b..ff5c001c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml @@ -1,11 +1,11 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.50.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.6.2 creationTimestamp: null name: thanosrulers.monitoring.coreos.com spec: @@ -322,10 +322,71 @@ spec: The requirements are ANDed. type: object type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. This field is beta-level + and is only honored when PodAffinityNamespaceSelector + feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace" items: type: string type: array @@ -417,10 +478,66 @@ spec: requirements are ANDed. type: object type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied to the + union of the namespaces selected by this field and + the ones listed in the namespaces field. null selector + and null or empty namespaces list means "this pod's + namespace". An empty selector ({}) matches all namespaces. + This field is beta-level and is only honored when + PodAffinityNamespaceSelector feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object namespaces: - description: namespaces specifies which namespaces the - labelSelector applies to (matches against); null or - empty list means "this pod's namespace" + description: namespaces specifies a static list of namespace + names that the term applies to. The term is applied + to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. null or + empty namespaces list and null namespaceSelector means + "this pod's namespace" items: type: string type: array @@ -514,10 +631,71 @@ spec: The requirements are ANDed. type: object type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. This field is beta-level + and is only honored when PodAffinityNamespaceSelector + feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object namespaces: - description: namespaces specifies which namespaces - the labelSelector applies to (matches against); - null or empty list means "this pod's namespace" + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace" items: type: string type: array @@ -609,10 +787,66 @@ spec: requirements are ANDed. type: object type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied to the + union of the namespaces selected by this field and + the ones listed in the namespaces field. null selector + and null or empty namespaces list means "this pod's + namespace". An empty selector ({}) matches all namespaces. + This field is beta-level and is only honored when + PodAffinityNamespaceSelector feature is enabled. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object namespaces: - description: namespaces specifies which namespaces the - labelSelector applies to (matches against); null or - empty list means "this pod's namespace" + description: namespaces specifies a static list of namespace + names that the term applies to. The term is applied + to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. null or + empty namespaces list and null namespaceSelector means + "this pod's namespace" items: type: string type: array @@ -633,8 +867,8 @@ spec: type: object alertDropLabels: description: AlertDropLabels configure the label names which should - be dropped in ThanosRuler alerts. If `labels` field is not provided, - `thanos_ruler_replica` will be dropped in alerts by default. + be dropped in ThanosRuler alerts. The replica label `thanos_ruler_replica` + will always be dropped in alerts. items: type: string type: array @@ -643,6 +877,31 @@ spec: 'Source' field of all alerts. Maps to the '--alert.query-url' CLI arg. type: string + alertRelabelConfigFile: + description: AlertRelabelConfigFile specifies the path of the alert + relabeling configuration file. When used alongside with AlertRelabelConfigs, + alertRelabelConfigFile takes precedence. + type: string + alertRelabelConfigs: + description: 'AlertRelabelConfigs configures alert relabeling in ThanosRuler. + Alert relabel configurations must have the form as specified in + the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs + Alternative to AlertRelabelConfigFile, and lower order priority.' + properties: + key: + description: The key of the secret to select from. Must be a + valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object alertmanagersConfig: description: Define configuration for connecting to alertmanager. Only available with thanos v0.10.0 and higher. Maps to the `alertmanagers.config` @@ -690,10 +949,11 @@ spec: CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will - be unchanged. The $(VAR_NAME) syntax can be escaped with a - double $$, ie: $$(VAR_NAME). Escaped references will never - be expanded, regardless of whether the variable exists or - not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + be unchanged. Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the variable + exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array @@ -702,10 +962,12 @@ spec: The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference - in the input string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). Escaped - references will never be expanded, regardless of whether the - variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: + i.e. "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. More info: + https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array @@ -722,13 +984,15 @@ spec: type: string value: description: 'Variable references $(VAR_NAME) are expanded - using the previous defined environment variables in + using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the - input string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). Escaped - references will never be expanded, regardless of whether - the variable exists or not. Defaults to "".' + input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. "$$(VAR_NAME)" will produce the string + literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists + or not. Defaults to "".' type: string valueFrom: description: Source for the environment variable's value. @@ -755,9 +1019,10 @@ spec: type: object fieldRef: description: 'Selects a field of the pod: supports - metadata.name, metadata.namespace, metadata.labels, - metadata.annotations, spec.nodeName, spec.serviceAccountName, - status.hostIP, status.podIP, status.podIPs.' + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' properties: apiVersion: description: Version of the schema the FieldPath @@ -1177,6 +1442,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -1345,6 +1627,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -1354,7 +1653,7 @@ spec: type: object resources: description: 'Compute Resources required by this container. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' properties: limits: additionalProperties: @@ -1364,7 +1663,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -1377,13 +1676,14 @@ spec: resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object securityContext: - description: 'Security options the pod should run with. More - info: https://kubernetes.io/docs/concepts/policy/security-context/ - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + description: 'SecurityContext defines the security options the + container should be run with. If set, the fields of SecurityContext + override the equivalent fields of PodSecurityContext. More + info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: description: 'AllowPrivilegeEscalation controls whether @@ -1480,6 +1780,30 @@ spec: to the container. type: string type: object + seccompProfile: + description: The seccomp options to use by this container. + If seccomp options are provided at both the pod & container + level, the container options override the pod options. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. The profile + must be preconfigured on the node to work. Must be + a descending path, relative to the kubelet's configured + seccomp profile location. Must only be set if type + is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile + will be applied. Valid options are: \n Localhost - + a profile defined in a file on the node should be + used. RuntimeDefault - the container runtime default + profile should be used. Unconfined - no profile should + be applied." + type: string + required: + - type + type: object windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext @@ -1496,6 +1820,19 @@ spec: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. This field is + alpha-level and will only be honored by components + that enable the WindowsHostProcessContainers feature + flag. Setting this field without the feature flag + will result in errors when validating the Pod. All + of a Pod's containers must have the same effective + HostProcess value (it is not allowed to have a mix + of HostProcess containers and non-HostProcess containers). In + addition, if HostProcess is true then HostNetwork + must also be set to true. + type: boolean runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified @@ -1514,8 +1851,7 @@ spec: can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. This is a beta feature enabled by - the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be @@ -1622,6 +1958,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -1909,10 +2262,11 @@ spec: CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will - be unchanged. The $(VAR_NAME) syntax can be escaped with a - double $$, ie: $$(VAR_NAME). Escaped references will never - be expanded, regardless of whether the variable exists or - not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + be unchanged. Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the variable + exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array @@ -1921,10 +2275,12 @@ spec: The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference - in the input string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). Escaped - references will never be expanded, regardless of whether the - variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: + i.e. "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. More info: + https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array @@ -1941,13 +2297,15 @@ spec: type: string value: description: 'Variable references $(VAR_NAME) are expanded - using the previous defined environment variables in + using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the - input string will be unchanged. The $(VAR_NAME) syntax - can be escaped with a double $$, ie: $$(VAR_NAME). Escaped - references will never be expanded, regardless of whether - the variable exists or not. Defaults to "".' + input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) + syntax: i.e. "$$(VAR_NAME)" will produce the string + literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists + or not. Defaults to "".' type: string valueFrom: description: Source for the environment variable's value. @@ -1974,9 +2332,10 @@ spec: type: object fieldRef: description: 'Selects a field of the pod: supports - metadata.name, metadata.namespace, metadata.labels, - metadata.annotations, spec.nodeName, spec.serviceAccountName, - status.hostIP, status.podIP, status.podIPs.' + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' properties: apiVersion: description: Version of the schema the FieldPath @@ -2396,6 +2755,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -2564,6 +2940,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -2573,7 +2966,7 @@ spec: type: object resources: description: 'Compute Resources required by this container. - Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' properties: limits: additionalProperties: @@ -2583,7 +2976,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute - resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -2596,13 +2989,14 @@ spec: resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object securityContext: - description: 'Security options the pod should run with. More - info: https://kubernetes.io/docs/concepts/policy/security-context/ - More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + description: 'SecurityContext defines the security options the + container should be run with. If set, the fields of SecurityContext + override the equivalent fields of PodSecurityContext. More + info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: description: 'AllowPrivilegeEscalation controls whether @@ -2699,6 +3093,30 @@ spec: to the container. type: string type: object + seccompProfile: + description: The seccomp options to use by this container. + If seccomp options are provided at both the pod & container + level, the container options override the pod options. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. The profile + must be preconfigured on the node to work. Must be + a descending path, relative to the kubelet's configured + seccomp profile location. Must only be set if type + is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile + will be applied. Valid options are: \n Localhost - + a profile defined in a file on the node should be + used. RuntimeDefault - the container runtime default + profile should be used. Unconfined - no profile should + be applied." + type: string + required: + - type + type: object windowsOptions: description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext @@ -2715,6 +3133,19 @@ spec: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. This field is + alpha-level and will only be honored by components + that enable the WindowsHostProcessContainers feature + flag. Setting this field without the feature flag + will result in errors when validating the Pod. All + of a Pod's containers must have the same effective + HostProcess value (it is not allowed to have a mix + of HostProcess containers and non-HostProcess containers). In + addition, if HostProcess is true then HostNetwork + must also be set to true. + type: boolean runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified @@ -2733,8 +3164,7 @@ spec: can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. - This cannot be updated. This is a beta feature enabled by - the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: description: One and only one of the following should be @@ -2841,6 +3271,23 @@ spec: required: - port type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs + to terminate gracefully upon probe failure. The grace + period is the duration in seconds after the processes + running in the pod are sent a termination signal and the + time when the processes are forcibly halted with a kill + signal. Set this value longer than the expected cleanup + time for your process. If this value is nil, the pod's + terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates + stop immediately via the kill signal (no opportunity to + shut down). This is a beta field and requires enabling + ProbeTerminationGracePeriod feature gate. Minimum value + is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer timeoutSeconds: description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: @@ -2963,8 +3410,9 @@ spec: additionalProperties: type: string description: Labels configure the external label pairs to ThanosRuler. - If not provided, default replica label `thanos_ruler_replica` will - be added as a label and be dropped in alerts. + A default replica label `thanos_ruler_replica` will be always added as + a label with the value of the pod's name and it will be dropped + in the alerts. type: object listenLocal: description: ListenLocal makes the Thanos ruler listen on loopback, @@ -2976,6 +3424,14 @@ spec: logLevel: description: Log level for ThanosRuler to be configured with. type: string + minReadySeconds: + description: Minimum number of seconds for which a newly created pod + should be ready without any of its container crashing for it to + be considered available. Defaults to 0 (pod will be considered available + as soon as it is ready) This is an alpha field and requires enabling + StatefulSetMinReadySeconds feature gate. + format: int32 + type: integer nodeSelector: additionalProperties: type: string @@ -3106,7 +3562,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount of compute resources - allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -3118,7 +3574,7 @@ spec: description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object retention: @@ -3242,7 +3698,7 @@ spec: support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". - If not specified defaults to "Always".' + If not specified, "Always" is used.' type: string runAsGroup: description: The GID to run the entrypoint of the container process. @@ -3292,6 +3748,27 @@ spec: the container. type: string type: object + seccompProfile: + description: The seccomp options to use by the containers in this + pod. + properties: + localhostProfile: + description: localhostProfile indicates a profile defined + in a file on the node should be used. The profile must be + preconfigured on the node to work. Must be a descending + path, relative to the kubelet's configured seccomp profile + location. Must only be set if type is "Localhost". + type: string + type: + description: "type indicates which kind of seccomp profile + will be applied. Valid options are: \n Localhost - a profile + defined in a file on the node should be used. RuntimeDefault + - the container runtime default profile should be used. + Unconfined - no profile should be applied." + type: string + required: + - type + type: object supplementalGroups: description: A list of groups applied to the first process run in each container, in addition to the container's primary GID. If @@ -3334,6 +3811,17 @@ spec: description: GMSACredentialSpecName is the name of the GMSA credential spec to use. type: string + hostProcess: + description: HostProcess determines if a container should + be run as a 'Host Process' container. This field is alpha-level + and will only be honored by components that enable the WindowsHostProcessContainers + feature flag. Setting this field without the feature flag + will result in errors when validating the Pod. All of a + Pod's containers must have the same effective HostProcess + value (it is not allowed to have a mix of HostProcess containers + and non-HostProcess containers). In addition, if HostProcess + is true then HostNetwork must also be set to true. + type: boolean runAsUserName: description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified @@ -3380,6 +3868,217 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + ephemeral: + description: 'EphemeralVolumeSource to be used by the Prometheus + StatefulSets. This is a beta field in k8s 1.21, for lower versions, + starting with k8s 1.19, it requires enabling the GenericEphemeralVolume + feature gate. More info: https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes' + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC to + provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC will + be deleted together with the pod. The name of the PVC will + be `-` where `` is the + name from the `PodSpec.Volumes` array entry. Pod validation + will reject the pod if the concatenated name is not valid + for a PVC (for example, too long). \n An existing PVC with + that name that is not owned by the pod will *not* be used + for the pod to avoid using an unrelated volume by mistake. + Starting the pod is then blocked until the unrelated PVC + is removed. If such a pre-created PVC is meant to be used + by the pod, the PVC has to updated with an owner reference + to the pod once the pod exists. Normally this should not + be necessary, but it may be useful when manually reconstructing + a broken cluster. \n This field is read-only and no changes + will be made by Kubernetes to the PVC after it has been + created. \n Required, must not be nil." + properties: + metadata: + description: May contain labels and annotations that will + be copied into the PVC when creating it. No other fields + are allowed and will be rejected during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the PVC + that gets created from this template. The same fields + as in a PersistentVolumeClaim are also valid here. + properties: + accessModes: + description: 'AccessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'This field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the + provisioner or an external controller can support + the specified data source, it will create a new + volume based on the contents of the specified data + source. If the AnyVolumeDataSource feature gate + is enabled, this field will always have the same + contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is + required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to populate + the volume with data, if a non-empty volume is desired. + This may be any local object from a non-empty API + group (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume binding + will only succeed if the type of the specified object + matches some installed volume populator or dynamic + provisioner. This field will replace the functionality + of the DataSource field and as such if both fields + are non-empty, they must have the same value. For + backwards compatibility, both fields (DataSource + and DataSourceRef) will be set to the same value + automatically if one of them is empty and the other + is non-empty. There are two important differences + between DataSource and DataSourceRef: * While DataSource + only allows two specific types of objects, DataSourceRef allows + any non-core object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed values + (dropping them), DataSourceRef preserves all values, + and generates an error if a disallowed value is specified. + (Alpha) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is + required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum resources + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: A label query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement is + a selector that contains values, a key, and + an operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. This array + is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required by + the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem is + implied when not included in claim spec. + type: string + volumeName: + description: VolumeName is the binding reference to + the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object volumeClaimTemplate: description: A PVC spec to be used by the Prometheus StatefulSets. properties: @@ -3437,20 +4136,52 @@ spec: type: array dataSource: description: 'This field can be used to specify either: - * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot - - Beta) * An existing PVC (PersistentVolumeClaim) * - An existing custom resource/object that implements data - population (Alpha) In order to use VolumeSnapshot object - types, the appropriate feature gate must be enabled - (VolumeSnapshotDataSource or AnyVolumeDataSource) If - the provisioner or an external controller can support - the specified data source, it will create a new volume - based on the contents of the specified data source. - If the specified data source is not supported, the volume - will not be created and the failure will be reported - as an event. In the future, we plan to support more - data source types and the behavior of the provisioner - may change.' + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified + data source, it will create a new volume based on the + contents of the specified data source. If the AnyVolumeDataSource + feature gate is enabled, this field will always have + the same contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to populate + the volume with data, if a non-empty volume is desired. + This may be any local object from a non-empty API group + (non core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only + succeed if the type of the specified object matches + some installed volume populator or dynamic provisioner. + This field will replace the functionality of the DataSource + field and as such if both fields are non-empty, they + must have the same value. For backwards compatibility, + both fields (DataSource and DataSourceRef) will be set + to the same value automatically if one of them is empty + and the other is non-empty. There are two important + differences between DataSource and DataSourceRef: * + While DataSource only allows two specific types of objects, + DataSourceRef allows any non-core object, as well + as PersistentVolumeClaim objects. * While DataSource + ignores disallowed values (dropping them), DataSourceRef preserves + all values, and generates an error if a disallowed value + is specified. (Alpha) Using this field requires the + AnyVolumeDataSource feature gate to be enabled.' properties: apiGroup: description: APIGroup is the group for the resource @@ -3480,7 +4211,7 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true description: 'Limits describes the maximum amount - of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -3493,7 +4224,7 @@ spec: of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object selector: @@ -3715,16 +4446,19 @@ spec: type: object maxSkew: description: 'MaxSkew describes the degree to which pods may - be unevenly distributed. It''s the maximum permitted difference - between the number of matching pods in any two topology domains - of a given topology type. For example, in a 3-zone cluster, - MaxSkew is set to 1, and pods with the same labelSelector - spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 - to become 1/1/1; scheduling it onto zone1(zone2) would make - the ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - It''s a required field. Default value is 1 and 0 is not allowed.' + be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, + it is the maximum permitted difference between the number + of matching pods in the target topology and the global minimum. + For example, in a 3-zone cluster, MaxSkew is set to 1, and + pods with the same labelSelector spread as 1/1/0: | zone1 + | zone2 | zone3 | | P | P | | - if MaxSkew is + 1, incoming pod can only be scheduled to zone3 to become 1/1/1; + scheduling it onto zone1(zone2) would make the ActualSkew(2-0) + on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming + pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, + it is used to give higher precedence to topologies that satisfy + it. It''s a required field. Default value is 1 and 0 is not + allowed.' format: int32 type: integer topologyKey: @@ -3737,17 +4471,20 @@ spec: whenUnsatisfiable: description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule - (default) tells the scheduler not to schedule it - ScheduleAnyway - tells the scheduler to still schedule it It''s considered - as "Unsatisfiable" if and only if placing incoming pod on - any topology violates "MaxSkew". For example, in a 3-zone - cluster, MaxSkew is set to 1, and pods with the same labelSelector - spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod - can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) - as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In - other words, the cluster can still be imbalanced, but scheduler - won''t make it *more* imbalanced. It''s a required field.' + (default) tells the scheduler not to schedule it. - ScheduleAnyway + tells the scheduler to schedule the pod in any location, but + giving higher precedence to topologies that would help reduce + the skew. A constraint is considered "Unsatisfiable" for + an incoming pod if and only if every possible node assignment + for that pod would violate "MaxSkew" on some topology. For + example, in a 3-zone cluster, MaxSkew is set to 1, and pods + with the same labelSelector spread as 3/1/1: | zone1 | zone2 + | zone3 | | P P P | P | P | If WhenUnsatisfiable is + set to DoNotSchedule, incoming pod can only be scheduled to + zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on + zone2(zone3) satisfies MaxSkew(1). In other words, the cluster + can still be imbalanced, but scheduler won''t make it *more* + imbalanced. It''s a required field.' type: string required: - maxSkew @@ -3943,12 +4680,15 @@ spec: this volume properties: defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a value between 0 and 0777. Defaults - to 0644. Directories within the path are not affected - by this setting. This might be in conflict with other - options that affect the file mode, like fsGroup, and the - result can be other mode bits set.' + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal value between + 0000 and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. Directories within + the path are not affected by this setting. This might + be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits + set.' format: int32 type: integer items: @@ -3968,8 +4708,11 @@ spec: description: The key to project. type: string mode: - description: 'Optional: mode bits to use on this file, - must be a value between 0 and 0777. If not specified, + description: 'Optional: mode bits used to set permissions + on this file. Must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires + decimal values for mode bits. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other @@ -3997,8 +4740,9 @@ spec: type: boolean type: object csi: - description: CSI (Container Storage Interface) represents storage - that is handled by an external CSI driver (Alpha feature). + description: CSI (Container Storage Interface) represents ephemeral + storage that is handled by certain external CSI drivers (Beta + feature). properties: driver: description: Driver is the name of the CSI driver that handles @@ -4045,11 +4789,15 @@ spec: properties: defaultMode: description: 'Optional: mode bits to use on created files - by default. Must be a value between 0 and 0777. Defaults - to 0644. Directories within the path are not affected - by this setting. This might be in conflict with other - options that affect the file mode, like fsGroup, and the - result can be other mode bits set.' + by default. Must be a Optional: mode bits used to set + permissions on created files by default. Must be an octal + value between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal values, + JSON requires decimal values for mode bits. Defaults to + 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' format: int32 type: integer items: @@ -4075,8 +4823,11 @@ spec: - fieldPath type: object mode: - description: 'Optional: mode bits to use on this file, - must be a value between 0 and 0777. If not specified, + description: 'Optional: mode bits used to set permissions + on this file, must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires + decimal values for mode bits. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other @@ -4143,6 +4894,239 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + ephemeral: + description: "Ephemeral represents a volume that is handled + by a cluster storage driver. The volume's lifecycle is tied + to the pod that defines it - it will be created before the + pod starts, and deleted when the pod is removed. \n Use this + if: a) the volume is only needed while the pod runs, b) features + of normal volumes like restoring from snapshot or capacity + \ tracking are needed, c) the storage driver is specified + through a storage class, and d) the storage driver supports + dynamic volume provisioning through a PersistentVolumeClaim + (see EphemeralVolumeSource for more information on the + connection between this volume type and PersistentVolumeClaim). + \n Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. \n Use CSI for light-weight local ephemeral + volumes if the CSI driver is meant to be used that way - see + the documentation of the driver for more information. \n A + pod can use both types of ephemeral volumes and persistent + volumes at the same time. \n This is a beta feature and only + available when the GenericEphemeralVolume feature gate is + enabled." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC to + provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC + will be deleted together with the pod. The name of the + PVC will be `-` where `` is the name from the `PodSpec.Volumes` array entry. + Pod validation will reject the pod if the concatenated + name is not valid for a PVC (for example, too long). \n + An existing PVC with that name that is not owned by the + pod will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC + is meant to be used by the pod, the PVC has to updated + with an owner reference to the pod once the pod exists. + Normally this should not be necessary, but it may be useful + when manually reconstructing a broken cluster. \n This + field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. \n Required, must + not be nil." + properties: + metadata: + description: May contain labels and annotations that + will be copied into the PVC when creating it. No other + fields are allowed and will be rejected during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the PVC + that gets created from this template. The same fields + as in a PersistentVolumeClaim are also valid here. + properties: + accessModes: + description: 'AccessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'This field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the + provisioner or an external controller can support + the specified data source, it will create a new + volume based on the contents of the specified + data source. If the AnyVolumeDataSource feature + gate is enabled, this field will always have the + same contents as the DataSourceRef field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API + group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'Specifies the object from which to + populate the volume with data, if a non-empty + volume is desired. This may be any local object + from a non-empty API group (non core object) or + a PersistentVolumeClaim object. When this field + is specified, volume binding will only succeed + if the type of the specified object matches some + installed volume populator or dynamic provisioner. + This field will replace the functionality of the + DataSource field and as such if both fields are + non-empty, they must have the same value. For + backwards compatibility, both fields (DataSource + and DataSourceRef) will be set to the same value + automatically if one of them is empty and the + other is non-empty. There are two important differences + between DataSource and DataSourceRef: * While + DataSource only allows two specific types of objects, + DataSourceRef allows any non-core object, as + well as PersistentVolumeClaim objects. * While + DataSource ignores disallowed values (dropping + them), DataSourceRef preserves all values, and + generates an error if a disallowed value is specified. + (Alpha) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API + group. For any other third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum resources + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. If Requests + is omitted for a container, it defaults to + Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: A label query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required + by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem + is implied when not included in claim spec. + type: string + volumeName: + description: VolumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object fc: description: FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the @@ -4477,12 +5461,14 @@ spec: and downward API properties: defaultMode: - description: Mode bits to use on created files by default. - Must be a value between 0 and 0777. Directories within - the path are not affected by this setting. This might - be in conflict with other options that affect the file - mode, like fsGroup, and the result can be other mode bits - set. + description: Mode bits used to set permissions on created + files by default. Must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML accepts + both octal and decimal values, JSON requires decimal values + for mode bits. Directories within the path are not affected + by this setting. This might be in conflict with other + options that affect the file mode, like fsGroup, and the + result can be other mode bits set. format: int32 type: integer sources: @@ -4515,13 +5501,17 @@ spec: description: The key to project. type: string mode: - description: 'Optional: mode bits to use - on this file, must be a value between - 0 and 0777. If not specified, the volume - defaultMode will be used. This might be - in conflict with other options that affect - the file mode, like fsGroup, and the result - can be other mode bits set.' + description: 'Optional: mode bits used to + set permissions on this file. Must be + an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML + accepts both octal and decimal values, + JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set.' format: int32 type: integer path: @@ -4576,13 +5566,17 @@ spec: - fieldPath type: object mode: - description: 'Optional: mode bits to use - on this file, must be a value between - 0 and 0777. If not specified, the volume - defaultMode will be used. This might be - in conflict with other options that affect - the file mode, like fsGroup, and the result - can be other mode bits set.' + description: 'Optional: mode bits used to + set permissions on this file, must be + an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML + accepts both octal and decimal values, + JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set.' format: int32 type: integer path: @@ -4648,13 +5642,17 @@ spec: description: The key to project. type: string mode: - description: 'Optional: mode bits to use - on this file, must be a value between - 0 and 0777. If not specified, the volume - defaultMode will be used. This might be - in conflict with other options that affect - the file mode, like fsGroup, and the result - can be other mode bits set.' + description: 'Optional: mode bits used to + set permissions on this file. Must be + an octal value between 0000 and 0777 or + a decimal value between 0 and 511. YAML + accepts both octal and decimal values, + JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can + be other mode bits set.' format: int32 type: integer path: @@ -4713,8 +5711,6 @@ spec: type: object type: object type: array - required: - - sources type: object quobyte: description: Quobyte represents a Quobyte mount on the host @@ -4865,12 +5861,15 @@ spec: this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' properties: defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a value between 0 and 0777. Defaults - to 0644. Directories within the path are not affected - by this setting. This might be in conflict with other - options that affect the file mode, like fsGroup, and the - result can be other mode bits set.' + description: 'Optional: mode bits used to set permissions + on created files by default. Must be an octal value between + 0000 and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires decimal + values for mode bits. Defaults to 0644. Directories within + the path are not affected by this setting. This might + be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits + set.' format: int32 type: integer items: @@ -4890,8 +5889,11 @@ spec: description: The key to project. type: string mode: - description: 'Optional: mode bits to use on this file, - must be a value between 0 and 0777. If not specified, + description: 'Optional: mode bits used to set permissions + on this file. Must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. YAML + accepts both octal and decimal values, JSON requires + decimal values for mode bits. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl index b762f221..2c4bc646 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl @@ -96,6 +96,39 @@ Allow the release namespace to be overridden for multi-namespace deployments in {{- end -}} {{- end -}} +{{/* +Use the grafana namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-grafana.namespace" -}} + {{- if .Values.grafana.namespaceOverride -}} + {{- .Values.grafana.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the kube-state-metrics namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-kube-state-metrics.namespace" -}} + {{- if index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- index .Values "kube-state-metrics" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Use the prometheus-node-exporter namespace override for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack-prometheus-node-exporter.namespace" -}} + {{- if index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- index .Values "prometheus-node-exporter" "namespaceOverride" -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + {{/* Allow KubeVersion to be overridden. */}} {{- define "kube-prometheus-stack.kubeVersion" -}} {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml index 488e4c15..5ebc70e6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml @@ -81,6 +81,7 @@ spec: {{- end }} {{- if or .Values.alertmanager.alertmanagerSpec.podAntiAffinity .Values.alertmanager.alertmanagerSpec.affinity }} affinity: +{{- end }} {{- if .Values.alertmanager.alertmanagerSpec.affinity }} {{ toYaml .Values.alertmanager.alertmanagerSpec.affinity | indent 4 }} {{- end }} @@ -103,7 +104,6 @@ spec: - {key: app, operator: In, values: [alertmanager]} - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-alertmanager]} {{- end }} -{{- end }} {{- if .Values.alertmanager.alertmanagerSpec.tolerations }} tolerations: {{ toYaml .Values.alertmanager.alertmanagerSpec.tolerations | indent 4 }} @@ -140,10 +140,10 @@ spec: {{ toYaml .Values.alertmanager.alertmanagerSpec.volumeMounts | indent 4 }} {{- end }} portName: {{ .Values.alertmanager.alertmanagerSpec.portName }} -{{- end }} {{- if .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} clusterAdvertiseAddress: {{ .Values.alertmanager.alertmanagerSpec.clusterAdvertiseAddress }} {{- end }} {{- if .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} forceEnableClusterMode: {{ .Values.alertmanager.alertmanagerSpec.forceEnableClusterMode }} {{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/service.yaml index bbcc60f2..b7e892e6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/service.yaml @@ -44,7 +44,7 @@ spec: {{ toYaml .Values.alertmanager.service.additionalPorts | indent 2 }} {{- end }} selector: - app: alertmanager + app.kubernetes.io/name: alertmanager alertmanager: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager type: "{{ .Values.alertmanager.service.type }}" {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceperreplica.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceperreplica.yaml index 0f12ae87..d44a2ebb 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceperreplica.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceperreplica.yaml @@ -38,7 +38,7 @@ items: port: {{ $serviceValues.port }} targetPort: {{ $serviceValues.targetPort }} selector: - app: alertmanager + app.kubernetes.io/name: alertmanager alertmanager: {{ template "kube-prometheus-stack.fullname" $ }}-alertmanager statefulset.kubernetes.io/pod-name: alertmanager-{{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} type: "{{ $serviceValues.type }}" diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/core-dns/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/core-dns/servicemonitor.yaml index 6acbb5ee..a456fc80 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/core-dns/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/core-dns/servicemonitor.yaml @@ -31,6 +31,6 @@ spec: {{- end }} {{- if .Values.coreDns.serviceMonitor.relabelings }} relabelings: -{{ toYaml .Values.coreDns.serviceMonitor.relabelings | indent 4 }} +{{ tpl (toYaml .Values.coreDns.serviceMonitor.relabelings | indent 4) . }} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-api-server/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-api-server/servicemonitor.yaml index 33a57b82..c37a6738 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-api-server/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-api-server/servicemonitor.yaml @@ -24,7 +24,7 @@ spec: {{- end }} {{- if .Values.kubeApiServer.serviceMonitor.relabelings }} relabelings: -{{ toYaml .Values.kubeApiServer.serviceMonitor.relabelings | indent 6 }} +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.relabelings | indent 6) . }} {{- end }} tlsConfig: caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-controller-manager/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-controller-manager/servicemonitor.yaml index c0846eaf..809beb1b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-controller-manager/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-controller-manager/servicemonitor.yaml @@ -42,6 +42,6 @@ spec: {{- end }} {{- if .Values.kubeControllerManager.serviceMonitor.relabelings }} relabelings: -{{ toYaml .Values.kubeControllerManager.serviceMonitor.relabelings | indent 4 }} +{{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.relabelings | indent 4) . }} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-dns/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-dns/servicemonitor.yaml index 1f1b0dec..c2da09a1 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-dns/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-dns/servicemonitor.yaml @@ -44,6 +44,6 @@ spec: {{- end }} {{- if .Values.kubeDns.serviceMonitor.relabelings }} relabelings: -{{ toYaml .Values.kubeDns.serviceMonitor.relabelings | indent 4 }} +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.relabelings | indent 4) . }} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-etcd/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-etcd/servicemonitor.yaml index 689e1fdc..2ddac928 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-etcd/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-etcd/servicemonitor.yaml @@ -48,6 +48,6 @@ spec: {{- end }} {{- if .Values.kubeEtcd.serviceMonitor.relabelings }} relabelings: -{{ toYaml .Values.kubeEtcd.serviceMonitor.relabelings | indent 4 }} +{{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.relabelings | indent 4) . }} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-proxy/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-proxy/servicemonitor.yaml index bc3b7be1..28f2a26a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-proxy/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-proxy/servicemonitor.yaml @@ -32,10 +32,10 @@ spec: {{- end}} {{- if .Values.kubeProxy.serviceMonitor.metricRelabelings }} metricRelabelings: -{{ toYaml .Values.kubeProxy.serviceMonitor.metricRelabelings | indent 4 }} +{{ tpl (toYaml .Values.kubeProxy.serviceMonitor.metricRelabelings | indent 4) . }} {{- end }} {{- if .Values.kubeProxy.serviceMonitor.relabelings }} relabelings: -{{ toYaml .Values.kubeProxy.serviceMonitor.relabelings | indent 4 }} +{{ tpl (toYaml .Values.kubeProxy.serviceMonitor.relabelings | indent 4) . }} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-scheduler/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-scheduler/servicemonitor.yaml index a9a454bc..e6b54409 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-scheduler/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-scheduler/servicemonitor.yaml @@ -42,6 +42,6 @@ spec: {{- end }} {{- if .Values.kubeScheduler.serviceMonitor.relabelings }} relabelings: -{{ toYaml .Values.kubeScheduler.serviceMonitor.relabelings | indent 4 }} +{{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.relabelings | indent 4) . }} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml index bcc62d81..8493d9dc 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml @@ -27,7 +27,7 @@ spec: {{- end }} {{- if .Values.kubeStateMetrics.serviceMonitor.relabelings }} relabelings: -{{ toYaml .Values.kubeStateMetrics.serviceMonitor.relabelings | indent 4 }} +{{ tpl (toYaml .Values.kubeStateMetrics.serviceMonitor.relabelings | indent 4) . }} {{- end }} {{- if .Values.kubeStateMetrics.serviceMonitor.selfMonitor.enabled }} - port: metrics @@ -44,14 +44,14 @@ spec: {{- end }} {{- if .Values.kubeStateMetrics.serviceMonitor.relabelings }} relabelings: -{{ toYaml .Values.kubeStateMetrics.serviceMonitor.relabelings | indent 4 }} +{{ tpl (toYaml .Values.kubeStateMetrics.serviceMonitor.relabelings | indent 4) . }} {{- end }} {{- end }} -{{- if .Values.kubeStateMetrics.serviceMonitor.namespaceOverride }} + {{- if ne (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack-kube-state-metrics.namespace" .) }} namespaceSelector: matchNames: - - {{ .Values.kubeStateMetrics.serviceMonitor.namespaceOverride }} -{{- end }} + - {{ printf "%s" (include "kube-prometheus-stack-kube-state-metrics.namespace" .) | quote }} + {{- end }} selector: matchLabels: {{- if .Values.kubeStateMetrics.serviceMonitor.selectorOverride }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml index 9933ecb1..eb5e5669 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml @@ -18,6 +18,9 @@ spec: {{- if .Values.kubelet.serviceMonitor.proxyUrl }} proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} tlsConfig: caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt insecureSkipVerify: true @@ -29,7 +32,7 @@ spec: {{- end }} {{- if .Values.kubelet.serviceMonitor.relabelings }} relabelings: -{{ toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4 }} +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} {{- end }} {{- if .Values.kubelet.serviceMonitor.cAdvisor }} - port: https-metrics @@ -55,7 +58,7 @@ spec: {{- end }} {{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} relabelings: -{{ toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4 }} +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} {{- end }} {{- end }} {{- if .Values.kubelet.serviceMonitor.probes }} @@ -68,6 +71,9 @@ spec: {{- if .Values.kubelet.serviceMonitor.proxyUrl }} proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} honorLabels: true tlsConfig: caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt @@ -79,7 +85,7 @@ spec: {{- end }} {{- if .Values.kubelet.serviceMonitor.probesRelabelings }} relabelings: -{{ toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4 }} +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4) . }} {{- end }} {{- end }} {{- if .Values.kubelet.serviceMonitor.resource }} @@ -92,6 +98,9 @@ spec: {{- if .Values.kubelet.serviceMonitor.proxyUrl }} proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} honorLabels: true tlsConfig: caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt @@ -103,7 +112,7 @@ spec: {{- end }} {{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} relabelings: -{{ toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4 }} +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} {{- end }} {{- end }} {{- else }} @@ -114,6 +123,9 @@ spec: {{- if .Values.kubelet.serviceMonitor.proxyUrl }} proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} honorLabels: true {{- if .Values.kubelet.serviceMonitor.metricRelabelings }} metricRelabelings: @@ -121,7 +133,7 @@ spec: {{- end }} {{- if .Values.kubelet.serviceMonitor.relabelings }} relabelings: -{{ toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4 }} +{{ tpl (toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4) . }} {{- end }} {{- if .Values.kubelet.serviceMonitor.cAdvisor }} - port: http-metrics @@ -132,6 +144,9 @@ spec: {{- if .Values.kubelet.serviceMonitor.proxyUrl }} proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} honorLabels: true {{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} metricRelabelings: @@ -139,7 +154,7 @@ spec: {{- end }} {{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} relabelings: -{{ toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4 }} +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4) . }} {{- end }} {{- if .Values.kubelet.serviceMonitor.resource }} - port: http-metrics @@ -150,6 +165,9 @@ spec: {{- if .Values.kubelet.serviceMonitor.proxyUrl }} proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} honorLabels: true {{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} metricRelabelings: @@ -157,7 +175,7 @@ spec: {{- end }} {{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} relabelings: -{{ toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4 }} +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4) . }} {{- end }} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml index 5615ba7e..02db6b23 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml @@ -13,13 +13,13 @@ spec: matchLabels: app: prometheus-node-exporter release: {{ $.Release.Name }} - {{- if (index .Values "prometheus-node-exporter" "namespaceOverride") }} + {{- if ne (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack-prometheus-node-exporter.namespace" .) }} namespaceSelector: matchNames: - - {{ index .Values "prometheus-node-exporter" "namespaceOverride" }} + - {{ printf "%s" (include "kube-prometheus-stack-prometheus-node-exporter.namespace" .) | quote }} {{- end }} endpoints: - - port: metrics + - port: {{ index .Values "prometheus-node-exporter" "service" "portName" }} {{- if .Values.nodeExporter.serviceMonitor.interval }} interval: {{ .Values.nodeExporter.serviceMonitor.interval }} {{- end }} @@ -35,6 +35,6 @@ spec: {{- end }} {{- if .Values.nodeExporter.serviceMonitor.relabelings }} relabelings: -{{ toYaml .Values.nodeExporter.serviceMonitor.relabelings | indent 4 }} +{{ tpl (toYaml .Values.nodeExporter.serviceMonitor.relabelings | indent 4) . }} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmap-dashboards.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmap-dashboards.yaml index ce5dd068..526f053d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmap-dashboards.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmap-dashboards.yaml @@ -10,7 +10,7 @@ items: kind: ConfigMap metadata: name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) $dashboardName | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" $ }} + namespace: {{ template "kube-prometheus-stack-grafana.namespace" $ }} labels: {{- if $.Values.grafana.sidecar.dashboards.label }} {{ $.Values.grafana.sidecar.dashboards.label }}: "1" diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml index c7c82efc..580508f3 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml @@ -3,7 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ template "kube-prometheus-stack.fullname" . }}-grafana-datasource - namespace: {{ template "kube-prometheus-stack.namespace" . }} + namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }} {{- if .Values.grafana.sidecar.datasources.annotations }} annotations: {{ toYaml .Values.grafana.sidecar.datasources.annotations | indent 4 }} @@ -15,6 +15,10 @@ metadata: data: datasource.yaml: |- apiVersion: 1 +{{- if .Values.grafana.deleteDatasources }} + deleteDatasources: +{{ tpl (toYaml .Values.grafana.deleteDatasources | indent 6) . }} +{{- end }} datasources: {{- $scrapeInterval := .Values.grafana.sidecar.datasources.defaultDatasourceScrapeInterval | default .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} {{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} @@ -23,7 +27,7 @@ data: {{- if .Values.grafana.sidecar.datasources.url }} url: {{ .Values.grafana.sidecar.datasources.url }} {{- else }} - url: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} + url: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} {{- end }} access: proxy isDefault: true diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/servicemonitor.yaml index 1e839d70..56aacf70 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/servicemonitor.yaml @@ -12,9 +12,11 @@ spec: matchLabels: app.kubernetes.io/name: grafana app.kubernetes.io/instance: {{ $.Release.Name | quote }} + {{- if ne (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack-grafana.namespace" .) }} namespaceSelector: matchNames: - - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + - {{ printf "%s" (include "kube-prometheus-stack-grafana.namespace" .) | quote }} + {{- end }} endpoints: - port: {{ .Values.grafana.service.portName }} {{- if .Values.grafana.serviceMonitor.interval }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml index 3bc451d2..19c43e57 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -6,12 +6,11 @@ metadata: annotations: "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - app: {{ template "kube-prometheus-stack.name" . }}-admission {{- if .Values.global.rbac.pspAnnotations }} - annotations: {{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} {{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-admission {{ include "kube-prometheus-stack.labels" . | indent 4 }} spec: privileged: false diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml index e5cb3177..0ab89413 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml @@ -1,4 +1,5 @@ {{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- $defaultKubeletSvcName := printf "%s-kubelet" (include "kube-prometheus-stack.fullname" .) }} {{- if .Values.prometheusOperator.enabled }} apiVersion: apps/v1 kind: Deployment @@ -40,7 +41,7 @@ spec: imagePullPolicy: "{{ .Values.prometheusOperator.image.pullPolicy }}" args: {{- if .Values.prometheusOperator.kubeletService.enabled }} - - --kubelet-service={{ .Values.prometheusOperator.kubeletService.namespace }}/{{ template "kube-prometheus-stack.fullname" . }}-kubelet + - --kubelet-service={{ .Values.prometheusOperator.kubeletService.namespace }}/{{ default $defaultKubeletSvcName .Values.prometheusOperator.kubeletService.name }} {{- end }} {{- if .Values.prometheusOperator.logFormat }} - --log-format={{ .Values.prometheusOperator.logFormat }} @@ -65,15 +66,15 @@ spec: {{- if .Values.prometheusOperator.alertmanagerDefaultBaseImage }} - --alertmanager-default-base-image={{ .Values.prometheusOperator.alertmanagerDefaultBaseImage }} {{- end }} - {{- if .Values.prometheusOperator.prometheusConfigReloaderImage.sha }} - - --prometheus-config-reloader={{ .Values.prometheusOperator.prometheusConfigReloaderImage.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloaderImage.tag }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloaderImage.sha }} + {{- if .Values.prometheusOperator.prometheusConfigReloader.image.sha }} + - --prometheus-config-reloader={{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloader.image.sha }} {{- else }} - - --prometheus-config-reloader={{ .Values.prometheusOperator.prometheusConfigReloaderImage.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloaderImage.tag }} + - --prometheus-config-reloader={{ .Values.prometheusOperator.prometheusConfigReloader.image.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloader.image.tag }} {{- end }} - - --config-reloader-cpu-request={{ .Values.prometheusOperator.configReloaderCpu }} - - --config-reloader-cpu-limit={{ .Values.prometheusOperator.configReloaderCpu }} - - --config-reloader-memory-request={{ .Values.prometheusOperator.configReloaderMemory }} - - --config-reloader-memory-limit={{ .Values.prometheusOperator.configReloaderMemory }} + - --config-reloader-cpu-request={{ .Values.prometheusOperator.prometheusConfigReloader.resources.requests.cpu }} + - --config-reloader-cpu-limit={{ .Values.prometheusOperator.prometheusConfigReloader.resources.limits.cpu }} + - --config-reloader-memory-request={{ .Values.prometheusOperator.prometheusConfigReloader.resources.requests.memory }} + - --config-reloader-memory-limit={{ .Values.prometheusOperator.prometheusConfigReloader.resources.limits.memory }} {{- if .Values.prometheusOperator.alertmanagerInstanceNamespaces }} - --alertmanager-instance-namespaces={{ .Values.prometheusOperator.alertmanagerInstanceNamespaces | join "," }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/csi-secret.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/csi-secret.yaml new file mode 100644 index 00000000..89399cec --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/csi-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.prometheus.prometheusSpec.thanos.secretProviderClass }} +--- +apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 +kind: SecretProviderClass +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +spec: +{{ toYaml .Values.prometheus.prometheusSpec.thanos.secretProviderClass | indent 2 }} +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml index 5f0dffa8..7b5e12c4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml @@ -301,7 +301,6 @@ spec: disableCompaction: {{ .Values.prometheus.prometheusSpec.disableCompaction }} {{- end }} portName: {{ .Values.prometheus.prometheusSpec.portName }} -{{- end }} {{- if .Values.prometheus.prometheusSpec.volumes }} volumes: {{ toYaml .Values.prometheus.prometheusSpec.volumes | indent 4 }} @@ -356,3 +355,4 @@ spec: {{- if .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} allowOverlappingBlocks: {{ .Values.prometheus.prometheusSpec.allowOverlappingBlocks }} {{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml index cb12c344..8be9f182 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml @@ -366,14 +366,16 @@ alertmanager: bearerTokenFile: - ## Metric relabel configs to apply to samples before ingestion. + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -396,7 +398,7 @@ alertmanager: ## image: repository: quay.io/prometheus/alertmanager - tag: v0.22.2 + tag: v0.23.0 sha: "" ## If true then the user will be responsible to provide a secret with alertmanager configuration @@ -596,7 +598,7 @@ alertmanager: ## PortName to use for Alert Manager. ## - portName: "web" + portName: "http-web" ## ClusterAdvertiseAddress is the explicit address to advertise in cluster. Needs to be provided for non RFC1918 [1] (public) addresses. [1] RFC1918: https://tools.ietf.org/html/rfc1918 ## @@ -718,6 +720,10 @@ grafana: # configMap: certs-configmap # readOnly: true + deleteDatasources: [] + # - name: example-datasource + # orgId: 1 + ## Configure additional grafana datasources (passed through tpl) ## ref: http://docs.grafana.org/administration/provisioning/#datasources additionalDataSources: [] @@ -737,7 +743,7 @@ grafana: ## Passed to grafana subchart and used by servicemonitor below ## service: - portName: service + portName: http-web ## If true, create a serviceMonitor for grafana ## @@ -751,14 +757,17 @@ grafana: # in grafana.ini path: "/metrics" - ## Metric relabel configs to apply to samples before ingestion. + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -789,12 +798,17 @@ kubeApiServer: component: apiserver provider: kubernetes - ## Metric relabel configs to apply to samples before ingestion. + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## relabelings: [] # - sourceLabels: # - __meta_kubernetes_namespace @@ -839,7 +853,9 @@ kubelet: resource: false # From kubernetes 1.18, /metrics/resource/v1alpha1 renamed to /metrics/resource resourcePath: "/metrics/resource/v1alpha1" - ## Metric relabellings to apply to samples before ingestion + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## cAdvisorMetricRelabelings: [] # - sourceLabels: [__name__, image] @@ -853,7 +869,8 @@ kubelet: # replacement: $1 # action: drop - ## Metric relabellings to apply to samples before ingestion + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## probesMetricRelabelings: [] # - sourceLabels: [__name__, image] @@ -867,9 +884,10 @@ kubelet: # replacement: $1 # action: drop - # relabel configs to apply to samples before ingestion. - # metrics_path is required to match upstream rules and charts + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## + ## metrics_path is required to match upstream rules and charts cAdvisorRelabelings: - sourceLabels: [__metrics_path__] targetLabel: metrics_path @@ -880,6 +898,9 @@ kubelet: # replacement: $1 # action: replace + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## probesRelabelings: - sourceLabels: [__metrics_path__] targetLabel: metrics_path @@ -890,6 +911,9 @@ kubelet: # replacement: $1 # action: replace + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## resourceRelabelings: - sourceLabels: [__metrics_path__] targetLabel: metrics_path @@ -900,6 +924,9 @@ kubelet: # replacement: $1 # action: replace + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## metricRelabelings: [] # - sourceLabels: [__name__, image] # separator: ; @@ -912,9 +939,10 @@ kubelet: # replacement: $1 # action: drop - # relabel configs to apply to samples before ingestion. - # metrics_path is required to match upstream rules and charts + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## + ## metrics_path is required to match upstream rules and charts relabelings: - sourceLabels: [__metrics_path__] targetLabel: metrics_path @@ -967,14 +995,16 @@ kubeControllerManager: # Name of the server to use when validating TLS certificate serverName: null - ## Metric relabel configs to apply to samples before ingestion. + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1002,14 +1032,16 @@ coreDns: ## proxyUrl: "" - ## Metric relabel configs to apply to samples before ingestion. + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1041,14 +1073,16 @@ kubeDns: ## proxyUrl: "" - ## Metric relabel configs to apply to samples before ingestion. + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1057,12 +1091,17 @@ kubeDns: # targetLabel: nodename # replacement: $1 # action: replace + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## dnsmasqMetricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## dnsmasqRelabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1119,14 +1158,16 @@ kubeEtcd: certFile: "" keyFile: "" - ## Metric relabel configs to apply to samples before ingestion. + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1177,14 +1218,16 @@ kubeScheduler: ## Name of the server to use when validating TLS certificate serverName: null - ## Metric relabel configs to apply to samples before ingestion. + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1229,14 +1272,16 @@ kubeProxy: ## https: false - ## Metric relabel configs to apply to samples before ingestion. + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## relabelings: [] # - action: keep @@ -1261,18 +1306,17 @@ kubeStateMetrics: ## Override serviceMonitor selector ## selectorOverride: {} - ## Override namespace selector - ## - namespaceOverride: "" - ## Metric relabel configs to apply to samples before ingestion. + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] - # relabel configs to apply to samples before ingestion. + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1320,7 +1364,8 @@ nodeExporter: ## scrapeTimeout: "" - ## Metric relabel configs to apply to samples before ingestion. + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - sourceLabels: [__name__] @@ -1329,7 +1374,8 @@ nodeExporter: # replacement: $1 # action: drop - ## relabel configs to apply to samples before ingestion. + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1350,6 +1396,8 @@ prometheus-node-exporter: extraArgs: - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ + service: + portName: http-metrics ## Manages Prometheus and Alertmanager components ## @@ -1499,6 +1547,8 @@ prometheusOperator: kubeletService: enabled: true namespace: kube-system + ## Use '{{ template "kube-prometheus-stack.fullname" . }}-kubelet' by default + name: "" ## Create a servicemonitor for the operator ## @@ -1589,7 +1639,7 @@ prometheusOperator: ## image: repository: quay.io/prometheus-operator/prometheus-operator - tag: v0.50.0 + tag: v0.52.0 sha: "" pullPolicy: IfNotPresent @@ -1601,26 +1651,29 @@ prometheusOperator: ## # alertmanagerDefaultBaseImage: quay.io/prometheus/alertmanager - ## Prometheus-config-reloader image to use for config and rule reloading + ## Prometheus-config-reloader ## - prometheusConfigReloaderImage: - repository: quay.io/prometheus-operator/prometheus-config-reloader - tag: v0.50.0 - sha: "" + prometheusConfigReloader: + # image to use for config and rule reloading + image: + repository: quay.io/prometheus-operator/prometheus-config-reloader + tag: v0.52.0 + sha: "" - ## Set the prometheus config reloader side-car CPU limit - ## - configReloaderCpu: 100m - - ## Set the prometheus config reloader side-car memory limit - ## - configReloaderMemory: 50Mi + # resource config for prometheusConfigReloader + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + cpu: 100m + memory: 50Mi ## Thanos side-car image when configured ## thanosImage: repository: quay.io/thanos/thanos - tag: v0.17.2 + tag: v0.23.1 sha: "" ## Set a Field Selector to filter watched secrets @@ -2013,7 +2066,7 @@ prometheus: ## image: repository: quay.io/prometheus/prometheus - tag: v2.28.1 + tag: v2.31.1 sha: "" ## Tolerations for use with node taints @@ -2425,6 +2478,13 @@ prometheus: ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#thanosspec ## thanos: {} + # secretProviderClass: + # provider: gcp + # parameters: + # secrets: | + # - resourceName: "projects/$PROJECT_ID/secrets/testsecret/versions/latest" + # fileName: "objstore.yaml" + # objectStorageConfigFile: /var/secrets/object-store.yaml ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. ## if using proxy extraContainer update targetPort with proxy container port @@ -2436,7 +2496,7 @@ prometheus: ## PortName to use for Prometheus. ## - portName: "web" + portName: "http-web" ## ArbitraryFSAccessThroughSMs configures whether configuration based on a service monitor can access arbitrary files ## on the file system of the Prometheus container e.g. bearer token files. diff --git a/charts/kubezero-metrics/charts/prometheus-pushgateway/Chart.yaml b/charts/kubezero-metrics/charts/prometheus-pushgateway/Chart.yaml index 011c7917..59e59470 100644 --- a/charts/kubezero-metrics/charts/prometheus-pushgateway/Chart.yaml +++ b/charts/kubezero-metrics/charts/prometheus-pushgateway/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: 1.3.0 +appVersion: 1.4.1 description: A Helm chart for prometheus pushgateway home: https://github.com/prometheus/pushgateway keywords: @@ -13,4 +13,4 @@ maintainers: name: prometheus-pushgateway sources: - https://github.com/prometheus/pushgateway -version: 1.10.1 +version: 1.13.0 diff --git a/charts/kubezero-metrics/charts/prometheus-pushgateway/templates/deployment.yaml b/charts/kubezero-metrics/charts/prometheus-pushgateway/templates/deployment.yaml index 0e7b1999..8b8f5985 100644 --- a/charts/kubezero-metrics/charts/prometheus-pushgateway/templates/deployment.yaml +++ b/charts/kubezero-metrics/charts/prometheus-pushgateway/templates/deployment.yaml @@ -61,11 +61,12 @@ spec: timeoutSeconds: 10 resources: {{ toYaml .Values.resources | indent 12 }} - {{- if .Values.persistentVolume.enabled }} volumeMounts: - name: storage-volume mountPath: "{{ .Values.persistentVolume.mountPath }}" subPath: "{{ .Values.persistentVolume.subPath }}" + {{- if .Values.extraVolumeMounts }} +{{ toYaml .Values.extraVolumeMounts | indent 12 }} {{- end }} {{- if .Values.nodeSelector }} nodeSelector: @@ -79,13 +80,18 @@ spec: affinity: {{ toYaml .Values.affinity | indent 8 }} {{- end }} - {{- if .Values.persistentVolume.enabled }} {{- if .Values.securityContext }} securityContext: {{ toYaml .Values.securityContext | indent 8 }} {{- end }} volumes: - name: storage-volume + {{- if .Values.persistentVolume.enabled }} persistentVolumeClaim: claimName: {{ if .Values.persistentVolume.existingClaim }}{{ .Values.persistentVolume.existingClaim }}{{- else }}{{ template "prometheus-pushgateway.fullname" . }}{{- end }} + {{- else}} + emptyDir: {} {{- end -}} + {{- if .Values.extraVolumes }} +{{ toYaml .Values.extraVolumes | indent 8 }} + {{- end }} diff --git a/charts/kubezero-metrics/charts/prometheus-pushgateway/templates/ingress.yaml b/charts/kubezero-metrics/charts/prometheus-pushgateway/templates/ingress.yaml index c009a056..1a6bad3b 100644 --- a/charts/kubezero-metrics/charts/prometheus-pushgateway/templates/ingress.yaml +++ b/charts/kubezero-metrics/charts/prometheus-pushgateway/templates/ingress.yaml @@ -2,9 +2,13 @@ {{- $serviceName := include "prometheus-pushgateway.fullname" . }} {{- $servicePort := .Values.service.port -}} {{- $ingressPath := .Values.ingress.path -}} -{{- if semverCompare ">=1.14.0-0" .Capabilities.KubeVersion.GitVersion }} +{{- $ingressClassName := .Values.ingress.className -}} +{{- $ingressPathType := .Values.ingress.pathType -}} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1beta1 -{{- else }} +{{- else -}} apiVersion: extensions/v1beta1 {{- end }} kind: Ingress @@ -18,15 +22,28 @@ metadata: name: {{ template "prometheus-pushgateway.fullname" . }} namespace: {{ .Release.Namespace }} spec: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} + ingressClassName: {{ $ingressClassName }} + {{- end }} rules: {{- range $host := .Values.ingress.hosts }} - host: {{ $host }} http: paths: - path: {{ $ingressPath }} + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: {{ $ingressPathType }} + {{- end }} backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $serviceName }} + port: + number: {{ $servicePort }} + {{- else }} serviceName: {{ $serviceName }} servicePort: {{ $servicePort }} + {{- end }} {{- end -}} {{- if .Values.ingress.tls }} tls: diff --git a/charts/kubezero-metrics/charts/prometheus-pushgateway/values.yaml b/charts/kubezero-metrics/charts/prometheus-pushgateway/values.yaml index c3f78ea1..fbb49040 100644 --- a/charts/kubezero-metrics/charts/prometheus-pushgateway/values.yaml +++ b/charts/kubezero-metrics/charts/prometheus-pushgateway/values.yaml @@ -10,7 +10,7 @@ fullnameOverride: "" image: repository: prom/pushgateway - tag: v1.3.0 + tag: v1.4.1 pullPolicy: IfNotPresent # Optional pod imagePullSecrets @@ -84,7 +84,9 @@ ingress: ## enabled: false # AWS ALB requires path of /* + className: "" path: / + pathType: ImplementationSpecific ## Annotations. ## @@ -218,6 +220,14 @@ persistentVolume: ## subPath: "" +extraVolumes: {} + # - name: extra + # emptyDir: {} +extraVolumeMounts: {} + # - name: extra + # mountPath: /usr/share/extras + # readOnly: true + # Configuration for clusters with restrictive network policies in place: # - allowAll allows access to the PushGateway from any namespace # - customSelector is a list of pod/namespaceSelectors to allow access from diff --git a/charts/kubezero-metrics/configs/dashboards/apiserver.json b/charts/kubezero-metrics/configs/dashboards/apiserver.json index 1c940dcb..bd6c74b2 100644 --- a/charts/kubezero-metrics/configs/dashboards/apiserver.json +++ b/charts/kubezero-metrics/configs/dashboards/apiserver.json @@ -56,7 +56,11 @@ }, "gridPos": { }, "id": 3, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -128,13 +132,14 @@ "fillGradient": 0, "gridPos": { }, "id": 4, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -236,7 +241,11 @@ }, "gridPos": { }, "id": 5, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -307,13 +316,14 @@ "fillGradient": 0, "gridPos": { }, "id": 6, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -406,13 +416,14 @@ "fillGradient": 0, "gridPos": { }, "id": 7, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -488,13 +499,14 @@ "fillGradient": 0, "gridPos": { }, "id": 8, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -594,7 +606,11 @@ }, "gridPos": { }, "id": 9, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -665,13 +681,14 @@ "fillGradient": 0, "gridPos": { }, "id": 10, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -764,13 +781,14 @@ "fillGradient": 0, "gridPos": { }, "id": 11, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -846,13 +864,14 @@ "fillGradient": 0, "gridPos": { }, "id": 12, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -940,13 +959,14 @@ "fillGradient": 0, "gridPos": { }, "id": 13, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": false, "sideWidth": null, "total": false, @@ -968,7 +988,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1021,13 +1041,14 @@ "fillGradient": 0, "gridPos": { }, "id": 14, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": false, "sideWidth": null, "total": false, @@ -1049,7 +1070,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1102,6 +1123,7 @@ "fillGradient": 0, "gridPos": { }, "id": 15, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -1130,7 +1152,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name, le))", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1196,13 +1218,14 @@ "fillGradient": 0, "gridPos": { }, "id": 16, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -1277,13 +1300,14 @@ "fillGradient": 0, "gridPos": { }, "id": 17, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -1305,7 +1329,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])", + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1358,13 +1382,14 @@ "fillGradient": 0, "gridPos": { }, "id": 18, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -1452,7 +1477,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1470,7 +1495,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(apiserver_request_total, cluster)", + "query": "label_values(up{job=\"apiserver\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -1490,7 +1515,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(apiserver_request_total{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "query": "label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/cluster-total.json b/charts/kubezero-metrics/configs/dashboards/cluster-total.json index 86e147d2..9cab13f7 100644 --- a/charts/kubezero-metrics/configs/dashboards/cluster-total.json +++ b/charts/kubezero-metrics/configs/dashboards/cluster-total.json @@ -1619,7 +1619,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/configs/dashboards/controller-manager.json b/charts/kubezero-metrics/configs/dashboards/controller-manager.json index 373d6ab9..56bffc9e 100644 --- a/charts/kubezero-metrics/configs/dashboards/controller-manager.json +++ b/charts/kubezero-metrics/configs/dashboards/controller-manager.json @@ -36,7 +36,11 @@ }, "gridPos": { }, "id": 2, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -106,6 +110,7 @@ "fillGradient": 0, "gridPos": { }, "id": 3, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -134,7 +139,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name)", + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} {{name}}", @@ -200,6 +205,7 @@ "fillGradient": 0, "gridPos": { }, "id": 4, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -228,7 +234,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name)", + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} {{name}}", @@ -294,6 +300,7 @@ "fillGradient": 0, "gridPos": { }, "id": 5, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -322,7 +329,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name, le))", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} {{name}}", @@ -388,13 +395,14 @@ "fillGradient": 0, "gridPos": { }, "id": 6, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -416,28 +424,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -490,13 +498,14 @@ "fillGradient": 0, "gridPos": { }, "id": 7, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -518,7 +527,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -584,6 +593,7 @@ "fillGradient": 0, "gridPos": { }, "id": 8, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -612,7 +622,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -678,13 +688,14 @@ "fillGradient": 0, "gridPos": { }, "id": 9, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -759,13 +770,14 @@ "fillGradient": 0, "gridPos": { }, "id": 10, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -787,7 +799,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}[5m])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -840,13 +852,14 @@ "fillGradient": 0, "gridPos": { }, "id": 11, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -934,7 +947,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/configs/dashboards/k8s-resources-cluster.json b/charts/kubezero-metrics/configs/dashboards/k8s-resources-cluster.json index 7e9f5851..f84c0ea2 100644 --- a/charts/kubezero-metrics/configs/dashboards/k8s-resources-cluster.json +++ b/charts/kubezero-metrics/configs/dashboards/k8s-resources-cluster.json @@ -24,10 +24,12 @@ "id": 1, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -47,7 +49,7 @@ "steppedLine": false, "targets": [ { - "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "1 - sum(avg by (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", cluster=\"$cluster\"}[$__rate_interval])))", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -60,7 +62,7 @@ "title": "CPU Utilisation", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "singlestat", @@ -99,11 +101,14 @@ "fill": 1, "format": "percentunit", "id": 2, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -123,7 +128,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"cpu\",cluster=\"$cluster\"})", + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -136,7 +141,7 @@ "title": "CPU Requests Commitment", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "singlestat", @@ -175,11 +180,14 @@ "fill": 1, "format": "percentunit", "id": 3, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -199,7 +207,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"cpu\",cluster=\"$cluster\"})", + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -212,7 +220,7 @@ "title": "CPU Limits Commitment", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "singlestat", @@ -251,11 +259,14 @@ "fill": 1, "format": "percentunit", "id": 4, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -275,7 +286,7 @@ "steppedLine": false, "targets": [ { - "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{cluster=\"$cluster\"})", + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -288,7 +299,7 @@ "title": "Memory Utilisation", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "singlestat", @@ -327,11 +338,14 @@ "fill": 1, "format": "percentunit", "id": 5, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -351,7 +365,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"memory\",cluster=\"$cluster\"})", + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -364,7 +378,7 @@ "title": "Memory Requests Commitment", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "singlestat", @@ -403,11 +417,14 @@ "fill": 1, "format": "percentunit", "id": 6, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -427,7 +444,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"memory\",cluster=\"$cluster\"})", + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -440,7 +457,7 @@ "title": "Memory Limits Commitment", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "singlestat", @@ -490,11 +507,14 @@ "datasource": "$datasource", "fill": 10, "id": 7, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -528,7 +548,7 @@ "title": "CPU Usage", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -578,11 +598,14 @@ "datasource": "$datasource", "fill": 1, "id": 8, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -741,7 +764,7 @@ ], "targets": [ { - "expr": "sum(kube_pod_owner{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -810,7 +833,7 @@ "title": "CPU Quota", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -861,11 +884,14 @@ "datasource": "$datasource", "fill": 10, "id": 9, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -885,7 +911,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -899,7 +925,7 @@ "title": "Memory Usage (w/o cache)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -949,11 +975,14 @@ "datasource": "$datasource", "fill": 1, "id": 10, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1112,7 +1141,7 @@ ], "targets": [ { - "expr": "sum(kube_pod_owner{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1130,7 +1159,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1148,7 +1177,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1166,7 +1195,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1181,7 +1210,7 @@ "title": "Requests by Namespace", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -1234,10 +1263,12 @@ "id": 11, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1381,7 +1412,7 @@ ], "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1390,7 +1421,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1399,7 +1430,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1408,7 +1439,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1417,7 +1448,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1426,7 +1457,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1441,7 +1472,7 @@ "title": "Current Network Usage", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -1492,11 +1523,14 @@ "datasource": "$datasource", "fill": 10, "id": 12, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1516,7 +1550,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1530,7 +1564,7 @@ "title": "Receive Bandwidth", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1568,11 +1602,14 @@ "datasource": "$datasource", "fill": 10, "id": 13, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1592,7 +1629,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1606,7 +1643,7 @@ "title": "Transmit Bandwidth", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1656,11 +1693,14 @@ "datasource": "$datasource", "fill": 10, "id": 14, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1680,7 +1720,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "avg(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1694,7 +1734,7 @@ "title": "Average Container Bandwidth by Namespace: Received", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1732,11 +1772,14 @@ "datasource": "$datasource", "fill": 10, "id": 15, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1756,7 +1799,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "avg(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1770,7 +1813,7 @@ "title": "Average Container Bandwidth by Namespace: Transmitted", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1820,11 +1863,14 @@ "datasource": "$datasource", "fill": 10, "id": 16, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1844,7 +1890,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1858,7 +1904,7 @@ "title": "Rate of Received Packets", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1896,11 +1942,14 @@ "datasource": "$datasource", "fill": 10, "id": 17, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1920,7 +1969,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1934,7 +1983,7 @@ "title": "Rate of Transmitted Packets", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1984,11 +2033,14 @@ "datasource": "$datasource", "fill": 10, "id": 18, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2008,7 +2060,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2022,7 +2074,7 @@ "title": "Rate of Received Packets Dropped", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -2060,11 +2112,14 @@ "datasource": "$datasource", "fill": 10, "id": 19, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2084,7 +2139,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2098,7 +2153,7 @@ "title": "Rate of Transmitted Packets Dropped", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -2149,11 +2204,14 @@ "decimals": -1, "fill": 10, "id": 20, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2173,7 +2231,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\"}[5m])))", + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2187,7 +2245,7 @@ "title": "IOPS(Reads+Writes)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -2225,11 +2283,14 @@ "datasource": "$datasource", "fill": 10, "id": 21, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2249,7 +2310,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2263,7 +2324,7 @@ "title": "ThroughPut(Read+Write)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -2313,11 +2374,14 @@ "datasource": "$datasource", "fill": 1, "id": 22, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2465,7 +2529,7 @@ ], "targets": [ { - "expr": "sum by(namespace) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2474,7 +2538,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2483,7 +2547,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2492,7 +2556,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2501,7 +2565,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2510,7 +2574,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2525,7 +2589,7 @@ "title": "Current Storage IO", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -2578,7 +2642,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/configs/dashboards/k8s-resources-namespace.json b/charts/kubezero-metrics/configs/dashboards/k8s-resources-namespace.json index d24d5f59..db4905ff 100644 --- a/charts/kubezero-metrics/configs/dashboards/k8s-resources-namespace.json +++ b/charts/kubezero-metrics/configs/dashboards/k8s-resources-namespace.json @@ -22,11 +22,14 @@ "fill": 1, "format": "percentunit", "id": 1, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -46,7 +49,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -59,7 +62,7 @@ "title": "CPU Utilisation (from requests)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "singlestat", @@ -98,11 +101,14 @@ "fill": 1, "format": "percentunit", "id": 2, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -122,7 +128,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -135,7 +141,7 @@ "title": "CPU Utilisation (from limits)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "singlestat", @@ -174,11 +180,14 @@ "fill": 1, "format": "percentunit", "id": 3, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -198,7 +207,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -211,7 +220,7 @@ "title": "Memory Utilisation (from requests)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "singlestat", @@ -250,11 +259,14 @@ "fill": 1, "format": "percentunit", "id": 4, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -274,7 +286,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -287,7 +299,7 @@ "title": "Memory Utilisation (from limits)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "singlestat", @@ -337,11 +349,14 @@ "datasource": "$datasource", "fill": 10, "id": 5, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -414,7 +429,7 @@ "title": "CPU Usage", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -464,11 +479,14 @@ "datasource": "$datasource", "fill": 1, "id": 6, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -648,7 +666,7 @@ "title": "CPU Quota", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -699,11 +717,14 @@ "datasource": "$datasource", "fill": 10, "id": 7, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -746,7 +767,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -776,7 +797,7 @@ "title": "Memory Usage (w/o cache)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -826,11 +847,14 @@ "datasource": "$datasource", "fill": 1, "id": 8, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1004,7 +1028,7 @@ ], "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1022,7 +1046,7 @@ "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1040,7 +1064,7 @@ "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1049,7 +1073,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1058,7 +1082,7 @@ "step": 10 }, { - "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "expr": "sum(container_memory_cache{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1067,7 +1091,7 @@ "step": 10 }, { - "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "expr": "sum(container_memory_swap{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1082,7 +1106,7 @@ "title": "Memory Quota", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -1135,10 +1159,12 @@ "id": 9, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1282,7 +1308,7 @@ ], "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1291,7 +1317,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1300,7 +1326,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1309,7 +1335,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1318,7 +1344,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1327,7 +1353,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1342,7 +1368,7 @@ "title": "Current Network Usage", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -1393,11 +1419,14 @@ "datasource": "$datasource", "fill": 10, "id": 10, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1417,7 +1446,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1431,7 +1460,7 @@ "title": "Receive Bandwidth", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1469,11 +1498,14 @@ "datasource": "$datasource", "fill": 10, "id": 11, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1493,7 +1525,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1507,7 +1539,7 @@ "title": "Transmit Bandwidth", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1557,11 +1589,14 @@ "datasource": "$datasource", "fill": 10, "id": 12, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1581,7 +1616,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1595,7 +1630,7 @@ "title": "Rate of Received Packets", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1633,11 +1668,14 @@ "datasource": "$datasource", "fill": 10, "id": 13, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1657,7 +1695,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1671,7 +1709,7 @@ "title": "Rate of Transmitted Packets", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1721,11 +1759,14 @@ "datasource": "$datasource", "fill": 10, "id": 14, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1745,7 +1786,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1759,7 +1800,7 @@ "title": "Rate of Received Packets Dropped", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1797,11 +1838,14 @@ "datasource": "$datasource", "fill": 10, "id": 15, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1821,7 +1865,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1835,7 +1879,7 @@ "title": "Rate of Transmitted Packets Dropped", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1886,11 +1930,14 @@ "decimals": -1, "fill": 10, "id": 16, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1910,7 +1957,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m])))", + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1924,7 +1971,7 @@ "title": "IOPS(Reads+Writes)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1962,11 +2009,14 @@ "datasource": "$datasource", "fill": 10, "id": 17, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1986,7 +2036,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -2000,7 +2050,7 @@ "title": "ThroughPut(Read+Write)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -2050,11 +2100,14 @@ "datasource": "$datasource", "fill": 1, "id": 18, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2202,7 +2255,7 @@ ], "targets": [ { - "expr": "sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2211,7 +2264,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2220,7 +2273,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2229,7 +2282,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2238,7 +2291,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2247,7 +2300,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2262,7 +2315,7 @@ "title": "Current Storage IO", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -2315,7 +2368,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -2336,7 +2389,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -2359,7 +2412,7 @@ "multi": false, "name": "namespace", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/k8s-resources-node.json b/charts/kubezero-metrics/configs/dashboards/k8s-resources-node.json index e7878449..7cc77908 100644 --- a/charts/kubezero-metrics/configs/dashboards/k8s-resources-node.json +++ b/charts/kubezero-metrics/configs/dashboards/k8s-resources-node.json @@ -21,11 +21,14 @@ "datasource": "$datasource", "fill": 10, "id": 1, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -38,12 +41,32 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ ], + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], "spaceLength": 10, "span": 12, "stack": true, "steppedLine": false, "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, { "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "time_series", @@ -59,7 +82,7 @@ "title": "CPU Usage", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -109,11 +132,14 @@ "datasource": "$datasource", "fill": 1, "id": 2, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -293,7 +319,7 @@ "title": "CPU Quota", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -344,11 +370,14 @@ "datasource": "$datasource", "fill": 10, "id": 3, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -361,12 +390,32 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ ], + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], "spaceLength": 10, "span": 12, "stack": true, "steppedLine": false, "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, { "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", "format": "time_series", @@ -382,7 +431,7 @@ "title": "Memory Usage (w/o cache)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -432,11 +481,14 @@ "datasource": "$datasource", "fill": 1, "id": 4, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -688,7 +740,7 @@ "title": "Memory Quota", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -741,7 +793,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -762,7 +814,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -785,7 +837,7 @@ "multi": true, "name": "node", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, node)", + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/k8s-resources-pod.json b/charts/kubezero-metrics/configs/dashboards/k8s-resources-pod.json index 1b29f34f..9db65a29 100644 --- a/charts/kubezero-metrics/configs/dashboards/k8s-resources-pod.json +++ b/charts/kubezero-metrics/configs/dashboards/k8s-resources-pod.json @@ -21,11 +21,14 @@ "datasource": "$datasource", "fill": 10, "id": 1, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -72,7 +75,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "requests", @@ -80,7 +83,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "limits", @@ -94,7 +97,7 @@ "title": "CPU Usage", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -144,11 +147,14 @@ "datasource": "$datasource", "fill": 10, "id": 2, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -168,7 +174,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[5m])) by (container) /sum(increase(container_cpu_cfs_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[5m])) by (container)", + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{job=\"cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -191,7 +197,7 @@ "title": "CPU Throttling", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -241,11 +247,14 @@ "datasource": "$datasource", "fill": 1, "id": 3, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -425,7 +434,7 @@ "title": "CPU Quota", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -476,11 +485,14 @@ "datasource": "$datasource", "fill": 10, "id": 4, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -521,7 +533,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -529,7 +541,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "requests", @@ -537,7 +549,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "limits", @@ -551,7 +563,7 @@ "title": "Memory Usage (WSS)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -601,11 +613,14 @@ "datasource": "$datasource", "fill": 1, "id": 5, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -779,7 +794,7 @@ ], "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -797,7 +812,7 @@ "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -815,7 +830,7 @@ "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -824,7 +839,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -833,7 +848,7 @@ "step": 10 }, { - "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "expr": "sum(container_memory_cache{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -842,7 +857,7 @@ "step": 10 }, { - "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "expr": "sum(container_memory_swap{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -857,7 +872,7 @@ "title": "Memory Quota", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -910,10 +925,12 @@ "id": 6, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -933,7 +950,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -947,7 +964,7 @@ "title": "Receive Bandwidth", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -987,10 +1004,12 @@ "id": 7, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1010,7 +1029,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1024,7 +1043,7 @@ "title": "Transmit Bandwidth", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1076,10 +1095,12 @@ "id": 8, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1099,7 +1120,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1113,7 +1134,7 @@ "title": "Rate of Received Packets", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1153,10 +1174,12 @@ "id": 9, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1176,7 +1199,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1190,7 +1213,7 @@ "title": "Rate of Transmitted Packets", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1242,10 +1265,12 @@ "id": 10, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1265,7 +1290,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1279,7 +1304,7 @@ "title": "Rate of Received Packets Dropped", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1319,10 +1344,12 @@ "id": 11, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1342,7 +1369,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1356,7 +1383,7 @@ "title": "Rate of Transmitted Packets Dropped", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1407,11 +1434,14 @@ "decimals": -1, "fill": 10, "id": 12, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1431,7 +1461,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[5m])))", + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Reads", @@ -1439,7 +1469,7 @@ "step": 10 }, { - "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[5m])))", + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Writes", @@ -1453,7 +1483,7 @@ "title": "IOPS", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1491,11 +1521,14 @@ "datasource": "$datasource", "fill": 10, "id": 13, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1515,7 +1548,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Reads", @@ -1523,7 +1556,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Writes", @@ -1537,7 +1570,7 @@ "title": "ThroughPut", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1588,11 +1621,14 @@ "decimals": -1, "fill": 10, "id": 14, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1612,7 +1648,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceil(sum by(container) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m])))", + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -1626,7 +1662,7 @@ "title": "IOPS(Reads+Writes)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1664,11 +1700,14 @@ "datasource": "$datasource", "fill": 10, "id": 15, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1688,7 +1727,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -1702,7 +1741,7 @@ "title": "ThroughPut(Read+Write)", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1752,11 +1791,14 @@ "datasource": "$datasource", "fill": 1, "id": 16, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1904,7 +1946,7 @@ ], "targets": [ { - "expr": "sum by(container) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1913,7 +1955,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1922,7 +1964,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1931,7 +1973,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1940,7 +1982,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1949,7 +1991,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1964,7 +2006,7 @@ "title": "Current Storage IO", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -2017,7 +2059,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -2038,7 +2080,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -2061,7 +2103,7 @@ "multi": false, "name": "namespace", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "sort": 1, @@ -2084,7 +2126,7 @@ "multi": false, "name": "pod", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/k8s-resources-workload.json b/charts/kubezero-metrics/configs/dashboards/k8s-resources-workload.json index 75b2039e..bfc912fd 100644 --- a/charts/kubezero-metrics/configs/dashboards/k8s-resources-workload.json +++ b/charts/kubezero-metrics/configs/dashboards/k8s-resources-workload.json @@ -21,11 +21,14 @@ "datasource": "$datasource", "fill": 10, "id": 1, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -59,7 +62,7 @@ "title": "CPU Usage", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -109,11 +112,14 @@ "datasource": "$datasource", "fill": 1, "id": 2, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -251,7 +257,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -260,7 +266,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -269,7 +275,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -278,7 +284,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -293,7 +299,7 @@ "title": "CPU Quota", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -344,11 +350,14 @@ "datasource": "$datasource", "fill": 10, "id": 3, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -382,7 +391,7 @@ "title": "Memory Usage", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -432,11 +441,14 @@ "datasource": "$datasource", "fill": 1, "id": 4, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -574,7 +586,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -583,7 +595,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -592,7 +604,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -601,7 +613,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -616,7 +628,7 @@ "title": "Memory Quota", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -669,10 +681,12 @@ "id": 5, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -816,7 +830,7 @@ ], "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -825,7 +839,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -834,7 +848,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -843,7 +857,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -852,7 +866,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -861,7 +875,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -876,7 +890,7 @@ "title": "Current Network Usage", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -927,11 +941,14 @@ "datasource": "$datasource", "fill": 10, "id": 6, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -951,7 +968,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -965,7 +982,7 @@ "title": "Receive Bandwidth", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1003,11 +1020,14 @@ "datasource": "$datasource", "fill": 10, "id": 7, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1027,7 +1047,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1041,7 +1061,7 @@ "title": "Transmit Bandwidth", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1091,11 +1111,14 @@ "datasource": "$datasource", "fill": 10, "id": 8, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1115,7 +1138,7 @@ "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1129,7 +1152,7 @@ "title": "Average Container Bandwidth by Pod: Received", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1167,11 +1190,14 @@ "datasource": "$datasource", "fill": 10, "id": 9, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1191,7 +1217,7 @@ "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1205,7 +1231,7 @@ "title": "Average Container Bandwidth by Pod: Transmitted", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1255,11 +1281,14 @@ "datasource": "$datasource", "fill": 10, "id": 10, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1279,7 +1308,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1293,7 +1322,7 @@ "title": "Rate of Received Packets", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1331,11 +1360,14 @@ "datasource": "$datasource", "fill": 10, "id": 11, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1355,7 +1387,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1369,7 +1401,7 @@ "title": "Rate of Transmitted Packets", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1419,11 +1451,14 @@ "datasource": "$datasource", "fill": 10, "id": 12, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1443,7 +1478,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1457,7 +1492,7 @@ "title": "Rate of Received Packets Dropped", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1495,11 +1530,14 @@ "datasource": "$datasource", "fill": 10, "id": 13, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1519,7 +1557,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1533,7 +1571,7 @@ "title": "Rate of Transmitted Packets Dropped", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1585,7 +1623,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1606,7 +1644,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -1629,30 +1667,7 @@ "multi": false, "name": "namespace", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "", - "value": "" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "workload", - "options": [ ], - "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload)", + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "sort": 1, @@ -1675,7 +1690,30 @@ "multi": false, "name": "type", "options": [ ], - "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}, workload)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/k8s-resources-workloads-namespace.json b/charts/kubezero-metrics/configs/dashboards/k8s-resources-workloads-namespace.json index 78ade7a5..3e995a54 100644 --- a/charts/kubezero-metrics/configs/dashboards/k8s-resources-workloads-namespace.json +++ b/charts/kubezero-metrics/configs/dashboards/k8s-resources-workloads-namespace.json @@ -21,11 +21,14 @@ "datasource": "$datasource", "fill": 10, "id": 1, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -98,7 +101,7 @@ "title": "CPU Usage", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -148,11 +151,14 @@ "datasource": "$datasource", "fill": 1, "id": 2, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -329,7 +335,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -338,7 +344,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -347,7 +353,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -356,7 +362,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -371,7 +377,7 @@ "title": "CPU Quota", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -422,11 +428,14 @@ "datasource": "$datasource", "fill": 10, "id": 3, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -469,7 +478,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}} - {{workload_type}}", @@ -499,7 +508,7 @@ "title": "Memory Usage", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -549,11 +558,14 @@ "datasource": "$datasource", "fill": 1, "id": 4, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -721,7 +733,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -730,7 +742,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -739,7 +751,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -748,7 +760,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -757,7 +769,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -772,7 +784,7 @@ "title": "Memory Quota", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -825,10 +837,12 @@ "id": 5, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -987,7 +1001,7 @@ ], "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -996,7 +1010,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1005,7 +1019,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1014,7 +1028,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1023,7 +1037,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1032,7 +1046,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1047,7 +1061,7 @@ "title": "Current Network Usage", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "transform": "table", @@ -1098,11 +1112,14 @@ "datasource": "$datasource", "fill": 10, "id": 6, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1122,7 +1139,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1136,7 +1153,7 @@ "title": "Receive Bandwidth", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1174,11 +1191,14 @@ "datasource": "$datasource", "fill": 10, "id": 7, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1198,7 +1218,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1212,7 +1232,7 @@ "title": "Transmit Bandwidth", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1262,11 +1282,14 @@ "datasource": "$datasource", "fill": 10, "id": 8, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1286,7 +1309,7 @@ "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1300,7 +1323,7 @@ "title": "Average Container Bandwidth by Workload: Received", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1338,11 +1361,14 @@ "datasource": "$datasource", "fill": 10, "id": 9, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1362,7 +1388,7 @@ "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1376,7 +1402,7 @@ "title": "Average Container Bandwidth by Workload: Transmitted", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1426,11 +1452,14 @@ "datasource": "$datasource", "fill": 10, "id": 10, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1450,7 +1479,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1464,7 +1493,7 @@ "title": "Rate of Received Packets", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1502,11 +1531,14 @@ "datasource": "$datasource", "fill": 10, "id": 11, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1526,7 +1558,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1540,7 +1572,7 @@ "title": "Rate of Transmitted Packets", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1590,11 +1622,14 @@ "datasource": "$datasource", "fill": 10, "id": 12, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1614,7 +1649,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1628,7 +1663,7 @@ "title": "Rate of Received Packets Dropped", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1666,11 +1701,14 @@ "datasource": "$datasource", "fill": 10, "id": 13, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1690,7 +1728,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1704,7 +1742,7 @@ "title": "Rate of Transmitted Packets Dropped", "tooltip": { "shared": false, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -1756,7 +1794,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1777,7 +1815,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -1787,34 +1825,6 @@ "type": "query", "useTags": false }, - { - "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "deployment", - "value": "deployment" - }, - "datasource": "$datasource", - "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "type", - "options": [ ], - "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, { "allValue": null, "current": { @@ -1828,7 +1838,7 @@ "multi": false, "name": "namespace", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "sort": 1, @@ -1837,6 +1847,34 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false } ] }, diff --git a/charts/kubezero-metrics/configs/dashboards/kubelet.json b/charts/kubezero-metrics/configs/dashboards/kubelet.json index a5f43f84..e1a47c24 100644 --- a/charts/kubezero-metrics/configs/dashboards/kubelet.json +++ b/charts/kubezero-metrics/configs/dashboards/kubelet.json @@ -154,7 +154,7 @@ "refId": "A" } ], - "title": "Running Container", + "title": "Running Containers", "transparent": false, "type": "stat" }, @@ -294,7 +294,7 @@ "pluginVersion": "7", "targets": [ { - "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m]))", + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -347,7 +347,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (operation_type, instance)", + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -432,7 +432,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -517,7 +517,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -602,14 +602,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} pod", "refId": "A" }, { - "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} worker", @@ -694,14 +694,14 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} pod", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} worker", @@ -788,7 +788,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", @@ -875,7 +875,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", @@ -962,7 +962,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin, le))", + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", @@ -1047,7 +1047,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{operation_type}}", @@ -1132,7 +1132,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -1218,7 +1218,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1303,7 +1303,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1388,7 +1388,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1473,28 +1473,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -1579,7 +1579,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{verb}} {{url}}", @@ -1749,7 +1749,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1893,7 +1893,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1927,11 +1927,11 @@ "datasource": "$datasource", "hide": 0, "includeAll": true, - "label": null, + "label": "Data Source", "multi": false, "name": "instance", "options": [ ], - "query": "label_values(kubelet_runtime_operations_total{cluster=\"$cluster\", job=\"kubelet\"}, instance)", + "query": "label_values(up{job=\"kubelet\",cluster=\"$cluster\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/namespace-by-pod.json b/charts/kubezero-metrics/configs/dashboards/namespace-by-pod.json index cbe880cd..1d2e1f18 100644 --- a/charts/kubezero-metrics/configs/dashboards/namespace-by-pod.json +++ b/charts/kubezero-metrics/configs/dashboards/namespace-by-pod.json @@ -1137,7 +1137,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/configs/dashboards/namespace-by-workload.json b/charts/kubezero-metrics/configs/dashboards/namespace-by-workload.json index c34f9887..67f48514 100644 --- a/charts/kubezero-metrics/configs/dashboards/namespace-by-workload.json +++ b/charts/kubezero-metrics/configs/dashboards/namespace-by-workload.json @@ -89,7 +89,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ workload }}", @@ -184,7 +184,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ workload }}", @@ -441,7 +441,7 @@ ], "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -450,7 +450,7 @@ "step": 10 }, { - "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -459,7 +459,7 @@ "step": 10 }, { - "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -468,7 +468,7 @@ "step": 10 }, { - "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -477,7 +477,7 @@ "step": 10 }, { - "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -486,7 +486,7 @@ "step": 10 }, { - "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -495,7 +495,7 @@ "step": 10 }, { - "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -504,7 +504,7 @@ "step": 10 }, { - "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -578,7 +578,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ workload }}", @@ -673,7 +673,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ workload }}", @@ -794,7 +794,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{workload}}", @@ -885,7 +885,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{workload}}", @@ -987,7 +987,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{workload}}", @@ -1078,7 +1078,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{workload}}", @@ -1189,7 +1189,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{workload}}", @@ -1280,7 +1280,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{workload}}", @@ -1349,7 +1349,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1415,14 +1415,14 @@ "value": "deployment" }, "datasource": "$datasource", - "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "type", "options": [ ], - "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)", "refresh": 2, "regex": "", "skipUrlSync": false, diff --git a/charts/kubezero-metrics/configs/dashboards/persistentvolumesusage.json b/charts/kubezero-metrics/configs/dashboards/persistentvolumesusage.json index e9ab9c20..cf722b5e 100644 --- a/charts/kubezero-metrics/configs/dashboards/persistentvolumesusage.json +++ b/charts/kubezero-metrics/configs/dashboards/persistentvolumesusage.json @@ -26,13 +26,14 @@ "fillGradient": 0, "gridPos": { }, "id": 2, + "interval": "1m", "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -124,7 +125,11 @@ }, "gridPos": { }, "id": 3, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -207,13 +212,14 @@ "fillGradient": 0, "gridPos": { }, "id": 4, + "interval": "1m", "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -305,7 +311,11 @@ }, "gridPos": { }, "id": 5, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -388,7 +398,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -406,7 +416,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kubelet_volume_stats_capacity_bytes, cluster)", + "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"kubelet\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/pod-total.json b/charts/kubezero-metrics/configs/dashboards/pod-total.json index e854c415..e5c0b6e4 100644 --- a/charts/kubezero-metrics/configs/dashboards/pod-total.json +++ b/charts/kubezero-metrics/configs/dashboards/pod-total.json @@ -903,7 +903,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/configs/dashboards/proxy.json b/charts/kubezero-metrics/configs/dashboards/proxy.json index 8cf589a2..d7ad8acc 100644 --- a/charts/kubezero-metrics/configs/dashboards/proxy.json +++ b/charts/kubezero-metrics/configs/dashboards/proxy.json @@ -36,7 +36,11 @@ }, "gridPos": { }, "id": 2, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -106,13 +110,14 @@ "fillGradient": 0, "gridPos": { }, "id": 3, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -134,7 +139,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "rate", @@ -187,6 +192,7 @@ "fillGradient": 0, "gridPos": { }, "id": 4, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -215,7 +221,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -281,13 +287,14 @@ "fillGradient": 0, "gridPos": { }, "id": 5, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -309,7 +316,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "rate", @@ -362,6 +369,7 @@ "fillGradient": 0, "gridPos": { }, "id": 6, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -390,7 +398,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -456,13 +464,14 @@ "fillGradient": 0, "gridPos": { }, "id": 7, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -484,28 +493,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -558,13 +567,14 @@ "fillGradient": 0, "gridPos": { }, "id": 8, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -586,7 +596,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\",verb=\"POST\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -652,6 +662,7 @@ "fillGradient": 0, "gridPos": { }, "id": 9, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -680,7 +691,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -746,13 +757,14 @@ "fillGradient": 0, "gridPos": { }, "id": 10, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -827,13 +839,14 @@ "fillGradient": 0, "gridPos": { }, "id": 11, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -855,7 +868,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\"}[5m])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -908,13 +921,14 @@ "fillGradient": 0, "gridPos": { }, "id": 12, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -1002,7 +1016,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1020,7 +1034,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-proxy\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -1040,7 +1054,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\"}, instance)", + "query": "label_values(up{job=\"kube-proxy\", cluster=\"$cluster\", job=\"kube-proxy\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/scheduler.json b/charts/kubezero-metrics/configs/dashboards/scheduler.json index de71562f..b480c682 100644 --- a/charts/kubezero-metrics/configs/dashboards/scheduler.json +++ b/charts/kubezero-metrics/configs/dashboards/scheduler.json @@ -36,7 +36,11 @@ }, "gridPos": { }, "id": 2, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -106,6 +110,7 @@ "fillGradient": 0, "gridPos": { }, "id": 3, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -134,28 +139,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} e2e", "refId": "A" }, { - "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} binding", "refId": "B" }, { - "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} scheduling algorithm", "refId": "C" }, { - "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} volume", @@ -208,6 +213,7 @@ "fillGradient": 0, "gridPos": { }, "id": 4, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -236,28 +242,28 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} e2e", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} binding", "refId": "B" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} scheduling algorithm", "refId": "C" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} volume", @@ -323,13 +329,14 @@ "fillGradient": 0, "gridPos": { }, "id": 5, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -351,28 +358,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -425,13 +432,14 @@ "fillGradient": 0, "gridPos": { }, "id": 6, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -453,7 +461,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -519,6 +527,7 @@ "fillGradient": 0, "gridPos": { }, "id": 7, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -547,7 +556,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -613,13 +622,14 @@ "fillGradient": 0, "gridPos": { }, "id": 8, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -694,13 +704,14 @@ "fillGradient": 0, "gridPos": { }, "id": 9, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -722,7 +733,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -775,13 +786,14 @@ "fillGradient": 0, "gridPos": { }, "id": 10, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -869,7 +881,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -907,7 +919,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-scheduler\"}, instance)", + "query": "label_values(up{job=\"kube-scheduler\", cluster=\"$cluster\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/workload-total.json b/charts/kubezero-metrics/configs/dashboards/workload-total.json index f1ce7029..9cf5daf7 100644 --- a/charts/kubezero-metrics/configs/dashboards/workload-total.json +++ b/charts/kubezero-metrics/configs/dashboards/workload-total.json @@ -89,7 +89,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ pod }}", @@ -184,7 +184,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ pod }}", @@ -290,7 +290,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ pod }}", @@ -385,7 +385,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ pod }}", @@ -506,7 +506,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -597,7 +597,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -699,7 +699,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -790,7 +790,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -901,7 +901,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -992,7 +992,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -1061,7 +1061,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1079,7 +1079,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)", "refresh": 2, "regex": "", "sort": 0, @@ -1099,14 +1099,14 @@ "value": "kube-system" }, "datasource": "$datasource", - "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "definition": "label_values(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\"}, namespace)", "hide": 0, "includeAll": true, "label": null, "multi": false, "name": "namespace", "options": [ ], - "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "query": "label_values(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "skipUrlSync": false, diff --git a/charts/kubezero-metrics/configs/jsonnetfile.lock.json b/charts/kubezero-metrics/configs/jsonnetfile.lock.json index ddce81e6..5a209575 100644 --- a/charts/kubezero-metrics/configs/jsonnetfile.lock.json +++ b/charts/kubezero-metrics/configs/jsonnetfile.lock.json @@ -8,8 +8,8 @@ "subdir": "grafana" } }, - "version": "c3b14b24b83cfe9abf1064649d19e2d679f033fb", - "sum": "YrE4DNQsWgYWs6h0j/FjQETt8xDXdYdsslb1WK7xQEk=" + "version": "199e363523104ff8b3a12483a4e3eca86372b078", + "sum": "/jDHzVAjHB4AOLkJHw1GyATX5ogZ1iMdcJXZAgaG3+g=" }, { "source": { @@ -18,8 +18,8 @@ "subdir": "contrib/mixin" } }, - "version": "3df272774672366beb02c5447782805ab5fec957", - "sum": "5XhYOigrKipOWDbIn9hlrz7JcbelzvJnormxSaup9JI=" + "version": "29292aa7bdafaf65cb5e054591fe0ff07b36f5ee", + "sum": "cdKL5kPYfpWSpTCu4qctmh+gWQqL+4YWom6rw9qLYJU=" }, { "source": { @@ -28,7 +28,7 @@ "subdir": "grafonnet" } }, - "version": "19b27b272abf4263af1365ec485784c49815a332", + "version": "3626fc4dc2326931c530861ac5bebe39444f6cbf", "sum": "gF8foHByYcB25jcUOBqP6jxk0OPifQMjPvKY0HaCk6w=" }, { @@ -38,8 +38,8 @@ "subdir": "grafana-builder" } }, - "version": "b7eae75972a369bf8ebfb03dcb0d4c14464ef85a", - "sum": "GRf2GvwEU4jhXV+JOonXSZ4wdDv8mnHBPCQ6TUVd+g8=" + "version": "b102f9ac7d1290ac025c2a7ac99f7fd9a9948503", + "sum": "0KkygBQd/AFzUvVzezE4qF/uDYgrwUXVpZfINBti0oc=" }, { "source": { @@ -48,8 +48,8 @@ "subdir": "" } }, - "version": "ff4641bcd83314c955150bea6b147df9ca335c4a", - "sum": "oUVGwcCbmdH8qz9B+lbRawI9s23GY9HeW7MwYZRbZ/0=" + "version": "9821d07e94e9a9916575a234fb699ae3331fa939", + "sum": "xubNXyvDwUw9GZzi9BRb6ob3bYzfoMr5F5zCVn2d7ag=" }, { "source": { @@ -58,7 +58,7 @@ "subdir": "lib/promgrafonnet" } }, - "version": "ff4641bcd83314c955150bea6b147df9ca335c4a", + "version": "9821d07e94e9a9916575a234fb699ae3331fa939", "sum": "zv7hXGui6BfHzE9wPatHI/AGZa4A2WKo6pq7ZdqBsps=" }, { @@ -68,8 +68,8 @@ "subdir": "jsonnet/kube-state-metrics" } }, - "version": "8dab6f7472c26987ab7f8899a4a2f753fed8e8a8", - "sum": "S5qI+PJUdNeYOv76jH5nxwYS9N6U7CRxvyuB1wI4cTE=" + "version": "e3056ae518d0234105276ec916296923968ad294", + "sum": "U1wzIpTAtOvC1yj43Y8PfvT0JfvnAcMfNH12Wi+ab0Y=" }, { "source": { @@ -78,7 +78,7 @@ "subdir": "jsonnet/kube-state-metrics-mixin" } }, - "version": "8dab6f7472c26987ab7f8899a4a2f753fed8e8a8", + "version": "e3056ae518d0234105276ec916296923968ad294", "sum": "u8gaydJoxEjzizQ8jY8xSjYgWooPmxw+wIWdDxifMAk=" }, { @@ -88,8 +88,8 @@ "subdir": "jsonnet/kube-prometheus" } }, - "version": "a2eee1803a074fb40cad109d690732c22f0130cf", - "sum": "kqVnoNBux2YF1s03m+O3w/5jreAnjXx2/NjvNP1Hoy4=" + "version": "9ca30579f61ec51e63d87927d19b9d2a433c7e25", + "sum": "EYlmVYtdVovF3ziMZ9dhV0trzXww6YSz8A2tH2YF9Zw=" }, { "source": { @@ -98,8 +98,8 @@ "subdir": "jsonnet/mixin" } }, - "version": "42fc15967e35e0cca68cf935f844086edbc82d0e", - "sum": "6reUygVmQrLEWQzTKcH8ceDbvM+2ztK3z2VBR2K2l+U=", + "version": "335ebbc2f6ecf10b699821fa8cebcbff4a718ca7", + "sum": "qZ4WgiweaE6eeKtFK60QUjLO8sf2L9Q8fgafWvDcyfY=", "name": "prometheus-operator-mixin" }, { @@ -109,8 +109,8 @@ "subdir": "jsonnet/prometheus-operator" } }, - "version": "42fc15967e35e0cca68cf935f844086edbc82d0e", - "sum": "sECNXs/aIEreFUma1BWVyknBygqh3AVJEB3msmrAYYY=" + "version": "335ebbc2f6ecf10b699821fa8cebcbff4a718ca7", + "sum": "Vr2IY6Uz1lYYyGDF7QaEAVkJwAtOEikCfuXJN2eAUM0=" }, { "source": { @@ -119,7 +119,7 @@ "subdir": "doc/alertmanager-mixin" } }, - "version": "e35efbddb66a73fd8723be5334477e76f21fbd19", + "version": "e2a10119aaf7777fa523d216e05897c5b719134c", "sum": "pep+dHzfIjh2SU5pEkwilMCAT/NoL6YYflV4x8cr7vU=", "name": "alertmanager" }, @@ -130,8 +130,8 @@ "subdir": "docs/node-mixin" } }, - "version": "0e6b23c338e98809c9872c70a2f5dfa8d6d370d4", - "sum": "MnfAA4+l2BkgJncnYfV8uHC7CxHZut8+ap8KkEqyB5Y=" + "version": "7dbf35891570f9ce3bccb25a55176ea4923b35dd", + "sum": "MlWDAKGZ+JArozRKdKEvewHeWn8j2DNBzesJfLVd0dk=" }, { "source": { @@ -140,8 +140,8 @@ "subdir": "documentation/prometheus-mixin" } }, - "version": "a05b510fc32c3ecc2fc369002576179ae1cbcc23", - "sum": "m4VHwft4fUcxzL4+52lLZG/V5aH5ZEdjaweb88vISL0=", + "version": "c965a7555b7ffcee1a127d782abd5bb478a16750", + "sum": "ZjQoYhvgKwJNkg+h+m9lW3SYjnjv5Yx5btEipLhru88=", "name": "prometheus" }, { @@ -151,8 +151,8 @@ "subdir": "mixin" } }, - "version": "360b39e1c6ab3ac8dcefa225a6205142f9362c68", - "sum": "Og+wEHfgzXBvBLAeeQvGNoiCw3FY4LQHlJdpsG/owj8=", + "version": "d1acaea2a11a3e4db6bb435c98dea63c517e3530", + "sum": "1Y1cPIeoPg2nCAEhKPCt8bAGuwuOP2eZ3kVF432mlMA=", "name": "thanos-mixin" } ], diff --git a/charts/kubezero-metrics/configs/rules/alertmanager-prometheusRule b/charts/kubezero-metrics/configs/rules/alertmanager-prometheusRule index 99d1647c..a18dcfdc 100644 --- a/charts/kubezero-metrics/configs/rules/alertmanager-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/alertmanager-prometheusRule @@ -4,6 +4,7 @@ "metadata": { "labels": { "app.kubernetes.io/component": "alert-router", + "app.kubernetes.io/instance": "main", "app.kubernetes.io/name": "alertmanager", "app.kubernetes.io/part-of": "kube-prometheus", "app.kubernetes.io/version": "0.23.0", diff --git a/charts/kubezero-metrics/configs/rules/etcd-mixin-prometheusRule b/charts/kubezero-metrics/configs/rules/etcd-mixin-prometheusRule index c54efb47..2aa76d4d 100644 --- a/charts/kubezero-metrics/configs/rules/etcd-mixin-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/etcd-mixin-prometheusRule @@ -64,10 +64,10 @@ { "alert": "etcdGRPCRequestsSlow", "annotations": { - "description": "etcd cluster \"{{ $labels.job }}\": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }}.", + "description": "etcd cluster \"{{ $labels.job }}\": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method }} method.", "summary": "etcd grpc requests are slow" }, - "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~\".*etcd.*\", grpc_type=\"unary\"}[5m])) without(grpc_type))\n> 0.15\n", + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~\".*etcd.*\", grpc_method!=\"Defragment\", grpc_type=\"unary\"}[5m])) without(grpc_type))\n> 0.15\n", "for": "10m", "labels": { "severity": "critical" diff --git a/charts/kubezero-metrics/configs/rules/kube-state-metrics-prometheusRule b/charts/kubezero-metrics/configs/rules/kube-state-metrics-prometheusRule index 832ed4ee..1557b6de 100644 --- a/charts/kubezero-metrics/configs/rules/kube-state-metrics-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/kube-state-metrics-prometheusRule @@ -6,7 +6,7 @@ "app.kubernetes.io/component": "exporter", "app.kubernetes.io/name": "kube-state-metrics", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "2.2.1", + "app.kubernetes.io/version": "2.2.4", "prometheus": "k8s", "role": "alert-rules" }, diff --git a/charts/kubezero-metrics/configs/rules/kubernetes-prometheusRule b/charts/kubezero-metrics/configs/rules/kubernetes-prometheusRule index ec6402e4..04e6b568 100644 --- a/charts/kubezero-metrics/configs/rules/kubernetes-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/kubernetes-prometheusRule @@ -19,11 +19,11 @@ { "alert": "KubePodCrashLooping", "annotations": { - "description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf \"%.2f\" $value }} times / 10 minutes.", + "description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason: \"CrashLoopBackOff\").", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodcrashlooping", "summary": "Pod is crash looping." }, - "expr": "increase(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\"}[10m]) > 0\nand\nkube_pod_container_status_waiting{job=\"kube-state-metrics\"} == 1\n", + "expr": "max_over_time(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\", job=\"kube-state-metrics\"}[5m]) >= 1\n", "for": "15m", "labels": { "severity": "warning" @@ -123,7 +123,7 @@ { "alert": "KubeContainerWaiting", "annotations": { - "description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} has been in waiting state for longer than 1 hour.", + "description": "pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting", "summary": "Pod container waiting longer than 1 hour" }, @@ -232,7 +232,7 @@ { "alert": "KubeMemoryOvercommit", "annotations": { - "description": "Cluster has overcommitted memory resource requests for Pods by {{ $value }} bytes and cannot tolerate node failure.", + "description": "Cluster has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit", "summary": "Cluster has overcommitted memory resource requests." }, @@ -249,7 +249,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit", "summary": "Cluster has overcommitted CPU resource requests." }, - "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"cpu\"})\n /\nsum(kube_node_status_allocatable{resource=\"cpu\"})\n > 1.5\n", + "expr": "sum(min without(resource) (kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=~\"(cpu|requests.cpu)\"}))\n /\nsum(kube_node_status_allocatable{resource=\"cpu\", job=\"kube-state-metrics\"})\n > 1.5\n", "for": "5m", "labels": { "severity": "warning" @@ -262,7 +262,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit", "summary": "Cluster has overcommitted memory resource requests." }, - "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"memory\"})\n /\nsum(kube_node_status_allocatable{resource=\"memory\",job=\"kube-state-metrics\"})\n > 1.5\n", + "expr": "sum(min without(resource) (kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=~\"(memory|requests.memory)\"}))\n /\nsum(kube_node_status_allocatable{resource=\"memory\", job=\"kube-state-metrics\"})\n > 1.5\n", "for": "5m", "labels": { "severity": "warning" @@ -332,7 +332,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup", "summary": "PersistentVolume is filling up." }, - "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\n", + "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n", "for": "1m", "labels": { "severity": "critical" @@ -345,7 +345,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup", "summary": "PersistentVolume is filling up." }, - "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\n", + "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n", "for": "1h", "labels": { "severity": "warning" @@ -468,7 +468,7 @@ { "alert": "KubeClientCertificateExpiration", "annotations": { - "description": "A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days.", + "description": "A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration", "summary": "Client certificate is about to expire." }, @@ -480,7 +480,7 @@ { "alert": "KubeClientCertificateExpiration", "annotations": { - "description": "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.", + "description": "A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration", "summary": "Client certificate is about to expire." }, @@ -490,11 +490,11 @@ } }, { - "alert": "AggregatedAPIErrors", + "alert": "KubeAggregatedAPIErrors", "annotations": { - "description": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.", - "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/aggregatedapierrors", - "summary": "An aggregated API has reported errors." + "description": "Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors", + "summary": "Kubernetes aggregated API has reported errors." }, "expr": "sum by(name, namespace)(increase(aggregator_unavailable_apiservice_total[10m])) > 4\n", "labels": { @@ -502,11 +502,11 @@ } }, { - "alert": "AggregatedAPIDown", + "alert": "KubeAggregatedAPIDown", "annotations": { - "description": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.", - "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/aggregatedapidown", - "summary": "An aggregated API is down." + "description": "Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown", + "summary": "Kubernetes aggregated API is down." }, "expr": "(1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85\n", "for": "5m", @@ -530,9 +530,9 @@ { "alert": "KubeAPITerminatedRequests", "annotations": { - "description": "The apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.", + "description": "The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests", - "summary": "The apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests." + "summary": "The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests." }, "expr": "sum(rate(apiserver_request_terminations_total{job=\"apiserver\"}[10m])) / ( sum(rate(apiserver_request_total{job=\"apiserver\"}[10m])) + sum(rate(apiserver_request_terminations_total{job=\"apiserver\"}[10m])) ) > 0.20\n", "for": "5m", @@ -748,53 +748,71 @@ } ] }, + { + "name": "kubernetes-system-kube-proxy", + "rules": [ + { + "alert": "KubeProxyDown", + "annotations": { + "description": "KubeProxy has disappeared from Prometheus target discovery.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeproxydown", + "summary": "Target disappeared from Prometheus target discovery." + }, + "expr": "absent(up{job=\"kube-proxy\"} == 1)\n", + "for": "15m", + "labels": { + "severity": "critical" + } + } + ] + }, { "name": "kube-apiserver-burnrate.rules", "rules": [ { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[1d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[1d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate1d" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[1h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[1h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate1h" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[2h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[2h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[2h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[2h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[2h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[2h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate2h" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[30m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate30m" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[3d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[3d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[3d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[3d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[3d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[3d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate3d" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[5m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[5m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[5m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[5m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[5m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[5m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate5m" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[6h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[6h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[6h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[6h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[6h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[6h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n", "labels": { "verb": "read" }, @@ -932,14 +950,14 @@ "record": "cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d" }, { - "expr": "1 - (\n (\n # write too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n -\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n ) +\n (\n # read too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"})\n )\n ) +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d)\n", + "expr": "1 - (\n (\n # write too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n -\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n ) +\n (\n # read too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"})\n )\n ) +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d)\n", "labels": { "verb": "all" }, "record": "apiserver_request:availability30d" }, { - "expr": "1 - (\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n # too slow\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"})\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n", + "expr": "1 - (\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n # too slow\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"})\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n", "labels": { "verb": "read" }, @@ -1008,7 +1026,7 @@ "record": "node_namespace_pod_container:container_memory_swap" }, { - "expr": "kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", + "expr": "kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", "record": "cluster:namespace:pod_memory:active:kube_pod_container_resource_requests" }, { @@ -1016,7 +1034,7 @@ "record": "namespace_memory:kube_pod_container_resource_requests:sum" }, { - "expr": "kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", + "expr": "kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", "record": "cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests" }, { @@ -1024,7 +1042,7 @@ "record": "namespace_cpu:kube_pod_container_resource_requests:sum" }, { - "expr": "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", + "expr": "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", "record": "cluster:namespace:pod_memory:active:kube_pod_container_resource_limits" }, { @@ -1032,7 +1050,7 @@ "record": "namespace_memory:kube_pod_container_resource_limits:sum" }, { - "expr": "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n )\n", + "expr": "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n )\n", "record": "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" }, { diff --git a/charts/kubezero-metrics/configs/rules/node-exporter-prometheusRule b/charts/kubezero-metrics/configs/rules/node-exporter-prometheusRule index 85ddccb7..332a2962 100644 --- a/charts/kubezero-metrics/configs/rules/node-exporter-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/node-exporter-prometheusRule @@ -6,7 +6,7 @@ "app.kubernetes.io/component": "exporter", "app.kubernetes.io/name": "node-exporter", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "1.2.2", + "app.kubernetes.io/version": "1.3.1", "prometheus": "k8s", "role": "alert-rules" }, @@ -259,7 +259,7 @@ "record": "instance:node_num_cpu:sum" }, { - "expr": "1 - avg without (cpu, mode) (\n rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\"}[5m])\n)\n", + "expr": "1 - avg without (cpu) (\n sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\"}[5m]))\n)\n", "record": "instance:node_cpu_utilisation:rate5m" }, { @@ -267,7 +267,7 @@ "record": "instance:node_load1_per_cpu:ratio" }, { - "expr": "1 - (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"}\n +\n node_memory_Cached_bytes{job=\"node-exporter\"}\n +\n node_memory_MemFree_bytes{job=\"node-exporter\"}\n +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n", + "expr": "1 - (\n (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"}\n +\n node_memory_Cached_bytes{job=\"node-exporter\"}\n +\n node_memory_MemFree_bytes{job=\"node-exporter\"}\n +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n )\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n", "record": "instance:node_memory_utilisation:ratio" }, { diff --git a/charts/kubezero-metrics/configs/rules/prometheus-operator-prometheusRule b/charts/kubezero-metrics/configs/rules/prometheus-operator-prometheusRule index be7b9988..ddc63b4f 100644 --- a/charts/kubezero-metrics/configs/rules/prometheus-operator-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/prometheus-operator-prometheusRule @@ -6,7 +6,7 @@ "app.kubernetes.io/component": "controller", "app.kubernetes.io/name": "prometheus-operator", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "0.51.1", + "app.kubernetes.io/version": "0.52.1", "prometheus": "k8s", "role": "alert-rules" }, @@ -110,6 +110,24 @@ } } ] + }, + { + "name": "config-reloaders", + "rules": [ + { + "alert": "ConfigReloaderSidecarErrors", + "annotations": { + "description": "Errors encountered while the {{$labels.pod}} config-reloader sidecar attempts to sync config in {{$labels.namespace}} namespace.\nAs a result, configuration for service running in {{$labels.pod}} may be stale and cannot be updated anymore.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/configreloadersidecarerrors", + "summary": "config-reloader sidecar has not had a successful reload for 10m" + }, + "expr": "max_over_time(reloader_last_reload_successful{namespace=~\".+\"}[5m]) == 0\n", + "for": "10m", + "labels": { + "severity": "warning" + } + } + ] } ] } diff --git a/charts/kubezero-metrics/configs/rules/prometheus-prometheusRule b/charts/kubezero-metrics/configs/rules/prometheus-prometheusRule index 638ce0d9..04587ae0 100644 --- a/charts/kubezero-metrics/configs/rules/prometheus-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/prometheus-prometheusRule @@ -4,9 +4,10 @@ "metadata": { "labels": { "app.kubernetes.io/component": "prometheus", + "app.kubernetes.io/instance": "k8s", "app.kubernetes.io/name": "prometheus", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "2.30.0", + "app.kubernetes.io/version": "2.31.1", "prometheus": "k8s", "role": "alert-rules" }, diff --git a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml index fa35e552..84f51024 100644 --- a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml +++ b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml @@ -16,36 +16,36 @@ binaryData: node.json.gz: H4sIAAAAAAAC/+192XLjyBHte39FmbbvtGa0cF8c4QeJlDy67kVuqWfCywQDAosSLBDgYJFa3bf97TerAJDYQRILATAfZlokQKAqazvnZFbWtzeENKZTSVmaht74C/k3fCbkG/8/XFGEBYVvG5Pb6c2nj+8v736+/HzbOHYuy8I9ldn1G01dUOORmvr64ozqoiYtDUlV2C3rC8brkj90JhiCrpqaSNfXlrL5ICnXM3Z9GfJQ6/oHu1iu1/IbvsP/fzu2qqTR301JoyGVct6/FBQo/erREn/pg2A+uMrjGOBv3q+fqaY79bLefBz+jgdNmAuKEHiL/+vVe/wXXG8anHZOB/GvC6+SJiwfQ1/l/nrjKoU1mxTZYEqwqcJe2TptnjZ3qJsuKQ8y1Q3BCL7yNuRasJarLiMoigr3wlXWZ6zXN2RJN1Y9aF0ouPKnPz0K+uPf6St7lnr/Xyoaf2k1e8PVu+Cee1OSjWv2tpbrW5cBww0D91BFuJfZdUMzqev7R2kW8q0kqspYlVWNPVB7uBfeNo9Ju9WC//V6x6R15H60Y57zdX3J/yHnMtUMTxHW7a0/3quCNmvY177zf397YzdVg84kw1faxoNCDT6KW8N+0/qGdbc7VZUNaQnfW1/yRlRMWbY+GVQT7Pmi1W91+81es9XvD/v8qiwpTyGjmdWdFZN+gV8rgkzYfa7JRnjgP/pt/Y1kyNYAkIxH8z44L3kfYGp8hns0jKX+l7OzB/6jU1FdnGlzTZDkzpk9mE9WhtLDO3KKkvpnhc2Kav2Kl9X++2xdxjPWMv4xwMdYiJFFVZaFpU5Zc80FWafrWT6pM0PDS7MbdT2mrH7sGxMv8LnddX3xxekj9mc2zJpOB/ROAu1+a70+rCqwNqNGl1QwXB3Na9x/mJL4RMY3n8kZeU8X8P+JpD8Fba2pL+HtKgriI72TFlQ1A29JNo9vmbww9VfC5ixK1DkRZJmXTFRhJSOG+sB+ra1/PJeoPBurylx68Np3RueCKRteq1sNyWcJ95fw9ULl00rDeIT3PKqy04V99rYeYeqGumDP8H6/EJZLmHF113TpnzTdrdY89n+tsnmh8ddG4IIBI4Zd+nB2HnLRap5W4MKzIJu8UqxBGp6r312ffvNVgnW8VrPp/VZS/CXmT/2FveK9bTv+Gs8tLmtGWly411XZNKivYg3doMugKYPGdLWpNfP3YOpvDdrHpAszf/N0NDgK2MxlHFZo39Xvx9u+st0ZwDvbI3hnk71zOIp957CX/o2scr2u9R9UMvZ9o8D7PJ9/i+zppiLxfrekMH4VY92HXLc1VIAUMMNZQPO3wAwVOft1fbNfJ2Hya0VMfs01GncWyPXUpy69eMb6UpOgNs5C23iEz19V+MLde2HSnJki/Rjyez7jyWKwcwId0I0PqvHBN968I4xPWbqHEaxbS3fWl1BTw1h6uXPG1DvGPXT/euS/672gPQHks6FJwIQWnfjFj7Fda7IGU663qh5L0C9L3ivfvn0rqqZi2P9XYHRPxaU51Sks+jN9agDUkr9JCszsikj/+p/Gn9gt/2kc/1e9Z5/gn/80vh+R+1fyFn53dHRETojw/PBWNxf8SzZfHL0FeESjHs7u+OsP0kymPxwnvejffwKCBM8C4gdYBEz/2xF75Y9s5juCFTDbungb2oawvmZj/dkuy5UgGnyse+b0hkwfqDK7UrUFX8x93Uejc4s1epcIPo9ybNF84x/+IUCLrbVsBQ6u/hYvLMf6T972CCxJ0D8ALz/QI8QDiAcQD9QcD3Q2wwMtIPwICPYPCNjazVdOWRVmvQ1WyzOS8aprreZem85Xq6cB6xU8G1pez2Z5zmYBvn3VyTuwGCxxC4Z/jsq+ErdwKcalGJfig1qK+xsuxSNcicu2ErdKsxSXZZ1t7X2h/aCy9VOQZOZEIp/O35MFXajaaznX0hkVpYXAu3czcpHFlavwlatZ8MrVrOLKNUpcudi0xAb2R7sEAYeftbT1cWkrherM1yZrupy+p4s7tjJN718NqieuT+SE+H59pVG64Y/ZIpnq5Wn5qTcCIttl8zje6lBqsF3A9ufOErapCVZqewo7HhXP7y/S4A62un9mcQR7Qxvs7eT2RVgiVUeqXmHA0yoY8LQrSdVb7Q3d6C0ENKUDNGyWToFo2M/TQJqtXx+OaYrh87+e35RhYf2kqga5usW1FddWFBNqLia0epu6pLu4uJZgcfXw1rkkU/0VxvBiyrXXzda44wVTxJcqrGnw3Rl8MdfZ9PYH+KTB1D/Xg8TW9SZd+kozfNH25LeQtdheA7NdjvnsciGITw8aWCYYh8+v/2LPJyGXvP0jeY7aZN6MnM4DttkaTHDcBRZY3FONud9Xrvc0yGINC3aZJ9fdC0at5tpTZLWopwiwSP+y4mnuSF5Yp50LTd98EDJNGIkTibHhJBI5ybf9mzD8k/xww0l+Pcc7g8zfhUPmfhuR3fkQkvv7mInN2cnEp2ViqITDsOCM7Sp1tLrnPEwTlIeEh7VjZgFo+Ql09hs2c+ne5mfXbqj2iTf1WkBnNuK3h8I0/unOgpceay5V3ZhLX7xbK+0vr2CpvIW5ll3sNf/sug4zXOA3/LvIn3B7vBeWMe0w1/iwCsOXEbi4YahB3Bs0pr6EXg3dxje6YEWRvdvdOi2Yg6CrQncdsUmoNfTMU3P2nuDgYU92P8d6TJvNZ6OO5wHu8Rno/3zzGzzHXPg3um66MucZbu0ajY1GWeKt3bTDbTB3HPbYO90762fYjk8+MCN7ML8a34PDiF10543gbcF68hudzcqiqWkrjI5Lv3vpX3nUW1F44JYDSGIuGbQrBQao6frv34PZbu7iMnZ75xwSWDw+CN+iPep6R3OmKCLilb3wV+aINdKjCh1hRcVhBYcSNhfmWGJjBf0eKOx2v0yAFu0CwcNn3yKRK3KIGPADb8UODl/EarybA4wgTEmHL5rxegNTbtI5L7KCF1wkO1SJoRPho+2gxIASA2KB3bFAflJ8RbaZeaDDoHk8ajbC9furW8KXBNQfDkp/SMIH5+8RHBQLDgL6QwQ4GPQQHJQSHAwQHFQDHOwWgN7Y0+LdCI8tx0UbF23fos1CI3HVLnjVbm+2aq+pP67auGrjqr3zqr1trHqJlm0eun5Y63Ypsun2orLpdtJk070QdEn0ZNP9QI3tc+oKsiToYweIuKrKs/PBL/94eX4x7Izcw40n7LlWXwQrJvqPw1HzqtkO3KHyFLrshtbVYDhuuW+4nlm1+GOz177oBS6RE/IrPB06JYF1nei8edgnmPgfYfGhSsSPH0yqW4UanY+7Q3cXa0jR75TWlQlWV9J+51curloXnj0nDUWy+s8fx63euDXwTEnq3HB+eNnuTroe+8CAtgKF/ng1vmxPLj3XuG7IL/aaw773h6Zum7TXGvXGl41A17oXND2kt+uP76jyYPD+2/R8T/UdBocLFLYj0zqtOqikzNVsgKE3bmoFWHbCixJfdLqeL/6mCTMWD+8Bu5EzwcA3E7TaCTNBPyykY0aVW0u3Dc0AMBisERoPFPOWA8bvg3Ku39l54f1rqPD8EPKtM2kGr1ibXgLf8k0v/m816eER1oYwedlZkX3p83W4+Vdpxrthu+dB2CzQ1zvV2d9OoFcFr/BYvZD3hm4c+O7GuFT3h82wL1/sUrXi4fAOADF014XADgFY7UuI3JBgbTYRHgKRPklbFbgTgfVlk7245/0+2NGgM8yoZs0sc1l1YQDLo/DRPZTC13q+jqyyudmLw3HYfrPVepGI+VfP5KtFxMMGlxft/mSLh/ECWmFuUc+014Atn/lZd2WM9z2xed7tDMbbPvGjJwm975H9Seuq307A5yINm/cB54lP/j7FQOaSzt5ZeN7bQTaGyk4WZQcWb5BJ+T/2igfIeZdsyrlnX2zHButG9qYM82DsYNQfGFD4obIGDQ6lbbJi5GBOCyAWatBd7BY2764sN96H5f73n8bpjwCEix3dO9nu0z/0CMtNMrFcorH+4GSVtz9YY9j+YE2Sq49Oh3Q+ar+v77SoR+k7a8jqtrL4ZWZ9Nf98/sWYLIiBVsa6SqsWeY5CWtArSzD06Q4L+ok+2FDW94PbR2kerVTwXP+MDLpkidXxVN/cnEHQ6CyENVjcoOmH+VNH3JCUmfQszUxBDvJh1/lw7kPYvghfJB8mvzfFJwvj+LjGKiOCO9h1LUn7OUso81nxkhAm+ip82Tq4vt2J6m/ePZP2Vmz79EDf1+oDNEsgY4LNARuBPm2xwIbvW1d9dwvab3d3qIrX6ImVCdxu1SXwdYiQHDZ4XoO9h/PwMALOvn9Hn1el9pzttrEad77kQiug/vbo0qtdXZjzueVagqut7mTk7hqNMXNduvmC/9rMlpNag65H9Rqri4VkGM71gBLFEr64qY3r0s+CNnuBscx2lmnmEp5BTsg5j25jW4yZ+9h4FAz4HyVPVFOoTAA8KoY0l+BWQWcbkO3fsa3tBnlRtSdJebCktqur86srj2CowAwqPVul6Q273YFHibsB/szlEd0ubq953vLfMI29A0o8XdX3snk1mnjesM4JEy4T3kKvtYhvs395MfBfmq7b6LI56LUnnhtYArWIx8Kl2NfCddezgxolv2FVr/ZVb9DzKJGfFdN5upcRl1dutLw0xNSFB3qQkmMgD9dOmuOwnppjx6M5Hp6G6G3qEouIwQgjv0Lnn4TXw6vpA8OW1BWZSCa6AHzqJD8Ra4GNVPYsZ9J2j+aTblaC5vkz+/M+WiKdXE7Or1IYqwzCYmaRcwVKWRG9OENlcF9ZiVl6I/evLSS56Y9/8rzZxq8lyMOb3JqeTENZ65L7MWjeukXSTJqhPLl7fy7CCME5P0N9cZ/pPPM2XjDl5tZa4T7lr/cWJUEFbGcFLCq0d3dxK0IPSqNtoWwVIlt9ouKzNY1MgdO33fj2OPQuWY2SYNg9M01drh/Uv5w0L5vhN9nPCWo1/BYgIq4CBZWT9U32c8bj8473ljtNUPTkqrlvi6ybdVNS5Vx3RdbOuiepeq67IuunbdAmmrtNmlbU3ajbbZ6H32Q/Z9QfdLzRgpq7TZoRhdb8bRKMOjS8bdKMaBNjkzYxPG0SVTkj0CbB2hmeNomqnhFoE2+UTPGSW4i6plCDSbE8oI8sqUb4Sj4HMojhfeyGzi5aW7dIrY2RocvF0niNuPYvqqnFS3QWvFkVOzwC0Pejvah3qNN5paez0x/5xHX645kvyp99yyARNxJ9EJiD5uSfqaLVvNXIRFVahwXYU9sUlhkKRbWXh81SQoYEBfw4zJsVsfWQfPs2o8+SSL9/z01iCpqIt+1CMspvI15Sl5FII8OTlopjkB/sZRdw23wOyzBSySyo5FLfB0fbkksulznS3bLzyD26mTtRQJhtryJ8qSLMOe6cWL5O+KMTHujgzkh/yO7nMExMGUL0blJMxsmtXrtGTumNEG9gNzk6rPcGhPeLU7M4UcQCQX/43w9WLrEfsjk+JPDU3N0B376t86Slwb37RHR8Fbnlqwg/zQsRXQaIzjmbKQtUl0f0awk8BN5yFgfs3BvxPctQkfvw282ojfi9kI34McpHiKcj9c71pN3rCTvYE3exx+9kT9jNnryjPWlXe+LO9rjd7b5BFcEH4jhBDC/YrBtGhqGGcYRG8IyzEPQfxwDiWEAUEwhYKYoRBM3pZwax7CByVK5HZttXjJewL78ET370ZDAKFjOOJKyHcycEPQTL6aML/lPOV4wh5MKaNIRctKblsAt8Yg65EMMeolfM+O300Vvq47fVx7EM7zrtWUj87RRGOKJJRwTxiCYfGxCQSBKSSESCtYnchb8JJ4njJTHcJJ6f2FdD0sMkspdYBhMZwplEZCLJjH/9TLNDL/t94hsSFA9JsbBhzA3hLCY8sMk6a+eEwIIjUl2nOqFfqGhy9CApzt4YDrADD4ugOlF0J2R1yKphst1rvkujbGFztrMcLP6BfSGTZbjhWYWSzH6xd7MzvFYVs3+AsjKzwz+zNFYf793qme4TztnqGxKSaHNP9m/ubBMv5G1wXlqfya/PPjJji+piKVMjrndf7t/cGeYNyNvW2u9gaGADz5JozSDwCLZtVI+x8NXeLZx1doacrXxrFddjabsKcXb+2/7tzMSIyliZFRZszHJMEx3mZIMtiDyBIlGBgAh8MnEcjC+PVCGaqSj2wimQZ0kzTKC0X2F1pQp8UpWFXwr1NtDPe28gLnZVpYH+xgrrbSCnAVbW57kO2YQvEF63QMvFNMf15s3h+uSldRH+izgfRqwfI9aX4fVnQM39CYSDHow4L0aEJ2Mjb0aQsod7NSI8G7HejTgPR7SXI0G3CXo7glUIej1CB1+cT8EbL+KSMYI3hfsY4vwh0T6RSH/DJpPJxvUJGnyjeoT+LNxnEh0REz8MX8N7WIQjJdaZ4n1R7EbgKJ9EdOIQfjk2eQi/IyaByPp6RBIR64b4RCL8nuhkIvxysQlFLF9ObFIRfktSYpHVTdPEu+ITjPBbopOIrC7HJRKxbopMJrK6zIAe+8eTSCP+J3E5RtY3RecZ4fcEc42E3gDFYw9aFU8RdB3Gjv1L/158dBvV020U+DILv1G7e5COow46jopwHAU73d48R3HbPEJBmXu7R+hS/OPpWRh/CUsmEo+dSuKZ2mNajd0TOfh+vF0GCV8GAAYmdvrlGhPtnntgq5LvLAI8Rk6cuykEDGhDZd47uAkGxj3/VzuxwsWF5VKWRMGaWop056Vqn9LYd11yn5UNlSwAsN5T44VSZSXGCMqMLB9fdTC57MBFYTbTuOOpUL/efvr3LiqkU7i1hTmteqIU+jasQ+ITo1tLaAnduvIoPFOwPJh9Tg1WK8IOmyI6YxD3psG51is1rFsW6owTskLdeztOa6Xp9qzMwWnFRXSZZ4m3GSMjRIfFUTRM6OVciZQMnagv3LVK3vLbdCJLT5RIrNLHhJFk7fWYQOMdFeoG3HmlKk3DOAPlhp3sN+MbgKwmeMv/FFV4pGIcWY1TqM8vo1xQ+zSuXWww74Wswqxj7TIgb+npwylMOtpsJulPyba9ytW2KVJM7dO2SUJKkU690qSt2usU75Ld1jsLC/Xdue3oMLwVwataB89NLa6t+85GF7ecYKMfb69+vGCKM68fL+p6GtcXevBK7MFzMpLAysC6dkxmq9DbnVxhoTt2Vqk8Nnx44H7v00vmeEEXynY7b1q9lC6U4eG4UMrk7ECXxtYujbCTaHrDQSu4XLmdH3dhua7WE2NcvqsM/B5Bo2Tu+CgqH1bOQqYr49MJ+WTVoFDJvbCkWUUa8s6uQjbieRl5kC/RFlKhfVCh8Kl52I3r9/5kXj7KxGVw0yBvT1i2E0khb386ahTHPbbmUBEm6MWZANnWjmyrZCFiHQwRyy1ELI7gBJOSbUF8Wr3+4QSPReYu88eIhT0Xg8Qqy6jKSVmyzF3meJSyyrBWLFSPTIu2M9spI0b3pU5DjI7uCnRXIICuB05ul8IP0B6k3UrRHu0VDrtOskA3AcLdvPdEfKLCLIVX4HjzN+kzYRr6pujjY7d/xX3SK7yHge/yCjHpFfbxVyleMUusxdWw2xmneQVNfEXALb51c7cSXmFvTU3zinZSNS7Ou53zVO0tdJLecdWd9EbDVN02yVT21ts0r2gnWcraJFvyV3SSuq21eTjN+E5qi1G/PRmmGhlikqHs7expXpFkKDtLb5pZKslQ/d64N7lI9YokQ12NRp3uZapXJPao8363n6rT0lZip+W75nM11PjyfJyquWknsS0u25N2dcMGWFz+VAM8pE+dfGezzZzdtnr21//BF+wp1kc93Au+gWLW3TZKAMq8ytE221O0ADffiyaxEIE922/L4IBfeaE3MuBFDQTI64/+g5mSpcdQ5oja4+7ao6TGuv2vPxI2E9lefz6qyu74RzUy4+BpSSW8CwcP1A32P3T919j1n1rT7LYPzcOPCWBQ7AySGLakhuZ7yUPszF/rzF/qzF/pzF/o3EXnrCJr3SY6OyXh2tPmWA9puzVFloh/DhPgq4WVI9z1xZJfgyqH3BC2CRLbouI8mqXF/8zSS3LtBRgalxAwniccpsViqnyC7kf92IDzi2Vi3E/l+He4GfpxZljEWWGvdUWKjhS9Suf+jVIy9FZ7gBQdKTqG3++RREnq1EbR7gM/dgbwpHAEH75qp8L39Y3F5xDekGTpK09riti9NNi91ey24gaKPe+ZihR3ZsifTWjcEgfvR9S8HbtPthxwvRwutcAJ8KGnynrO+QEqZ6cLOyMfqAH/Zw7x9XntzsDToJolPgu+FXUWfD+bs+Dx3BU8d2V9U/K5KzkdolKVo1Cy3v/STMNEB820TLTTx7NESs0325nxTdbx34WpMtsRzfKSyGKP23Dm/xKmx3eKtk7H/vIoiY8EMKiV4B7WGiZ5s3aWrVztp4RcG0TSCdxOCYXhLsFQZznb7ym7UxagjDOeqt067HNpaktVp/s7IOK8rNY/99nePn/ANj1Pac2NvLI/OwDC1IEkwt8Me6zNbSq8pYR7XZVNg7LrlLnfBO21vu4222y2Gc/ICmgVRNpFc2HKPCYEg1hxAz2GrCJzROaIzLHCzDE0zDgD6thD6nggrspqUcfNA0ZX68f0/NY+HXLn4FPrWe8k+F/UQZOBvdFBVmLNEc2KBah6Dsly2bSE7MzX5OuV3j5faAm8x6ZlsqyKAlvYVYUv/1aitb0RXlcHK61dne6/NqtjV3suZ3ZlieqEtbqwvZ3rQXAtixWXGA55LfJa5LXIa5HXIq+trUd0mJrWjlpIa5HWokc0hUeUZaWu2lmcnsKzM2fhn5N7dnjczKEwQE8kx7P67tNnIku6sTc2uCqvoKhKZY3NCs9wCfzzCthE575Qfii7dVR4mMmP4StRNmcARoixWM518lZ/BKscZcMYd3dFV7Hbn2/Q6Tfv8uMizFzFDn++cXe3jS1TQTdOnBCBEx418BZa4Ch8CMS0yaSe0QBkQg3BF/ON4gmKJzuLJwk1KVY8qewe2fO1AjAadbvN8zj1pDdsdZvjvagnE0kzXt1pTaLUlYvB5OL8IlbqsHOG+m55LzCkbQkYvfZFr5WjFuJXMiwBIQcl487mhFGm59eZuBRT79U95CeyKlPcraubwtW2Xxbcg/B5LZ/4U7iifLJxWEB6/aTTPEj9pEJbmDGkvKy+c55YhvGvEjp4V2XzR5VLuk1a5FdyTxklWeUFYncbKpm5990VLZSsyn23WJbZrFC8tWU537t/JVefby950L1BF0tVEzQr6ZLVDvc2lNqX7MExVAlNyssV0ktfBMngjFklMEQDvZR5whJ66rgWXHo9kpn8YCFh3BCPNBpjEDAGAWMQMAYBYxDSxSCM0nPoIcYgYAxC5ULrw7LcdFudkPw+q8D5W+a5/dl8oGzC1YMEyEKcjjPSdvRy3Gq5ftfBw+RFMh7JIzyKLNmzNg2uP96pTsNS16lSGwYsobyEPNIqGLTfZ33dXJJCFtb3vEUIP4F9tb/+mVq7vBdwD50dE92ErwHIyNI9kHdpj9vleacsoZF5uRwbB0YG/3hPjRdmVB2QtibIZKmpbBM81Y8BOZq6yxfOECWj8Pr+YkG8g78og2eWDLDIuaugsBFepZvFrNh5JtsWWRWfE6gVg7Jbw2qGM94CR07b2GE80GYbtUE9wkRuLYOw/mj7XzE+ZA/CVjhoasamJ6+6BBZR50FsikcUyzDoBINOMOjksINOMhDMWhh0gkEnuGtnB/nl9rNiJ5wrozqwKhwQnxtB46yHLTzHludIFBTmF/LkKFQVdy4H3dTo/kSXT1ap2HxTRuu6ihdu3wWbpjzmXWtafMuCXvs0DdyBh+ERGB6B4RHI+JDx1ZHxIZnLLvih1UxL5ga4gQCjH3ADwU5kzp7Jxo+m8lS1PdLusgMbecfsoAPFUxVDAiBv6uQeLj8xhvJs3UoAbgvr+O45W0z3RfXs0vNVp6KWt1bME2L9q0tfqdvWNqdmJt+bc921UFfUxjzMwR0EG96VeRxm3fch2CZBco3kGsk17j3AvQe49wD3HqT1pabn363eCHcfIP9G/r0D/76A9UUso6vPKpgvnpdtLLc4tXUCOLm37kreWH5eCwJiGQVDRJF/IP9A5x4699C5h+wi3rnXSk0u2iP07iG7qPaxYasctKGnfEFZFZ2BAm4t+sAh38k/GxXeossSFhe+p3Gb9Mru8q1JjqQQYZVqeaNdcDkfWQ2FKbMJg+YDmqjZG515nO1qTyHf9lz/M6id3oM+KuSIyBHRR4U+KvRRoY8qtY8qPY3sN9FHVWoWifv9ykrm/s5XklujnGmmXaWDxdD65MQecoOdkrtHSWdBcmzF09Y72PZG66DPi0uzcolloNjjm89s3x9wPPaXE+G5SuEzewUCIInwxSvjfLIqzPhWQaAM5iHQP7v3nTHroJcQGSAyQPQSopewjF7CcPIY6iUMvxW9hBl6Cdup6V03W3oXNjWhmxDdhPUkeGtPHVtbSsXuvEUD7vHzyivH3E1M6FwCtLZkT6aMMo73So01Kdkby1sX/ZP+PCupVVnRvFZlMZ7WtjKBiByuLGAQsfNuHJOSOaw2a9M/Crqdw1iY0WNybxrQCM7NMID5DaxJVjftbTvgut63prYsaZOwokV3dOFefab8I5/L2bUzlt/4TH/Vz54XZ4o2ZY7rDDPolphvr134Y+YDgUkZ3a57Jd0JVLUR0S2rR7obDaTcSLmRciPlrqpLNTXnHrSQcqNPFSl3SsrNJ8ByUpFAapcVC4G/SxAe+7NDdKB4JfRMu4vnEDpmV27O2vtC173o1l9dJGboDUVihsQMiRnumERfqI+XddLvmCzwMNiweYtlwrtcLPnAjbr+L6qpEZeR16ErtbK8DhYsKhqwqLT+VkJG4iqdZ+OI5WayjykVdOB5kp7EUvKldquStt9XLk7WVfbUVr4oxsrdp+paufv31Fauhzd0ZREk20i2MfQYN59uSbZx8yluPkVPaR6MvDPAzadIqJFQ70CoPyss06hR0gMRXaXzLIam63t7l6Rz/OQP/HhE/cUiKKpp8IhbgTwLUCrjlf1aozB7Kvvzq75nKV7zPzFjB3u/f8dLBra+tV3SFt2zCsyClBdOulpAFhyKLNi1t0dEfwULLQjbnFp736u7WwrKjNhmwy2pyAuRF6IbFt2wtXTDWreubgpXEtBXm6GvtpueGbYxuy1SQ6SGu1DDD1e3QA/1snJDKB5xiufJkcouWLRFt3dWMp4CbwbzO/snrU2s1v5Li9jwe+DdrF/U/ZwNsBD6sJCrIFfJiau88b0O5o8lFfxDMzAs4R9JmauN9Q32wNCgbm9creKUuCGqsiws9cAITYaBobit4VvBOVxrd11f+KAah2nt9htfn7HAV7u/kuUbS0GhsneAxNG4MmDkUkHhdictEh70DwYJI9QtF9Td/CAH1TTqe4CDJhj0LQfYzwt4rTFdPiwfJCURVP/7T9Mp++3UAcm/HRWJs/kmI4DVJ9bmKvhLhX4lWD2zSBdKqP2gx1TCgMwRZVuQ/bmRCevhLLE2qV0r5Ix8NA3kHRXIIMK76NuTI2gyGO5vfzoqcNMXMpHivSYHDbdDg5JS4+12G+E2wm2E22WC2/rLsgJwm5fSxor6KpAJhu1+sTYUqwpY2ypmwHyqaRwI0Gb+aUTbiLYRbVckRgn3ruDelSyjonDvSmFemU56r0wPd64giawcifzTnx5hhP+d8qBV9f6/VDT+0m+1hkE4sKKbNw9zwZRXIvhC+C/blqHMCHQ1+Mu6GCeL2yO/GQLVXZwyS5Ia6ElFuIS4IUpPs9I2ZrFuIihfRczqlJQFtHGjMoawjT0virDnzsYkUK9qt4+krNqHd/Vt22dcG6mBXLFa67jpaR8SQ/gC3OnEjYYkMWIebM9yxRxGVLqbotKoUmyoUuBOKvsW3EmFO6lKLVSE+rNTKxWd5gB3UqFUgTupdqb2qrqYPsH43wvJySwtINSCsFpQjUjKs30EnU7qu2Xq48f35O+8wuhMLQvT6XUGqZiOGHKaXfmpDtQaqc52VGfHjVi/8Bm7cvuwOhH7sEadKu/DglVe1CQOVQKrWMk2aXXTBo32mxUD2dWICkW3XTD28xdBkzh4+/E0JAZUZFOAFc0wal4121UL/uTAm6GZL1Oqw7/spOop2EnVACqLqjLLMZfBYyTnzAOQXzrVI7x6LITMrmHx+e0sg6vzuU6Nutn5DopCrKoxG99T44Wdt81O4Zad5HfMAwo2hhEL9SUiy4xXfO55qxFgKq9ld38PWHFhLrbp7BV3ufGOd/uqiJqqSF9hnE809hMko/uN7I2hl5FdsoJhvDlxZdw5V3eCFL6tLjVDGrWQISFDqp23wIXcZFVd8j+nsIowzGZU6NSgm0dYEk7sdNGsJoS9nwiz/5q6UV8nAcdoN+/ekfNgRRGclW7bVfGYBvdZITYrt3jdSQvNWv0hQjOEZiheZwkG9VdFnDJnpFmlsyOvdUv8JKz4j45uZahEIBqFZmQZe618vORti/yVvFL9mDThD0U92pdqPdfo7yZVxNepBVYXMCCmPJ6/QoZ/x9Voy/arCpF1heqbAyGgkt7yQYNIHJE4InFE4pVSSTvpVdIuQnGE4nVWSQ1JfMrft505QLu1SrwKH7CgGqvM3sI1DEEqLGQje6bBfqJwKgjA99xQF5JIOBZ8e3d+fWQHbNQc9r6XdBGBLsYDIMgtKIp8NfxurbgvNgqZ1FG5iPFueMR4Zw1L8eSOlLr6ICWW77cRyucA5XHjZKlB/lJTRX16bwUSlOrYOZh0RKrrFDC8HebwIkgGOw2bnTN+ffaR6dyiuljK1KDFY3rLcJqpKFCkkhpOUggroOUGMIRYM1Ucn68rjXp0ybLl9JrNw9tDCrVu4R7Smkrc5VKx0wLfTquHyBeRb1mQb/7JQFf4jQMGHvVBKyTFfvtmgTny/Xt9A329eI4inEM4h3AO4Vz9ZcxWWjjX6mJ4MMK5QxIy1xngYEJ+0qe81xSa/W2jrfk7SXi8Rvbe+0NAe+SKVRjhXmngXr+bCu5Z/feM9eCqQb5+FyEfQr4iFLwMMN8IMR/GoaZAdptvCXsvfDn98Sz62KAqnl5ri4HTZ0kzAAxMFzw95fT+1aB6NkCy2ORNazxl14hYNSK69JXyrFqsZoU6zR0TA8bkJxI6NoapY0M7lyUjlrA6btFnXOFZkGTubN/AwjmdH4Q92WfncaF23rw3l9DW2XbwSV0oqZWrGCnpfuO+Z1SM6HErrhl1vSwsE7kjcsfMU2Gnj/7oIHVEd8EeSGWY6DZojuKOtr2e6ESWFpIRl4vkqt0d9ca1Iai+aJWlVKl9gx/MxT3VGJZkjbefMG3LcAyarz5VKdNfTKevTfQ2q6PdVVhu73eB6iLixpgfjPlBEF9PB1B6FN/EGG5E8WVB8d1RvxMD489Of7S38oU6kqBOis6mWW5V+sDPIj/5Z6OKjiWOQXV23jI/mtLeLujk/9hDpFLm8HR885l8+0bEpUm+fycnzjEhRF9C1yV2hYlAbOhdKAMIawK75x1OE9y/OrZnDeHeQ2s8SjqBH9d+VyjhzW/au2B18snulWfkV8scSDXKQjW6w3481ah2+peIKiPPQJ5RBM/opPcWYMJD5BnV8BbcAe4QZgfvMDAsM1RI8T6X2WGP7KRNu+z7dhtUz4QJfb82WRntaqLrAF0H6Dqod+7HVUBe5ZI/9sKTP7b7I0z+mFEY1DAlr8Et08hqMmA1Gl1SIRSyVHAztajCyvvFmOovkiE+0n1K5duI4VapiVPqPTkcWOeqwUZ0nvZdM5eGXl8S4fSYW7vHkDMSUW1kEgdyrhFG/aOQX2ohPy3eHSDeRbyLeNeRe2VVmLUKTOLd3eCASWFGWovilW9mil75TNFbFLqXet0rSmiLVqwxxtVG47baxyqK2Ls0Kn47VYhKVVX8dr+NKj7i+fzl61ZaPN/uYQKoPBA9Rt/4gmrOTn8ca5IhiYL84+lZXFTNZbs76bY3jao53ioBVdKrr4bdzrheKaocVdnWJesQt/7tG2Gwygpahw+SMldrndR+LSuTCTUESUaEi+oyqsuIRsuWj3SYejtqH+EoCsy5ANXq7PZkS58uSyKt3y7D+mK0W2er4Lr1CP1CRZPFo9+/EiqIj8wICN0qs2VwLyAPtw0iHqxLjsG0cLDVayEcRDh4QHCQI0GwjqYuX6ertMnTe8nQS3VW+KVVRFdmZ0MlmqDM1AXMo3xTFfyAAihlk2NtQZ9tBsR05dk21h8e4rax/gghHUK6AiS+dnqJD/dMIaY7OInPOfdDXJp1SiF2B2+z8oXVF+QxAdNY1ZMdYWLqdtIA3Qo6tHe84aYfTAeGuh6CwLrrep20GLCPycAw6DC/oMNQitxpJ6QWTg4PHI6aV812bcIDHUjKMlXNd8pSldtJciogLQJmpQQaXNSkZXYy4pbHTrKSlMU6H7e0SsX311+xmk4iaoroep8qa6dziCprp4Mq63YAO5CcK3Tjb2AT33tJFyuXrasfnq2r0+xitq6sAgr6qQMKUHxG5lEw8xi0+/HMYx8boyJKOkjLkWq2hYrHYzy+LFRlCgvTcirC/A1drEIZfVno7aO0ZJul4E+dKoCB2QdWneKzRLhNCb1+KsiCtsjfqnZ6s+AsWqTFnWFOzlmli09L4bd9ffqyY9lssluksekjANhD684/Szo8K8H+k9ztz2Ssw7A8rMIxlr6sturzs6DNXgSN8gWKBRCa8Dc0ssSshgpQWRSgQS/Wv2oPxBgNyNW8VdOBBj10tKKjtZBou37qaLsmCh4oeBQqeLT6vR7KCEGsKgJeYSeIzuizJMJHU5uy7b40d7Qan9U7u42+1kTAsCpDLAypSgpZJ2spXm3wWZwxhJwtnodHfBeDVv98JtWAv4ndgsRqQcT/ZcH/rf5geIB7p6Ha6AJG6F9EjGV65N9G5I/IPwXyL/MppeoLBfRsLpfy61RVWCPWBUUDvHPXjoE+q4L13WBzw+pLrPoixEOIhxCv4lF+QWdOJSP6BhHnb476GNGXEcxtNdMmMB/hkUQIc+sHc115IXlM9GyqqwxbTAVRpEuDztgxmwosWKwJapLU2xE5x+ua1ThfpNWs5Fa1ICPiXszonQrMIkRFFTL7jN7p8RnuuEB8ll8Awiqs4EqQZDqLiym4andHvXFjt/NjrhWAMtIzjX3B1WjYae74gvPExw86F1f90Y6Pn1BefsGQlIf4OowvBpM0dUh8xXh41R63G1mGcATGUz6yswPFTWWDfJzHPNQAPq4NX6rIgrjGyjswI4UhafmMSIvfy7GbAWe0pH0xfnbKexPHbsac8wWnVGaMWgPz3oWxmwElpYQDOnqdr8sGC0d4+Mwai9yyxkDx4RDFh0xdaag+7OQgswdj5VxjwwjX2KBZZdeYk1YJplcrCpU6yezfCnMYgGRBgTboR0Sdk+uzj0Sjv8NkALOoqC6WMmUn/SxZJANPIElgSBB4SWjs6lyi8mysKnPpIYSf07lgykawL3KxQzf40vDte3AUhXH6wNBtqB7vR+QsVi4/YSulDrXfY45ZCNG/qKZmpVFp0sOjcRsZl3TQKasFmWrGnYOvkmqTJHLJ5oOk/EI13Z4TBqedU2+SjHyEsNBESbtJZGFRLu1mJyEd4icqzE5/DNmkA1VVdLbc83ahDwKDyyf/bKSOvIEydePLpM+EaWiZXCLZ5UW7P8mkML2kwtwnFeby/GLYGWVSmH5SYcSkwvQvJ83LZiaFGSQVZpZoGWt/VxaFGSYVhiYWxkqsk0VhRokduJVQmN6w2x0MsihMt5lYmHaSaS7Ou53zTHpwt5VYmk5Saa66k95omElp2omDO6mhmue95nkrk8J0EgvTTmqnq9ZFM5PB3U2cg+87SeOpOei1M5mDu4lzsJjUTKN+ezLMZHB3k+fgpGbqt7qTUSeTwiTOwWJSM43Ox93heSaFSZyDZ0nN1O+Ne5OLTAqTOAfPkprpajTqdC+zKEwvcQ6eJY6m8363n8nQ7iVOwbSVOLSvRpNMVqdeO30zjS/Px5l04F7iDEw7iX3msj3J1qtZaJjhTNKfphqAfn26UjOyjCsMaPHdDUICLenk+3dyQhgfcQkthfosfVZ60SSDFmWm9nZm+pWXbSM7VXyP+ASaglx/XOpkHFrZDfwYodIJOjK2dWSEzqmtYT/Oqyepy7jsUNdMXBVm5O3JETkjfMCRtz8dVW0zEVhhgJuJahqpGe0tUOfk/pVNw7wPz+H3RNV4Lzaowk7KXbsFXA4D9BCkC1VN6yLodNBHgD6CCvkIdg+WzV3fL0C1L0CLL0BhL0A3L0ANL0DjLkK5LkKPLkBlLkA7LkARLkDnLUC9LUCTLUBpLUA/LUAVLUDrLEDBPFRdcsop1R63OqdRLi0+uG/REthneay4g7DJ2HOSIeugan46+5VMBENAPbM0emanH5sV6CJWzXSJQRUXNDuDJuqZh6NnCs9UEx4oYYOUxzOvIp4lXTfpzKdlwqd7SmAFeKazU3L3KMF9iiib8GR+G3+MvmRJou9f+Tfr5ykE/jLhlcrMfy97oCSyNMBwYXFaVZ20U44zFFPLpAOUSd0r66q4IZfZ6cWosaLGihoraqyosaLGihoraqyosaLGWlqN1ZbwWDhKCpGQnJHiAkuzTM++pbKrTYUXQTL2HouaW7PlGeladFp9V7u9JLZbHVTkc1u8+RWqSu78MivKyfuUk7vd+CNE4w5OhRqe1kJM7obtwkMxue5isqXzyvzNLFTWowMbj4JBXqhGQxVmDJFNd5xsau23h9ovar+o/aIii4osKrKoyKIii4osKrKoyOajyEqqJey9UIZT6Sy9wtcoUiCt79k9jrL2D85kb6WvKK2VR1rrtTpxqpKihh2lulLXhN/NE/1riY/5Ca9yF4W0wxHS1rvMuQrMIiYtDXilovE8tZ5ktGtdjctvqKtlGFLZSSurtXDrOepqqKthTCUqeKjgoYKHCh4qeKjgoYJXl3yaFhsrU5ZIviXdKlYJEmmWzz48i2aygeqy1/x9sKKoYe5Vwxz20iTPPPuoVy0WsBefLhQlzPpImDcrWYJJmNQ6zcza7z0zNbbN++VREh+9J2xFhgSSt/eCMuPaDDENSZa+CuxFvtO3jk7JxLpdFwxTs25RRdHU4NGPLCkn26zORzW8hYiyqvPt7K1m88/8UdZjdGsvOhRxVTAmZgiy/HpKyIVpeO4N3iwpZClocDuVj4luQiUFnXw6v54QQdOEV51ruWyu0hRyezvRj61y2YLvTIWHKqoBz5vLMGhY/SSNqbx8oMCiSWRpIRk6bpFPFyeZWtDt9FHPRT0X9VxUWVFlRZUVVVZUWVFlRZUVVdZ84ySrFR5JTsj1x32qr/A/UdBmFbWdXfr6qrNsxza55QnwJipTEQKyHmq0+9Ro+73Yo31ssmYqUoxu2fgzk6wqFm3a7w1Rqj3EaFPVNNiyMPNImgKXIYm1Ylh/6wJLF0JeBJ0YwhNVTsm1Imp0AcOBzpjiuf6xRsmD9GwdiiQsl5q61CRYV4gOPVU0nBun69SgM+p5ErzulcwlRdIfTzGINU0Qaze15tlFzRM1T9Q8UfNEzRM1T9Q8UfNEzRM1T9Q8c9M8FfWlHEezX38kUJb6bve+tritoFDV1HHTdxnFuEF7VK6AybxFuEEHD+I5EBEOda1UwXzpha1mC5WtqotTTCz+yoyIItWWK0272QnJLlKAnBVRmF5SYXIRviIK008qTC4SWURhBkmFyUVMiyjMMKkwuchuEYUZJXbgPAS68MJ0m4mFyUXKiyhNK7E0uYh+EaVpJw7uPOTBiMJ0EguTh5AYUZjEOTgXyTGiMIlzcC7iZERhkufgPGTMiMIkzsG5CJ4RhUmcg3ORRiMKkzgH5yKihhemlzgH5yK3RhQmcQrORZiNKEw7fTPtIOFGFCZxBj4MsdeOdczpIJ584zQndtnJquwliHnNM+dAPkcWrcx4EEkJrj8u9ZCeQ87Cqo/K+z6V93Yrfud+kvRevVQFUOMBau9bae9vfK+D2WpJBf9EsJ4EbmGOZPkB2GTQWF+2B5oGNXvjaj2nvGzZt7IJ+EZCsu4eqkA3fGIkF57brvXPH0xpHbIzeuPrW5aM3B6sHtVYCgqVvQOu/E4JUVoIvNAd9FbkFYXbaqd0VnT3m0kWvREYKruBF6KcrIvTBBj5VH+FZy+mwrMgydP7V4PqSRTh2ILqf/jfD5qqGnP9h+9xCMHeccdH8m7nom5FIxaqqRi8+TiVOGfV4gM/iCeooUliOIPJkaT5DT/XKN2X3YMTTUZmv4JKZUPbdrSqLn2tn1UDcVheq46rTYavVq1H+CRJhNCxuwEDjt6nggQ4HQHuDNuxBJiPuBgGHHW9zNFnUGekwBh+hoQuOfwsPaPD6LPy8T1kdZVldfxPTjHqROoYuyCsaoRVRK/vZgwGickHVscgo0IcvFccPEpzCl0jrvOWHAyPegiGEQwjGE7ybrTTYuH2sGLeDfReIM4tGufOZ1NoUGlhLhIDm3bGsztEhb0XvhB1SRWOUvX9+BfANIIsq6Jg0FmBxknG8x83McxFDZD7xF5PwS4Yw7UP6L4dIC8zFkeEjQgb5ebMIXZrhHoz6s2IwzPVm2slNa+EOmINkUPQmnfJ94Nac/GA1WouRK11Rq3eR5/x02w/KjLX5P84HDWvmu3ogbP36PnAdIMAOEOJuZM63GL/aawvF0vjFdEx6tflwc2BNsodNrOj01WY1AtEzlmA49VatM/4ecskU2hyVdvafsdznYFL+MJYLDOxZ4Y7on3Wts/05RWtt3BuR4FLyqqHkTNyGag3spJ9RsD0B2mScjZE1rmptgt5aYTkECko+qU/aGH0y1YsZ9fd0Ot5oGp7ojvNqD3R7V33RPtwLBWp9EyngGXYdDKFqjTdiQeP42+XVXfGOT/S1QRFX0jGpg8P3O99esmYabnYWzMle+s3D2b7sw4/+tVmSZ1ms6bOC9aJ34XJFduxr9IdAXTHJonQ3FV8+mDrJ7cUfRAMmKlO/lndFFYKNV5U7Wnqn3QzzL20P0LiStH0yarfntJbOVYOrD51M/OdXcH68r0PVlOyms7nkkjuX8mN1ZjI9fbrgVrG5rKyRxw7J5O8PTkCgg50/e1PR2VOcIVErRZBVOUKQkoN41vddmm8MGH9x+WGCbtcHhpQm7NIkUMgh3BxCLDXPpFtSopgOSz0sjCFqhrTLn+yNWvGBy6D9UUugFwAuQBygdIp+t30VKCDVACpAFIBpALhVGCmqcsKUwFW/LIQgYqa0vEIJNmyZjRg4q8tkgAkAUgCkASUzyGQAQtoIQtAFoAsAFlAOAtgZ4QBMtOzPdOtYC6wrkRZGEGlzbriBRvZtWbsYBxeZ+QIyBGQIyBHKJ2joJeeIvSRIiBFQIqAFCGcIixMAD6ioBsVZgirOtQ36ZIfyL4PrTLiWMSxiGMRx5ZP684AyHYRyCKQRSCLQDYcyM6luVphDMuKXxZ9u6KmXCnbCbasmaZ95a8tsgBkAcgCkAWUTs3upycBPSQBSAKQBGyUhK43CDmCHdmCxRY0gA57wLjZn0MQxiVY5Q5HC78KVBcR8B4TfvaGo7jhUT+oHGqEURPzfyKmLkBZTw2q2x2MIkdQfZiguiqwdR16LEDhqVZBdfbWABKhG4DX/JU5HKQ6DqkwYtXSHZmVIuU8yrMIJasqzw7SI0lMUIhIEuVZjNGI3jkny3q1N81BBQ4IsAaqi3AV4SrCVYSrJVA+U+PVfutgzsVBUFgNn32/34nx2X+4AgCisKKLT0SWQs/7ENmM4T70OYiZrFHcrBistBDlfLqywBRamJk8EUkWCRs9TWQXsPhjT8PMNOUdprzGiujPtQnbvQI0bVUVAXVZYhX6g2Gas0kjx3cqxJr34aT9wQhjExCh5y8oD9MLyk1E6IjQDy4AgGM4QVuWEuF++0YsZZRwafT80022OLeM4I1V8jKkkgjeSqeGXpYekyHUQqiVuRiaHmsNh4i1EGv5Cn0gWMtxVC8Mc3r/apQbcV2wAtYXa72/+4wYax8Ya0ZFaQEDKmg2T+++D+9+KwAWdR3hF8Kvuipdo/ToC5UuRF8Hjr70JaWz8uOvW1bM+uKvYPUQgSECQwSGCKzUAlgGEGyEEAwh2GFDsNVOjd9NCsu0zItXZjBGyDV7xhxsudpYTHjhiVX4+gK1f/BavgvWEvFaufCaoio0OecNAjYEbAckmbWaqQHbqInbjTeb83C7cc3i1raP+m132vHZINnJ6QAsD2Nnsq7ODcC806Wmivs+yZPdmAk0Ht98Jt++iUuTI+Mbp2p7SivvmHhm9av6GdgeMPXdsnJrtSC5CQGoyDH2uG2l3Ryk2bZiEQ6qEzY07TSb9kRYxVyb7eYQ97MgZSlAY07PWTot5CzIWXCvTZk5AQMB+lT/3aT0ax1x661dsxqHMNjA9aNpEHVO/mFCEyJ6rQl6rUC6I0SpiFL3J6y3UoPU5ghBKoJUBKnljNEwl99UsBa83mBw1FwCFi1XhIbkhGRwvPkRCiuwziHIhBeafL7ZQ0ok23p20vk8Y1rYg43IU55iE43ePL7qkgiGYgPRslasvlydlKLuXsDOAjBRT8a8ohiYkiN+fuN7HcwbSyr4R2tk8t/G+g57cGhQuTeuZnGKzHIOysJSD4zSZPgcCoQbPlzC8W+76/rCh3057u203vg6jR0mMuisvlgKCpW9gwS5xTbcopM25WoHmUUWzALJAZIDD7zVVfGJwcXp3fhmKsiyKuYftZ2ZPr0qM8snxf4F2DsjrEqhcbF5UwWPLSXF1GnFbMnLzI4tEJeOFYk9L8qv7ERNuJxN2MrOZl3QRX5GtU/ZDc6qeRkcagPm/gwIiMCfqvZKoKTEEJcxVh4XYGVVWz4KSsV6r1VoJhxYfyTPA5MCTGm8VMyMxgvz9dmj/wWot/JARFmNHfmXVff7WQ1GoP6oLhyiuoC7XtA5V+4QsrQMuuUWQpBCI4VGCp01hf48uXl3fXdZOernKTfjI7OlLBl0FwqYN7OGolbRvG7TlpFZsyLmyqxzMWo4ezZnGbHnUjMVqD8yFWQqyFSQqZTL1ddOT1T6SFSQqCBRyY+o2Ah0CqBzViHMd+sqtkujzoeelBv96Zhqs+zwL9L9hPAP4V99herU+K/dxq3OiP8Q/+Uc67Wgi6JSpmcGAFdlZjFK45uVkgq4z3gEWJS07+G8GBm10naFOmxt14uawOr3lop8K32liK73i64xVz1ia5RWvdC6k15a7SG0RmiN0Do/aH316fxvlYtQWBcaAOCVJjyUMfSDF9Ly81fNtHZ0QjBWYQ623mPYx6fzXyvXV1dlBnN+El526al1iQFhfYucEbAIspWyJKlq9Qbt8mWpypLZRNS6g1mqtuJAu+6yd0Z/5bbZt6O22Xdxm31WBLEdTRApo0KBVXhjn0wLiWMWxLE2mb3ApDdU+6QG+yeephG3UA4H/fjjND6axmEcpaHA7xgJuV5efoH/Kx9FA76pdsJcpxaMm1CRQgvNiMq/2dNhGl4jQ9/ao5W3sOOqoCzMCwZ6shEr7oL6YLUTub4h1wpwOpaB+GOwzkjt9knthsNRGmrH+rAIXVqFtrUOz5CUKp6b0RqOmkj20OFVQDBZboRmiHwG+Qx60crNC6YACF8EbSYpD9WmBet6AJ4FiDdffa7vXgUXor0Kry6C2X2C2VEv1WkabM4Dyy30qnkqRj08T6Ou4NV1rnpgYiqVUN/NC9e2WvuP8CoLcEWp/WCk9sMW0MXFcnqtvNcfKq+eszoARCbvqa5Dj9PJy6MkPhKY6gn0Qcl4JZotq5+SD6pBre0JxqOkEztwgkiKKJvQaYggy3BF1al9aUbuX4kEtrpWLjWNrVp7luRZs0FnrX672ZWAhgu0GzSM3XCCYdDFkjWDoRIdfp+iBeGFiU1YcYfA9fj92hWAtKncW70XTrevkrSPnAc5T6Fifm6kp9dEzoOcBznP4XEeCwVWnfVYtQiDzwHaQ+4BYcwovARmD/go6ORReGaOBQYYT/QlrAZzSSTUeuTbe2HGrxDxkYpPurnQjwn7TuaNekyoIZ4e1dcJwaseQhUQQyOGRgyNGLpqfoNebhAa3QYIoRFC7wyhjzd/0a0yOyys/nnGoPrECRioOlpfVQQA+/pvB6Dv2anAbA09uSbGdtfEY209ODPXSPtnKbhQ+q8GbVnFQSFvQd5y2MH6uZGTVnOE7ATZSf3YSbUQfM3Edoay1oiSh6GIqinPiKIa5J6SGZUB0GtWwIqgEGG5lCVRYB2vBCD/g3oDY73irWFXItAYK2+HCmYnS7iHvEjGI7QMkSUwpxKWICfHrFmhTfBOAqtmOyriDRzZl1lJLBvyv7Lv1ONGhj33k/h8b87rMJm4a2L3Yfg8p5rjfttAFJgUNF/cKrOaWN1dk1Crs+DC+p5VzuqLHs1ybabqtjvtMm6myj0VAFQc876hV3TPXtF+bmnPRig8oPCAblF0i+YgqtyJTFS5pYXv/mHvMsJHYz6SC6sjT+T1sIC+v6ZFx/ZWHxa+aO3zWRF/SbHQ/Cm5W20NWm8M0v1PYkLBOgE0hRreyxIsfzP4paIAUuGDdr+qDWtvGE17aPDsvbL+FmX+2GBrrhvF3Qw8hJV+8d4K1w1BUtgXqiKzeFc+7heSYWySQq/iDl92lhU6fNHhiw5fpGQJlEwXNYlj0CCnKZc3uJ+fN7iLuduQtKG3uMTEhiUbfsc9dawGULuXYjHvY+RoyQMQ+2rKDieFwul8/5blryS/m9QEQDwngn1gDVHt2/ceMupprommLuvfVLyWjL/884POHKHvrm/vLj+sjz19UFQttl0uimsX4Aa3r8oniw9Vmzh6qmKZ/wT+Ozsf/91F+HiL3GtUeCIz9UVZX9F1Th8BbMOPjqHH6MYZqwDD4a6fZ3PwUyqab1cxO6q/XWCA6/Ve2cXNqU+s6ABJP+bzlGIu7qnGZih+sLLzG/cvPBwdJjONLGCYkKVGnyXV1IG1b8XZJ40MLc5jIfYXhREqb61ELPKWnj6cWhtvmXVXe3GPYt3U1dAt0BddBdWi/MeKoUKBIelZO4YHuaWPx+2yqCHUz/EbGuLT68YfWPVe+DJWFYX8eBrivRXZ5MRu/ONw1LxqhkVJWRNGs2L+3AACHcOAvWSOx/zO0i2WKq8qxPQMhlpdLjx2dvE6R43j5GO2oISlfJTga41c3t6dXwC3/vlywtjC+N3H28sT8uv59V3xx0i7m8rusXVpKGcAnpB3EpAv5nZlrcLnXh+vc7ehK7uQKChEN5c80P7tRAJUK4msHf/TOGn9p3FUb8frOCJCAFnMPiNq+6N0pyjHRH2U+nyC7gDP1kLv7d69t7kxpxEevIXOWyRecYGwUNqDi4Rlnq7bV0VU1Sew3JUgyXRWYzekv6rsWDAFHiHNmG+L2NfKklQo0D6fqPh8EK3DKmo5Krdpk4t9tAmbNg6iTfh50N42SUgHNa4+SYXqj3llkaJWwtGGwcFILw/a9TbMzfXWRtcbMkAM3811X+K5yLjlxyWtesihqyIhbiweAfcoPFOyEGaUCGQmAbw3rDg2id3DYhGZo4SHKl5+uLN9XHOYgvn33Kc1sb4twf7CG0HX69Fw7pqkbrlP418m/pZzYn4TWq4GDq6JZZq7lWmQQ5SdQ1TBe4WsIYY1vPG9DiaUJRX8Y3U9Tj9Q40XVnsgHazJvrO+wx4cGlXvjahanyCzCRhaWemCgJpOGUPjf8IE6jvrbrh13fsTP0X6n88bXaewEKoNVApXGUlCo7B0n6LDLim71utF0ayNa1d3vGXrb86YDcJwhN4rhRgHz5hPiBxOAsKRTNstSBl6nM1Pj6TIBCsM6PdPrEkX27duqjt+/M32dV5w41a3vGXEfoM3I5RcW/EY1p9p3fvyIMLl4mBwDke3Bh6o6quoYtJUZCGz1BrVBgQjkqrG/pDUKC/V1B0RZieB+jN1dctXujnrj/cRM7QuJ6qYoUr3uANSuZfE7Rgz6xYCpfGV43g1rbmynzs6+9Rb5K+/eNd4CEgb/EfmXZR9IqzvsptsIkse+99zzqkO1e7gNJF/F3T3u4/T2N/ZD2YzHZil2tdO0ZvCGLj7ShfAL1XSLH7T71tfGq/WWmaA9WXcCNFuPgcaTeU81hRowLlbPN+hiKQNAUR5WBmqw9FGukePS6ddA+psXk7FZnc9KIcADpndeKjoXTNmH5vhIdl8NY2sNvjAEOoSzern9Ana23nM5BIV702naJMx1eQHvl0J+Zs9Qjcnt9ObTx/eXdz9ffr51/3ANo92LQeN3k2p8fC3DWJ6rZVuebx/oF9+K29CfpOVnTWbhqCHlc3qPq1Z+x4irCQVZ/sU2us+crsZ1G38DxspaT1KkELqaedP9X/V+mzb7r/f2pJbir5lay4sV6GCyJ00lZa4eE3jYUS4taK/b7qfAyOXNpP/DKVvDezVQCfZd+M12/7Bq6bpg6vTOepB7Nt1Tp4kz/TcP2mU5ny08fJRnT/tZ1Y2/bNPXWKGz6mwb1Rj7X7oV6d/Cydfffvp/yvOC/rt5MvrtJ8X6J2KJirx9hzWrvW139D0mqS/OJP1pRp8l0cP63F3yTRQ6dNsuTOLaxnjbmc8NrUJHzAbv3HCtFE3dAHb3xv1axiW+W6BI4la04dDc4oEwvl9OWg4bAoJmf9fw/GwpAQ/S1j+2x+gq3M1t+0bPFRfRaro+dNwfWov13z3X3y33h07TfWXN2Rpt19+tmVXj35w6MG0g2Cei3+J+cN/9YPdb2l33h/WepcZg5i6vUxaP+b6qXL5q3AMU1m2IHEWbrxw9s2FyIbmh/XM2m8k3v9rg93kFkHv9N9//P4On2tnk2wUA apiserver.json.gz: - H4sIAAAAAAAC/+2d62/bthbAv+evINQNSAa39StpXKD3Is2jK5C2WetuH9rCoCXa5iJLKkk58bLcv/0eUpJNSVTitE7sJBywNuJLFHkePx2eqBcbCDm9Hg2iWHDnJfryrZaUMPI9poxoZTgIQoEFDQNZeAFFUOhTLlQTuLxUzYhHBe77BEoH2OdEFQ4DIt56UBTEvp+UMByNumHoCxpBeV0VjqhH9sNAsNDnuf4019enwak2sQgHRLX/ouaUzAzKXRiJBHJ6TndE0KfjD2iTEzahLkE+mRAfhf2/iSvohGwhHHgoFCPC0JgIRl2OPMojH08JlAdIjCiUYD7qh5h5CDOCBiFDNIA/x2pRsI+imEUhJxw6+NNnTi2biIcF5mHMXKI9RVJDuMtoJPuvfppDRr2TcL65qnAEl83a/PpMXre1gvNs99LrqbxOLy9nY6sNnI3jjENProUzxuzUC8+C+SQ47CbUNOZtBRVKmpz3oYA1mTcV00iVC3IuHFV4CX8mMsHIAIRXTt5p1bmTlIVnRinxfRzxnLzmKrxSTUHg9OGSntgdkS4dkzAWhR3PRg7Za+yeDlkYB6Xx523+xH5MqqvzU1DlT5qdjtvecWr5Yjbs481m60UNFrZTQ+16DdWf7Xa2iu2eeO02bmFHK/2Wu29OlJ0TFoIYjkjMnXwr4tIxVkvUKlTk5P338AyNcTBFEYERA4HCAZKGh3DB0WYfBB0usaek/oxRAQpAA9Sqg4hPORrhCUF9QgKo52eEgQrw2HUJ5wNY8anqNcBcIBKE8XD03/wcE42Qs0hvHgdU5JsMcTwkOXVIhBefZxvTqNcLKzimQVZZrOKj8My0mVKUR1JaQ987xn2St32mVu9AbYjafsFiojW6zM9/rtD5CqWN+Z2hYCvZBPsmec3bW20doogGw26iho2qurKU5tcTWgd4rORpIlcOiRApla4Vm02yVc9VXNYWG5zhYLjA4M384JWaAFJwAMpwEsLK8bIoOHIVVeW71NaBOwrAjIM9KbXryhkZFh7MtBhQaWEdx1RxBB7uE/1HDb9d/7XQBmygsa8qv7KrWqp3YPyu37oBmAA5hpx7eVFF8mDO++d7hspw1nHBNU+dQ7tYyE5BRA2aOqC+vy8tpdp/aQNbDTCBjV34Y7cjbWBjt2QDpfGoUEB5F328ZLgmWNNGp1UaSNf3Sh1VrARjxuOgvFUCsyERC+wCOY/UnHBEJToQ1kvN6Es8wdTHfepTMW3VvQuo67/66mDf/+rUkOvHHPQeCn5Jf/zqXJZ3am4rBXi1HtyCEl5ultmQI+wKtUbNUhOfDEngHc3GKw8C0qlI0dlbVCpmZpEbVjCDhz1tHdAmrMQW+g/qdJ7V60XhFzMovSjuJ/CUd/2OpljCwf75sAW44FWUodHVb7c4A9ViMfUL5TydVzfUvMzW3Uj5VKf3qTHFk+G8n/b8OQ7CPsV8PyOVgg/qY2b0c47E12MSDMVIWdVSHanqtnQyid0RIowBQPdjDxQRAVz7ZCCQH4ansLsIA7HEDMxIp1P/FemqhoYxBisqCOFF7qDKvBQeTJa+YdijyQtL/QZ+PG8NEwUrCy/sxTDY493iq5neBPbUXOPGjCUTM9WCJ6yqoUFFDaPDESiAVzWX1HZKvCnWQKe/qKeko+gwlfYKXGW+lQTzaxVYWnluuLmqOEtv3ViEkEoMUPKSGXzioZnzI9lbykUsh98u15mVAXbLAyBWPmHghwULxEhEsBE4Esv+AZwEiBwxPBE4W5dUaWfqnnfzhQJedIyTBHcTRcQ7Tlx3uf7G3g8QDP2GNpfiBNHTRK3hv627dIjK3iTmZum+sbCXcupHCb6VBEHWfRrRgTBXpk71UE72dWIbl+FTi6oeMmF4j1Jq3MvcLA08OqFejHOOzOySVdApP7NzfE55eV792D1NZM9gY7LohVykIvSlbxuGXtUmbWaXVBjN/AhTfL7Im1SVf1vwlTcRSfkOanwGWRkOX2NeeuHTHIGxW+IJjFXautzove4hPaquvxvF0pnWmg13WvoWTA1O8cVQ/TE8ywVgZ/PppsqcD3HNVPxgFkyEEZx8fQawo51yUI5B6w1tG228bW3jbdjTgm7Hbz91a28OuzbStrxI27aNtNlI272MtLVspO1nI23Svj6uUNtH6VHK8TYbZXtEUTYzdlSyBoAB4gTstoc8cBsjgmb6xZF8u+xPkQvW/bZjaTs2lmZjabcXS7vayihFlnd53nz27DktGxQ3c4hPtnf2dtqvnRvR2Xz41nXDHzX39+v7Pzp8+7rhW80X7YPOjw6/fd3wh/XmUbt9A9K5NqLZMkY0SzK99IAmj8fS8m1K07eV/AW4kdjflyUC6Sl9fcmwINvjhfDjTiObFxfKhKPLy/UPbCqE+XT8Fj1FH1N3ZQOaKwlozuURpDzi9zZaeW+ewxyKvP/suUjIS8NQmTrJiIhZQDx0RkV6EgzNt8/Ptyo4dFkY+sJiqMXQdTrSba3qSDcjoIx6fo6CZNdX//vqAEJWMxF6jpZ71/Vgr2zu942/1Bmzpa9V09eKjlHrtwBg6/8oD5XBErbiMo9Oxvg6HcCqdDeon/yqirSZMrVOUdm/isowGtIJCWYW7Jbha9fCl4UvC1/yHDWBld73GCsFNWCPFydJJ71UtV+OKBchOMbxrNfaHYTdYxg6SJfb4tCKceghBKK4zYez+XA2H64QHFS/Z6pFB08+AIeefIb/97r7v/97cHh82D202XHLy47r2Ow4mx1ns+MeZ3acsraPKz3uL+VgbH6czY8rYciC7LEu2XKFkWyozIbKbLqcTZe7z+lyV/OIzZfzr2IamzBnE+ZswtwDTJi7OZeuNH2u0bBcarnUHuEuL38uw6I7TqBbLxq7V4fGOpHZFDqbQmdT6B5NCp20miqFrgLT7jifrtG0NGZpzNLY0hPq1ufs7D7DkU2psyl1NqXOptRdnVK3hoS4dE5rPWBOM1ZZUFtzUNtZYdhsU4aiNs9CdgpgBu4bex5PglQXf4fq68EZu0kEowHMLnBVcOyX7KKSzb5sj79tbakIWda2hqQjvuOQVnbzy0t0cSHvfy/gDbYE/SH3BO154DNgmyy7rZjdwrs9oLuNkNb6PsJ9D2UtHVTaFlQsqFhQqQAVj0RiZBllXRjlQG6HBZRVB5dGcvXuOaKs80NYSClAyvaPQIpBHRZhFEO3n0UU08HWrR55XfkbhJZPfo5PCmewdwgo5eOrTfnvKtWQgVxSfimcgPUS73OLRFNDPtmyWHMjrDmGrQvcqQUbe2pmT83sqdnj5Tz7TwbYWNRasV57VagXsVB+9EQmecPsA9Ebk3HIpr3+VBBuwLeb0duq8Gz9qeydWmZLYismMSXmD4DG1vw5bKCpACD2Y/EWQCyAJF/WEmQzoxA3imcRpKq0nR+IH1kMqcKQ/ZPPKOZSkC2J2MMue9j1mBjEfjPdMohlEOVnwt4wZGEs5AbaoMet0cab2SJb3HhsuHFbx1Dr/RwP+ShqI30Ch7sjMsZ/EsaTWSa5vmCmp8mtPMxO1UBgmYdzKXVO4z5hAZGRq9lYgowjH8tfhZ9pHThJLjTZnkuDxhc5esu+YOqRAY5900dU9VoTDDqjBDE041IhU7NPRGsUp9WG6msARWvqfI8Jm6bx9zLtScMt7XBeQqF0SIrfZJ7ti3b7jcLjaAuGfT/7wnjhMbSl1NdhATbN1krzVmB2wR96ZM/0NeTZQma/Wq2PNYYdoYY+2Sobuly3xOp2vcSKblZ8w2bmyrfM29C8bhtSZ6RvFwi7Wmv+RzYTJ19bmrIsMzdONzl5Jq0i5qSbDKSj6V3vfL1q5/PerEqDrtvzDHGWvemm7CQj0c2Tlh6GeGxk/ugysbl0PP/++fyj7OHZ00aGQ9n31hPjP+8WUQAhNu+cLkkvQ1mdSZztuew4jbp20dIvGuP5z9vazw39olXXa+bQ5jS1nxteYgq/Zc8g4VsTnGvvog+8ow+s36XZ1i/mH+h3Xnj6fLO55Jbvn1C9wjifu/upg8zc897JW5RIZVIRq3d2p94h7i7GDdLp7Hg7g4Hr7ew2XvT7eDBoe/1GP2k8mbni+sbl/wFvRsHeLqIAAA== + H4sIAAAAAAAC/+2dW2/buBKA3/MrCLULJAu39S1pXKDnIE2TboG0zTbu7kNbGLREW9zIkkpSdrxZn99+hpRkUxfHztaJ7YQLbBvxJooznPk0nKjXOwhZnQ71w0hw6xX6+r0SlzDyI6KMaGXY9wOBBQ18WXgNRVDoUS5UE7icqGbEoQJ3PQKlPexxogr7PhHvHSjyI8+LSxgO3XYQeIKGUF5VhS51yHHgCxZ4PNOfZvp61L/UJhZin6j2X9Wc4plBuQ0jEV9Oz2q7BF2cfUK7nLAhtQnyyJB4KOj+RWxBh2QPYd9BgXAJQwMiGLU5cigPPTwmUO4j4VIowdztBpg5CDOCegFD1Ic/B2pRsIfCiIUBJxw6eOPnViWdiIMF5kHEbKI9RVxDuM1oKPuvf5p9Rp3zYCZcVejCZb0yux7J66ZWcJVKL7key+vkcjIdWwlwOo41CBy5FtYAs0snGPmzSXCQJtTUZm0FFUqbrI+BgDWZNRXjUJULciUsVTiBP2OdYKQHyisnbzWq3IrLglGplngeDnlGXzMVTqEmp3D6cHFPbLukTQckiERO4unIAXuD7cs+CyK/MP6szR/Yi8j86uwUVPmTeqtlNw+sSraY9bt4t954WYGFbVVQs1pB1eeHrb18uydOs4kb2NJKv2fum1Fl65wFoIYuibiVbUVsOsBqiRq5ioy+/xaM0AD7YxQSGNEXKOghaXgIFxztdkHR4RI7SutHjArYANRHjSqo+JgjFw8J6hLiQz0fEQZbgEe2TTjvwYqPVa8e5gIRP4j67n+zc4x3hJxFcvPIpyLbpI+jPslsh1h58VUqmFq1mlvBAfXTynwVd4NRmTClKrtSWwPPOcNdkrV9Za0+wLYhSvyCRURrNMnOf7ahsxVqN2YlQ8FWsiH25ILUBtl18EifKDXNLQT2aN8/4u3E3Mu55PWO9l1xQR2yaKpZk64tdRhSv9+Od3ptXl1xI2RnCq19PFAqO5TCQSJAympU8s2GqWAzFZPKcoMz7PeXGLyeHXzuZgNFewv77TwA4fCitlnSsKjKD4k5BY/ng6cAk1Vo15YzKrFF4AlEj0ojblllFafgRC/o32r4/eovuTZgZkv7qvIbu6ql+gD2dbHoemBl5Bhy7sVFFfGDWR9fHJVUBtOOS6554n+a+UJ2CSpaYgx61POOpTFW8pdmtlEDK1s7hD8OW9LM1g4LZlbapzl7XN5FHy8erg4Gu9ZqFAbSTcrcvaVwDMaMBn5RVAKzPhFLSIFchWpOOKSSTgjrJJb6FR5i6uEu9agYN6rONdR1X38D6+B9syrI9iIOpgUKniY/frMmRUnNzLEAx9mBW1DCi81SM3WKbaHWqF5oElur0+l4xUFAOxWMWkfLasXU8vKSFUz55EhbB7QLK7GH/oNarefVal75xZR7r/PyBGRzFks0IR8O9s8DEeCc41KGRt9+h/kZqBbLbb9AztN6fcudl9q6W20+1eljYkzxsD/rpz1/BrXABWF+nMJQzs11MSt1pZYk5DPi94WrrGqhjszrtnL4iWwXEcaA0buRAxsRAb97pCeQFwSXIF2EAYoiBmak1ar+gvSthvoRBisqCOF5tKHKvOQeTJa+Y9ih8TtR9Rao0LwHVJDiLjeJdsRYPOeyWnCS82qoP6cmjyXlVrWsBvr8SR2lN3lXqva1wPMMu9JtvnBrS/vPS26uKkbJrWvLsFOBDgr+MyVf3C9/yQhlb6kxkRx+v1hXvk1AWA7QuPIWPS/I2SZGQoJLUSS2+Z/AfYAykpInAjdsk3n7NnHch9lCAW9ZpZMERxSGxDmLnXqx/tZ+EeAM/Yp2V+Ie0bN4w8N/e/fpKpUlig3Ryr1mTpZy6qcx2BUUQdZduLQnyisTd3siJ/smtpqr8Lb5rR4wUfISp7ZxJ3XA1HfokDoRzri4cmetIl7ZmV3hK8qL8+pG9mWseyU2Jg2dyEXK42DyHlLSa75Jm9olFcMrf4QxvlrmHWue51vyfTtWSfkCXPoMsjLov8G88Cqo+YHSbrEjKK3S1uVWb3wP6VH1/buTL53u2nLDnZS+B1ODE7Apqf4cjDLR3+l82slmzsbXplv87TSSCSNY2foUbd2DYkSQQesdTYwm2LexwT7saBG/s/cX7cq7k7YJ860uzLdvwnwmzGfCfOVhvoYJ8/1smE+a8McV5/ssnVYx2GdCfI8oxFdONnNxBtgDcQJ220EOuA2XoOn+4ki+wHbHyAbrfteBvAMTyDOBvHUE8m62P2qLy7u8qD9//oIWTY2duson+wdHB8031q24bTZ8Y9Hwp/Xj4+rxvx2+uWj4Rv1l823r3w6/v2j4k2r9tNm8BQMtDKc2SsOpBZ1eeTSVRwNpE3elUdyL/wIQiS3zqwKbdNR+fcWwIPuDpcDkXsOq19fKuKPJZPOjqgpuLs7eo2foc+LITDR1LdHUmT6Clod8a0OlW/Mc5XHQ7afSZeJtGqDKpFFGRMR84qARFckBNTTfv7ram0OoqwLUlwZQDaBux0lzY10nzSkbpTz0c3wku77+3zcL4HI+LaEXaLV33QwqS+e+bWSmjr4Nl62by9Z0ulu9AzTb/Ed5qHQWUxeXiX8yLthqAXAl0qBe/Os70mbKXEDFa/8oXsOoT4fEn1qwO8ayQ4NlBssMlt2IZQnGdH5EWG3dEiByojhLppNs+lcu5SIAlzmY9tq4Y7UtxqS3yXIbUFozKD2E4BU3CXwmgc8k8OUCiuq3crWI4vknINTzL/D/Ufv4t3/enpydtE9MOt/q0vlaJp3PpPOZdD6TzndH6XzKoD+ufL4/lQ8zCX0moa9AOkvizaak9+VGMnE6E6cz+X0mv+9h5vfdTComwc+7iXZMhp/J8DMZfg8ww+/2xLrWfL9azRCrIVZzsnwfCX8pMN1zxt9mcdpWnWXrrGZy/kzOn8n5ezQ5f9Jqqpy/OQB3zwmAtbrhNMNphtPuMQNwc07ithmbTA6gyQE0OYAmB/DmHMANZMeVE1zjURJcqR03CLfhCHewxlDbrgxf7Y4CdgnIBo4dOw6PA1vXfwXqy80p1Uk4oz7MzrdVQO1pejGX2r4+7XTk8J10+33f21MxtrRnBUmHfc9BsfTmkwm6vpb33wrIAwGh36WE0JEDvgVW1TDemhkvuN/Dv7sIim3uI2x7MGzlQNM0QGOAxgDNrYHGIaFwDctsJsu8lcIxILPuYJUrV2/LUWaTH8LATA5m9jcEZkq6bd3x2qLfWTQk8xMkkzsJvkeUKR6V7cp/JauCShgnIZ3caVsn9kz3xj4V5JE9A0C3AqAzWFbfHhsEMud15rzOnNc9XiI0/1aDiW5tCRM214WEIQvkl2NkSjrM3hedARkEbNzpjgXhJZh3O8pbF7htPq99UMtsGG3NjKbU/AFw2oY/hwlW5dDEfKXfoIlBk5vRRAWkUj6xw2gag5qXSvTTESiDK/Nw5fj8C4q4VGtDLOZgzRysPSZWMZ+uN6xiWOVmVukHnX7AgkhIAZqwyZ1xyLvpIhsQeWwgcldHXJv9HA/5mGsneQKL2y4Z4D8I4/Es48xkMNPj+FYOZpdqILDM/ZmWWpdRlzCfyNjXdCxBBqGH5a/+T3cdOEkuNN2eaYOGFxmuS7/y6pAejryyD83qtWWYaLkxYWjGZapTlvy8NbqIUU6rT7+nrYGeVhuozyDkzar1IyJsnITyi0AoLbg0yFlVhdI+yX/Aeiog7fY7uefSVg57XvrF96xG62uqL8gS+Joumua2wP6CY3TIUdmno2crmjhPfawBiIaW9ElXuaTLoiVWt+vE5nQ3CouOfjL16HvlQqgvEkLik3Rhgc6rlea/p/OwsrWFCcuy8saJiOMn0ioiTtrxQDqh3rfcq/PknnVqc4zzQomnpLNSkc9huFlC1MPQhJ3UA01iK0sHs6/Czz5VH4ye1VIASr9CH5v7WbeQAvqwWedkSaZxOJ1CrP2Zmli1qnbR0C+0F1FrX/u5pl80qnrNDNOsuvZzzYlt3vf0GSRuazqy8C76wAf6wPpd6k39YvbPFlgvHX2+6Vwyy/d3oF5arC/t48Qlpg756Pw9irUyrojU+7tVbRH7EOMaabUOnINez3YODmsvu13c6zWdbq0bNx5OnW91Z/J/pLorRNWkAAA= cluster-total.json.gz: - H4sIAAAAAAAC/+1d62/bthb/nr9C0HqHFDeeLT/yKFAMeTYF1q63STfstoVBS7StRZY0ksqjQe7ffknqRUmkbCdO7MTchy4+lI/I8/yRPKRvNwzD7PddP4wINt8YX79vxRQE/4lcBAUa8P2AAOIGPiPeUhIlei4m7BH+yUiovGUQuR5579NGayunOoAAHETIhrTB/ISCCSRjGGFTeAb6YOCxdoIiKNDHriOhunbgHwZegBhDNBqAzdaW0bYs+k+vt2VYr0XWPpjwF+/nYzF+NvY9iEihC+Qm5M85AI8HAUCOmbTd8f9/p//ecaFAxyWl3pojH5L3DqX4kefFFATC8XkQeMQNKb3FiWw4h4FPUOAxgQ6Bh+Pvu4Xveq5/IWghBD7kz8ciTwVu2oHngRBDkVOhwam0jJDrfApyZcZCLinsin5udwXCdTqA5PMN+5xKJ+PNx9DOPua9/p7REAwhIMJQBep7AhFXj7z5c3BVEDBvwuPg6twlFcsxSUI0DyOEoE+MA+A7V65DxmbxmTP3B39uvC00JJaAgqvYBpIxZoIHngswN0AuyVwEA8Apha4we/oN+iPCxdwq0CGuaGiqt5hD1/MKkmaEdwg4Lh2oqCm1tvdK2rbaU7RtybXdyT56cAR9p/gmKqaRv4/P5a4NLkflwTPTjfUljwTHk5DcyJv+C1FQbZmAa8k7Jq4voSJ3NCZn0njDzExCpc/+yU2qaJWsKUBsCNloSk1HENtVfoSGJ0/SsUvgRTA1q4oeaKyQWBGjXiV9s0TqRckjqSzOqKcWPN5ko/kUuD75EHBxcIIpeLYNZRYdQmq1PgGjakQKGTdmohF7f69Ir3afCs2BCPL4PvSCXICKCIIhciH+/RIiavGwNESs6i+uDBwTYF9UeoMJDEPo/EZlWmkjAI0gwUI+FDMiyxbXIR8G03uf9s3exNFk06WhDm7SNEYA5Yr6NH1cBeiCpmAbupewP7ghEPe5RdzaXkR7gN5+M18lf34zt1hW4wN7+79v5i///mbefX1FRQkRtZY3r2gSD7yIBdPvr18bgxtjM3v+tZgcWfAI0IQLlMbDCezHkiw+kvI9ATbhOdcqNMeuf5Lxub3NXnZ3V+SE4JAHcXO/SGcS5rrJiHfJX7kayZiOahx4Tkm9rNsnNEyWbYLRz8bukFQbSrnhM9WFEQyNAyZ043OsAkfIB1kWvxVjAkA8wZajQuz67bID99Os4vqOe+k6EfX1ijOnz3DwkHfgGly7pSg+iOyL2O5KoWeSuGxFkSkQKoeqOLYpg85XUU9pPMto3ytDuAHXsMYdcnM7CEtm5oEB9CodZA3B6ABwmFM0vDi6Vx6Pw3uraGB5BM9NbGv1u7ghinn9YEiBUINDuhqHaByiccjicAiVkY8nLtFAZHlA5DzRAdFYRGORVcciNnXziV8jbAKv+TjOqTcUB8LVmzVNFUnK6A/2NeOnfQWzrHlehgf1DA/mZnhYz/BwboZH9QyP5mZ4XM/weG6GJ/UMT+Zm+K6e4bu5GZ7WMzydnWGWYxQc8/aanDIz3M5B1JAm7XT1cq/1L3N+uD11jddSLPL2KqivMNV4BNBnAGz8YNB6BmiGbRRwWRU6xULaKQQO/0axJc6SgrBoMCtJw4mRMs9MFZnMBexuvLqUxKd08jBps4leIpNKELfTWWAu3dSuYA6I/qL/NT58aBwdGaenbyaTN7iUUxxouxPAV+3bxZxClVfNzElDvsthHiFqpoYTXPlm5bEviEnVLDaEgOIb5MuHrEBaBXhCJ10OLL0t8l0+XvpdRKZ7cib0ymZBdTFoPVQhz62zqMOPJgPqYlJ1MAjzEGXIAPFa6eNgmfrYp7NaOqXWziGHdctWxto7x9Ey9ZHO3FOXMD6BeE68lqo4XqwqwnupQnCI9dbGySpoo+wYxhEK2GLmeqrk3Yo6yHpr5XSZWvkoX0ZYBdkXV5XvJXqnuTvYAbuDTnvb2QHbQ2vY6ra73db2dmd7197eAcPmRUQl6UMCcSPZhXD9USNbPWmEgYN/DdDovfPW+hnBIVPO204L/3wJUP7Y21f9vg2FXaKyrhXLNQ9VdnmqWd01eEGFInwjtbQzgwmQbJfKNm3atZs2996n2Xr++15LkOvB4uQKLkfaXBOxHj6NWNfOXI+eJgyk5hrGsGgNBHv8xPF1fSR7shyTdWIkvwYCfrck010fCZ/eo0jmwcUwZwQQcQM0ReGxAFRFCOmBmMIOn3geptCwuOMwqq3Sbcl5mJqZoKSaU1XRWVfVWVPZOeuRLEmFZ22Vp1Kesu1nacWnTLa5fC2BVpQNl/OOxK6r3aitBFVXg9ZWhE6pCq2vDFVXh6orRKdUiaorRadUi06pGK2vGq2rHFVWj1Z1Ka0iVRcVKAoLlMUFM1SV1lWWTqkuraswrakyrS9nUFebTq04rS1OkBcoqKtPp1WgKhc5yoHuOcweZyxInaEodc7C1LpFD0XuFfNvIQdPX9eS5+jaPF3M1ek25JQTNMrK1brqVUUF60xVrNXAIq9mVVS01la11le2qqtbaypc1VWu8kpXsdq1OtRq1avUBdWlpbXlpXUlpjVlpqpSU2W5aWVcz3QIGzKV1e4GaAwoI84CAnc1CNQgUIPAlwkCn36tW6PAh6BAZUGaBoIaCGogqFo4fforciq1pIu4ImepdxNZimPieyt2OVFevXvqYpo2bp7icqLyOf/VvxZgur7bcn0L47nvvQAl4mNcC1Am8sBSIi7yUoCHne9XnvRqP/ykFwV9PrQL551f/hn/4lk0fdXQizzhnyyJSnPsczzRT4pn9erP85fGkONbfWL/cbv4oNuDXiBOaCtwoWVpnKBxgsYJ+iqgpQOFdNVMIwWNFFbobp9lldV1FGV1VnuhdXWy3dHnvala3M2pKazrTNlTtToL3VSVNDz2nqqsQYKdHnVHdXGborLDeFVstZg90SrO0hujUjz22PuiK3ZWRW+MzrYxOtutEC9yP5SUL3OaZTdUMs4qBFzAPmf4/Pc5w/UpeFtjcNZtTQFnXQ3ONDjT4GxlwdkyzrtqdDYfOpt6UZQGaBqgvXiAtgKFaGXnW0z52ZMvWao2GXt6yXIxqLg3DRVva1SsUbFGxc9myfIJb9LQ4PhhS5fyGyI1QtYIWS9harAmA2t7U8DajgZrGqxpsPZ8ljA1WntOS5kasGnApgGbBmyLA2y7GrA9d8BW7/ExkDjwgPT2/2Lm+XMMiOFi4/zwk/EZpljh12qciOIfAhgTEuI3zSawbei4wP/FDiZNyFJ3iFwMcXNAY0YzwR6NGHI0vADjBkq5Y0yRA24A32k4Uei5Nk1+DfqgH1x50BnBCWWHm+aMiVmj1WeJVjlY9almGE6lnSD9czvsJxZ4BkdYCkyVMNRoGnKOv0dkfm4xqI2v5X16TJu+94VA2kJgwUYQEUYGnmdgdk0whiPu8BrfLhnfxqGO/x7Kc8e5Kz8UjXefEO/uabyr8W6Od2+MSUAjQ4CMs78+GihPTdNB78gl42jAIW+YeUaTYw6Kcmj2gahJ4S0VRtNqtTs/8b/p0yzBNbqtXau929nuaWS7fsj2+Jr+7/DT2Y2fYKEFwdt7A2YNcRcLcVksUcBcIcRokKtB7pqD3BWoUz1GiGHp+5SpbiQ9N5MfsGStnVbMiz2WxxkT22M4AX9AhOP+x+uuNITdxL1wALqIv0ezcG7FZv47mmb2NgInoUcF4Y8yr6TgCBPB9gv33Xj8F1Gr1zSAiASSG2EYuW8HEcewnVa5Jda8abXE+CWg3sJshcBrbp+9STGmXCY9Yg2yKc8sMw+Olkslu65Pk50D9z3ZVTdyPzAnEY27kseTOGXmiVJ8eRAySjnelEM39GIkJofKqXQ6rcpds5mAWJsq5KnfJrvJW64KlTJmflXdwKyx8l20aUMWHoS0bP4TQXSTCGGrN9kqsBOcrl2gjuB16devTHzhhl+QRwGXLVF0kkTF2nDqhNxp8H/SLpjF1jKE4DT5w1kWjqGR2BZheB7z4n0ql5qvkxe3H9mLZeKf14drvKqrtvTuPJbeXV8Tn2J8DhwCqmOFBaatUjOUJAqFFaXWIhiu3F6kygulBp4r0ZqmxFSQwuvvFRMEUa6oOyZz0nmky9/Uj2cCm1F4+3cwoLNbGziXLjVvOrvdMhKur+/rQ7GXtB7XS+IBTXeRjRQr38Woz+WSS/DeMJ6rmn5w1cjSoskzAaOZha+FLp2qofzLiUj6qb+KEdDsCVhYTA8FlGJaeSoQ04JpiR86LbFFuHqsLfxtObGNf0/HwNYXqpFZ/RaR8bbIWHxLuyt+EK4a3HHE/qZ9KYjvR8DXhswv54cJRE/nDh+zn7Y3msahYNJmxFebzeFwu9MDrXZvYA8h2BnYHcfpDnut3b29FujAvfjhy2xa0Nq4+z91+P/u0acAAA== + H4sIAAAAAAAC/+1d62/bthb/nr9C0HqHFDeeLT/yKFAMeTYF1q63STfstoVBS7StRZY0ksqjQe7ffknqRUmkbCdO7MTchy4+lI/I8/yRPKRvNwzD7PddP4wINt8YX79vxRQE/4lcBAUa8P2AAOIGPiPeUhIlei4m7BH+yUiovGUQuR5579NGayunOoAAHETIhrTB/ISCCSRjGGFTeAb6YOCxdoIiKNDHriOhunbgHwZegBhDNBqAzdaW0bYs+k+vt2VYr0XWPpjwF+/nYzF+NvY9iEihC+Qm5M85AI8HAUCOmbTd8f9/p//ecaFAxyWl3pojH5L3DqX4kefFFATC8XkQeMQNKb3FiWw4h4FPUOAxgQ6Bh+Pvu4Xveq5/IWghBD7kz8ciTwVu2oHngRBDkVOhwam0jJDrfApyZcZCLinsin5udwXCdTqA5PMN+5xKJ+PNx9DOPua9/p7REAwhIMJQBep7AhFXj7z5c3BVEDBvwuPg6twlFcsxSUI0DyOEoE+MA+A7V65DxmbxmTP3B39uvC00JJaAgqvYBpIxZoIHngswN0AuyVwEA8Apha4we/oN+iPCxdwq0CGuaGiqt5hD1/MKkmaEdwg4Lh2oqCm1tvdK2rbaU7RtybXdyT56cAR9p/gmKqaRv4/P5a4NLkflwTPTjfUljwTHk5DcyJv+C1FQbZmAa8k7Jq4voSJ3NCZn0njDzExCpc/+yU2qaJWsKUBsCNloSk1HENtVfoSGJ0/SsUvgRTA1q4oeaKyQWBGjXiV9s0TqRckjqSzOqKcWPN5ko/kUuD75EHBxcIIpeLYNZRYdQmq1PgGjakQKGTdmohF7f69Ir3afCs2BCPL4PvSCXICKCIIhciH+/RIiavGwNESs6i+uDBwTYF9UeoMJDEPo/EZlWmkjAI0gwUI+FDMiyxbXIR8G03uf9s3exNFk06WhDm7SNEYA5Yr6NH1cBeiCpmAbupewP7ghEPe5RdzaXkR7gN5+M18lf34zt1hW4wN7+79v5i///mbefX1FRQkRtZY3r2gSD7yIBdPvr18bgxtjM3v+tZgcWfAI0IQLlMbDCezHkiw+kvI9ATbhOdcqNMeuf5Lxub3NXnZ3V+SE4JAHcXO/SGcS5rrJiHfJX7kayZiOahx4Tkm9rNsnNEyWbYLRz8bukFQbSrnhM9WFEQyNAyZ043OsAkfIB1kWvxVjAkA8wZajQuz67bID99Os4vqOe+k6EfX1ijOnz3DwkHfgGly7pSg+iOyL2O5KoWeSuGxFkSkQKoeqOLYpg85XUU9pPMto3ytDuAHXsMYdcnM7CEtm5oEB9CodZA3B6ABwmFM0vDi6Vx6Pw3uraGB5BM9NbGv1u7ghinn9YEiBUINDuhqHaByiccjicAiVkY8nLtFAZHlA5DzRAdFYRGORVcciNnXziV8jbAKv+TjOqTcUB8LVmzVNFUnK6A/2NeOnfQWzrHlehgf1DA/mZnhYz/BwboZH9QyP5mZ4XM/weG6GJ/UMT+Zm+K6e4bu5GZ7WMzydnWGWYxQc8/aanDIz3M5B1JAm7XT1cq/1L3N+uD11jddSLPL2KqivMNV4BNBnAGz8YNB6BmiGbRRwWRU6xULaKQQO/0axJc6SgrBoMCtJw4mRMs9MFZnMBexuvLqUxKd08jBps4leIpNKELfTWWAu3dSuYA6I/qL/NT58aBwdGaenbyaTN7iUUxxouxPAV+3bxZxClVfNzElDvsthHiFqpoYTXPlm5bEviEnVLDaEgOIb5MuHrEBaBXhCJ10OLL0t8l0+XvpdRKZ7cib0ymZBdTFoPVQhz62zqMOPJgPqYlJ1MAjzEGXIAPFa6eNgmfrYp7NaOqXWziGHdctWxto7x9Ey9ZHO3FOXMD6BeE68lqo4XqwqwnupQnCI9dbGySpoo+wYxhEK2GLmeqrk3Yo6yHpr5XSZWvkoX0ZYBdkXV5XvJXqnuTvYAbuDTnvb2QHbQ2vY6ra73db2dmd7197eAcPmRUQl6UMCcSPZhXD9USNbPWmEgYN/DdDovfPW+hnBIVPO204L/3wJUP7Y21f9vg2FXaKyrhXLNQ9VdnmqWd01eEGFInwjtbQzgwmQbJfKNm3atZs2996n2Xr++15LkOvB4uQKLkfaXBOxHj6NWNfOXI+eJgyk5hrGsGgNBHv8xPF1fSR7shyTdWIkvwYCfrck010fCZ/eo0jmwcUwZwQQcQM0ReGxAFRFCOmBmMIOn3geptCwuOMwqq3Sbcl5mJqZoKSaU1XRWVfVWVPZOeuRLEmFZ22Vp1Kesu1nacWnTLa5fC2BVpQNl/OOxK6r3aitBFVXg9ZWhE6pCq2vDFVXh6orRKdUiaorRadUi06pGK2vGq2rHFVWj1Z1Ka0iVRcVKAoLlMUFM1SV1lWWTqkuraswrakyrS9nUFebTq04rS1OkBcoqKtPp1WgKhc5yoHuOcweZyxInaEodc7C1LpFD0XuFfNvIQdPX9eS5+jaPF3M1ek25JQTNMrK1brqVUUF60xVrNXAIq9mVVS01la11le2qqtbaypc1VWu8kpXsdq1OtRq1avUBdWlpbXlpXUlpjVlpqpSU2W5aWVcz3QIGzKV1e4GaAwoI84CAnc1CNQgUIPAlwkCn36tW6PAh6BAZUGaBoIaCGogqFo4fforciq1pIu4ImepdxNZimPieyt2OVFevXvqYpo2bp7icqLyOf/VvxZgur7bcn0L47nvvQAl4mNcC1Am8sBSIi7yUoCHne9XnvRqP/ykFwV9PrQL551f/hn/4lk0fdXQizzhnyyJSnPsczzRT4pn9erP85fGkONbfWL/cbv4oNuDXiBOaCtwoWVpnKBxgsYJ+iqgpQOFdNVMIwWNFFbobp9lldV1FGV1VnuhdXWy3dHnvala3M2pKazrTNlTtToL3VSVNDz2nqqsQYKdHnVHdXGborLDeFVstZg90SrO0hujUjz22PuiK3ZWRW+MzrYxOtutEC9yP5SUL3OaZTdUMs4qBFzAPmf4/Pc5w/UpeFtjcNZtTQFnXQ3ONDjT4GxlwdkyzrtqdDYfOpt6UZQGaBqgvXiAtgKFaGXnW0z52ZMvWao2GXt6yXIxqLg3DRVva1SsUbFGxc9myfIJb9LQ4PhhS5fyGyI1QtYIWS9harAmA2t7U8DajgZrGqxpsPZ8ljA1WntOS5kasGnApgGbBmyLA2y7GrA9d8BW7/ExkDjwgPT2/2Lm+XMMiOFi4/zwk/EZpljh12qciOIfAhgTEuI3zSawbei4wP/FDiZNyFJ3iFwMcXNAY0YzwR6NGHI0vADjBkq5Y0yRA24A32k4Uei5Nk1+DfqgH1x50BnBCWWHm+aMiVmj1WeJVjlY9almGE6lnSD9czvsJxZ4BkdYCkyVMNRoGnKOv0dkfm4xqI2v5X16TJu+94VA2kJgwUYQEUYGnmdgdk0whiPu8BrfLhnfxqGO/x7Kc8e5Kz8UjXefEO/uabyr8W6Od2+MSUAjQ4CMs78+GihPTdNB78gl42jAIW+YeUaTYw6Kcmj2gahJ4S0VRtNqtTs/8b/p0yzBNbqtXau929nuaWS7fsj2+Jr+7/DT2Y2fYKEFwdt7A2YNcRcLcVksUcBcIcRokKtB7pqD3BWoUz1GiGHp+5SpbiQ9N5MfsGStnVbMiz2WxxkT22M4AX9AhOP+x+uuNITdxL1wALqIv0ezcG7FZv47mmb2NgInoUcF4Y8yr6TgCBPB9gv33Xj8F1Gr1zSAiASSG2EYuW8HEcewnVa5Jda8abXE+CWg3sJshcBrbp+9STGmXCY9Yg2yKc8sMw+Olkslu65Pk50D9z3ZVTdyPzAnEY27kseTOGXmiVJ8eRAySjnelEM39GIkJofKqXQ6rcpds5mAWJsq5KnfJrvJW64KlTJmflXdwKyx8l20aUMWHoS0bP4TQXSTCGGrN9kqsBOcrl2gjuB16devTHzhhl+QRwGXLVF0kkTF2nDqhNxp8H/SLpjF1jKE4DT5w1kWjqGR2BZheB7z4n0ql5qvkxe3H9mLZeKf14drvKqrtvTuPJbeXV8Tn2J8DhwCqmOFBaatUjOUJIrUiswjaqLGWWyjErMRLFhuOFIthlJLz7VpTdNmKlHh9fcKDoJMV9Qvk8npPNLlb+rHU4LNKLz9OxjQaa4NnEuX2jmd5m4ZCdfX93Wm2F1aj+su8YCm+8pGCprvYvjncsklwG8YT1pNP7hqZPnR5CmB0czC10KXztlQ/uVEJP3UccVQaPYEUCzmiQJcMa08J4j5wbTED52W2CLcQdYW/rac2Ma/p2NgCw3VEK1+i8h4W2QsvqXdFT8Idw7uOGJ/074UxPcj4ItE5pfzwwSrp5OIj9lv3BtN41AwaTPiy87mcLjd6YFWuzewhxDsDOyO43SHvdbu3l4LdOBe/PBlNj9obdz9Hx+Hf4PapwAA controller-manager.json.gz: - H4sIAAAAAAAC/+1d3W/bNhB/z18haBuQAG4nf7XxgD6k6VoUa7q09baHrTBoiZa1SKJKUomzwPvbd6S+KIlKXHdNnIYvhnXHj+Pd8e6nMy1d7VmWPZsFcZJyZv9k/fmxl1Eo/pQGFCs0FMeEIx6QWBCvgATEMGBcNoHLtWyGvYCjeYiBukAhw5Lox5i/9oAUp2GYUShKllNCQh4kQHckcRl4+JjEnJKQ1foHtb5hEJ8pglG8AEGXQLCHDrMzGrmQLaSUmaxAdUkYooTVZKsxvBYnQTEOq6HU4bKeyF3iaRBhknJFxopPQkKfI/fMpySNW+NXbX5HYYq72XURJP27wWTijp7YvTqZ+nO0Pxg+7Vn9waRnjZye5Tw+nBw0233njUZoiGyF+rE2r4c4YiSlrhDLPqUkwnyJU1YbyF4QGiGxdDsmMa7zfJT6uHSWkhyhVbHavuM0xIqCuGA2WWxJLnQaAhZfChcgofcGzXHdeXStThA9w1KnnKZYabSuy08D75RId68zpD8O6qSYY3qOQp0T1B1W0UOSBLE/vUykKrp4bdPX9QmtYxRJI50LzVmcWByveMPgVs4WU9UY695mg1MU+xsMPqgP3ule4AUvwMNOCWiOtV3BFlqUzBPiyfldEsfY5bBJW+2mQiKN4hPC+CJYid62jvESgs2H4B85/Nj5odEGAou2r6Rf21Wq6gQiys2mW8C+krsHZG8rlWcLs9/+eKRhkrLjhjpnEM5angtEegYuqtmpiyAMj0X4kfYXgWXYh7jSP4SPw4kILP3DVmBZCIH0G1DMoo6XDTeAENWfDFsDqfu9c4/KZANjplHcNhVH1Md8AyvgVSJlYmm0nyZXbpgy2M7P/rK/z7/+Zfesv8kcKGfpHD9ysywVYvooQjHyRYP1QdtEVXTkkCNmDNMAs3azIni8RC6Xyhm0moTYx7H3shyvPQi4pcyx9tGm7lDGQ6ZRXcBlErd/SxqMMmlfNc2FaJVBuw2WhTubQXgLMQNIUR9fxhF1dx02d5dssdnuIkJO+9lnbqwilH3W3pKd3uaxErJY1U9Zfw07oDBA7LjI7o0UM0dUm8YgLbPlGxz7fCmDZouHu7ptlM0DuXf7LeIrirwAx7yZlm/IkcN6HpQ+3PYcUIQfH7FpjhtFTm6EAnTud4QUN6U0k0vTDXJMRzdhID2HBv4SfM/rkCQPSjoO9Pkj8KRdmplI7huOuuKi9J0b4YiInkwzt2Rc5DP3N0Eerdzayj52gsFVYo58PShNRG/hE6kYftzm6b0QTOVhimWsXYSksfUpTjDSJvIscP56jim4G9asCJKYi7u2RZ72mlQOsFwrJQT8JMHemywntvlbpRWKON6/IPTsU4pTPEOex2bSJ7bLNj0riGEFsYuf/QsdiwtIQ3+Oo48HB9b80trPh6va9iyB5m41UV0V61uvraurQhB5IWRZr//3TNZwDbGglxnKavmV4H1YBguuZ+Yp8A+wmvVOmM068jzrPVhyq5zYDBiEcs1tjgwGsyJNBrEXnAdeimqJSJ9S5U11XbIVWgWsLdc8dc8yD9ZEqigPCkI5TUyW3wxoenUHxjK6yTKBfgmXaLXJjU7lpiTRuGco7v604gkm8Z8j1rrVUhKFtluWKbQsZcmfdUd1T1ah7ra9JrXcY/qonVNfw76XNSM9+z25qJWFSnmm+darF2PKDfkCcM6cIAp7EWSv8wvYuHyiMPINQqH1nmKhr1kZegDobmTQnUF3O4PuBjuE7jyc8KUBdvcI2L0QFjOo7o5RHUxH+W0iIucrgLpdXoTBdAbTXYvpxgbTGUxnMN0yYJxA5o1mn1IU8yDE+87jyaRnacBeDvnSLCoCxAJY57FZlo1vHwT2rBAfGCS4NRJ8A9aN3UuDBe8aC34D9T1T3TNI8L4iwSfbIEFdFNwECmoR3xdiQW1E/qpg8PqTDwYNfhkaHN15gQ8QCJ+5odhM8oQsXBY/4m4L6HoumFDQBo8fl/DuNsHbYLX6DHjW2yGNDe9IY8NrNfZ8hzU2uiONja7V2PEOa2x8RxobX6uxF7tx0/QL6NU6On1tvc+0bk5GmJMRD/hkxMM95vnU3CqYW4VdulU43N26sQat3FLtuGeBUkXX018/TOulZMHoWSkN76J4LCaXxWKY/z7UiiEO8hLy5NViazLhS+tdbnODgR5S7di574Vjc37AVI3/Lyh4aM4PmPMD5vzAvcCBr342MHB7GPgKGxRoUKBBgQYFGhTYRIETUxA0BUFzduAn8bQY4mLGAOIJJwGsF+GI0MvZ/JJjtiWw6zobert4rTrnuftY7UQq3QCyOwZk0um/gR8nd3wd5ufJOhppyGfgiIEjDxWOyNJTgUncJC0rTV/yKJpr/6tiQEkXKDk+/c1KmXBrg0vM344f4N+OHzAi6RtEYhCJQSQiz5CZTyhJuTCgKYjc1o9XpcoN+Hho4ONr/dd1t9fxLf9mtZevwGbuEkfod0xZJmV/lJH5ZTaVh+hZ9oYESF6Vl8rQSmMsqlrlWBxHSQjLjf32+x72mt6goI0aliues+zhBUpD3aOeVa4OGsq3QtSDS4dPlU+qVzCdwiVJ8fIKNZran1JML/NKfRv7KW+X6NeoPm4+Gr60izL9XmM5isJQGBYvOmgsQ1GlqocNkGqhKyVbQdiFROrhI91D2UtF2nmyVceKwCKBpk+hZU2Xm1Qsp5tlUVQ8Z/2mJ6r3rHySA71JBjeZJE9MqunA8aXe2btCKrvObYkvaPrGucGz9SmMlOFpNpAKWm/bC5wuL6hntq7ddJP9C7jzJQ6w3YP2q5NN34Zf7BVJKXuFjkQrZcgtXxBBLh71C0xUvPshywBVtyQANESrzrlKZgWeVYGJPa6cxu47ysVQvehH1fex8r2vXgwdlVMhN3ugfO97WTz8WKxBIHDFY26cRR34iTqwOstgpF5ULwuxn3qqvIUsNfX9Q+LsnQfT4zxLlrW60g+tk8wPswapvJ23nw6wg53xHC/GzmSCxwtnNJk744XnjgYT7I2yxudlXnb21v8BquF9oXtpAAA= + H4sIAAAAAAAC/+1dW2/bNhR+z68QtA1IALeTb208oA9puhbFmi1tvO1hKwxaomUtkqiSVOIs8H77DqkbJdGOk7aJ3fAliM/h5fBcP9K0dL1nWfZkEsRJypn9k/XXx05GofhTGlCs0FAcE454QGJBvAYSEMOAcdkEPi5lM+wFHE1DDNQZChmWRD/G/K0HpDgNw4xCUTIfExLyIAG6I4nzwMPHJOaUhKzWP6j1DYP4XBGM4hkIOgeC3XeYndHIpWwhpcxkBapLwhAlrCZbjeG1OAmKcVgNpQ6X9UTuHI+DCJOUKzJWfBIS+hK55z4ladwav2rzBwpTvJpdF0HSv+uNRu7gmd2pk6k/Rfu9/vOO1e2NOtbA6VjO08PRQbPdd95ggPrIVqgfa/N6iCNGUuoKsexTSiLM5zhltYHsGaEREku3YxLjOs9HqY9LZynJEVoUq+06TkOsKIgLZpPF5uRSpyFg8blwARJ679AU151H1+oE0XMsdcppipVGy7r8NPBOiXT3OkP6Y69OijmmFygUmuhGdT2E2MfS9g1FoDDw4yM2zuNFyNI0ZuDP+RmExU2i1mNCUXWSBLE/vkqktlfx2t5VlxRaxyiSfnAhjGNxYnG84A2fsnK2mKrGWHY2G5yi2N9g8F598JUeDI72Cpz4lIBxWNvbbBGtknlCpIYh1OIYuxzyQKvdWEikCfCEMD4LFqK3rWO8hnx2Fvwrhx86PzTaQO7S9pX0tV2lqk4gad1suhmErgxQkL2tVJ4tzP71xyMNk5QdN9Q5g4zZCg4g0nNwUU0ymAVheCwynLS/yF39LqSu7iH8ORyJ3NU9bOWumRBIH+NiFnW8bLgeZMHuqN8aSE0pK2NL1jMYM43itqk4oj7mG1gBLxIpE0uj/TS5dsOUQcZ48bf9ff7v33bH+odMgXKeTvETNyuEIaZPIhQjXzRYHrRNVCVgDmVowjANMGs3K/LTa+RyqZxeq0mWpl6X47UHAbeUZdw+2tQdypTLNKoLuMx79u9Jg1HiguumuRCtivRqg2XpzmaQ3kLMALXUx5d5RI2uw2Z0yRabRRcRctovbhlYRSq7VWzJTr/muRIKZdVPWX8NnkCFQey4ABCNKjZFVFspofKz+Tsc+3wuk2aLh1d12wgwBDJ2uy3iG4q8AMe8WflvKMP9eyjD6MJfkW7clNJMZk03qD8rugnj6TnNiq9PWDoO9Pkz8KTNmlVKxhRHq3Km9Cu2AcQoG7UYl/nM3U1QSavutiqTnWBwo5gjX4+JE9Fb+Esqhh+2eXoPBVN5mGKZh2chaaQFihOMtEU+S6q/XWAKrog1K4IC5+JVIZOXxCaVw65AKyUUgyTB3rusXrb5dyo5FHG8f0no+acUp3iCPI9NpE/crRJ1rCCGFcQufvEfdCw+QIn663vYPcJkkyIUPx4cWNMraz8fvOrZsQTuu9eSdl2sdrm0rq8LQeQHIcty+cVrXsNRxIJeZ3is5WWCdzYPZlzPzIvln2BD670wonXkedYHUPWdqmczfRDKNXsumRomRUENYi+4CLwU1UqWvvjKHX5dsgVaBKwt1zR1zzN/1uStKE8RQjlN9JZvGzS9VqfJMtfJMwv9Eq7QYpMtUeWmJNG4Zyi2olrxBJP4LxFrbcqUsqHtltUNLUtZ8q32XjuyCjXa9prUMsb0OTynvoW4lwdYevYHclk7oyrlGeehVz8ZKgPyFSCiKUEUYhFkr/MLgDl/pjDyAKHQek+x0Nc8pnoEOHBgcKDBgTuBA3tbhAM9nPC5gYA7CwFfCfsZ/PfA+A+mo/w+sZPzFeDfNi/CoD+D/taiv6FBfwb9GfS3Dv3NA8YJVOVo8ilFMQ9CvO88HY06lgYW5uAwzTImwC8AgB6bZJX6oeFixwrxgcGMd8aM70DXsXtlUONDo8Zv4MzQnBgazLirmPHZlmBGLTTcNdC4/kaGQY2fhxoHD35kCNiET9xQhJm8HAwfiy+Q7wr8Oi6YUNB6T5+ugIH3CfJ6i8UtYFxni/TX3wr99dfq7+UW62+wFfobrNXf8Rbrb7gV+huu1d+r7diG/QJato5O31ofMhuY+xvm/sYjvr/xeK+tPjebD7P52I3Nx+H2nlhrUM09nVp3LFCq6Hr629l43SG2aNaxUho+xLG1mFweU8P8u3BKDfmSl9AoP6e2RiM+t97nHmCw0mM6tXZ2/cja3HEw59VfCjIemjsO5o6DueOw84jxzc8GMH4pwPgGG7xo8KLBiwYvGrzYxIsjc8RojhjN/Ya1gDGhxMWMARQUTgKYMMIRoVeT6RXH7I4AcNU91/tFctUt1e1HcSdS6QaqPTBUk07/DXwRuuXrMF+F1nFKQz4DVAxQMUClCVTk4VWBVtwkLc+qPuchPrf4RY4BL6vAy/Hp71bKhJMb/GJ+hv0If4b9iJFL1yAXg1wMclmLXHwy8QklKRcGNEcq9/XFWKlyA0seGyz5Wr/13e51fMvfh+3lK7CZO8cR+gNTlknZHWRkfpVN5SF6nr0CA4pX5aUytdIYi3OxciyOoySE5cZ++4Uee01vUMBGDeUVT7n28Ayloe5B2ypXBxrlaz/qyaX0KVs8v986y4Cdwi9eGKDAPoVLkuI1JWpatT+lmF7lh/5teKi8R6Rbo/q4+YT+0kDK9HuNdSmaQ2FYvNKi7tGqTlWFbABmC6UpZQvyL1RUDx/pno1faTSvuupYEZgm0PQptKzpcpOK5XSTLJ2Kx93f9GD7jpVPcqA3Se8mk+QVSjUdRIDUO3tfSGXXuS3xBU3fODd4tj6FkTI8zgZS0et9e4GzygvqJW5Fqr7R/gXu+RwHuNv7DqrLVN+GX+wV1Sl7WZKELWXuLd/TQS6fdAtwVLyCIysFVbckAFhEq865SsoTOxWh2MPKaeyuo3zoqx+ULas9VP7vqh/6jsqpIJzdU/7velk+/FisQUBxxWNunEUd+Jk6sDpLb6B+qN7ZYj/3VHkLWWrq+5fE2asnxsd5uSyP80o/tE4yP8wapHLHbz/vYQc7wymeDZ3RCA9nzmA0dYYzzx30RtgbZI0vygLt7C3/B8u4cmNlawAA k8s-resources-cluster.json.gz: - H4sIAAAAAAAC/+1daW/bOBr+nl+h1XYXydaZWI7tHECxaJxmWqCdZtq0g0VbCLRE29roGorK0SDz24ekLuqw49hOLNtvPsWkRFLvxUcvH5F3W4qiItf1KKKW5wbqsXLHilihbQWU/fr2g/28b/DLsGlR1LcxK6UkxKJs6GL6zmQlbmjbUQlB/ujC82xq+ay8KQpHlol7nkuJZ/MuBsgOovtty70MRDfiJ8EDgoMRK1D3m4EalXnX4goxrGhwrNTwbBv5AZZbExUjbA1HfOSq1mz6N2pa4SMX21lLcmuiHtkWCnqe7REhhfuGXNlHJCh2JSpMFIzeY3dI+aC1ZqkOj7uNosALicEfQD0nnoPpCIeBmrtqYNk2bzZf6BEHiQf0MbvfpaFr0fx9llm6y3IpJlfIFoJx8pfbeIhdM1V9JpGrYdXgufRDQljPY2oddDOuxnLH1AQj71oyLKmGMtu0x9zFHijMJCzV5bXHrExcVWxdVFxbZqS8Yo1kmGkxN/Nzjwnzg2cK1fECBQXKT0y8vFRj7aAhrrQAn7dCkGmFvJtOua7acJjYTUww4X0PbK+g+AATCwcfrzAhzOMqxh/4yMDj7JVVcvW08oUUGZeVQwko9n1svmcyrKyniAwxzXtc0evEhfjGF8+jKbsKs7ltgijedpmAdcMP9QAbnmsGurCDO4cVv/rOLNzG39WGYtghGwZhJS/if7+r999e6DpvQ0+M/sfOjtoo9pr5EbUcrEeiK19muUwCwtTLtin51RkyqEeK0otVNhAhUn2t5qrupV95LdERj4KebXLhqQfNxmEzr2c+4jMWNKTAm6v7PLIGtLqSigCu9s6/KF+oxaK8CPyF5tPwfVf0UkSwOc6FPUKTiF/0UZ3e+qJfyzWtK8sMmUuPddfk2sByhzZm4i9Y+Q26sYLy4PqhcRnZW/GxeeSJ/ZVLp6Bk1UUOrr5rfFRKI4+YIKuf4xbd4CnMPzNE1h2hZRO0UR/blQPkld7wBIl5UCvVRYG48rYoEjdL5dIT5421UduHqKyKn6M4LxS8bqvi8dYaFLRg4oeJv3YTfxA62zwIi0HySf/4Muxj3fdMnc39FLGOiM6mROEb7J8/me5pcMxuu6sEADvKnsLbFK0IIMEnkTDQkW17BhJvMXdJe+xW1iMDE9VNAXAoA4dPsQaUnuc4FnV4RAAAAQACAMTaA4h9ABAAIFYcQNgWm7QAPiwNPrwX8gfwAOABwMNGgYc2gAcAD7VcduDT/bGY6h3seORW/4Cd11fIsvlcr/dvKZ4CLxTuv+AWFN17B+BgGnDwQcgO1iUAGQAy2Cxk0AFkAMig5mmFaGZ/5qWJqFNILzwOQcACBSAJQBIbiSS6gCQASaw+klj0GgXgiFlwBKxUAIoAFCGjiK2CQzL39TEqOlNc+o45f5TGq6z+5F3nvh1JB3QRu2H+y47UOd9iZEbzXr7us/VT1I+6kc/EunjMNyOtzrp8M9IsQ6MDgEYlaNQEaBRBI60aG5WE9zTQiAOYDB/lMNFxho745yBhwGSb/yiEoyTd4l98jMNK/Vslg1/zgJ0H0UzkVWdpe3d3ab/39+qYy5kwL8cGaCZvobRZwFHBDhb02QjXQC2BkPj2EDAQYKA1wkCyPeX8EMDPlJkjgX0OAftAWuiR2OfxeaGA3trThHHhOvyBLspTTGTsOIMQ/2N/ux8+7J6eKm/fHjvOcVABTXxEWaBxx7eZzJEjyzSxqz4qYKfDPffMir4NHgNi86gMsEYSJPK6mfVZTWxYDhKhqTwp2RGWqsxG8boLgU9PbOReVrtYcl0KadRTwsKKYnrXrkI9xa+UAb/lCxGf1++Ze4cd1Om2mgeHhjk4YH9HGCENmWZ73zCwoeG9y8NgN8nOBbspRPzvFSK7Wah79SL7/9+8KoG3CbgVhendr17ouoFtW9cm2sdXHkKUf76usJHx2DFnQ27o9JlDlupFSj6d1mczsT88cml7COzserwg8saGDg8G/aZ5pB1ho4lRZ9DttPe1VhtrXaPdQp2CsaXtLsfsTuppdtUvNss2u9Y4s5tgT/PY3URTm0a7vfpqN+EEgILnUfBp/RWs/AtUPI+K3zydiuWl+9kVHa3KgZLnUfJZff04XnQFL55Lwb/W2Yt/S7Djauh4k18mJ9qaO16PdYgmNbcuSZB7v/xnb3YZBpRY7vBxMhy/gDTLAl5KZWJ2jckCluHERpNPwDYqrs+pk/hI0y7GNaYTlOGFLt3mO9xlS53J2zgX3nFeiscEi4WQSdJMbm8omVxXTc4ni5ZzzVaUayDi3pOIeKE7Nq2WQE9X32aTj1Y3WItvlusWD3N8V0ucZxvoFOumw1/VFWAg/R4y3daLgUSQG3BNVusxReoVVcBQAobSejKUCn4KPKVpSNpHQFQCknYtSdoZ+on3uCFB9cY2DSW99B+sAhjYi8Q/yR45HOsq29d7nmIgY1QQKRCynx/uiI2eAO5sKNyJ3BKwzmM42YWeAOsAKRtI2UDKBlI2kLKBlA2k7Py7DrDBasrLjt575lIwULNrTs1eoI6B17nG7Ozctkmg55oStBfgzcDRBo42cLSBow0cbeBoA0cbONozUfuea0F5/QnYC9qaHjjYi7TJEpd0I7X0ZumWDzTr+ts90KifmUaUpuOYXKvf9IBSDZRq4BjVj2NUXi0BstE0ZKNCWRzuxQGQjgpEJCAiARFpTiJSL/ICFqAMbF1h5QS5ZmRNsEowxyrBE3KCTvxgTl1fcJTGsDsoezHKPqmjsj8xHSjeIHFsUzlHEZIGVdeTk+PPrerErylou/bsHH/hjq2cEo9DGVB5Xak6T+HgoPWaE3dm1zqQOYDMAWSONSJziD1tpFUbF1POM9BJNJPrguMX7YEzZhknM7y/vqu/vPyu3n9jFsib1ZPk2I+djeeDTJI1jafQTRX2yXMKOzFsP8Iqmyft3lJMe2PFfbpM4zYjIL55Un+zVCPfWLGf1X7Xtzjd/FukujqeQQl0BaArAF1hap8F0kKJtFCxG1xh7RmYCbAdnMRM6C5xN7g6JiBg57jJlM9JtAzYMu758RFfkQd0tAR01FgLbLAP2ACwwSphg6UmzAEcTAIHD/D4AB0AOoDcyfPlTspeCPmSqTBRGzARYKK6YSK+nQTkS1YNEr1mxsY32e8lOsuwUe6r2eOUPguQCSATJFRWFzx0ADwAeFgl8AAJlTVAD9KXGAAgAEBAzmVJOZdpPRZSMo9DVV1AVYCqVpHCshzuO8CqiSSWabYiAOT0/MjJB+QEqZfZQcIBgAQACSvJZQGUUF+U8NAuNgAUAChAiuX5UiyJXxZ9ETIpU4GkQwBJAJJWOZOynA+sASvNklGp3A0MABMAJsisrBRoOALQAKBhpTMrgBpWKsMCwAGAA2Ra6pVpKfkkZFyyq7ItQ3e1aWFVqwmwCmBV3WAVe4O3txm2YjhHPpBvu4C0BoFOMDLTNav8gX4NpfIIvm8d58eO8lIpt3VNLIl9PH1jOwDAZgdg7z6ef97+xLX48g8h/x3AW7DdHaRqVjhV09IAUwCmqGGqZlo4kfsKaTGgYsYmAVnMsRXMiHjhcHQeUoEvIngB6AI+TYJszrKyOZ9ZKOKfJr37CAmcxxz624Ltc8uIajMO9o2mmoKimTuwwnZBuKwBoxzY7uGg4Kc6KDjLXezU6sSoXQ3OBl7IOWBCwVVpKdBwjQ4EnlvDwoXZ2xtouubnAc+j6fh10I9fB3fgdNB5lHxax/O9JRVXvOyDjutz7u9CdMzdOInaoOq5VH1WR1XDYb9w2C8c9rs+h/0+GZtjI4/zfViaM/NZNvHA3hWhGm3i6b5PvGy7kUf4Th091lWob1bPUJ+dXwAn+k5/om/F0mYtGAZwnC/wG4GDUMlBUHaVU4u/FfVD0csDlISteNxqYIywg75iEkRjiw5riVY7+R0mIpeiMeZVw8wy1cuQvcO6mAVqNW2LYse32UO6w9TT2It9QCV7zoxAWsbPcRkovhE2YeIBCu3CErNwIbm2im3K10Fx3lLHmFLixjKnQar1fC7KYqRV/wwxuRXfclVxH3i45zE6b5isdIhvCvNDGrCk7rcKjyMJDNn21/j5C4/xgCjHyLBaeFPQOxIBS5Mhi99sNjbxa7uCLTFO+g7ToFVxeaKVeH5/jEpET3oUabdD/+7/Xp8hBQOZVxabhhhSSAHETrXOWg/pLJ7NZN0yzxCKCX5PhqHma0vj5WXVF8cWET2QVBEG+CJqSI5PUQziMek+8kBLSC72vUEEBlTXu97VEkoem8TjMjV3m2+xqZBkN8ciSb/UlWcltSNtE6Q1pR/78g/Nyf7vSP9r8o/9plwjna3Vkv7XzMhYfyTPwNmiki082IvccFduWO6l1ZZ/SN/lHZjyeJOx5MT30xO8EfXLRS8Olymo8hw/pFj5lOQrlT2lJ1m2GgpalMoi2mF3YGpm09BaGmp12+12v7vfRfuDTvMIHUYXX6WRurl1/zdx3YRnKiwBAA== + H4sIAAAAAAAC/+2daXPbNhrH3/tTcNnsjt1YtSRb8jGT2fERN5lJGjd20tmJMxyIhCTWFMmCoI+47mdfADwEXhKtw6Kkx68sgudzAD8CfwKPG4qiItt2KKKmY3vqkfLINrGNlulR9uvbd/bzaZvvhg2Too6F2VZKfCy29WxM3xtsi+1bVrCFILd/5TgWNV22vS429k0Dnzo2JY7FL9FFlhccb5n2jScuI34S3CXY67MN6m7dU4Ntzp3YQ9xWcHNsq+5YFnI9LJ9NFPSx2evzO1cb9bp7r8YFLrKxNTyTfDZRjiwTeaeO5RBhhadtubCDiJe+lCgwkNf/gO0e5TfdqGfKcNFhFHmOT3T+AOoFcQaY9rHvqYm9uqZl8dMmNzpkgMQDupgdb1PfNmnyONPIHGXaFJNbZAnDDJK7W7iHbSN2vWyRnn3sXaWcLu9x28t7PO4fnxB2bwWlA3RfVGLaBSWE+/WSBVL+rXh95y6/hLLgtgpOyiziD10klSXdz8JU7JU+uyi4M43A++kSKbLjzTxPLhzmjY+OeBKxQUGe8gMTJ+mW0L2oh3NDyOVnIcgwfX6ZVrYsP/KYVwxMMOHX7lpOKnI8TEzsfbrFhDBL59y/5yIdFwU8K+TeayY3UqTf5N6KR7HrYuMDs2FuOUWkh2kyZdNpK3bE9654noZSUzx/sMnCUuk8KJsDZuMtZZMgijdt9r+mu77mYd2xDU8TYfH4p9N5c63ywho7i0NYllyr2wo/8s0/1yyTLPy36dwhk/7NbhdZvFC3fPY/Yce9Cv+9Vp++vdI0fiEtyrTvW1tb6nb6VofZS80B1gJ7Z3czbWY2kT7ZgJay+Rzp1CFpk4d+7oqKWT1WE0VP0q+ka2mf172OZXCLq/v17YN6Mjj4HZ+zqkqq7hNll32zS/MLqahB1NOLL8oXarK2RTQ3qdPHjcZjOrURwUZBBnvMZ1kDBImt0QdXXNe0DfPWNHxWDxTmeLSvZ9o9CzPzp1LjHt2bXvbmOr5+EwRp+rF5bRYmObdOysmqjQY4/6jiqiyurkSznP8cD+gel8iZYSCyyxGaDUELdbCVe4O80OmdINH6NjJlQeWee1hQu9cz26UnTgbrdmUfIrcofI50Y5LKuo2cx1tpFGkCigCKrBeKcAzhNby4Sc4dRzd+B2uuY2gMPyhiFyIaa29F4rF//mK+p94RO+wxFy+2lB2BNuIsgmV4C+V7GrIsR0fixSyEGb5LjZfiGktlYuoeg5boUqyc3QzbkH8VAJYssHwOnaOcOoOBSQe8LgFwAXABcFl5cNkFcAFwAXApBhfLZC0iYEsVseWDcA1AC0ALQMtaQcseQAtAy3oO/BwJxBjggUMetI94cHyLTItHsdZ5oLgEp6SOv+IRFBxbMFIEPFKGRz4Ki8LQD8AIwMh6wUgLYARgZJ17UAKYqM7oT3A/QC7PIxcYAwKCAYJZS4JpA8EAwQDBVGUYCPhlEn6BwSCgF6AXmV42UgnJ0tfFKJ1M4db3LPmDbsvc4s/OXeIjqPiGrsI0TH6iFCfnO4yMoElMll2aP0R5vx3kTOiL53z81GytysdP9SyS7QOSvSyS1QHJAiRr5DNZxnjzQTIOTkMuS7DY0ZDK+EdOvsdsm/zUidOZZvJPlIoYjX8yFZ9+GpIai0pBWp7H53t8jK/79KQW7M6MeVNY+zN7C6dNQl6pOJjRZ07cA5WkLPGFLgAWANYKAZYcT4k8BLIq2R0mwOoAwAr6uqoEVs/v7PLog1WmjRB5yR/oKtt+BZmEh3zyP/ZX+/ixdnamvHt3NBgceTnc4yLK8sUuPmfUAPdNw8C2+qzWIL7dC8fIubbOK5gwPHJrbz2qgZK+mfRZDaybAyTqvWyLZwWgltuPxsuuBPyeWMi+yU+xaL+Yl9QzwuosxXDubIU6iptrA37IFyJqqx1j56CFWu1mff9AN7r77O8QI9RAhrG3q+tYb+CdmwOvFvUrerWYP/97i0htWI++eTX8/z+8KGLniJzFxvjoN680TceWpTVGxsdXXoUoPx3nxEgxmCZiyPYHHZaQmXIxiBEzw2Qh9odDbiwHQZzdFRsiGWzoYL/bqRuHjUOs1zFqddutvd1Gcw832vpeE7VSwRafdzFhd1LNsMt/a1p02DWLwm5EPE0TdyNDrYx3T6vr3UhFAQ6exsFn1Xew8m9w8TQufjs/F8tih8kdHYwngpOncfJ5dfM4HC6GLJ7Kwb9WOYt/i9hxOXy8zi+TI2PNLvZjFWqTikeXZMidX37emdyGHiWm3XueDYtHpyYZHYz1WSyuMRmlplKmH/8T3btz0FClBwbVUSqrsqOA2+WMqDu+TfnckZL2LXpT54Y9Slr4iGAxAjNqNDU6fFsZ2nXZ7HwyaztXbCi7AiY+nYuJZzr72HIZ9Gz5Yzb6IHmNvfh2sWkxXtS8XOY8X8OkWDUf/qougfTpd5/5tlrSJ4Jsj3sy348xxecUgTQKpFGrKY1K5SkIpMpIzw9BIQXS8/WTng/RKpwciXjRjEg6Mm5N1lIXdSyxrdHB/2IFoDSfJW5FEytxtFY273YcRUd6P2VSEJ6/PF2JOcOArtaUroK0BLR6jvY8TVuAViA+B/E5iM9BfA7icxCfg/h8vmEnv0iB6q2i+vPgpWoqB4MEveIS9Bn6GPSrK6xCT0xsBX6uqBB9BtkMWnTQooMWHbTooEUHLTpo0UGLPnMJ4+LGtldfej6j1Q9AfT7fKM3oatfSb28XngsgOV/GTACR+QurnuIOPmbX/HdHEJyD4BwkUdWTRGXHX0AbVUYb1QBtFGijQBtVZW3UaZAkrIbTsXmLlRNkG0E0wcDFFAMXc5QpnbjelL6+4pjH4B+cPRtnn1TR2Z+ZDxSnGyW2oVygAMXB1dWUCblTuzrKawrerrxgyJ15YitnxOEoAy6vqnpoHgkOXq+4lmhyr4O+BPQloC9ZIX2JmDJIGgiyMeXyBo0ELbkmZIfBFEOlR4aGofjPtfrL62v16RuLSX4hLepv+7619sKUUdanYaMK5p+rXmVc8LsBz4D956RuGRv+4ID5SmHKJoARAD34YU7SltKJAI6YQBGzqAn8wq7t3wJnVnEdU9BWgLYCtBWlcxYUFhmFRc7EfqlxbpBYwMx+LyWxaC9wYr/l6EmBSQBHy2FHKU5g9r+XxzEuNgAYWwCMba8EiuwCigCKAIpUc1gBWGQUi4xRRAKMAIxAz9DL9QxlsxB6g0oh2B4gGCDYWiEYn2wEeoOWn8COWfjx1SBOIy8OUSzxvfRRrHsGQgNCg+6i5WWVFrAKsAqwCnQXrTKsSF/sAK8Ar0CP0oJ6lMpmLHQ4PQ/i2gBxAHEw5lfRb1mA4kYKkMrMkAGg9vKg5gKoQcfS5EyyD0wCTAJMUtXvOwFKykDJuLmcgEuAS6AD6eU6kKK8TOci9BOVYrIDYDJgMmCyik/5AGg2SX9R7hR8wGfAZ9BvtFSMcgiMAowCjFL16ZAAUibsPwJOAU6BfqRq9SNlchL6k4Z7DacFrjXKUlyzDhQHFLdWFKdj09pkKMcgSl4SdDMFdl1PIxgZIwYAk4uMFixynoNvymsle607Yo4Usk96sS2gw8np8P2ni8vNzzwKXv8h/LMFMAhzYEK31RJ3WzVhkU0AnrXrtirLOmO+ppsH8czpkoA9U0zv1CeO3+tf+FTAT8A+gD7wQR70gy2qH+ySVUX8g7z3n6Dr6zlrqjdhwu8Xxr31WFM9aMdSkcJyjW3cSxmXnUDP1ppPsEb7vNZoH/babFVqsb5aA5Zln8kSjMLBeR1y4OEKrcU+tYdFCrNXR/B0xZdin8bT4bumG75rbsHCzNM4eY4rsJ9MvDCz5OKcngTwcXWWXJ+Jj3kaR7U2uHoqV59X0dWwzjqssw7rrK/OOusL08Gs5brq4609NyXQOq6jviYir3Vcon3BY9pruSp76dprXY3+dvUCvXLiDVjgvfwC7znjxpWQb8Dq7qBsBYFHrsBDqSlnJn8r7PjiKmP0Hhvhfaue3scD9BUTL7i3YHWrYLSXH2EgciNOxrKqN4xM9cZn7/A2ZhW5Gp+L4oFrsYe0e3GmqRa7KSmeh0EgqRwSQhGK70VMGLiLfCs1xC5SSC7N0xnzcWCcjNQ4lNQzRJFyGWhDpPIwn2XliFTquNym6SpX/cvH5EF8a5inMOH1Pq+skxHKtvbwfaqhiGsu6fIbqeeSLIcs62toiGQgj7NpgTHzrVhCRBNZWqraWUXOGmkDH1s5spGCjFYHzJVmzu6RV8Jm/zkuEVfSgip303cznPEUc8VWvs+a43wWNmuyb1mKCMd4v0e3oSZLM/fLt+XvHEZE8EBSge/hq+BEckUVVEa8cnoKUtEUlguTsBtQgWo7d7VGJHxkrXm4TU0c5pqsTSTDg0OTxKAlN09qS5qkq1GXfuzKPyTRlNqS/m/IP3brcom0KmFT+r9hBMH6PXoGrsmVYmHsVeQTt+UTy1dp7sk/pO9G9w35fqN7SZjvhyMENOqXq9Ow3ozpyhm4PsXK56jjVtlRTqXIVn0hPlNZ1XbQ7hoNo643mg3UbO/t7XXau220223VD9FBsPNtXGXXN57+DxMbrcusNwEA k8s-resources-namespace.json.gz: - H4sIAAAAAAAC/+1da1PbuBr+zq/w8fbskLOwJIEAZaZzpkDZdqbdskB75sy2k1FsJfHg2K5sEyjD+e1HF9uRbTn3i5O8fCKSLcvvTY9ePbKedzRNR47jBiiwXMfXz7RnWkQLbcsP6K+/v9OfL3vsMmxaAWrZmJYGJMS8rOPg4INJS5zQtkUJQV73znXtwPJoeZUXdi0TX7hOQFybPaKNbF/cb1vOvc8fw38S3CbY79IC/bDq66LM7fMreLdE52ip4do28nwst8YrutjqdFnP9Vq16j3qSYWHHGwPWpJb4/XItpB/4dou4VJ42ZMrW4j42UfxChP53Y/Y6QSs07Vqrg4X3RYg3w2JwV5AvyZuDwddHPp66qq2Zdus2XShS3qIv6CH6f1OEDpWkL7PMnN32biDHTNR7+CtHzqqDjIJh4TQ1gtqe+ixqMZyCmr8rtuXjEeqCaj92QV3PSA7HEhRqktriFoSvyrbOq/oW6ZQULZGMr6kmJnytWs5wSfX5OphBRrytZ+YuGlBRxpAHazUssdaIci0QvaYRr5ObRxU7CYmmLBnt203o1wfEwv7nx8wIdSrFP33PWTgIpuklUw9h+nCABn3yq74AfY8bH6kMlTWB4h0cJD2qqxn8Qvxo8ffxw97uw6Va9NBPcx72vRcs2nQ4IDoQ8hZ8l/T8MJm6FPZNn1MS02/yc3kjDbRtAgK8LNhh7SD5M03/VX07zd9T0taZuXJj2/6S0U70Njz78NW5qlNGnW4N9J/flBrC/wJ297T4hZoBe04e5q+lxXCwHUDq8dei2kyf5nlUIVwz8u7Cq+mnaFecYWMwGVCrecuoXGUR2X9rZ6qepF+pY0m6LLA69om06V+Ut07rabNjvX4isYpKdan6m67VjtQVwZ8zNAvrr9oXwKLDix8rNF227Q5LRZ4JfO4ZAR5zgYRRLBZFGFcEsSDTjaENIMnj/fDckzrwTJDGnEKo0l8rW85HZt2D2Wc8BE9Wn6+c63QuBfukBUDC4xROGHSyihdZ6akvqs4aCaBkY/R6vd4Qo94DO8cGCZ9HAnyJmmjFraVHWSVbucc8aG4lqsT44TyNjFQVHPl0hunjXevtC+hrIreIztsZbxwR/F6G41L6oBLAJcALpkQl9hWzwJUsjRUIsQNmAQwCWCSLcAkh4BJAJOUEpMMoEAP91zy1Oy75J4OAHSYDpqtpwBPjAmSFv9Ba9iVVo/qRPxYWopEvAzgkTwe+cQlA4kSACUASrYblBwBKAFQAqBkWfkRgCQTQxLIkgAgAUCSlCaOSd3YwyjrVFHpBxoEuBOpq2/cfopQk3ToLnLHNN0lcdL3GJliCE3X3Vo/eX33WPhMpItJiDT1xqYQaap5lNUAlJVDWdXNRFnDgxA3W9bWj5CKWdtPJtz5cGQw42aX/nJVP3rduMhfkVisEghE5pgPoF3LNLFzK+BFwc2MSzfg1ykvSaxYXStpOo9AUlByoiieE6CAB0PFd/X69LC6JeIrhHDj4P1aXQn4c268YQujrSdtlz5uFkg+EnMLdV8l7T0/0ye+vOgFF1LR3hcCByp9rsKJHCcRtYFsRMQ0J56YcF+aeFrDQDAtpIDbpD+lSU4c036ffTV4YrmOjqsbIGAR81Yp3qKoO7Vwx5t6ZtDBnBbmWTQp5fSS09xhZgkzyw2aWcr2NMwPYWI5PMHP55XHMK+E7P2EaH7y9L0fPNkTTW7v8gONMHY8QBL/pX/7nz7tX15q79+f9XpnvgJOeCig4cYpbjMeKcWkTJ9uKqkGAYNZZGQjylhrxJEiraBpX9jEhtVDPD7VVZPCe7UDRZV3fN51bqNR1yXwRr8kNLhoptt3dOWlXwjzWX2oar4y79V+eatQTzF4S6nPCXst6gu5er5omYyr02v3ZniWBRQ8loLPy69g7Z+g4llUfLE4Fcs8hOkV/XFYug+UPJaSL8vrx0K94MWzKfhdmb342jXXQ7tFifkFKvfAPDhunNbw0TE+wg3jpIWOqqh6cnx0fPi6gdsnrfrhwf2pvx/nAv19zzX//YDI/mAm+ebV4P9fWVWcdoyTjrxwkHUc5Bx5BW3wzatm08B0zjnM1DyVGssQRkpuXJIID37/18H0MvQDYjmdyWRYnGXdwBUb/hmRBdCnsjlxfRjBat6LC4wYJ0R0lojkjEveC89on60HfLYAXv0ayfx8ETJfvalH9ERQ/lDlX5TO4abijK6RxC/B3bZV9e/0NVjd/YutWJdrdZcgx2eaVOsxAXiKKlj9hdXfDV39TfsprP6OQys+geVfoBUDrRhoxUArLp68zX0boTZiHyHQiRdEJ57L5klgFA9hFK9WwptCKo63srLMhLbbP3A1AxldXAGO8YpnmTzcwyxzS2eZslvCRHMSmvEpzDOBZgw04xlpxsr4A/ymcjGNBUSYScFANi452XiOOgam4gbzjSNdA+W41JTjOXgzsI43nnWczsnc3N5WQNmzKPuq1A4dqfkin3YDRU+o6D/WQdG3feSBnmfS8/ty6hl2ixSrFXaLwG6R9dktsvTv+cImkRxxWUgetgosbJ/Iyox8KF8d1L6SHSLjix02iWyhr8H2kHkonfizqnmNRHq1FJFyttL2CPWPpQjV7yNve2T6Xl8Pjh7sDoPdYcDbWwveHmwQm5i39zpdFEV41latpwOnDzh9wOmb9dOhwgu0G2xgOt/TzpFjCmuCxahyEvzOPX9GXd8xhEan7qDs0jP9plf2DdWB5rZjxza1ayRQNKi6pHS/mVUd+3UA2i496c+bu2Nrl8RlUAZUXlYO4CIcHLRecjLg9FrfcuaQFria55o+MIiAQbQhDCL+ST9pZcXBAVvybRIxlIs1X/EJwJErLf/LLKv+TW2QNd+Ms2TfK1vOICqSdhCNotsr7vNlijs2bk8Alm2U98VKzHuLBX65SgM3BSLfRrm/W6mhb7Hgr0r/Jdso9/ynUF4ZzywF3gLwFoC3MLbPAoEhR2BQfOE2UwY0hQ3+xO3kNIXjFX5XtJzJiNJ/VnRF+Gk4PwO+xbh8bMSW5gEZrQAZ7W0ELqgBLgBcsE64YMVpcwAGamAwgswHyACQAeRMlpczyXsh5EnGwkN1wEOAh9YxT7KqZVYARAWZknGI7wCLlg+LPIBFkDCZHiAcAkAAgLCWCRNACOVECKP2SwFIAJAAuZPl5U5iv8z6ImRQxgJIRwCQACCtcwZlVfxdwEmTZVKUe04BLAFYgozKWgGGBgAGAAxrnVEBxLA2mRUADQAaIMNSrgxLzich0zK4avBNiv3a2JDqGCAVQKqyQSo6e7d3Ka6i2EYc/7CbwVdtv0kwMpMVquwpEkpgNQRXNXrfK9pvWv4pfWJJ3OF5PKYCQG0aoPbh8/Xt7g3T+W//4TqpAC6DvdaQzlnndM4JYA/AHiVM54yGHakdRYsGH3N/GCCQqfYtdYkbdrrXYcBxiIAhgEJg7xJkh1aVHbqlQYidRv7hMySEJjmlpnYKyCuHvLbjKBox1GQUTd2BFh5lhEsbMPKB7QWOtlnU0TaDHEelVJ843q/BaTZz+XA1V7AqfQUaLtERNjNrmLswndGBpkt+gs0smo6mg140HazAeRazKPmyjCdSSSpWTPZBx+U5qWYuOmZuHEdtUPVMqr4qo6rheBo4ngaOp9mg42mWSg/ZyiNphkl4AdSYbTyGZiM5Ttt4vs0KVo238libMWLS9gn63WYYdIloEHBGzvhn5CjWYUtBh4ADcoC0CYQJJWFC29cuLTaDa4X8KSP4EztRv3Xf6OIe+oqJL/omPuIhlmbZHSYi97wx6lWdgWXq9yGdbzuYhnA9aSvAPc+mL+l0Ek/TbdopyZ4HRiBxDlLEiwA/cpswcRuFdmY9nLuQXKui0LJFW5y21AJTit1YJmBIta7HRJmNtPqPEJMnnrFQETVYuGcxOm2YtLSDHzPjQxKwpMfvZF5HEhiy7a/R+2deY4QoC2SoFt4YXJRYwNJgSOM3HadN/NZWUDuKpN+jGrQUl8daiUb+SVTCn9QUkXaX2WiTIp6m5bTdBFNU1Mqqj1JWNIzJSqUuwTXi/xU/X0/X5jrKytQXR6Yg3kSqCH18JxqSA9Na2Ed1wfaRIMD5WIhy+/aLtH97MyxnJx7GXkTQtrgwo3DdFvhRd9z+fi2mnFLcF5Xpqds8i6InMrg5Ekmym10GMnpD+pxWrSr9OJR/1HqD/xvS/zX5x2FVrpE+dF6X/q+Zwn6/x+/AeNCScYx8itzwsdyw/JT6kfxD2sd6Ysr9jfuSEt9Pl/Oi9C93F9EIm+Bwt+eFAdZu4qy8dqD9GVuhtnvtxowiPeT8P/20gRrH9erJqWG2T+jfa4xQDZnm0aFhYKMmHER/SEb56s7L/wELa7j8UxYBAA== + H4sIAAAAAAAC/+1dbVPbuBb+zq/w9fbegbtkSQIBykznToGy7Uy7ZYH2zp1tJ6PYSuLFsV1ZJlCG/371YjuyrbyTxCSHT0SyLfk850iPpEfW45ZhmMjzfIqo43uheWI8siSW6DohZb/++s5+Pu3yy7DtUNRyMUulJMIireNh+sFmKV7kujKFoKB74/sudQKWXhWJXcfGZ75Hie/yItrIDeX9ruPdhqIY8ZPgNsFhlyWY+9XQlGl+X1whqiUrx1It33VREGL1aSKji51Ol9fcrFWrwb2ZZgTIw+7gSerTRD5yHRSe+a5PhBWedtXMFiJhviiRYaOw+xF7HcorXasW8vCw2ygK/YhY/AXMS+L3MO3iKDQzV7Ud1+WPzSb6pIfECwaY3e/RyHNo9j7HLtzleBSTO+QKw/Syl7u4gz07hV61SMd7G97kQFevuOvoXo/jExHC6jYkt4fuh+U43pAcwnG9Zo6kr0rY9fv6HMqc2x3yUGaRaACRkpeFn7mpuCr/dJHRd2yJfj5H8ew0mcfJpc/Q+OSLNxEJBgqNn5j4WVhieFEHa10o4E8hyHYiXkyjmKf3PIaKjQkmvOy26+c8J8TEweHnO0wIs7Sm/mGALDzM4VkmR28/m0iRdautSkhxEGD7I7OhNp8i0sE0G7L5sBUX4vtAvE8Y9bY9Ztemh3pY1LQZ+HbTYi0PYoWQk/S/phVEzShktm2GmKXaYVO4yQl7RNMhiOJHy41YBcmbb+ar+N9v5q6RPpmnpz++mU87xp7By7+NWrlSm6xJE6HO/vnBvI2Gj3/7LXY/v7TCzENxhYU/cayQlzBdubtG8nSWwV6K18TczRto0GZQp8dfmaNcvMzxWG1E0BbDSGlDLpBFfW7weuES1oCL7sB8a2aynpRfWYeiXd7i+67NcTaPqrvH1axL8hpfsAZS6WQyedddp031mVS0W+bZ5RfjC3VYjyY6OWO7zR5nJGDs5IpLu67HfAODCLaHtCOhT2jRILJ5adKHQNTD8WznzrEj1hoNbWmSa0PH67iYO0e2evfo3gmLlWtF1q0MlbwZeJsaNzXcWjnQTe5K+ruGN6hpoynIgf49HtA9niByB47JiiO06JIuamFXW0Ge6XdOkeAAtUKe7GK0t8k+plpIV94467y7pX0JbVb8HvkuLReFW5rXW2tCVAdCBIQICFGZCJHr9BygQ6WgQxIKIENAhoAMbQAZ2gcyBGRo88jQgIP0cM8nD82+T25Z78I4AG22HihOyIiF7DuHdWQzUJC0jH+wHH6l02MoyR+lmCaSrw7UqEiNPgnLwGQR8CPgR5vNjw6AHwE/An5USn602FkjYEdTsyOYOwJuBNwoTU0Dk4VxgFE+qOLUD6wREEGkz77y+xlhVVqhmzgcs7KnNEjfY2TLDjebd+38FPndQxkzMRbTCKrqjXURVFWLhK8BhG+5hK+6noRvdAsnYoI/60fEzGxU0omFYltn8cjhl/5yUT943TgrXpGGg5ZlxL5ebJ27jm1j71pylyE3c8HmQMSpvSQNA32ugnSR3mRY7VRdRMGAknuMNN/F6+P96oaYbyg/nGToUatrxx6FMF6zderWg7HNipuH748l9BLui/R5j4+sxKcnc8iFzLS3Q1kJs76AcKrASU1tIRcROb5KRj0ilh6nHTNxhs0SGZu32U9lBJW0ab/NvwA/tV3Ht6trYGDZ5q3SvMNa3ZmNO9m4NscOnkkLwVuTUo5dxV4KGLbCsHWNhq2qP42KQxi1jl7IEIPWQxi0wirFSlYphgwVpl+mCOmDO9XI+abYi8lIwgOa8j/2V/n0qXJ+brx/f9LrnYQarhIgyuLFG/7MpBuWIz5ztnGqnmEMhqixj2gbcitphrIAzfrCNracHhKNX1034rzVB1CceSMGdacuGnddyp3Mc8JaLsP2+56pvfQLEW3VSGi+8ug1fnmrgWc4M8zA50W9FouFQr5Y+U077dnRvRo9hQMATwTwafkBNv4JEM8D8dniIFbFHLMD/XHUXCKAPBHI5+WNYwkvRPF8AL8rcxRf+vbLQHfYrP8Cwd2z9w4bxzV8cIgPcMM6aqGDKqoeHR4c7r9u4PZRq76/d3scVpKJxrAS+PZ/7hCpDIapb14N/v8Xz0rmNJMZTZE4mNIcTGiKDPbAN6+aTQuzAe0oVwt0MJahGSm5cykm3Pvt33uz2zCkxPE609lw+BTuGi4HifmSBQi/8hPu5ihp2HOvXHABoDTRSWqSE2H5IDphdXbu8MlEGxfW1uani7D56l09Fl0C+CPBPytdwMVK2LW1+DmE26ZC/858AUvHf/Ll8HItHRPkhRxJPY4pwdNkwdIyLC2v6dJyNk5haXkSQfQRrC2DIBoE0SCIBkH0agTRy9iLaYzZjAnS6AVJo59llymoo0eoo1dr4XURSCd7fvlEiLHd3/MNC1ldvAN66RUPakUHAIPaDR3UqmEJ49ppJNPHMKwFyTRIpsssmdY2bqDVKpdqWvKPuQAG4XTJhdPPiDGoLtdYOx1jDfLpUsunnyGaQUG99grq7ITP1fX1DoA9D9gXpQ7oGOaz4pweAD0l0L+/BKCv+ygAnOfC+X05cYadL8NhhZ0vsPPl5ex8KcE3mGELTEGWLbGAjRAL2wVTIrcfqc8HR1jJjpjJzQ6bYiD6YIPMM7kBCZ8f+Bdk5IulGFlIqjbZzL8vxcxhHwWbbOX35ssQG8KuOthVBwLEFyFAhI11UwsQX4MAEQSIIEAs9TdbZZAYV9jCbJhpnCLPlt4EK2flVCOeBuGcWN9witdzKIBdelni7GBfMQwMv50Etm1cIknDAeqSahPnhjqJawpol16hGDx7YBvnxOdUBiAvq2BxEQEOqJdcuTg76hsuczKobwS+HYLcCeROayJ3Et9SVBZrPEz52nOTyK5cLj7Lby/OvXjz9BdzSl5eM5l3+76z4WqnYeancbcK9l+s7Gmc+weS0gAAi5IbjQ0AQGDB2qNJQ8CWrB6AWJT6Z+JQACRmkQit6kPF8Qz3HxLNMp53C/IKkFeAvGLimAWdRUFnofmAcT4NlBbwBeMlKS0OV/jZ2GnmU5bG00r/idgVcbPRChP4rubyeRcXFwDrWgHr2l0LzlEDzgGcAzjHqEUEIB0rJh1jpI7AOoB1wFzP8uZ6ilEI8zsTca06cC3gWsC1RgoGgGyteoZnki0HQLmWT7kCoFww0TM7+dgH8gHkA8jHaLEcsI+SsI9xu+CAgAABgTmf5c35JHGZj0WY+ZmIfB0A+QLyBeRrIp08cLCSzgBpdykDEQMiBjNBL4qMNICMABkBMjLZXjFgI+WdEQJCAoQEZobKNTNUiEmYIRpcNfhCSqU2MV07BLoGdG2j6JqFHXebcTZGnOQBK9s57tYOmwQjO121y5/ToiVt03A241ejWGSfOIoo/NnL3AF+OAs//PD58nr7invDr/8VAO0AHYRPAcAM1UueoToCygOUZ9NmqMazncymtKVynsWWDMRnpl1xXeJHne5lRAX9kewHyA/sjIO5sFXNhV2zRoj1ecaHzzD9Nc0RU7VjIHzLJXybccaU7MdynsJijSUe5IzLHmAVW80nOLNqUWdWDeZtdkr17fJKDY6pepYv0guAdVNygHCJzqaaG2ERwmzsCEiX/GiqeZCOx5pBPNbcgYNq5gH5vIxHzSkQa2YSAOPyHEH1LBjzME5abYB6Lqgvygg1nDsF507BuVNrdO7UpGIYzeEWS16wWdOTp0YBkJEGAQILOnuqTCEwXiEGbrCgE7CmWygHGBZ0DNYE7SHgkMPh3UaEw6RKEnCJl3Qcl2ZxuxQaEziLCwS4oELRqlCMinHu8JFrKxKljBGlbMX1NkOri3voKyahrJv8fo1ckuZ32IjcioexqOoMPNO8jVpssI1ZC2+mz6K4F7jsJb1OGmmmyyql+PPACRQpRkbNQvG98Akbt1Hk5nQAIoTUXJ0cmi9W46ynpq5kniOKjGspYFHy43hW5S1Krh9wm+abXPNHhMmDmLLRyWB4u88b66yHstQOvs91FGnLpRS/lXsvxXLIdb/Ghsg68jibDjGm3ooTKH0SSytNO2vIWe9t47euRtsyJKLNHoPS0VyeoBLzgWkgESU1ZZO7HQUxAeFeW2E9PsUV9jbEsUJGJVLqsaNHrz4OvbiDU1FmwSIgCv9MKmRmcws152n6i2PfkK+mZEQhvpEPUpusF+Ew1QU7TMoUZ3YZ7inN9DFN7jNR2Ay6rMsZ4Ut6FvukfEVgPXxsK+kKn2TD7wizx01+W3JQ0/P7lVqiBWbcMU4zM7cFDmNgZHBzbJKU1qtkyGwoX6OrVZUf++oPRUdoNpT/a+qP/aqao5xvUFf+r9nS078n78AF6oobjS1FffCh+mC1lPqB+kPZTn1kq/VN6pIx309faMrMLzdncS+dcnm/F0QUG1fJioaxZ/yReKGxfeknaiwzEsJM87iBGof16tGxZbeP2N9rjFAN2fbBvmVhqyZDybxLmUJ16+n/ZJnATaMfAQA= k8s-resources-node.json.gz: - H4sIAAAAAAAC/+1cbW/bNhD+nl+hcRuQDO5iO0mbGeiHNlmaAs2W5qXAsBYGLdE2EUrUSCp2Gni/fSQly5REJU6cLI5Df5J4FF+e5+54Jk+6XvM8AKOICigwjTjoeNeySBYSzIW8+/ubvJ00VDUUYAF7BMlSwRKkywYREh8DWRIlhKQlDMbDM0qJwLEsb+rCIQ7QHo0Eo0R10YeEp88THF1w3Y2+ZajPEB/KArDV5CAtoyNdQw8rHZws9SkhMObIbE0LhggPhmrkoL3TjMcgF8QwQmTWktmalkOCId+jhDKNwqRhCnuQ8XJXWhBAPvyEooFQg241KzJU95iAnCbMVxMAx4yGSAxRwkGhVh8TUm0WK7xbhSKCBigKcvJmc7oc2LpX+CWMoUjUSEM4rpPgqEbCh3RkqIYhEVK7SM1Tl5AkM4wMWRF/qSe6Vrl1LRjhQMPfLEsM1cqLlaIeUxyJIxpo8FWBB7n3HTFahD9Gkp9IwAGychirVhgMcKK62anK7NRL2APEEFN99wkVxT45YhjxPy8RY9JmLOPnMfRRncZJoaKn1S6WCuhf2MDjAsUxCj5JDK1DFZANkCjaTNludEU0jvV8eBKuRxLXbgRDpEfajWnQ9aXpQ9kJ6+RXXT9OugmX2HY5kqUB72o16cgmuphBga59ksgBsrdfwU/Z5VfQ8FTzb/+VZeriK5hseL0rb132sgEa5WH1KQuh9gUCh6ojhW21mqQKMamIB9AXVM2jXamSGthB3t71texxMgE1FSWiF4ZTLNqJBF0zVxBMjLsi4WKoXCIlgUUZ1LQOpPOw9KVlp0PcF3ah0I4c7B2fe+eKhqIaityBX5etHDIU1LkAykTZDqc23hVXse4RRwG+xEEiXUKtuU/r6sWkOLIxHGNeHVcv8S9SVa2CDsLM1BUkJcqA0lT7U/UOLXdaenW0T+EKjtEcljNTUdkdExaFgj1E6nSJ0MF7qBfBVkWW+nDrY6kTb1ZVczbjonI2lnYSdhNL51FeUkpWtlYuzW1L+ugYwbLhZKUfpbPQwZJdfEJHhYAoH9BZZnKmPt1kh6nsFH/X8uHr1FoyKl5mHFQNg9ouDKqEQS0XBt0YBlXHclscxMUVmceZa9NREzqrLjSpsqNZCPGX/L06Onq1v+8dHnbCsMMtoUkMhXQ3UX2b05VS/sMKUATu5Lbz4dqDgKl/oSzTEauv9aeeokjQfSccIB+HUPsnSxSWhlUWA8qEZzpgfU/gbfXy8AbsM+lcvICOImCtes6UzYIbqfmirNf78Z2FnvrgrUBflIQ9aQsVeRJhY129P7sn6B/pXwR3BC9C8PvlJ9j72VG8CMV7j0dxto7q2/sT/QmH2NnxYiTvL68dp/Q6K16M4N+X2YqPaeDYvRe7sQ25ZbDcJefTgHDz1182748hFwxHg7thWL+xuTq7y/pYzLKvLP9y6s2F6m7A/badq30w1NfbTcDy56Nmp7kxP9oZMp0c8I4GPE46csz4EnUukl6JhK7UJL2hIy/SkHTloH7/GFA/mWJ7m56juo7qvaWzKqIDxJUDet/Z1Msg+nfwDE5EPydSAZbrRJTBiCsm7TzmEZpF5E5M3Ynpip6YFu3UnZjOkzm25Y5MXebYc84cC1FI2VV3RNkFjgYyZhPd3pVAc4ZKDS9v6QdZ02WQLRovHWk60vNjb320ST0f+kO04RLKnjg80kbhwqMXGh6ZZukipLvklG27AMnllLmcsgVzyqz+xx13LldaWRoiLESwyyxb8syyB+TYpaWscHJZxrXLL1vq/LIHsGaXYrbyKWbFPZmT09MNR/YiZB8stUFnNO9Vt90c0Xck+sNzIPp0BGPH80I8Hy4nzy412KUGu9TgpT4+vOfp4cvKEE4xdknCT5TQ+GgqfmNeoyP9/0wXnh9tlzH84gzsxecOPxrljK9+oHDw9DDrFKKVB/rD0wPNRzBeeZwPwfPIoXOvHbjXDlxe3bPIq5vrzYO1bKiAy9UshF8Q4+lwWttpscoBUk8EkF2kX7sVcDBTRqBiPBahbEMxbUugMCZyXtGg+rHetTLvRjJaISFPoLFWgwD1YUJKiVLaakzpmkW/9Dd9i8pZoz1TyzUT8wwpjadfHjbdKpD/ENmV3tmzJfAZXwpuFUoHaFxaDHIfZXS/VpqOARgk5Es2/9I0boGyBkM7eHPkKE4BNlY+6bLl+hygd8SS81eHfigZxJbqU1ayFf8ulOieuqlzXc//h+CoTxte1tyGnaz2bWRlK5dJqjQJzQj/PO0fFKWVgaoye+VMFdKZGIKEo7O0IdMXPQv9aD6QfhSXz1w9VPD3MLphDTMnaZy5GuqyNl2u0g+m6ygm99H9NEKUgI5etabvIMj4LisDhcdiLKMkNns4g6Q7jYvNgAXszNQCtJrGzZZ50wpn1zvGdcu82WqaktmbEqBtXLeCVGm/TeegXo0x9OLWXsyGX5sNm720t82b2ekOeBOY452OpQDfd6qzbsH52V62rOZvDdIwTgTyTrKtEu5ten9IBfTWj2nAUzUEic4DB+1mE/q7/aDX7/XebPe2foP9/u5uq7WLtoOW3/bTypf5qt5cm/wHvjWxiFBfAAA= + H4sIAAAAAAAC/+1d/1PbNhT/nb/C07Y72MFIAmxd7vZDC6PdXbuxQnu3W3c5xVYSDdvyJJlAOfa370lyHDmWS0hoCUH9KX5P1pf3ed8kPdPrjSBAOE2ZxJKyVKBucA0kIMZUSHj66294vNlWzUhEJe7HBKiS50TThimRv0ZASfM4NhSOs9EZY7GkGdBbmjiiETlkqeQsVkMMcCzM+zFNz4UeRj9yMuBEjICA9loCGRob6xZ6WmZyQA1ZHONMELs3zRgROhypmaPOQSu7RCUjwymJpz3ZvWk+jikWhyxmXEvhZttm9jEXs0NpRoTF6DVJh1JNut2q8UjTaxILlvNQLQCdcJYQOSK5QJVWAxrH9W6pkne7Skol4RdYNUbtpNpJTIYkjUpg7fUO0+fibAZSu8XF0DV5Jf2cc5LKBm6CL5s4NG3gcIXaKaiJeypixMZujgTVjRs6BYnkUwAsXhVcUELdarZ3zRjTSGPbmuVYeluSlRWcMEDjDdMr0YQAi+Aj4awKS0YA/FTiIXEqSKZ64TiiuRrmoM5z6xWgEhFOuBp7EDNZHVMQTon4/YJwDpKu2sKsPUxtQvUFkAYhznBI5RXanm0WKqtRzb4+7uz/dHBYb1GaQh3BqZ63agzwGxFJT/W0m15WvmXqb5xNShNwcy2UOzWukDg8d6gQKJH1VFUDAYIiTV4BmMoI2p0qtRimpoNCkiwj0WuYpBNxifmQyDmwJJeZxkjkyeZ53ie9FHS0B+PKXPQm2F6HcQ4j8p8/oG+Knx/QdqCa/vwf0NQPRQAvrb0XtAuz/AO62apjPmA8wdoTS5rASAbFWrOJ7zrGodRa1GnA77js79PaaFqDwM6tuDSDKck0MFU8t+cXoJZdihOioe5lLOqFEN8woMS75a8eyKaXC7BxWD1QI9HT7qoLXfQox5LMJ++braB/FWzCKF9UzNfXMOLNzT1KuNFi5EjFfRZHDqeqlnUMEdIxluadjuhAuplShzZ0ePIueKdgqLpDWXqN69logzmJGoKKYFzWxWdiTU9eZXpEmkb0gkY5hKbGsDNpqzOm6swu8SUV9Xn18/Dc2Hpd6CgpQo4SyQxkSGmq+63mwFoGT50CupdwhS/nCSNTFYXhuHQoFO6TuEmXYjZ8gXWm167xTKrhfM3kGvWYYq34Tub/gItwm5hZxy1xaWOWWtoW5AoZwbOGU1B/BWehdwRu9ls2rmT95YTOCpOz9elTdmh4p/Sj5o9+MNZSQPE0k/16rt/xuf6XzfXb65nr31uSWp/LbVmqkFfxnTYcZ/UoZiyJTPOTP+Hfzps3O0dHwatX3STpCkfek2EJ9pI29zkJw2avge4UE8rpujOM6eao0BGnIw8nbqgK0KILjkhIE6ydX8e11zl3G1DBPNPbiRcxvq1dmTuhIw6eK4jYOEXOpu+49lWfhOa9st7g6+cOeJozwwp8aZ70wRZq/DylVtBeHN235F/wL1J4gJcB+MXqAxx86yFeBuLDzwdxEUf14+JAv6YJ9Xa8HMhHq2vHBl5vxcsB/MsqW/EJizy6C6GbuSS3Cpa74nhaItz9/rvdxWUoJKfp8G4ybD41XeTsfzWPrvXJhOPQGrac+uzBeXOzwJl2fQxOBvosCzk2H8tfFBSS6ZYC72qBZ3kX5kwvSFdfxVRA6E0uV+CHSUnXTtQv0DrdyQS7gYe6CerDlbOqWCeIayfoI29TTwPoX9AjuG79IwcFWK3rVo5ToZB041hmaA6Wv47117Freh1btVN/HTtP7eWev4/1tZe+9tLXXj547WVCEsavfPllw7bEiKc3ZvycpkMQgez1ryQR80q67OkraOnLMJfdF7zRcJg6iWBzvMtAi8IR2fJVmQ+8DdBG4bcBT3QbYJul3wncpTBz328EfGGmL8xc5cJMp3PzNQOrVZtp8o+lAPblmStennmPGPvarjWu0Cyw9kWaK12keQ/W7Os0175Os3rg8/b0dMuDvQzYxytt0AXMh/UzPQ/0HYF++RiAPh3jzOO8FM6vVhNnX1/v6+t9ff3nra9f8m5ywavJp1Vmb2TsK+0f5/V7s4p/sjjYg/4la+7nl7Yvu39yBvbkC/A/G+RcrH+icPzwYtb1SWsv6JcPL2gxxtnay/kVehwFev7bHf/tji/aexRFe3N9vrNRTBUJiGYJfk+4MNNp7xuyqgFSb0SYn5s/ui7xcKqMSOV4PCXFgaLpS5Iki2Fd6bD+N+M3ZnG3atUq1X6SXGo1iMgA5/FMoZS2Gpu74dAv/QlCVTlL7UFHWOLg1BT4WfzChO3yP4vLsslfwrf9K4KtIr/SR3yuMkHrL9e3K9QhuZyJCqWzsobfmFmXJTkcx+8LQVR19zaZNgjTLcU5KiEnkra8OfhuCNQReR47iv8ajBglACV1NJ+gUoT+u0CiR+oZL7uZZ9f/sD7kBkprd9Q3FmQHVsNpKCBb2A6K/rfc6HVuQ6+IaTbKYCwaIvHHZEKoyq3NXNHcjQvdMEuzGLkgZ6Yj20s9CoVp3ZPCVANrqS8qLVxYWaYf49B0wJwZ6I1JQddDXzYmkcz8lx46wSnd98AkjyDR8U578u0DpH4FDVVeyygkUHz6ciGS3iRltnMZdDDVC9RuWQ979oNVNI0OrN9t+2GvZXOmX2igjvW7HRmt/XuyBvVJjqUYt45id/yD3bE9Smfffphe/KAfI3u+k7lUxPeR6YJc9O7ssIi45Ve5LMlySYK3xSmKCHaD30ABg80TFgmjhijX9eeo02rh8Nkg6g/6/R/3+3s/4cHg2bN2+xnZj9phJzSNL8qA39q4+R94i1XE8mUAAA== k8s-resources-pod.json.gz: - H4sIAAAAAAAC/+1daXPbNhP+7l/Bsu07dhvHkmw5iWf6IbbjJjPJxLWddDpxRgORkMQxr4Kgj3jc3/4C4CGQBHUfpLz5EhOgcOyzCzxYLMDHLU3Tket6FFHLcwP9SHtkSSzRtgLKnr59Z49PL/hr2LQo6tqYpVISYpHWdzH9YLIUN7TtKIUgf3DleTa1fJbeEIkDy8QnnkuJZ/MqesgOot/blnsTiGrEI8E9goMBS9D3G4EepXl34g3RrKhxLNXwbBv5AZZLExkDbPUHvOV6q93w7/U0w0cutoclyaWJfGRbKDjxbI8IKTy9kDO7iAT5qkSGiYLBR+z2KW90s1HIw2U/oyjwQmLwDujnxHMwHeAw0DNv9SzbLhZrcXk3M0k27mPXTMEb9um2r6qeyy8kBLu0JNdB92U5lluSEwy8O0k1pBzKtMsu+dUtssOhjKS8rPyZnoi38qWLjDvLFOJv5HMk1UqTuaKee5ZLP3mmED5P0FCg/cDEy4rfxwwfl6I+VmLo81IIMq2QV9Mu5qmhZ2I3McGE192zPZqtM8DEwsHnW0wIs5msuuZVdqi2vCyC/2WSpFkVSmzFE9X9fNY6eNM+Kb4RK1qjkMENd2jMRWxl3VPnSvi0CrkBRcaNAvwc/KO6bVuONabTZ29e7zdq0WnpKau1gY8MXDbOsExulM1WNjWupmAyAcW+j82PrJFKBaWI9DGdQPXwvS8kHITOtsusqeMiB4uWdnzP7BhswEesEnKU/tUx/LATBsyiOgFmqWbQEYPDESuiYxFE8WNaxh/X+i/pw7X+QmNl8jT2H38y7JB1hPCU+M9r/WlH6z5o22l1OwrQPeIgMT1Qy+Gt4OZWfI1ZLyZsbDpDBhVq1CpRgbO0vMfHtN6nJ73kdSb0G2m2zOkF9gW4UxmCDMK1y5Nuwm4OgA6bVMVk00kGiUeV+F5okwo/KY8lMUSZ4K/dnWt3ldIuH+2qIehoWKq9mMtG15mFXDrC0QFnfp5tKuZs3qszxpEUdYm8y4HVo+pMKviqfnL+RfvCx53sbEvTUf4xT2YQwWYZ0/EILU4cEZXp0Adf1Gi5pnVrmSFjPqWsJnlXcOZsy+7RvRUU29UNjZtobC4KXXdiRsNFkoNM51qm/lU5b0u5mVgEqLvwgO4nYSlDDWXVEapQKNTFdpku2V7/GAmu3yzkRVRV+bOIqxbnd6nHU40Aa+yE2sSifozhEVv51NS22CDqY5Q3nDj1AxsrxJpQnX3h3WXWfWmDrmKTk/VplB1GeZfWD5E/OIysJYYClnvxcq+1sOWews4j7VNlwGJv1Yu9mtF+yzUIZqPadpbiGz1G7AfEo8y0zQ4TreUlVH9ygp+U+BNLLGX839rO95087df2xrRt5S3ahIXIhNRttPKIZXlijQaxqGWw4aJsbV66ui7L83zBpxRzsxh7+LjxstUuZD7EbIsJhxHJSbu9EFZ6FZmJ5faBmq6ZmsZTQOhai+d2TWCn9WCnCnsEilqgqEWGug8bEgWO2gSOOpKjFtsyjqQG9MGealfkqjjhRMqOh4TpH/Zv99On3dNT7f37I8c5ChREzEeUjTlueZnJjDmwTBO7+my7GWo/VYE5KQdcIxkpsgDN2mETG5aDxPjUUnGwG7UBxZlXYjVxbKNx76U0Rz8lbHDRTO9O4eHkr34h3Gb1kdB85dar/fxWAU+5fzEDnxs6XWYLhXzBCtLJdXZ0L0Zv0wHAEwF8XH2AtV8B4nkgPlkexDLRnx3oj6N2ngHkiUA+ra4dR/CCFc8H8LtKW3HiJwOMZ8LYKJdfFay44qhKgtx7+dve7DIMKMm7D8fKsNyrudbon3niJaaL/hGRrAp3O1ubluyTzeaNL9ZBcE84p3TFKmX+0JRYbEepmI4EGn54xNps3eKjpYcH1Q6H42XgUE2T0PY0UJKZlOSkcsa6gBCz2qFwCqYKKjJaRd7pNYhF/CtkqlOtDV+C3IAjqcYxJZuKLIhVhFjFDd0NztopbARPEqt4ADvBzyVacQVH01JNVVKHZ3lwrXYiqe+xtiGndrDjkYfOnUduLLfPlgC0032geN4jPoWYUsthRh09wHE2nrTa42wRzHCibfkn2tYn6U051PZJSDCKF9K2/7683IEQ4jWvGMWUACvGZ7pilA0SFo3TRA+3Yc0I0cMQPTxn9PAYQgCBLdUJI46IwlwwQyRxxSOJF4gxhCFucDBxjDXEE1c6nngB1gwhxRsfUpylYBdAweYE+6zSBh3DfIKMAQag5wL6zzoAfXmHfMB5LpzfVxNnOAoCR0HWh+oGHQWp0Y758zoCEsEBAf5lUCzlFMjSrWGU8o+MIAd1qMx5j8mhgCMfdbHLqWapBRkqnPZYlnaQYGH6oP30hxbfHplNOv98WkP5n61E/gZ3sAACSgT+XAkCwR3yAQAlAO/1esQJwpkzOHMGEYS1iCCEY2fTHzs7zCbFQzwvrOnoEF5YCC+EC/Tj8MLDdd6fz+8NkNiWiylfibI1nYHZci9aisbX1I9jX/+p6Nd/6cru2y+dDq+tk5hGcks9y17xySJW4zpupl80r7qIQNKOkWtGVgXHL9ZLno59OHyxFur0YhMYxCtgEMAgNoVBiLW1Y1GgEFWmEFcxSsAhgEOA+2Xt7peiFYLvZRLm9BqYEzCnTfO9sObzqQyoU0W9L0yamtfTYi+MqZ1HeAGBWvdn9IBAgRNmZirxBqgEUImNc8IAl6gFl0jcMRToBNAJ8Mes3R+T2GXeFsErMwmVyqcBlwIutTluGZN4vHKgVPVyz2inEW7Aq4BXgZumvtyiCdwCuMXG+mmAXNTVXwP8AvgF+G2q5bcp2CT4b4ZvDe8F2m1OzL5awLCAYVWNYbGFvr3NaBbjNoLaaNs5utULOgQjM90Gy9+RoeRZk9OstsOI1UoJ1QXvzeo/2DOJoO+IJcV9117Sf4vu1J+5fvh8fgnUFM7Yg/Orzs6vfaBfQL8q6OAaz7wyp8GWwQqeAf2ajHltlKQ3hX5dDYgX9gfnIQUSBufswD+4Jv/gJRt3+IX6Hz5ru9qpxW+t7oa8lu1zz2RJfGDX/qdFo84OuA0X5TY8AN4KvLXKbkPpTtxVOA+zjEr7XVuFH22dbrRlfdV9jd60bbEK+D0/VwCtA98a+NZq6FuDT0EDR6mqb20SerJwv8/kJGXJVQNTWYTjSfCViK4AWwEnFDihqueESj8GCL6nUexNQd4OgbwVyFvzWZC3aBbKAc3MgSUe5ITLCjCKY97T1GQwFwWZsMFi38bRwYA+2JPMG+nnJK+Kc9psX4WUPvyoLjOZlAeWaWJ3xq9fDt0k1fp47W6zbl+vfVvNz5sKgFUeMEB4WoSPK4ywMGG2AASkF4H0STWRjleKfrxShM+NzwXy6fJA5mvHuSFW+AEA4ykxfldxjEU0QzxqA9RzQX1WRahTdwFgOxO2Rrn8qjAhVxxVSZB7L3/bm12G3AHm9qeTYbk3fYk7LGva4KjAp0bf6ssJZR4t9aWHwFRd7MdrEftGRjtVHeqTNUK91u3bCsj+dJ2j23MX/rvNVfzKhkxUAPazqn+8/CTaEdSGu7XwCXMIAYWwivqFVYwLpdiK260HxgA76CsmQdS26NhKtEvLf2EiciMKY1bVH2qmfhN22UIY84OKaVkUO77NOun2U0vTbdYoSZ+HSiCFH2RiMCi+Fzph4h4K7dzWuDAhOVcVkMv3b3FWU0tUKTFjORZDyvV8Lsr8SKv/G2LyIG7dUsVs8OGej9FZxWSpfXyfmx/SAUuqfivXHUlgyLa/xv3PdWOMKEtkqBbeBGEpiYClyZCN32zWNvFbWxHlUSZ9hyFoKV5PUIl5wDSQiJo60Ui7zXW0wwhCx3J7XsowdtRgtcaBFU9jMqjMJAQiwV9J/Xo2t9BQnqZ+OVaFqCdSRhjgq6ggeWCqhX40lqwfKR9cjIYor558ku6eBM3ZFM1hkC9RZ+TrSrPLliexbtkMPdpK6NBTNPlbQrTxtN+L1iG6693tNpMAZ7Z+iNP0zM98i7FwMvxxLJL0CleZEOtt6csUzYb0sC8/SPci623p76b8sN+Qc6TvkLakv5tmpM3fkz7w2HtJYcbWIhd8KBcs19I6kB+kyxtfmXJ7k7ZkxPfDE6F2+perk5ippes5z/FDirULHNldoO1p57Hq66GIItUP26+b+OAQH+C28aqLDhqo8erw4HD/TRv3XnVb+9HLtylBbGw9/R9muCx5yfQAAA== + H4sIAAAAAAAC/+1dW3PbNhZ+969g2e6O3caxJN8Sz/QhtuImM8nEtZ10duqMBiIhCWuKZEHQl3i8v30BkKJAEtTVNknp5CUmQOH2nQN85+CAeNgwDBO5rscQI54bmEfGA0/iiQ4JGH/6+zt/fHwlXsM2YajrYJ7KaIhlWt/F7KPNU9zQcaIUivzBpec5jPg8vSETB8TGJ57LqOeIKnrICaLfO8S9DmQ18pHiHsXBgCeYu43AjNK8W/mGbFbUOJ5qeY6D/ACrpcmMASb9gWi52dpv+HdmkuEjFzvjktTSZD5yCApOPMejchQeX6mZXUSDbFUyw0bB4BN2+0w0utnI5eGinzEUeCG1RAfMM+oNMRvgMDBTb/WI4+SLJWK8m+kkl2F6g8TLZnOYLsTBfezaCbBqf/vuu+AyA6n6xk1f13gx+iGl2GUFuUN0V5RD3IIcKlC74GKib0ow8G71OYyLrlNQKB+RcAyAkpcGlwuhfCtbusy4JbbEtpHNUeQ2SRZacOZxND57sicywUCB8QNTLw2Ljzn4LkN9rBUQX5RCkU1CUc1+Pk8vVxwVG1NMRd09x2PpOgNMCQ6+3GBK+UindSGrD2OdEGVR/A8fSZaWz5EierK6n09be2/3T/JvxFLcyGWIWWE8U+SxVYVXn6vg08rlBgxZ1xrwM/BP6rZDhmRKp0/fvtlt1KLTylNaagMfWbhoEuOZQmebrXRqXE1OZQKGfR/bn3gjtQLKEO1jNoPo4TtfjnAQDjddrk0dFw2xbGnH9+yOxVcTxCuhR8lfHcsPO2HANaoTYJ5qBx05ORzxIjqEIoYfkjJ+vzJ/SR6uzFcGL1Ok8f/Ek+WEvCNUpMR/XpmPW0b33thMqtvSgO7RIZJrDyND0QqhbvnXRrP1KbKYFKNWgQicJuU9PCT1Pj6aBa/zQb9WluKMXGBfgjuXIqggXLki6TrsZgDo8BVbrmSd0STx8F+vy4dOvLrNBYXhbb66UWIFRUP7ypgVmFFdPImjzUG5creu3JdEongmrAYI0ZS10hAUzcoLA1A4M7KBoKOeY2vWetGrU07cNHXJvIsB6TF9JpOMyzw5+2p8FfNVepVmyerwkCVBiGK7gOsEHmX50YsoUIfd+7JG4trkhtghZ0yFbGj0riTy6ZbdoTsS5NvVDa3raE7PD7o5jJmQGJIMZKaQMv2vivlewumkZaLvwj26m4XdjCWUV0eZRqBQFztFsuR4/WMkDZBmLi9iwNqfRRQ4zwuUHs81O5TYCb2KRf2Ywj82sqmJbvEJ1scoqzhx6kc+V0hDVZ997t2mjNGkQZexyqnyNEkPo7wL8kPmDw4ibYmhABs0tkFbFbFBNT+LZFeXARZoxSzQmtkixLUo5lPmZtrusHrc2hhQj/F5w+7woSXeyP6IuZiF7BvCV+l5uFZSx088sdAw+fuXTkeYN52RCn7fytoqxs6UtleuxatgXc3IKycLn/Q1jLTZooQRi083RQ6HQpdBUZ7nS7KnIQ5y7hLzzuvWfi7zPqaCfHA4y521209CmS8jNSNuH3hzybw5XkJClzw98WwCda4HddboI/DnHH/O0+dd2MJ5WQLdBAI9kUDn2zKNQQfs3plrH+kyv5pFmoTHbOw//N/258/b7bbx4cPRcHgUaFiejxjXF7e4zNFyPCC2jV1zsf0fvYcuR8u0s7k1mobSAC3aYRtbZIjk5NfSEbxrvQLFmZfS1Dl20LT3Eg5ltimfuQzbu9X4dsWrX6mcqyZC801or/HzOw08xZ7VFHxuOOxyXcjlS8qRrNyLo3s+eWMTAJ4J4OPqA2z8CyBeBuKT54NYtSIWB/rTpL16AHkmkNvV1eMIXtDi5QB+X2ktHjnhAOOFMLaKx68KWlxxVJWB3Hn9687iYxgwmvVNTh3DYpdpqfFSy0SKzBcvJV0YGl8+t00L9vgWc/Xn66C4Jz1fpsZKWT5gJx62o2SYjiQafnjE20xu8NFMAVVrhcPxc+BQTZUwdgwQkoWE5KRyyhoH3q0VCm1QVRCRySLy3qxBFOafIRedau0mU+QGAkk9jgnZ1GRBlCZEaa7oVnNaT2GXeZYozT3YZoY4zXqcFEzUQMtL1vIcYe2GpL6nDMeEfYiHHr3v3Hr0mrh9bl+wTvee4UATF7vcWapctCwZcjWPHuC8oUiqznnDSCjgyGG5Rw7LQ2FVTh1+liMYhTUZm39dXGxBGHXJhq1cXMCwXVPDVlVIsG3niaDeB9MWIqghgrrKEdRT2AYE91QnlDpiIUvBDNHUFY+mfkKMIRRzhQOqY6whprrSMdVPoM0QVr3yYdVpCnYOFGxJsE8rrdAxzCfIGmAAeimg/6gD0Be3yAecl8L5QzVxhuMwcBymPFRX6DhMrTf21+tgTAQQHHsoguJZzsaUoB+T1GFipD0ISGXOxcwOBRyNqa+mzrWSPZHqwjmZ55IXGjyjhBg//W7EH/1MJ519adcQkdMXQcQSjhrAZEZM/ngRTIJb5AMkM0LywaxHwCOc8YMzfhAKWYtQSDjmN/8xvwOIhXzZWEi4jiGOhTwo8zYG8aEHhcC5mAmTmJuSFuZWZmQTF15qsAih+19iYRZeYsCzX/iwFq+xjIsLnpqpnUeoGcfItSM1g5Mp5dKxYx/OpZRCxl6tAic5BE4CnAQ4ScxJpLk/JAxISa1IyWUMG7ASYCXgIirdRZTXQvAPzcLF3gAXAy4GXCzjH+LNF2shkLG6eIj4aBpez4g9RbZxFgEIlKzsmyCBkoGjaGFy8hbICZATICdZRxGwk3qyk5HLiAFBAYICPqPSfUYjvczqIniOZiFn2TRgZ8DOwHWUkDObeqJyIGk1dyEZ7QhIYGrA1MCVVF+20gS2AmwF2EqRLwnoysr4lICxAGMB31K1fEs5nQQf0/it8femtpsz87kW8Dngc2vF5yxMnE1O6jhxkrzJ2MyQu17QoRjZEzYGsx9t0bK6ZUjdi5K5c9HZl7/9ahYcbimZGMy/WkD8JXtbf1L98cvZBbBm+NQDePrq7OnbBWYIzHDdPH3TSeGU44XPzkjWgBnORgrXCYhVYYaXA+qF/cFZyIAfwilP8KqW5FW94POOuN7i4xdj22gT8Q35bihq2TzzbJ4k5n3j30Y062yBs/WpnK17QKmBUq+ts1X5kHUpLtdJbM74zSjF+1gd5+PDQ9KfVdjYFz7ITWm//JZdxoBxgkcSPJI19EjChfRAn9bSIzkLc3p+d9ii/KnclgGJegp3naRSEZMCIgWuO3DdVc91l1xoCh67ScRSwyvh4/4vzCuba8EroyUuIylc13jiXmZweQFWfkJ9nJunZsJ5R0Q137dpTDVg984si1Jy3+5lfsFc7Npc5WZcfZmjFX9AbBu7C14PPHYPVet27+1m3a73flfN+58lwDrPHyA8L8LHFUZYqjA3PgHpp0D6pJpIx2aoH5uhWxW/273iILefD2RhmC4NscbJABjPifH7imMsA0ziWRugXgrq0ypCnfgiANuFsLWKx68KC3LFUVUGcuf1rzuLj6Hwrrn9+caw2FX/jJs/1dxcqcD1ye/M54lNnwxK2YFDVUfluBRUIMKsdoJyUqKgVHnbugLQtMucWQGbidi8X1u1qWscSgWE5tSseDTKSbRRaoy3wKsViSI+ISYg1QOaGBWaLAj5hViVdYpVmRafshG32wysAR6ib5gGUduiE1TR7rT4hY3otSyMa1V/LJnmddjF1MXizGxSFsND3+GddPuJppkOb5Qiz2MhUKIyUoEtDN9JmbBxD4VOJiRAqpCaqwvAFvvWOC2piSiZbcSQcRHFsij5sT6rkS5KrueLMc1OueY/Iab38vt5uogYMe+LyTotoTy1j+8yC0UycynVb2T6pYwccpxv8UCkBXnamBYMpn4UZwj6GY20MrXziZyv5TZ+52jCXAo02hxyKInm9REqMTuYBxJZUyeacjdDP6YjQmq3+YrP8DbvDSVWwKlEQkS29Oi1pqEXL3AqylxZJETBn6MGmencXMtFmv7lWDairikZYYAvo4LUKasWAtN4ZoFJeOPCIiMkpZMU0xEyEwYdf8CXnAmypOe0j8r3Z0HGVkXGuB2ynHTxArj10vPmlqfizxk/SitpNWRsY0S3HiNyQeSwx7SiF9k5puvdbjdHUencPonTzNTPfMJZPh3/OB6SxHRUCbe5r1yv02woD7vqgxK2au4rfzfVh92GmqNc+NxS/m7akaR/H/VBHJhQhGlqLWrBB2rBai2tPfVB+brroa22d9SW1PD98GQIo/n18iRmgom96A39kGHjHEc6GRg7xlmsFmYoQ3/Ng/03Tbx3gPfwvnXYRXsN1Dg82DvYfbuPe4fd1m708k1CQBsbj/8Hik7lsdX9AAA= k8s-resources-workload.json.gz: - H4sIAAAAAAAC/+1da2/bOBb9nl+h1XQHycDZ2Hk4TYBi0SQNZoB2N5NJu1hMCoO2aFuIJGooKo8G7m8fPvSgJCpxHk4k9/ZLLVLm6557eXh1It+uWJaNgoAwxFwSRPa+dcuLeKHnRoxf/fmVX8464jbsuAwNPcxLGY2xLJsEmP3m8JIg9jxVQlE4PSPEY27Iy7uycOo6+JAEjBJPdDFGXqS+77nBRSS7kZcUjymOprzA3upGtiojV/IOOSw1OF46Ip6HwgjrrcmKKXYnUzFye3OnG17bWUWIAuzlLemtyXrkuSg6JB6hchVmHb1yiGhU7kpWOCiafsTBhIlB97qVOlz3NYYiEtORmIB9QomP2RTHkV24a+x6XrVZV6x3r1Dk4QkOnMx4+ZwuJ6buxfrFlOKA1dT66Lquxg1qaqIpudKgodUwji6v5luXyIvzNdLqiuvPcSLvKrcuK65cRy5/t1yjQSsrFkA9IW7APhFHLr4osFBkfcOUFJc/xNw+AUMTbLRhKFqhyHFj0c1Otc5ser7sDqaYir7HHmHFPiNMXRz99xJTyn3GMP4oRCNchzheKczT2yyWMjS6MC1exHAYYucjX0PjUBmiE8yKPlP2G3kjvg7lfKLYXz0PRFHAV3cQIB/L8Q5C4gxGPAAg3hXdzz4NRmE8iCO+woMI81InGkiw7POGBi5FDN+OvJgPk747t98kH8/tjpW1LMqzi3N7Jnr/xSLBalbY4X2vqUFNKInDgYfHbPWK0AuPIKdjpZ8G7CbEa3nLg6yCN7B/EQ/VLMiVmAHFHhpi74GjyzsTFelnvVwOQlSK/+V01qzhjbWq5mB3ygs/JtRHMtox1xeLKNBTvY2DEVPuasdoxIiw1GblFhVCjrP2bm95n7OZXXMjx8yFFvaLkYDDSmKzUDHTroqQZlMR9InnGOAupnXMw6OhL1n3x9QdM3Mlk1uVfXjy2fosIFZ0NJZtUbflOIYoduqCHKGsHGnSKCZtJ3p0A8e9dJ2YB73agJbeK7fL4siu0bUbVcc1jEcXyhmri277STATS1IymS0AaP5WfcjOwrLc/81TuEHXeI7YkEOUd0eZAVDCk+qw5JHJAZLbfK9Sp3Yp49fUNtWtQjOfcRGcncZOwuxiah7lTbPkZSvl0sy3+C4UYlR2nKT0Nx4sJB00V5+SqwLlywZ0lricjqe7/FDV/eF+k/XTvvKWxBQ/JtOrEr1NIHoVotcDoncn0auO5T6mF7Ebb55gLl1HTOisutEosOOcQvyf/1v/9Gn96Mj69dd939+PDNQkRIyHm6C+zXSn5GdIBwf2g8J2NlwzCUjjC6EJRoyxdpRGiqKBHjthB49cH8n4ZGBhilYZHCipPJOU/MBD992X0Rv7iPLgYjmcuNrGWz9T4bP2nab5IrzX+um9wTz15K1gviD2h9wXKvVx4Gr76uOte4r/4vGFRWDgpxj4oPkGtv4JJn6KiQ8XZ+JkH5WXjzf0R9d3wY+fZuSj5vqxMi948dMM/KHJXnxCnHZYt3qGWbhxN5yN/s7bHt7u4228M9odou0u6u72t/tbezt4vDvc3Nq4eButc1vKU2O0HhLn35eIrucnyXdv8s8/i6o0DZkmIWVhnoXMc5Cygjf47s1gMML8zHkX1EKTGZsQRhoOLm0JN/71y8bj1zBi1A0mD1vD+iwrJPPbk8yXz1kNaXx+wpe5HGPgekSWv9oHxWOZ3bMNZ72axH7noXjK1j/HThru+AdF8x9sjbQFXsGBCOh5XfQcLA49EI0WgKcN8M4fyDsPXye2e/LoB9hpNXaOILIvY2QH31wC3/xgt0CM83vMvbVZYhyKgkhY0mzH7DxuqAKxDoh1llSsU/RTEOvMI8veArUOyLIbLMvOiZ+PfUJvJMdygwln0mwwvGH44Qwwa/IfvEoUuD43jboAOTbIsUs7yyeJO1BkN4LkSY8HkveDkjyjKwLPm0OUvQ00D0TZIMp+oii7lgqAGqw5umxFEZ5kYJBmN1ya/Yw2Bl3nEquzE1uDQLvRAu1n8GbQaINGGzTaoNEGjXZrNNqQ2Qdtdhu02QqdAKCllWdDIGqnLBscE5TZAB8QZ0NcXyZRNrgl6LLnV+WANBuk2aDaaYVqB9TZD1btFEUiaYQXbfV8GxQ9oOgBRc9TX7OovMA6xSPsXmLrAAWOQhM8SGymvOcgjJ5o6zPB0PgxBYzdeJ3P4419ym1gkXHq2I51ghSLBlM3VOzzZFOnfs3A2o2X/ITP7tjWESWCyoDJm6oAWoSDg9Wfw+rHTbQ66L1A7wV6r7brvVbFcyP5jqPV/HFRgJl4TDKgag9XD+PUO5HufQ7zvfQqpD85CEXzgzQ99nXtPBDPjazKgyPTQ6OFPjP6XvvQ6PucT43yZ0Y/osjrTvCwhA0AepYDPQcvip409ISKRwJ82g6fw9cJPoCfJcHP0auGH0cdYwFGbYfRh9cNQ4CjJcHRceNfjJk8TfuPgmIT348ESixQYoESa26fBUlWRZJleGFmH3RXFd0VvDAz0V31X+l9mZBlfSFmCC/JNLPBu/Vz8KbMl2d6QjoFPO8VeF5nGVjOLrAcYDmtYjnwOBBozsJpzj3SceA5wHMgn/Vy+ayqF0IOax529xbYHbC7xrE7jjfIYQG5ezVy9577DXdX6zBFX87yxNqecOtmf3UCtA9oH6S3WkuA9oAAAQFqFQGC9BYwoEYwIO2PMIEEAQmC3Ncr5b7u91VIij2IE5bKgBQCKWyLsgv+CAlY4QK1XfO8Sgno38vTvxDoH+TAHs93esB3gO+0U+MFhAcIz+IJz30vFATOA5wHUl4vl/JK/bLsi5DfmovvbQLfA77X6vwWvN0CaN/L57mM75gF7gfcD/JdreI/W8B/gP+0O98FBAgI0KvkvYADAQeC/Fez8l8Vn6zJg60ko7aj0RT76AumkRpZb1sVi994E99wEL2QjfFtbJKj0hYhnfI9CcufclBtMeyHHp9iMMm8jNOHiGlYziGgUacCgWT4WiLCwWMUe6VtXbqPXmvio+LX33ARpzVASl1YJ5JaLQmFucph1f4rxvRG+p2JcIrXNYqAXIQlL53g69L7HbNgpXW/UpqOtmDI874k8y9N456lrFlD8+LNwanTBda2QR6b+Rbu4PeegaHWrb7PLegabk+tkpCCh5hE9jRQUXY1ox1uMCYdK2luzWyszfuMlWxRulG5S0iLRL+n/dvF2spARZn55gQKaiZaRRzhM9WQHpZagY/ugvGRccPnQYiRj840QgrIWRbkpOeFRwNnkb8jP8vPM4C4ZUGcXLMmok07VRcO1bPSqXo5oLiSMviZYqyutE7CVcfqpGwH5Gq9lx4e+YE2KbMLXwtdTrhp/uVkSbI8in6Cs3e0B9O9rnaxpV9oP+Bt72ife/rFVlev0d5ysql97jnKIb6mcxDpDQ1y9/aiN9zXG9Z72dzWL7Qzx66jjzcdS2H5vhGZLbM/nx0mx4vshcTED2N+qDlNf6fM2rD+p8drO5a5Wxv1+ttod9zd2tob7+3ht3sjjHZGw+1dvDfE/d6uuvkyO9p0V2Z/A6odrq4q2gAA + H4sIAAAAAAAC/+1d+2/bOBL+PX+FTtdbJIvkYufhNAGKQ5M02AXau2ya9nDYFAZt0bYukqglqTwaeP/25UMPSqITO2li2Zn+Uosj8zXfDD8OJ/TdiuO4KIoIR9wnEXMPnDtRJAoDn3Hx9Ps38Thel69hz+eoF2BRymmCVdkwwvxXT5RESRDoEori0TkhAfdjUd5ShSPfw0ck4pQEsokBCpj+fuBHl0w1ox4pHlDMRqLA3W4xV5eRa/WG6pbunCjtkyBAMcNmbUowwv5wJHvubu224hs3F8QowkFRk1mbkqPAR+yIBISqWRivm8IeoqzalBJ4iI0+4mjIZafbrZoMT/oaR4wktC8H4J5SEmI+wglzS28N/CCoV+vL+W6XiyKO6RWSL7vtsFxJgIc48nLFmuMdRu/ZeUWl5htXQ1vn5ewnlOKIT5CG6GaSxI8mSKjU2mcBE3tX2Ihc2yVcQDeYUKmYkaRQgCErK1eAUL1VrV0Jrn1P6bZVlRi4zYulFZwSoY1PRI1EFTiIOd8xJWW1xFgoP+JoiK0AiWUtFHl+IpvZrcvsuBJa8TDFVLY9CAgvt8kw9TH7zxWmVMy0pf8sRn08Cc5CKLXX3iqXctS/tE0e4ziOsfdRzKG1qxzRIeZlg6wapXoR38RqPCwJVy8iWRSJ2e1GKMSqv92YeN2+8C5INEUP8k/dfpx0EyZmuMuwKPVYV4HlQFTU9Sni+K4fJKKb9N2F+yb9eOGuO3nNsjx/uHDHsvWfHRKt5oXrou013akhJUncDfCAr14TehkQ5K072acuv43xWlFzNxeICg4uk54eBbmWI6A4QD0czNi7ojEpyD6b5aoTUij/V8NZc3q3zqoeg7tenfgBoSFSrpT7oZxEiZ76a5nzOUF9TqSmtmqvaB90ktd3dyfaHI/dCS8KzFwaa0rZEwhYKWyWBGPjqQxpPpIrCgk8C9zlsE6E77W0pWSfR/6A24VcOU336PSL80VCrGxoPF//7qp+DFHsTXBXjFBenz7txZTuZIt+5PlXvpcIpzfRoWXvqrW43LMbdOOzer96Sf9SG2N90t0wdWZySioqcyUA7d+a7LJzt6zIhX0It+gGT+EbCoiK5ii3AEpa0iQsBWR4iBSHaNdkehGzfk2vYq06NIsRl8G53thB2E1Mj6O6aFasbKVamtuWWIVijKqGk5b+KpyF4pp28Rm5LvHJvEPnqcmZeLrPDrXss/9dyUcdbS2pKl4njayzyC1gkS/LItvAIu9lkfW+PEQjGb8NplkplF3KAZ3XVzFtSbjgJ/8T/zY+fdo4PnZ++eUgDA+YhffEiAt7iSbXmS3DYvfr4cidaU3Iu2tnGJnzIjTFiNWR9zM3VFbQYwfs4b4fIuX8LBRPczaLAaXCc8X3DwP00Hs5d3KPqfBcjidYsWt99QtVvupe1XyV1uv8/b1FPZOZYUl9URL2hC3U5EnkG4v247V7hv8Q/oUzUPBTFHzYfAU7/wAVP0XFR8+n4nQdVY+PV/RHP/TBjp+m5OPm2rFWL1jx0xT8oclWfEq8xdBufQ/z7Mrd9DY7u2/beKeDd/Buf6+HdlqotdfZ6Wzv7+LBXm9re/PyLdsQulRbUrYRE+9fV4huFNvUd2+Kzz9JURbjzCKcqrAIcRYBTiUQFb570+32sdjQ3ge12KbGJriRhoPLmMLNf/68+fg5ZJz60XC2OZwcwoWTgsU5KVCBIMsZgdjhq1CP1XE94gih3gbFAxU6dC17vQmnBuuz4imf/wI7mbsTHzTNv/s/6Ym5k69uiFFzvBFiYQ59Jmd4Vk1ltQuBACkga77IOnw+ZIGnegY8bYLlguUqyz2az5oQqC0j4GppcXUMK8Iyrghgt0tutx/cBcgs+i0RltyszCKKIiY1addjvv+3iCDzCDKPljTzqGynkHk0TQL7NqQeQQL7a01gLxhniENCbxWB86OhoPC827vlmM1M1/Iq/yZEssAPhWr0AySuQ+J6Zdn6pHAHueuNYJDK4oFBvlIGaTVFIJFTpK/vAIeE9HVIX29y+vpEngF5c83JYNf840kKhiT2hiex/0AdQwbsEuexp7qGVPZGp7L/AGuGbHbIZodsdshmh2z2hclmh2MDyGJf9Cx2jVwA19ImsoOTWr4EdjBayGEHaEEaO6wHkL4OJgsZ7LOkGEESOySxQwrSQqQgQR77zClIu5CCBClIkILU6Bs0tZE4Z7iP/SvsHKLI02iCk89m5iMdxuyJuj6XFE/sgUDZjU9Meryyz4QOHDLIDNtzTpGm4aDqhmYnPVnVmV1z0Hbjc5TiH27YzjElksqAypuasvQcBg5a/xFaP2mi1iFBDRLUIEFt0RPUVuWhlLpqarU4i4owl+csXarXcH3Sp6+mSs+l+si78hmhs59GjX8XqJTtdbOA27e1i0ieRDm1oyjbMdR8TqH+nPIYqjiEeo1ZaveCiafsANC0nGg6fFE0Za4p1jwT4LRscDqaj3MCPC0pno7n6p48vQ0GWC0brD7M100BrpYUVyeNv+M0Pa37t4ZmE2+jglQxSBWDVLGpbRZyxmo5Y5a7TzuQNPaySWNw92maNNaZ09WnECKeEw2F+0/t1PP+ZEC4BPXlaaXMAwNSOQdSub4MlGoPKBVQKqBUcFAKnGpunOqBpHsgVUCqIFL3cpG6uhVCdG4aKvkWqCRQyddFJQUcIToHTLIxTPK9sCNhvs5RhsaCUsq5PRXKzf84CDgmcEwI3C0s29oHtgVsC9gWBO6AbjWabhl/mAuMCxgXRPXmFNV72FYh3DcTAa2WAQMFBgrZePBXkUBBXzQfb5q7vIBrvjzXjIFrQnTv8eSqDeQKyBWQK7gjANhVA9jVQ9dnAsECggXBvJcL5mV2WbVFiNxNRS63gFwCuQRyCRcGAcdsbgTPen0zEE0gmhDJWyiytQ1kC8gWkC24Rg/YVrMjekC4gHBBZK9Zkb2aTU6I8K2kvXZZf4RD9BVTpnvW3tHF8rca5Tc8RC9VZWJZGxaoVD/2TcUahdVPsui6OA7jQAwxGuZWJugE4waWCwgYTKvEVjm+UYjw8AAlQWWZV+ZjSm3kV/6KIy7jNAeSeyz4qfNZE1RDntqySV8NKYml3qr+1f0jwfRWGaCN5sr7UaVnLuNTlA7xTeVC1dxrGc2vVMZlzBwKgq/pRJRh/NCcTphM+yxOweSzmTbctnDSYin38PvAQl0n2LMbClX6ltczraTkYBaVqJa62t2uJvE9P1E/zinTml17Ww9pL128TC0LY1EqYr9lHXLL0lrPZZn95RQbemiGIGH4XFdkOqyFAEzrmQGTs8ZHQ0Yx1oLNSswkrBuPxIJzD5bs7HtscFrA2LJgTM3ZY+H1nPukcWU/BJBbFshlam0k7CbvwgvJckBxJdsGjDXt9ZV2UsI70NttNyLXG+1sByp2xWmZW/pa7AvWTosvp1OSx2LMbaC7a5zbt1vGw7b5YARJ3V3jc9t82G6ZEuPini3jc9vTBvEtG4OMkRhwe7AVs+KOWbHZytaO+WBsXPY8s79ZX0rT952oEJz75fwo3aPkt4eTME7Ezugs+9FCZ9P5r2k9bqKizS5qd3bQ3qC1vb0/2N/Hb/f7GO32ezt7eL+HO+09/fJVvj9qrYz/AtFMtIzM4AAA k8s-resources-workloads-namespace.json.gz: - H4sIAAAAAAAC/+1d6W/bOBb/nr9Cq+kOktmksZ2rCVAsGqfBDNDOdNK0i8WkMGiJtoXoKkXlaOD524eHJFMSFR9JbCl5/dKIlEjqXfy9x/fkuzXDMJHvBxRRJ/Aj88i4Y02s0XUiyq7++sYux5v8Nmw7FPVdzFopibFoG/qY/mazFj92XdlCUDg6DwKXOiFrb4nGkWPjbuBTErh8igFyI/m86/iXkZhGXBI8IDgasQZzpxWZsi24FneIZcnFsVYrcF0URlgdTXSMsDMc8ZWbnb1WeGNmHSHysTsZSR1N9CPXQVE3cAMiqDDeVDv7iETFqUSHjaLRB+wPKV90u1Xqw1WPURQFMbH4C5ifSOBhOsJxZObuGjiuWx7W4fRu55pcPMS+nTFv8k5XQ930nH4xIdinFb0euqnqcfyKnmgUXCuiofRQJl1uxVNXyI0nNFL68vRnciLuKo4uOq4dW5C/VexRRCtr5oL6KXB8+jGwBfF5g4Ei4wcmQZ78IWb88SkaYi0PQz4KQbYT82n2yn161jOy25hgwuceuAHNzxlh4uDojytMCNOZvLgWRXYitnys7zEjs7FlEPydUZTmRSnVmUBM+9NpZ/dwr1u+I5PYMhcn4tgqdTD1trH/WSy96mFuAiZmQXtLJsX6XoXTnVJvRJF1qRGjgiDNQkDX8Zwp5Ds9fLPTeiHkU67ymhSFyMJVto91ckPR7uRbk2lKahxRHIbY/sAWqVUaisgQ0xnUAd+EgkdR7K1f+IbhMy3v+cjDYrW9MLB7FtuIEJuIHGV/9aww7sUR0/RehFmrHfWE0Tpiw/Qcgii+s9yYLZK8vTBfJX9emJtGNjJvzy4uzPGF/4sR+OtZ0yabeYMvaEiCOOy5eEDXrwNy6QbI3jTSv3r0NsQbk1F7WQd7/Ogy7ss3CK756gl2UR+7c66sMBnv5f+LJW8Y/VujclkXflniBwHxkNhtqeNx4gkZLt3GjCEmzNSfIosKHepUSO9pNt7dXTr7eMyUcnIpFjMemxUjMBG6VPBIQcpxKER1LgORiZSFXETWBRcYShG7t7AZc3MgIfwIEZtdpmOxltR2v2byyDiysUyCT98/ngGBpW1fJXmrdpeFiVtpn+mIY+nAtTUoiL/dKUOdmrlE3+eRM6D6Tio8ALP76YvxhVvMPH6h2R51V4SHiGC7CjsGhJa3RgkOhb7zGR3fdq4cO2ZYshInpvcKLyS/sht040TldfVj61LuLGWim16CETlJCiwzucDpn6pGwhnaFW6V/hVu0c0suG8iqWw6QjUCxTeHKllyg+ExEt5Tu9Qnwb/2MYn+ywhGeeO5NH+FL6FXMfkeU1DQWrE10y1mwEOMioqTtP7GbIbwsvXdZ8F1zpPOFnSeqJwqT/fpoez77PwQ/aN9qS0JK16mA132nzvgP5fcgvbz9J8fzWspr2Wa2xLRW3cuJ/68vNFIYccTJPF/9m/r48etkxPj11+PPO8o0sCJEFFmbvzqMdOdUjqf5mIu81ns+44/ND4FdpXDnIiJ1txaqbHI82jRd7ax5XhImKiWzv+91OtQ0nkuXMxjF027L0M45glh9sWwmTtmam/9Qrjamvdy5ytXYOOndxoOVeO3HAf92OszdSj1x76jbK2LMViP8lbN3U7TuHtcX+6e3R8uBAbPxOBu/Rls/BtY/BAWnzwdixOgJC4XZ/SH++LWwOSZmPy+vnos2Qta/DAGn9ZZi/+XhLibweKqY6Yn5PC2vY3a+7voYNDa2TkcHB7iN4cWRntWf/cAH/bxfvtg+/JNtJVGfKOt9NTgv1eIbE2CBm9fTf7+mXelEeY0viwaJwHmSXhZdKSjvn3V61nYdUWjCDonDb3OvcJ4XcnoOlibVAyNcz4ZmJtFzE3utKqebK45ZxVibr/+ZXtxGkaUOP5wPhpWH6rMexBtBbFP11dzonv/ee69p18i0Utz7hVRJKKeWuO/wLFYeQ6CByIObmpCIg88X4ScgCXnBNRAio6fSooyyk8kJoUdvfQMfW4+KEfV8owaZGYFMtMFy9MAKdoGTXz2mniyCustU3RAYhopMe/Bdj8X2w162GA9PDUbkMD4J0/KrFcCI0F+xDmp52MW1NB0QYIjJDg+0wTHvJ5CgmMpwVFTIbgDGY6lDEeoEIQKQagQhApB1a8zjInr4WEvILcC6Tv+kHlwtNe/pXh+HyQb8l+sizc4HjMS8mLMZ63wSKA+EOoDZ6kPlIIKJYJPVyK4Wgo/lyrBj4KKUChYCz9abGXgR79QP1qriuBKl1zpsie9C550yUGAWkGoFZzPeYZawWdeK1iJ9SBruD7lghIDPojBUDFY84rBR+QxlBs9jMu1LhpMeA11g7WuG3wEbYbSQSgdhNJBKB2E0sEmmBsoHYTSQSgdrGfpICQLQOFg3QsHs5NryHtfgdh0wfRA5SCoIhQPgtBA/SDY7yZXD4IirlIRa19AmATXoYYQaggh97ERuY9QRjh37mM+1S618HystmdCXiTkRUJe5APzIrtSC4wzbGHnChvHyLelNMHxWT2TJI/D6IG8PucIjbknwOzaJ0wuzuwzxgMjGKSKbRufkETRwOp65k2GD2Z1qtcUuF3//MlHV2zjhAQcygDLa5pK+SQKDlyve3LlwlyHpMpr36CBEWpLtpqVXKlPpoPESkishMTKl5ZYuc7PD8XHWNcnB4c+plxAekSiOnkgKz/eOvVU7u/CN1v/YvaHD99LA6bfNuQZolE6RNQdIT7pCeLfi6WAbjTgyPDR0z7vFRSaYEGQlOZJyvFSJSU1KaH0GEBUmiQq3dUYFZCVBsrKyUrNii0DESAyTRKZ96s1LyAzDZSZ+n9bPznn/F2KXR2//wc5cpAjBzlyM+ssJMuVkuU039zfh4y4UkZcCzLiZEbc/oo+dA7RTh0K5D2v//OIuPAJv2/e/O9A35/cCB+DXj7Y43ltAPVWAPU2nwPQOQCgA0CnUUDn5Z7WAdJZJtKZktoPUAegDkS1lhfVKmshRLJmAXhvAOABwKsdwGPyBpEswHerxHfvmOowjTW6qQBOgB6ncpqnfZRVBwH8A/gHka7GAqFDAEIAhBoFhCDSBUioVkhIKZoFMARgCGJhK4qFzaiwECmbCyAW2gAhAkJsStLXC6wxAoi41LSvWT6BBTBw+TAwBBgIMbHFIU8bIA9AnmamfwHmAcyzFMwz7VuQAHsA9kD0a3nRr1Qvi7oIUa6ZIF8HIB9AvkZHuV7gpy4A+dUh2qX9QjDAP4B/EPVqFATaAQgEEKjZUS/AQICBVhX9AhgEMAiiYPWKgpV0siIatpas2oysEfbQV0wiubL2rmzmP9LHn7ARuRSDsZ1sOJFKk1t1wrYlLH6LQ45FsRe67BX9YaZlDEFEVJHliQgo6CmHISm+ERJh4wGK3cLOLtRH7dVBUv7zfTgvpxWClKqwiiWV3iDk7CqaVfN7jMmt0Dsd5uRfdeQGOS+WrHWIbwqfgcyMlTL9WuF1FIIh1/2avH/hNaaQsoKGeuLNAKtTAit7IbPNbBe38TtXA1KrqO8xDjqa21OuJLhgHpaImXrSyq5nyMPxB8GmkQy3oWdWZxqzki1KZSpTCcGR6M90fjPfW1oob9PfnIiCfBOlI47wuRxINUvzyQeKaaChM2/uWUEsJGenVeyRhtJst3K8n6q2oRvcevyOKs3NblhY/pj2O76T2NI8z5cMQsfF32E377dDT6EmhV8/mU9H6kSveZTx0gm/EPfzrW9pKFNGk41R1RWZ8qeW0UwoHseYawVxrEji8zDyayneHEt85QhiJshqIP060w+ut9qpq2MKQ8/bzNxjocPgIZk8nJAkc/xVf8PcUw5TVetv7qgXyu+Fm3vK3231Yqel9igf7egof7dtKb/f0nfgHrkiHFNnUQfeVwdWZ+nsqhcKQj6w1fWma8mR70cgwjvml/NuAoazr+wGXhgzCH6W/hKasW38nkqhsZ7WYERSIM1YRB5N9OZg0G/Zh+1DbLUw2hvs7+3utDu7uL1v7XbQnrz5KkPlrbXxP+5krB/a7wAA + H4sIAAAAAAAC/+1d62/bOBL/nr9Cp+0dkr14YzuvJkBxaJwGu0C7103SHg6bwqAl2tZGFlWKyqOB728/PiSZsqj4kcSWnemXRhyJpDgP/mY4Iz9sWJaNgoAwxDwSRPax9cCbeKPvRYxf/fmNXw63xW3Y9Rjq+Ji3Mhpj2dYLMPvN5S1B7PuqhaKwf0mIz7yQt9dlY99zcYsEjBJfDNFFfqSe973gOpLDyEuKuxRHfd5g79YjW7WRW3mHnJaaHG91iO+jMMJ6b5LQx16vL2ZuN/fr4Z2dEUIUYH/Uk96bpCPfQ1GL+ITKVRhu68QOotH4UJLgoqj/EQc9JibdqBdouOwxhiISU0e8gP2ZkgFmfRxHdu6uruf7xW49sd6NfFPAML1B4ma7Mch34uMeDtyMsfr79oL30eUYS/U7bnqmyYvVjynFASuhDtBdGcULSihUcO2Ci4l5KlGf3JopjIuuX9IpX5F4xACNlmcuF0J513jvknDruZK39XGKJrdZs9CCz4Rz4xORbyIbLBRZPzAlebaEmDM/YKiHjQISil4ocr1YDLNfpJnlinPFxRRTMXbXJyw/ZoSph6N/32BK+UrndWFcH0Y6Ifr6HvNltmoWxd/5irK8nKYKSeSwP5019472W8U7MnUocnEk6/UCgdsOFwcXcuplDwv7MrI5xlsyNTBTNU43C9SIIefaIEZjgjTNAvrewJuwfGdHb3frr2T5tKu8JkUhcnCZYeVEYUcazXxrMkxBjSOGwxC7H/kkjUrDEO1hNoU64LtQ8iiKB5tXgWUFXMvbARpgOdt2SNy2w3c5xAeix9lfbSeM23HENb0dYd7qRm1ptI55N22PIoYfHD/mk6Tvruw3yZ9X9raV9Szas4sre3gV/GyRYDNr2uYjb4kJ9SiJw7aPu2zzltBrnyB320r/arP7EG+Nem1nBP748XXcUW9AbsXsKfZRB/szzmxsMEEV/8spb1mde6t0WldBUeK7hA6Q3MqZNxCLJ2W4cFu6+Z0hh0kdapZI71nW38NDOvpwyJVydCknMxzaJT1wEbrWwM6YlONQiupMBiITKQf5iG5KLnAIJKGBtBkzcyBZ+D6iLr9M++Itqe3+hcsj58jWIhd88v6xBgusbPsyl7dsd5l7cUvtM+sLoE5814CCxNudcUhrGEvSLvpel5mJTGJRu/X5i/VFWMw8fmHZHvUwDg8RxW4JCowIZcVVVOBQ6rsY0Qtc78ZzY44lS3Fieq90cfIzu0N3XlScVyd2rtXOUlx0e5BgRLEkYyyzhcCZnypHwhnalT6b+RXu0d00uG8kqXw4ygwCJTaHMlnySe8ESdesUaAp38D4mHIOighGe+OZNH+JL2FWMfUeE1DQxnhrplvcgIcYjStO0vobtxnShTeTz8ltzk3PJnSZqJwuT4/poaJdeD8kvX+gtCVhxev0zovOeROc88U65431dM6fzSUqzmWSTxSxe3+mCMFlcRdTmoRHMOW//F/t06fa6an166/Hg8FxZMAqIWJcX4LyPtNtWHm29nz++HkcBF7Qsz4Tt8wbT8TEaMud1BLleTTvO7vY8QZI2r+6ybm+NutQQryU/uuJjybdl8En+5Ry42W53Nezjbd+odJcPcqdr0KBrZ/eGzhUDg5zHAziQYerQ4EeB562b8/HYDOEXDZ3m6vG3ZPqcvf88VgkMHgqBreqz2Dr78Dip7D49OVYnAAleTk/oz8+FhQHJk/F5A/V1WPFXtDipzH4rMpa/J8kfr4aLC47w3pBDu+4O6hxsIcOu/Xd3aPu0RF+e+RgtO909g7xUQcfNA53rt9GtTScHNXSI4l/3SBaG0Uk3r0Z/f0PQUrD12nwWjaOotej2LUkpL2+e9NuO9j3ZaOMaCcN7eajwnhbyugqWJtUDK1LMRiYm3nMTe4orJpsrjhntcXc+eXnnfnXMGLUC3qzrWH5ic2sp9wOiQO2uZzj4scPix89WpMhU8OhWsSQDIoajf8cZ27FMSjuyiC7bQiJPPHwEhIOFpxwUAEpOnkpKcpWfiQxKexopwf0D3+RDl9NcWuNvzPDtQHm1siJxJrPyiPtjFwdjoM8LUGeWmCVVkCKdkBLX7WWni7D6qu8IZCmtZOmD2Dz18Xmg46uqY6e2SuQjfmHyDCtVjYmRUEkOGnmYxZEMZAgWxOyNdc0WzOvp5CtWcjWNNRS7kK65mLTNaGWEmopoZYSail1h9KyRj7PAA8IvZduhBf0uOvI2p17hlPnx0Hujcdx5DwuTzbI3zhJNHgDbjbUxVDMo8QBgtpKqK2cprZSiS6UV75ceeVyV3hdKiw/yVWEIstKuO1ycwO3/ZW67UZVBM+94LkXHfc9cNwX67hDnSXUWc7mq0Od5ZrXWZYCSci4rk6ppQKYT2IwVFtWvNryGXkMpVpP43KlCy4TXkPNZaVrLp9Bm6HsEsouoewSyi6h7HIVzA2UXULZJZRdVrPsEnIToAxzsQU5L13glR2iQ8b/EkSqBYYKKjNBcaE4swLFmSBQyxSoD7ATQL0mqC2UbL5MeidUbULVJqR/rkT6JxRuzpz+uQ/pn5D+CemfVU7/bCklsc6xg70bbJ2gwFXSBKeE1cwFPQmjJ/L6UkA87vsAsyufFzo/s885DyzSTRXbtT4jBcOB1dVMDw2fzOpUrxlwu/ppos+u2NYpJQLKAMsrmjH6IgoOXK96DuncXIfc0dvAYsQKjZVpq5VDas4ZhPxRyB+F/NHXlj+6KQ4n5bd1N0enkgFmQkDaVKE6df6rvsX75FPg4Z/cIInx2mkI9tuWOpW0CseSpkPJSpxJ5k8kt1bgDPLZs10fFRyWYEOQnNWXnJOFSk5qckLlUYDorLLotJZjdEB21kB2TpdqdlwVyAARWmUR+rBc8wMytAYyVP2fW0jOUX9XYljFbzRCEh8k8UES39Q6C9l8hWw+w88wHEA632LT+eqQzqfS+Q6W9O17CNVOAznf/e/K/uWfzwhCX/CD96v/YfDHMzXh6+CLR5YiSQ9w5RJw5fY6oKpDQFWAqgBVwTkmwKplwqoJRRGAqwBXQbxucfG6ohZCjG4aNPkW0CSgydeFJrk4QowOwGSVwOR7rkpcg61WKpAjVClWOU2nP86KuABrAtaEGN7Koq4jQF2AugB1QQwPYNdKwC6tkBqQFyAviPItKco3pcJCDHAmNDreBnAU4Cgk6kGBI+DRRafqTfMNNsCci8ecIWBOiPbNj68agK8AXwG+gq8AAMCqBsCa9OVTwFiAsSCut7i4XqqX47oI8bup8GUT8CXgS8CX8KUggJmVjuMZP74NWBOwJsTzVgpv7QLeArwFeAs+qweAq/JxPcBcgLkgvlet+F5BJ0vifBvJrO3I6eMB+opppGbW2FPN4sc2xRMuoteyM76z9UZSKX+lnfJtCsvf1FF9MTwIff6KQS/TMo4oIqbJ8kgENLCVA6wM30mJcHEXxf7YTi/VR6ea8K/4GU6cl9NMkOxTDlGtC4VRNXqiyzqC1agkFHwbt6/29xjTe6mAJqQrPqMqLHNePnlrD9+NfXc1s1ra8Btj76WtHPL9r8lC5MV40pqWLKZ5FacA8+lKa2abG2m+m7v4vW9AryX6bA84Kz3D7SlXEnwwC0vkSG1lbjfjMIFKQmprHNsxXONvQz0n4nAhQ01bZu41J3Ev2bx0LnNlkSyK/kgnZOephZmLNvPNiWyoV9MIcYQvVUe6wVoJgam/sMBkwHFukclAqxd0ySPSY4bcQw3IvkKpQjEjBu6I5rZDYilvu/VxitqY7UY9JzETt4nQJ/cDcUfZTpHdMLfU8t3GC7xk787LyWJ9nuGY07O1BN0a+9Gs2dSqQss1iypee+EX6l/cB45hYRJFrVdDUTdSjDpUmMyTXEvQWFf5gnZAbmuN1D2ypbKKNjv3WOhxSElHDycLlsUKdB/F3teOlnUNtnf1Cy2IZ+9rfzf0i926TtE+O9PU/m64Sp2/pe8gvHhNFCeOond8oHesj9Lc0y80VH3o6vNN55Jbvh9EhojsL5etBEBnX8AmgzDmsP08/RVEa8f6PZVjazOttYmUuNqxDI3a6O1ht1N3jxpH2KljtN892N/bbTT3cOPA2WuifXXzTYbk6xvD/wPkH1LMs/YAAA== kubelet.json.gz: - H4sIAAAAAAAC/+1dW2/bOBZ+z68QhH1oAW8R23ESLzAPbXrBYjrTTNrpAtsWBiPRNieyqFJUYm+Q+e17SOpCWVKcqy0nJ0/RIcXLOYffd3gRfbnjOO5oxMIokbH7L+fbj46RCPozYYJaMhKGXBLJeKiElyACYcBiqbPA45XORn0myWlAQTomQUy1cBJS+W8fRGESBEYiSDT9wnkgWQTyXS2cMp8e8VAKHsSl91np3YCFZ1bDIhJSnf+bbpNpGch9IknME+GptrjHgs+onNIkdjtZjjGjgQ8Vjtkk75J5lY5JEsi4JK3UnItnJIpYOKlJkVPQ4ZQH/nJR6i3u65aR05gHiaRup5weSxrFmXKzv6tS8UnIlPrdkIfUzROudpbygraZf8zLbXCn8HhQFOdewPOe9TzP7JI+L9RzpWhtml7+WNWQy6Oy02ihxwMufks1cE6CxO6+8Y4slQhK7MS/kliy8SJPTiS3k7lgNDSOWpcsqJ949FNNm1SrSOAVnlSYncTydy5/Bwd0rZSyqbUzqZfdkiFN53J33qkxpCvpXJa6U9FyFCQTFn6lIk67dVB4sSRiQmW52aVe0Xkk1DtxMntxlpzSgMpRCNWNQjKjl14A+qTil+/uP9J/v7sd5y9+CpI093f36mW5U2MuZkS7nmQzOoopKD0uZ2EhFAWdf088yYXtJEandEJD/31eTvllQccaL9zXVbcuPEsyqYHGPUnCEAag86tprzXEpSBhHIELhdJGFJO2iPTbMXiLqSZVOELIA0LI3s0gpI8QsmUQIsygG0Xcj2+GIh2HhTDYQo/+8jdkzB4UvDifTpyGwkceT0J57xoeG8AuL7Parq4eBMyOQa8IZO0BssObAdkeAtmWApkHUw/CQij40eAsr+K5gtpRpgBEtvYgW7d3M2gbILRtCbSdg68BAsxISCYANpJLEoyM8L7g1nHUYKSQFyAkgWL144iPRxdcBP5WodJr3QPnq1YMYBNAMuJSi3Bp/2a4tI+4hLjUyWDJpzEDNW8zLr01XUBgaikw9W64Ln6AwLQlwCQAK8qL454eACMqBBf3nad9G8x+vNwe+DFj33mnuv6w4EMCRuIj5ePaxwpDnhIRV8rzSTz9SMOJVGOyu1uS07rsq5ENPBWKKgk+COIz053d20NDae5Ut2V2UA8NhwU0aDOWawI9TcLX8Zd0J1WKhFrFkvPJcufVIE1EapWl7DMyr8k+Y2GNVLDJVH5mfk2tANgXNVLI+x/maxNlO7SZOyh2r6kjH/mqqIp2ACnzxJLwIq2lex2oqiYccxg4GW4oQeEAEQXvABSc0Ir3ROot5QpJXJrqannV2UDVPhVUA8g44NYIETSiekDa6nDN+P50DngCGltqNQwoj9Z5Oowk76xSt6LAiPofQSnVMXhn1BMwzhUScVCS2ec38Vk9+C1j3/XQ55wunBd5wSMFFAVarhUYncvLcjvuiJV2BPPNxtAZfQ/Is2x+Jf88ZWNZTUhB91PWKueEWFEPjKLskMSlPRYBgf2a0ciFXMIgPdpGGTSz0GfnzIdZb5VMszw6yigaMCdztgSEp4l3ZpxsachnwZvqrh1dqJ3mau56RMnhIY/wiiYuyJxe49uF6/BoyWUCAp5aaYJK4JM3JKY2sFioWcluYLMitrpSF8q0uZWZbz9Xwq6sfjYw9hAZGxm77YytpysPTdyZtOOUqROJu0zcZsKE9I30jfT9WPTdW3VEtbtXT99Wf5C/kb83wN9TFksO6Dwb/UwIMHdAX+y+Gg47zk2I3U9S6oupx0M/HhkQf3SO7zgBfYlEXyb6zBjOcCinTmZNJP37kf42UD4SfusW2HvdBsLvIuEj4bdqwq5Os0N9Qlbp/JrzoPdg8/USN3TvBkzduYPSLrg4o+Jpas30rUFxb9Yb4hxz3/msHBRXMXAVA4OaDW5CNEY1PYxqMKrZjmWMtYc7G1iseJCY53YqbQqGHm1FaO1KbWdI9DZVOIZFuM6DIdH613kOGyKi/voiInVDzbtZJBf1Sf+lgmMYhWHUrRaHgPoFKKTY7FlftFRUqUig46Sf55hT9Zvb9FGN0aJSe9qyDfTZ2MvBA5u4VoKBQSvWSpoigz2MDDAyeEqRwdqOd2I8cI94AM+BYlSAUcGGz4H2Bw1BwQCDAgwKnuCuyw3WEa7bIbjt1+y3CR02fIh02+KHt3i8FLcdMI5ox7bDXq8hjtjHgxgYErTqeKk3ETyJ8tu1brON8GDsv6bvQtv5hciRNoCTGqDQiiNwLQDXApDDN7ZD0EjiB0jiSOLbcZpyFbuvZXKPn4ReR/g4X8f5OnJ94XeAsJ5gUXbnqDpKG7Ax9RZeQB16DhTgwKBXA5uLDc/yhw0BAt7TiAFCyz4iBUccCap+enF9U/z2X1778Af/P7774Bg94/wd5+/I6ZucvzfRM17KiPS8LV9DWrydkePT+3avfdyddQb5G+fjyN1rP4c32G/4zXC8jxGpewup++l+dt8+6vbxi3ukbqTuTVH3fr+Bum91s+Iy+a3i7gpN35G8l9Menr3tXy5D+t7YwjgwkBx5gRoIQNI/wTbyVp/INf6ipAcmULLeq1dr+xG33nz+gHcoPrpq+mtUTb9RNW/aqJq9Napmr1E1R21UzWCNqhk0qubtegPbk+Mj/CYU95EwoN1cQHvQ8Du9PbxUE9eiWr4WVUPK91uLut0RUFDVacdJRLCR05+qdv0PNKAtC1Unxgj4YyC4ZIUMvyaGP1xF8A2/3dnr44oVrlhtmuQjwT0ax0DfyobA4zM642IxOl1IGt9vA+n57RP9pnWHNHs/mtWutwVU25J2PnO6Pbwp3e4h3SLdbppu9bQ541wvSvJZ8kNdn/j8SPfo+E8niZVzIe/eb3o7VZrYgiluO9r5zHm3uGtoFfEOkHiReDdNvBM+mnDBE6n0jfPa21Hsh1xzyLHIsRvj2J3Uy9U4UC6uD+DsGp90Bb8o/N2NvSmdka9UxOY7f3PtPkDJwni0T8SZeQ/QsbCcHv0ipGpin9cm6SwKCLj/JPc2Vx0Ftux9WUdNJdVKOteK9emYJMGSarVr2al1ltLX8y4NonqbZG5tBwpWKteXH5TBAcQ/EyoW6ZLgckBR0nm3JJ1QZWXXzpoNWKv65RFTCgCCr2n/l7phqdLWw8rwp9CV/c0mCwHqffo6qGPpTJFuSgd2WTOwCKt5J9NyzSurVKyrGxlEeZFEl0vUc9Vx0kJf1pugt8oEKeDapgJH13qO/8ha4ZZTK81VsvrMqYFNf6yEJKZfTEG10c6arL7bZPUlRG8YPavsnbHznQ2efcIhklDHDPk9QtcuAzhVJ6m7CWF7vWQng/orA7tMqzsF3LEJXNyQX/yzm0UBEISkMrf0WsS8Mx2rpi+nKsk/cbWp2h1YQU1313ro2w/dWfH/wPq/az/0d+0UK1bpWf93fYOGP7I+GBfI/WdlLXbB+3bBdi29Pfuh+D1Q98C325u1paS+/3Edabt/fjlKOTILBH81zmekiZ7euf1u/3BMuoOBPxh2Bwf7w/GpT/3D4SHxdodjf2gyn+dUvLtz9X/j7g7C47UAAA== + H4sIAAAAAAAC/+1dW2/bOBZ+z68QhHloAU8R23ESLzAPbdIWi+lMs0mnC2xbCIxE25rIokpRSbxB9rfvIakLZUlxUieW3Bw/WSTFy+Hh9328SLrZsSzbcfwwSkRs/8P68q2nQzj9nvicGmEkDJkgwmehDLyBIAgM/FioJHB5q5JRzxfkPKAQOiFBTFXgNKTinx4EhUkQ6BBOotknxgLhRxC+qwJnvkePWCg4C+LS/X7p3sAPL4yKRSSkKv0XVSddMwj3iCAxS7gr62KfcDanYkaT2O5lKSY+DTwocOJP8ybpW+mEJIGIS6GVkvPgOYkiP5zWxIgZ2HDGAm85K3kX81TNyHnMgkRQu1eOjwWN4sy42e+2lH0S+tL8dshCaucRtztLacHavnfCynWwZ3B5UGRnX8H1nnF9nfVLer2Q15WsVdcM8suqhWwWlZ1GBbosYPyP1AKXJEjM5mvvyGIJp8SM/DuJhT9Z5NGJYGY04z4NtaPWRXPqJS79WFMnWSsSuIUnFd1OYvEnE3+CA9pGTLmrlTPJm+1SR+rG5e68U9ORtqDXotScipWjIJn64WfK47RZB4UXC8KnVJSrXWoVvY64vCdO5i8uknMaUOGEUJwTkjm9cQOwJ+W/fbV/Sf9+tXvW3+wcQtLUX+3bl+VGTRifE+V6wp9TJ6Zg9LicxA8hK2j8O+IKxk0n0TalUxp67/J8yjdzOlF4Yb+uunXhWcIXCmjs0yQMYQBav+v6GkNccBLGEbhQKExE0XGLSN0dg7foYlKDI4Q8IoTs3Q9ChgghWwYhXA86J2JefD8U6Vl+CIMtdOlv/4OE2YWEF+vjqdWQueOyJBRrl/DUAHZzk5V2e/soYHYCdkUg6w6QHd4PyPYQyLYUyFyYehA/hIyfDM7yIp4rqB3lNkZo6w609Qf3w7YRYtuWYNsl+BpAwJyEZApoI5gggaMD10W3niUHI4W0gCEJZKsuHTZxrhgPvK2CpdeqBdZnZRgAJ8BkxKUO4dL+/XBpH3EJcamXwZJHYx/MvM24dKybgMDUUWAa3HNh/ACBaUuAiQNWlFfHXTUAHMo54+tO1L784jiyBCcDlG8vtweMNBJYb6UhHheKSOCT+Eh6vPK4olvPCY8r+Xkknn2g4VTIEdrfLYXTuuSrcQ78FrIqBbznxPN1c3YfDhSlmVTdDtpBPVAcFkChurFcEthpGr6OP6Ubq4In1MiWXE6XGy+HbMLTXllKPifXNcnnflgTyv3pTJz5Xk2pAN9XNaGQ9t++p7oo27DN3EFyfU0ZOQ7IrCrWAdzMI0uBV2kp/bsgVlbhhMHAyVBEBhQOEFHwDsDEKa14TyTvkq6QxKWJrwqvOhuY2qOcKjiZBMwYIZxGVA1I0xy2Ht8fLwFdwGJLtYYB5dI6T4eR5F5UypaEGFHvAxilOgZ/GAM5jHOJRAyMpLf9tVqrh8JlJHwIEFrnC+tFXowjYaNA0o3CpHVzU67HDyKnqW6+mIg6p+8Ah5adQYafzfyJqEakEPwxq5V1SgxFBGMqO0FxY45MwGOvZmwyLpYQSY09JwNqP/T8S9+DGXGVaLM0SoEUFbgm1/4SLJ4n7oV2uSUAyISdbK6pPOQ2dDV1Pb7kYJGrv6KKC3JN7/D0wnVYtOQyAQG/rVRBRrDpGxJTE2YMDK0k1yBaCTaaUidzulzLzLefK31XVkYb+HuM/I383XX+VlOZp6XxLE3PKhMp0niZxvVkCskcyRzJ/KnIfLDqNGt/r57MjfYgmyObt8DmMz8WDNB57nxPCPB4QF/svhqPe9Z9aN5LUuqLqctCL3Y0iG+Y8XtWQF8i7ZdpP+saazwWMyvrW5QA60mAbRAASP+dW4of9Bvov4/0j/Tfqcm8PAYP5XFRJfc7DpI+Grdvlsahsffg7d4PmPCK8QvKn4MNdUsbzPhms/LnhHnWmXReXO/A9Q4UPC1uXjQqngEqHlQ827Hg0bIUamFZ41H00MMM3CSUNrSStHETd1MuHafmR8mE60Molza/PnTYoJaGm1NL8pU4b+eRWNRH/YdyhhILJdaDFpVACHAwSLFl1JaSKiogKaFnpQ8H6TP97W0dycqooFJ9urKZdKZ7z8IjobiqgjKhE6sqTTphD3UC6oSfSSe0dIAU1cEa6gBPmqJGQI3Q8knT4ahBIoxQIqBE+Al3a+6xxnDXXsJ6T9Y/REi0fEx129TEMR5gxQ0KVBXd2KDYGzSoin08zoECoVMHWN0pZ0mUv/frIRsOT6QFNvRUajefSDlS3WGl3VFYxeK4ToDrBMjore0lNFL6AVI6Uvp2nNBcxfUtTPzxgdS76B/n8jiXR+Yv/A7w1uV+lL0pVR7IDfwJdRduQC16CYRgwaCXA5vxllcAxg1yAd8niXKhY4+wgiM6nMovRrY1/e/+K3cf/2GCD2/fW9rqOLfHuT0yfJtz+yayxpdHIllvy9OXBotn5PizPx3YPSbPGoNsjnN1ZPKNn+Yb7Td8Bh3fG4lEvoVE/lwe8+8ekXv4hD8SORJ5W0S+P2wg8ge9AXKZClcxeYW0f5DKl+Men8vNL7Mhmbe2hA4MJBw3kAMBKPs79I140EN4jV/MdKELZNjg1auWPlI3uL5+xHc9Prmhhq0ZathoqDddNNRea4baazTUURcNNWrNUKNGQx1vVhCfnhzh86m4N4VCuD0hfNDw/eIBvhgUV7Q6vqJVQ9HrrWitc+QUDHfesxIetHLaVJau/kAFurLcdaq7BD99ggtfyPcb4vvDVXTf8BXTwRDXvXDdq23KjzhzaRwDmcs+BFaf0znjC+d8IWi83qbU89tt+kPZDml2PZpVrrcFVNuRej5zuj28L93uId0i3bZNt2oSnXGuGyX5nPlpXvP4/Cj46OQvK4mlqyELrzfZnUlLbMGEtxv1fOYsXLwFaRUNj5CGkYbbpuEpc6aMs0RIe+Ms92EU+z63HHIscmxrHLuTerkcB9LF1cGdXe2TNmdXhb/bsTujc/KZ8li/ZUB/LACgZKE92iP8Qt8H6Fj0nBr9PKRymp+XJug8Cgi4/zT3NlseLzb6+6aOmkqmFfRaGdajE5IES6ZVrmXG1vWUeo3w0iDK+sQ+BklgnWlNUPVvUzEYsUy9g6GMEhD8PaF8ka4ULiuLkvH7pdApld1tm0mzkWsUvzx0Skog+JwaYmlAGjY1DbJSBxVGMx8W9UPAfI++DuroOrdoygtmXnPoGr/mnszKNbesMrEqztHQ8iKJbpY46LZnpZm+rO+CwaouSJHX7CrweGXn+F9ZLexybKW6Mqw+cdrBuj1GRBLTTzqjWtmzoV7fber1JWhfNYxWdXzG14/Y8706iXJbbJ7/HP6wk6H7rUZaX9kzxdiJ1ip2yK5+7WfED7ojDbNLt0W+e6HkaXpzapJ8ZcRkZ3tk6Jj+rnExNC/68+L/yPjfNy+Gu2aMIU8Gxv++p3HvW9YGqRQNB1lZipnxvpmxWcpgz7woPmpqH3hmfbO6lMz3X6bEtf3Xp6OUFjPt97v2SB2aqBmdPewPDyekPxp5o3F/dLA/npx71DscHxJ3dzzxxjrxZc6+uzu3/wfcn4zpC7cAAA== namespace-by-pod.json.gz: - H4sIAAAAAAAC/+1dW2/bOBZ+768Q1G6RYpOpJV+SFAgWuW4LtJ1sk3ax2wYGLdG2JrKkoahcGmR/+5LUjaRIyU7SxBmrD21NUofkuX48JKWbF4ZhDodeECU4Nt8Z38/W0xIE/0w8BLkyEAQhBtgLA1p4Q4pIoe/FmDZhv4yslNWMEs/HHwJSaa2XpS7AIA4T5EBSYR6jcAbxFCaxybWBARj5tB6jBHLlU89VlHpOGOyHfogoQTQZgbXOumFbFvmr3183rDc86QDMWMe75VyM18auDxEWhoCvI9bOBfF0FALkmlndLfv3jPx9y5gCXQ9LozUnAcQfXFISJL6fliAQTU/D0MdeRMo7rJBOZz8MMAp9ytAx8OP0eU941veCc04KEQgga5+yPGe46YS+D6IY8pSECrdSM0GeexyWwkyZLAnskvy2e1zBVT6B7Pc1/Z1zp6DN5mAXP8tRnxVlCEYQYG6qXOkHDBETj7r6S3gpMJhVxdPw8tTDFc0xcVZo7icIwQAbeyBwLz0XT02xzYn3k7WbDriKTBNQeJnqQDbHkvHAmcJTbwbDpDIZh+rlHnDOJyhMAlclmxB9A34C1VUxZ1qk7KW9ve30BryiMoW3u5tE0e3tdaNHdL/z29a2oPQv3V4PdEGuwqUEGo3RdKHjzQCTXCFzcxyiGRMc4doMDmOIPMg9MwHJBIpKNQNX+SytDq88My/IK/hiKkqZI1QWU+KQpqHvfgQjKBqN3OITQOeQ8Y+qQkU9taq/Lam+ZTeovlWhPYXeZIp5Wqk1dMufAVHvC+DL6iKaesa5KPKCyWmqhZaqXFSScj6cu7ugPDZwaGB4hTnVMLIqbh7cTDTEEAgmDcTsklhF6YguHBC9Ow4JF2JRIag6nBBXIfDdpCxirT+FzP8T0wgC6GDi0YQ2p3Q4EkfDSIxXqf560Hd/V9Qwa/YdkaGpZAAJc1zRmTBtF45B4mOZWDrZdIr5n3Wp3gtEnRId1ivK8zgCxEClJkngMQvci2J+XLfCuDIlEVUq5csFRCgNpzfiM4UFVbkgzi33UWmYROcbEwRhII2TKbsLr6qT5LSFSkyou11fpN9r6PvEXWg7trQd9wux3KN7xOlhpW9b2/empm+tkrEHC59XMbDS/0RhjMce7d005cIjgjfyKNfv/I2rR7D6DCvTPsL8wCcCLfT+Z0xCCn2OSlj0FDg1VvPz212pIiweqHEiccVLkBJ0ThwolG3d90V02CXI0LK2yF9b2zRYWltCsBzTnquBhVLm6aRkbBJure2uQICPXRXZMKxI6CSzQOQ1BmgCcQ0r4VXE+o6T2ZpHsBFcI24QAzIsNCR48zJE5wSzO9C7gMPRNYbxEBOE6984fhKTaLPzw3yV/feHuV64lZ3//eCczA/z9vurPDi9e0UcARkpdZNnb96IUqqHAJkJxBgEskcW4t8RcDDjqGCipg8nMHCPii5EukQpGfYzd+sURHBjPJ/JcI9SrRQCBS0/mXpjXK2QwOMXwnsjHBt7lMnGl5TlbhUwpjCoKGbmy9vSFm9LrLbelkK6cjB35jSj3M00WhJr+DkL7U46xxbotkBXLFgM6fZapNsi3Rbptki3Rbot0v0lSBcTkcQzD7dQ9/Gg7mnGc7waaJepbs0I88FQRKwZDatqRBs5IYYPjZe6qRXVixLcqye4tzDB/XqC+wsTPKgneLAwwcN6gocLEzyqJ3g0P8EodDW0aE2N6javYWgEEKDzmDNAgsw4C5x7WdC49WNp9n76PNaHsbwnQwvTLRh+wKplQQGUy6FUgTItMEBs/IQo5OI28a0QQRZNxn5YQngzdlDImCUMiga09xC47AmxJkRYZBbxDxI3XBg72nDIgs5HGEzSGXfk+M5NLsbXft1SB/geiNWeh+G1jCeVUFQutCVMT7x8GYH+Q/5sfPq0cXBgvH//bjZ7F0vBjlsp22IsI8Kr4omsotz8NA8Q0VPDDS9FCM2afUV+NQxGgMQcFKinLKJ4cWZ5cJp6risB9mJZQZ5FuNl8C6YXe4fV1M9qiEAdpuYRQ5DMRsS0lGIQVncLCEEFSlZKDntPKYccIOamYBwD5xzieDVFsf+woojuJArOIFZbGgfLIA3ZMIwDFEbRqvqqwyU1kNWWytFTSuU4XEKuiycM78R09+0msLY6g00Hbve6oNcB0O73+r3BZt+xtsfjvvP2PCE8DCCG8UaW1vKCyQZZBf4jRJMP7o71GsExlcVOtxO/vgBoo8xllYksVkEe2nk1HDpQTj5y0q4sPO8raBk5VxNTy76nbYyujTXCF33Gjx3t1OT6KlpSyfXZd8v1CeVkdhG/zNbb1JPnTZeSnXu/kp25ckZpHFkFfu4/inquEEMPHlNB3RTorAJfDx9VUVeIsUeNjK0igXtvRZ1ggPlcdw5QUgbodnCe7kaEta1Oiw+W7ErEg16FYKh+P8fo5XELcwRQ5RATu1ujyojTcqhqPuf+hy0U/BMB14PMHjq/4Pi/rdn+2Cz3NJhViV0RRk2C3fg0uzkkb8GDi4miNN+urNbQm0SHswhfq+9M/ZduilRq0gM7Mqn0mI5ciug5r5P0FI1cl50CkKjHpPG/s/0dcRlnMhepIKQ89nE7/zaSPd820oLnrXT7NhEkmkjCwqTqXyJKkqpdEgsbYVF+EExsrt+lUruD9ATC79m5JmmeTftM/DkSTGJXZaOLeHMSxz6mx0vEgf4F1nP6ExwLHdO4uSG93N7eeQVXf4bj+0MEzizzqLzrVmQybngzBohFRNmQ051PWzbUYR4NvMD1Ljw3AcXBi9vq6RB6/7EcwBW48iTnO0rSVHXFW8wy08Tizl9+RlNqrXZGhWf5flYZ4jW4qttsLXWH7oyISkHP5qqyVH442QMxrGhR6nErzRUnI/l5NOLVZRpirtirCg8qh6Z1+GCrxQctPmjxwdJlKFcHIOT7YC1CaBHCsiAELmUjeB0+YyNUPFzCxtYkbLYVCZuaPU0FvNFCnDqYUwN15oM7SshTC3u0DC2ZKl/fkTGQirsFh7sd7fUdxmlp8gpcNAc20uOjeozUgJPqsZIeL+kxUxNu0jurBvxUh6HqrtfIMlHhKT2m0uAqLbaaF1/VYawGnFWHtWrwVj3m0uOuRuxVi7/UGEyLw5qwmBaPKa95LetW55zgbA6ANjdIqwNqGrBWf5mt9pSFGrzVAjgpyzPXAUwVnKuDdBpYNxe0qzoSNcTTwLxaqFcH9/SQr8GTVqFfdQpVCKi0o1JVo0ihojqsVYe3ajCX/lqtBnvNc+PzOUyBt7X5Tpq1qKySnZofllktLGthWQvLlhOWPcmJqRaXzYvLGm9jtNCshWZ/eWhWyWY//jkn2fju88LXp0tSdjW7iZbdZikfKEvZa4LD3RYOt3C4hcPPI0v5mMeyW1R812yl+v5lC41baNxmLVuUpsxaNsK0XgvTWpjWwrRnkrVscdozyV62UK2Fam0W86mymIcIUUR4lyTmi2zkZvZSEVrb7aS0aDPuE2OxM4Uz8A2iOB2/tZUW0/cS5i+DTp8jkbLUWrN8oYlZ9IbhLPIJI4LJPN+MKxGV8t2Z2dvHNe/PzGtV4Dr/dhyfUFWrWvHydw4Vc7Xla9Z5r2v+mUB0zZRa+UW7kueWUDqBV9JFa+7Lc0X3cs5XuCjif+PeLa68DcKvHub6/l7GK/5AtxeQQOzCXfUbo9WMnBFheIrmOYOz0L4Id1lPw9ShriXRzR/hiIAEB7gXHokmBAqsGxnVN2r2203sz4ISrydEyRmP43/lwzDF2sp4aZm6cSbcdEJcRRLD05SQ8nKLWuLmb3/naYCEvVG8clmIFA+dMGG60O3INalzNK2OoAINhkgtfSO+JoyeaYyRb6E0yHkUkVi0F3iZFxWFf58DikRLCvj4xqx3EoLiS5cG7qj3qi8dLKb5v2TyixjJuRd9Rf7JdeCoLnylJmQ9BxOSRPeIJtTXWU7/HgbTpMAP5LnLRZZGhV/oACBZ0/rpmlydTsi50+3I2LBkEK3TAWl9bwpgrhGFThhzd1U3MWuq7YtUvVCBUKU/IExY78/WBXLPzYSLjwO1VvwU+EvF/kVtuMaqenpN7y2i6b0VUXHpw9Is+1AsmYrPy4SXG4XN51+QSdd45WORR0AAKh/OeDbMB8OL1+xzy0le9wUXbFqlnvM6b1r8j26Hr+Fuc9rc/63sIwdn+Rxo+q2qdvpeeMIDnjDfi93jf3DXtTddfrz5WAT2/QxZCtT8erqfrXLz5ffn4jWdxlvjc46ijLXj0I1TLGUmbPfB3Bptgq1R1x64m2Awtsadnt3rdQaD7mDLGWyCcdr4olhid17c/h8c4vp8in0AAA== + H4sIAAAAAAAC/+1dWW/bOhZ+768Q1E6RYpJbS16SFAgGWacF2t5Mk3Yw0wYBLdG2bmRJl6KyNMj89uGihaRI2U7SLNfqQ1uT1CF51o+HpHT9wrLs09MgSjKc2u+s7yervATBP7MAQaEMRFGMAQ7iiBZekyJSGAYppk3YLysvZTXDLAjxh4hUOqtVqQ8wSOMMeZBU2IconkI8gVlqC21gBIYhrccog0L5JPA1pYEXR7txGCNKEI2HYKWzarmOQ/7q91ct541IOgJT1vF2NRfrtbUdQoSlIeCrhLXzQToZxgD5dl53w/49IX/fMKZAP8DKaO1xBPEHn5REWRjyEgSSyXEchzhISHmHFdLp7MYRRnFIGToCYcqfD6RnwyA6E6SQgAiy9pzlBcNtLw5DkKRQpCRV+LWaMQr8w7gSJmeyIrAL8tvtCQWXxQTy31f0d8Gdkjabg1v+rEZ9UpYhmECAhakKpR8wREw8+uov8YXEYFaVTuKL4wDXNMfGeaG9myEEI2ztgMi/CHw8seU2R8FP1m4yECpyTUDxBdeBfI4V44E3gcfBFMZZbTIe1csd4J2NUZxFvk42MfoGwgzqq1LBtEjZS3dz0+sNREVlCu9214miu5urVo/ofue3jU1J6V/6vR7ogkKFKwnMNEbbh14wBUxypcztUYymTHCEa1N4mkIUQOGZMcjGUFaqKbgsZul0ROWZBlFRIRZTUaocobKYEIc0iUP/IxhC2WjUFp8AOoOMf1QVauppVP1NRfUdd4bqOzXaExiMJ1ikxa2hW/2MiHqfg1BVF9nUc84lSRCNj7kWOrpyWUmq+Qju7pzy2MKxheElFlTDyquEeQgzMRBDIBrPIOZWxGpKR3Rhj+jdYUy4kMoKQdXhiLgKie82ZRFr/Slm/p+YRhRBDxOPJrU5psNROBoncrzi+hvA0P9dU8OsOfRkhnLJABLmhKITado+HIEsxCoxPlk+xeLPqlIfRLJOyQ7rFeV5mgBioEqTLAqYBe4kqTiuG2lcuZLIKsX5cg4R4uH0Wn6mtKA6F+S5FT6Kh0l0tjZGEEbKOJmy+/CyPklBW6jEpLqb1UX6vYJhSNyFsWPH2HG/FMsdukeCHtb6do19rxv6NioZe7D0eTUDq/xPEqd4FNDebVstPCB4o4hy/c7fhHoE68+wMuMjzA98ItDC7H9GJKTQ56iEZU+BubHan99uKxVx+UCDE0lrXoKUoDPiQKFq62Eoo8MuQYaOs0H+2tikwdLZkILliPZcDyyUskiHk3FJuHU2uxIBMXbVZMOwIqGTTSOZ1xigMcQNrISXCes7zaYrAcFGcIW4QQzIsNApwZsXMTojmN2DwTk8HV5hmJ5ignDDay/MUhJttn7Yr/L//rBXS7ey9b8fgpP5Yd98f1UEp3eviCMgI6Vu8uTNG1lKzRAgN4EUg0j1yFL8OwAeZhyVTNQO4RhG/kHZhUyXKCXDfvZ2k4JIbkzkMxnuAddKKVDQ8qNJMML1CgU8fiG8t+KRtUOZbH3hLPfrgJHDoLKYma9oSxuiLbHaZluK6crB3prTjAo3M9OSWMPPeWj3+BxboNsCXblgMaTba5Fui3RbpNsi3Rbptkj3lyBdTESSTgPcQt2Hg7rHOc/xcqBdproNIywGQxGxYTSsaibaKAgxfGi9NE2trF6U4E4zwZ2FCe42E9xdmOBeM8G9hQnuNxPcX5jgQTPBg/kJJrFvoEVrGlR39hqGRgAJOo8EAyTITLDAuZcFM7d+HMPeT1/E+jBV92RoId+CEQesWxaUQLkaSh0o0wILpNZPiGIhbhPfChFk0WQUxhWEt1MPxYxZ0qBoQHsPgc+ekGtihGVmEf+gcMOHqWcMhyzofITRmM+4o8Z3YXIpvgqbljogDECq9zwMr+U8qYWiaqGtYHri5asI9B/yZ+3Tp7W9Pev9+3fT6btUCXbCStmVYxkRXh1P5BXV5qe9h4ieWn58IUNo1uwrCuthMAEk5qBIP2UZxcszK4LTJPB9BbCXywryLMKzzbdkerl3WE/9LIcI9GFqHjFE2XRITEsrBml1t4AQdKBkqeSw85hyKABiYQrWIfDOIE6XUxS79yuK5FaiEAxiuaWx9xSkoRqGtYfiJFlWX7X/RA1kuaVy8JhSOYyfINflE4a3Yrr/dh04G53Bugc3e13Q6wDo9nv93mC97zmbo1Hfe3uWER5GEMN0LU9rBdF4jawC/xGj8Qd/y3mN4IjKYqvbSV+fA7RW5bKqRBarIA9tvTo99aCafBSkXVt43lXQKnKuJ6ae+p62NbyyVghfzBk/drTTkOuraUkt1+feLtcnlZPZJeIy22xTj543fZLs3PmV7CyUM+FxZBn4ufsg6rlEDN17SAX1OdBZBr7uP6iiLhFjD2Yyto4E7rwVdYQBFnPdBUDhDDDt4DzejQhnU58WHzyxKxH3ehWCofrdAqNXxy3sIUC1Q0zsbo0uI07Loa75nPsfrlTwTwT8ADJ76PyC4/+uYftjvdrTYFYld0UYNY620+P85pC6BQ/Ox5rSYruyXkNvEu1PE3ylvzP1X7opUqvhB3ZUUvyYjlqK6DmvI36KRq3LTwEo1FPS+N/5/o68jLOZi9QQ0h77uJl/G8mdbxtpwfNWpn2bBBJNJGFhXPcvCSVJ1S5LpY2wpDgIJjc371Lp3QE/gfB7fq5JmeesfSbxHAkmsau20UW8OYljH/nxEnmgf4H1nPkEx0LHNK6vSS83N7dewTWf4fh+H4Ezzzxq77qVmYxr0YwBYhFRNWS+8+mqhnpaRIMg8oPzwM9AefDipn46hN5/rAZwCS4DxfkOM56qrnmLaW6aWN75K85oKq31zqj0LN9PakO8ApdNm62V7tCdEVkp6NlcXZYqjMc7IIU1LeIet9ZcczJSnMdMvPqUhlgo9rLCg9qhaRM+2GjxQYsPWnzw5DKUywMQin2wFiG0COGpIAQhZSN5HTFjI1XcX8LGNSRsNjUJm4Y9TQ28MUKcJpjTAHXmgztayNMIe4wMrZiqXt9RMZCOuyWHux3j9R3GaWXyGlw0BzYy46NmjDQDJzVjJTNeMmOmWbjJ7Kxm4KcmDNV0vUaViQ5PmTGVAVcZsdW8+KoJY83AWU1YqwFvNWMuM+6aib0a8Zcegxlx2CwsZsRj2mteT3Wrc05wNgdAmxukNQE1A1hrvszWeMpCD94aAZyS5ZnrAKYOzjVBOgOsmwva1R2JHuIZYF4j1GuCe2bIN8OT1qFffQp1CKi1o0pVk0Sjoias1YS3GjCX+VqtAXvNc+PzOUxBtLX5Tpq1qKyWnZofljktLGthWQvLniYse5QTUy0umxeXzbyN0UKzFpr95aFZLZv98OecVOO7ywtfHy9J2TXsJjpum6W8pyxlbxYc7rZwuIXDLRx+HlnKhzyW3aLi22Yr9fcvW2jcQuM2a9miNG3WciZM67UwrYVpLUx7JlnLFqc9k+xlC9VaqNZmMR8ri7mPEEWEt0livshHbucvFaG13Q6nRZsJnxhLvQmcgm8QpXz8zgYvpu8lLF4GzZ8jkbLSWrt6oYld9obhNAkJI6LxPN+MqxCV9t2Z+dvHDe/PLGp14Lr4dpyYUC1UzaYvabeOOADWfCtOgMdCbfW+ddH92n9mEF0x7dZ+2q5iviOVjuGlcuNa+ARd2b2a/JVujITfhJeMa6+FiMuIuT7ElzNNPNkdRCQi+3Bb/+porfHaUyKVQNO8YHAe4xfhLuvplHvWlSy5/iMeErTgAf88IGGFYIJVK6f6Rs9+dxb78+gkKgzRdsbj9F/FMGy5tjZeWqZvnAuXT0ioyFJ4zAlpb7noJW7/9neRBsjYq8Vrt4ZI8akXZ0wXuh21hntJ2+lIKjDDIqnJr6VXhNFTg1WKLbSWOY8iEtMOoiB3p7Lw73JSkWhJiSPf2M3eQlJ85fbALfVe98mDxTT/l0x+ESM5C5KvKDy6ijzdzS9uQs5zMCFFdA9oQn2T5fTvYDCzFPiePHe12jKo8AsTEiSL25AvzvV5hYI73Y4KEisG0ToTojb3pkHoBlGYhDF3V00TcybGvkjVCx0a1foDwoTV/nRVIvfcTLj8SlBrxY+Bv3TsX9SGG6yqZ9b03iKa3lsSFVe+MM3SEOXaqfzOTHyxVtp88SkZvtirHksCAgJQ9XDOs9NiMKJ47b6wrhR1X3LBtlPpuajztiP+6HbEGuFapyv838m/dnBSzIHm4epqZ+5FJDwQCYu9uD3xh3Bve90Xx1uMRWLfz5jlQu2vx7v5crdYh38u39dpvbU+FyjKWjmM/ZRjKTtj2xD2xnAdbAy77sBfB4ORM+r03F6vMxh0BxveYB2MeOPzcq3deXHzf4qvyjuTfQAA namespace-by-workload.json.gz: - H4sIAAAAAAAC/+1dbW/bOBL+nl8h6HpFcufcWrbjpAGKQ16bAtu9XpN2cdcGBi3RtjaypKWoJG6Q++1HUm+UREp24vgl5n7oxjMyRc4Mhw9nOPTDlqbpvZ7t+iEO9EPt+3UjoiD4Z2gjyNGA63oYYNtzKfGBkAjRsQNMH2GftJjKOP3QdvBHlzCNRka1AAaBFyITEob+GXljiEcwDHTuGeiCvkP5GIWQo49sS0C1Tc898RwP0QbRsA+2mw2tZRjkn729hmbs8E27YMxefJSNRXurHTkQ4VwX8MRnz1kgGPU9gCw95j2y/1+Tfx+ZUKBl40Jv9aEL8UeLUNzQcSIKAv7oyvMcbPuE3mREOpwTz8XIc6hAB8AJou/bue86tnvDacEHLmTPRyJPBK6bnuMAP4B8SzmGVeIMkW199jJlRkIuKOyOfG51OMJ9MoD484R+TqSTts3G0Eo/Zr2+TmkI+hBgbqgc9SOGiKlHzP7i3eUEzFjByLu7snHJcnQcE/WTECHoYu0YuNadbeGRnn/m0v7Jnht1OUZsCci7i2wgHmMqeODYIGAGyCSZiaAPGCXXFWpPv0J3iJmYmzk6DEoaqp0t+sB2nJykKeEDApZNBsprSq7tdwVtG60abRtibbfTjw4cQtfKv4mIaegeBVfiqQ1uh8XBU9ON9CX2BGdjH0/ErP9C5JU5Y3AveMfYdgVUZA9H+FLob6iZCajk2d+ZSeWtkrI8RIeQjqbAOoWBWW4PE/fkCDp2C5wQJmZV0gPxFQIrotS7uG8GT70pzEgii0syU3MzXqej+ezZLv7kMXEwgs7NbBOKLNqHxGpdDIZlj+TT1qiJhvT9e3l6uftEaBZEkPn3geNlApR4kAAiGwb/uoWIWDwsDDGQ9TcoDTzAwLwp9SbA0Peh9SuRaYmHARpCHHDrIb8i0tXi3mfDoHrvkb6Z20E43raJq4PbZBnDgLSKemT5uPPQDVmCTWjfwl5/gmHQYxbxYDoh6QF6/0N/E//5Q2/QVY0N7P3/CD399EN//P6GiBQiYjWHb8hi7jkhdarXOz/cv2meq22nzzZ8zyLUIfJCv+fAAd6mXXA8YDWSP3rUFe5o6Vd6KYN89/Am7EP6R8+7I2M4RNABfThrfxta0ibl/OPvPIW9nrZD/0+GtqP1J1rayx3SeW5mUUfooTEzDuLbx7AXWUX+kUQ258DEDD8YOXbkxs7Tdh4e0s5oj4/5phAcsBVJP8rTqbkwQ0uJj/FfmU3iEVHNyHOsgq3Sfp8Tn180cEq/HNkDXGYUFrovxLA0b6AdUwvSvkT2ZHGLWwpJHngHBxBDC0UXF/mxVtEb9ZIl0nYt+9a2QuK4Sp4peYYhoawD9+DeLixJ/dC8iSZRwY+OY/9T0mSC6op+N3LUUg/6nddT4pxT2nVpCBNwDyvmdmZvx37BzthUKHWQMrzhMWCYLW950VJVejxaq5p5A8uWo8zEGqvfxS1ezJuHqXKEClDVUaBKgSoFquYHqoiM3GBsY4WqFKqaC6q6ig0KK2ClgNWqAyuT+KqxWyFsDO/ZOK7IbMgPhKk3ZdWKJGnoG/2a9pcjSWMpe9YGj6sbPJ65wZPqBk9mbvC0usHTmRs8q27wbOYGz6sbPJ+5wQ/VDX6YucGL6gYvpm8wWWMkDabsihVl6p1DhgcHBH8kUeV3zb/qs+8camPvhiT4vlcCsLld0wvgVw0E2k+6S5gCZQYm8piscp2iDu0CAot9I8+J1khOWMSVFaRhRaCfrUslmcyEUSdO1YLEdqdiJ2nSPWssk5ILN5MNbSbdxK5ghof+Q/7b/fRp9/RUu7g4HI8Pg8KKYkHTHgOWTWnlVxSivPK6HDOy7JN+ioiZahaBlHrpsa+ISlXPM3xA0A1yxUOW4KwcOCH7RwsW3ha6Nhsv+S7C9fM4FXopiVOOa22GKsQr6zTqcMNxn0wxoToogHmOMkRweKP0cbxMfRyRDToYQjU5xKBu2crY+Mlxukx9JPv2ZEpon0G0I95IVZzNVxX+k1TBTYjN1sb5KmijODG0U+TRuOxmquTDik6QzdbKxTK18rswiLAKos+HlJ8keeuX/dZBf7C/b5qG0e1arUHbaLf7g9beQad7sN81zV9oOgK5EMNgN86n2O5wN4md/NNDw4/We+MtggOqlfftZvD2FqDdLDORpSUYgyUfWOqBfUzTFG96PRNyCbGiLYiDOc+1heJOtJxSUOd7VjITxdLfhRxUgIEgyS1KT7Uq01NPTkg1VLZyw23keH42Am6Hyo28QhM5WYyJKDeyxjZyupilJnEjfrTLUEayVkZytmA8oqxkHa3kfDmuxIoCFspY1spYPizJpShrWUdruXjCCc1nn8S8xADz52+SKE8kANkJuKRONnfAhC+TzTHmVyVrvBOf1OkKymQrApGCughZbURVfURFjcS0ldqCWonKegmpPEWnn4S1EyLZpvJtNTlaXjZMzvsCuy53o7KmQl5XUVlbUVNfUV1jIa+zkNda1NRbyGsuauouamovqusvqmowpHUYZV0K6zHkZ9ok59qkZ9umqM+oqtGoqdOoqtWoqNeoPk0nr9uord2oPBsnPh8nr+Ooq+WQBtGLjk5FwRaLHaYs7ZiivGPWEo+qqLoESPBgIgco6pNAYsBRCTrywCM50lNTWCutAamqA5HUgkxVD1L2kuK6EEltSGV9SHWNiLxOpKJWRF4vIq4Z4etGykMt148I/Ym8SKOyUKOqWKOiYENWtCEt3CiNa02HsCVSWWVmXQFaEXEaRHugEK1CtArRvk5Eq5J2CtIuDdJKT6orVKtQrUK1spD24u80LBWZzONOw6VeJtmShMnfrdhtkllZz8XHgKwbk0XcJlm8y+gVXCfZPhDrmxvPFHcfFW+jqLv8qMhZ6O1HRd7M1x/V3mQkLoKurQRvTVcJnum0vK8gQNaFZu42lFd2nRE3+mTTkK9VV1dEbu5lRsnbVvsqozhiLUQN63h1Ec5fS1B9cVFhDBliV1cTvWwXn3Xn4ytAPqVLH6XQx1DQR0EfBX1UaZTCPnPHPkloU4EfBX5W6F7GZZ1KbUvCbUZrrsdSRfn4DTmX2qnJ4hvtp6TxRcmHafL4Iu5SE/ki/pMz+VMl43mEKM/Gi65UKKPF6ZLxeVuZBjluYka+ICURwnzphLyqkFzfjLwEfq5ZPn66W8peZRoeFy8XnSYJLxhnGdTOIb3ur3963d+cQ6MbAjeFp0Zr8WZH4U2FNxXeXHW8qS5bUIBzgYCz9i5WhTkV5nz1mHMFjnQWJ998DnIuOq7ckfwuibGn4spziisbNTi/q3C+wvkK569bXFldgKTg/uLjy+Jr5RXmV5hfxZlVnLn1FPy5r/Cnwp8Kf65dnFkBUAVAlxJvVhhUYVAVd15W3PkMIQp1nxJ23op7rsc//US57WbUFn0scyR6YI7gGHyDKIj6bxxEZPqjy+xnqAC6ib5HMEBmtXr221N6+jYMx75DBOEO01lIMEuAOVt/EFVACX8h3IIDEDpY8gPhCVe0a2BAshACF5taMsV5uM9xPZ8qteh19T9DiCbMqEXbAk7mRo46hPeFC5tTzXGvL0bpc3Vmzrd4/IVhcKLk5TDFJiaRFV81YbtkobbgkSOq65IIckyUYQseTwQcL/2zSJe9qRc51O3Qf/jD65PF3wTWrU1WE7L+N7S41R2x+Ft14o8XJd5OiJEzGQf/Trqh57ml/lKa+OFYudGAOEYYwKuoIWFt3FQaByH2RLWGhNwzvZDZQrtZ5ETOUTeaOROomYh0pu8GEyLosWQy8k8IJ+Q0hkhmtO3asRfNK/85Z3+JlaTwckevdhIvYfjpy59s+i8y+llmyY3tf0XO5cQ1RQWj0Rwy1ByqXsx8x5uMi5cZ8utZ+sBLzKDFbtkeC3u2Zcw7ZjlPnXIrJK55T1S12FVO1D3ZGrf3jKVtQSafhXckhr8l26rpAXSiuKA4pJlIp90s3cqXCojyZFte+dtEF7iKVSFTxtSvqhqYMZK+i7C2RNtFoRchQmjsjRu55tZtrU1iYWoWL2WnJBL/rHO4YlZ15JbemcXSOxti4ltJYOgxCnHYTEtxcGMQRV5117vbTee8zsyc0vTc13yboHWUfTmWWS/pDK9efY8L/PC2n3PBupHZOW/zusF/aDd5Dne5QYv727Ai3V8nY6DR8rLZyd/CN9zlG+bf0urwH7h7WfYtvr9JX3Li++mxNIz+9eokjkclgbLf0t8+137RfkuQlrad/Dx8hKf0kGVB9X6/3wL73T3QbbUBbB8Y7War2TX3rXcdYHSbg+jh2zQg1tx6/D9hH5uqlqcAAA== + H4sIAAAAAAAC/+1dbW/bOBL+nl8h6HpFcufcWrbjpAGKQ16bAtu9XpN2cdcGBi3RtjaypKWoJG6Q++1HUm+UREp24vgl5n7oxjMyRc4Mhw9nOPTDlqbpvZ7t+iEO9EPt+3UjoiD4Z2gjyNGA63oYYNtzKfGBkAjRsQNMH2GftJjKOP3QdvBHlzCNRka1AAaBFyITEob+GXljiEcwDHTuGeiCvkP5GIWQo49sS0C1Tc898RwP0QbRsA+2mw2tZRjkn729hmbs8E27YMxefJSNRXurHTkQ4VwX8MRnz1kgGPU9gCw95j2y/1+Tfx+ZUKBl40Jv9aEL8UeLUNzQcSIKAv7oyvMcbPuE3mREOpwTz8XIc6hAB8AJou/bue86tnvDacEHLmTPRyJPBK6bnuMAP4B8SzmGVeIMkW199jJlRkIuKOyOfG51OMJ9MoD484R+TqSTts3G0Eo/Zr2+TmkI+hBgbqgc9SOGiKlHzP7i3eUEzFjByLu7snHJcnQcE/WTECHoYu0YuNadbeGRnn/m0v7Jnht1OUZsCci7i2wgHmMqeODYIGAGyCSZiaAPGCXXFWpPv0J3iJmYmzk6DEoaqp0t+sB2nJykKeEDApZNBsprSq7tdwVtG60abRtibbfTjw4cQtfKv4mIaegeBVfiqQ1uh8XBU9ON9CX2BGdjH0/ErP9C5JU5Y3AveMfYdgVUZA9H+FLob6iZCajk2d+ZSeWtkrI8RIeQjqbAOoWBWW4PE/fkCDp2C5wQJmZV0gPxFQIrotS7uG8GT70pzEgii0syU3MzXqej+ezZLv7kMXEwgs7NbBOKLNqHxGpdDIZlj+TT1qiJhvT9e3l6uftEaBZEkPn3geNlApR4kAAiGwb/uoWIWDwsDDGQ9TcoDTzAwLwp9SbA0Peh9SuRaYmHARpCHHDrIb8i0tXi3mfDoHrvkb6Z20E43raJq4PbZBnDgLSKemT5uPPQDVmCTWjfwl5/gmHQYxbxYDoh6QF6/0N/E//5Q2/QVY0NjJLTDz/0x+9viEQhIkZz+Ias5Z4TUp96vfPD/Zvmudp2+mzD9yxCHSIv9HsOHOBt2gPHA1Yj+aNHPeGOln6llzLIdw9vwj6kf/S8OzKEQwQd0IczdrehJU2+/98P/R9/5yns7fR5+n8ysh2tP9HSTu6QvnPzirpBD42ZaRDPPoa9yCbyjySiOQcmZujByLEjJ3aetvPwkHZGe3zMN4XggK1H+lGeTo2FmVlKfIz/yiwSj4hmRp5jFSyV9vucePyieVP65cge4DKjsMx9IWaleQPtmNqP9iWyJotb2lJA8sC7N4AYVig6uMiLtYq+qJcskLZr2be2FRK3VfJLyTMMB2UduAf3dmFB6ofmTTSFCl50HHufkiYTTFf0upGblvrP77yeEtec0q5LQ5iAe1gxszN7O/YLdsZmQqmDlOENjwFDbHnLixaq0uPRStXMG1i2GGUm1lj9Lm7xYt48RJUjVECqjoJUClIpSDU/SEVk5AZjGytMpTDVHDDVVWxOWMEqBatWHVaZxFWN3QphY3jPxnFFZkN+IEy9KatWJElD3+jXtL8cSRpL2bM2eFzd4PHMDZ5UN3gyc4On1Q2eztzgWXWDZzM3eF7d4PnMDX6obvDDzA1eVDd4MX2DyRojaTBlV6woU+8bMjQ4IOgjiSi/a/5Vn33fUBt3NySB970SfM3tmV4AvWog0H7SPcIUGDMwkcdklesUdWgXEFjsG3lOtEZywiKurCANK4L8bF0qyWQmhDpxqhYktjcVO0mT7lhjmZRcuJlsZzPpJnYFMzz0H/Lf7qdPu6en2sXF4Xh8GBRWFAua9hiwTEorv6IQ5ZXX5ZiRZZ70U0TMVLMIotRLj31FVKp6nuEDgm6QKx6yBGflwAnZPVqw8LbQtdl4yXcRrp/HqdBLCZxyVGszVCFeWadRhxuO+2SKCdVBAcxzlCGCwxulj+Nl6uOIbM/BEKrJIQZ1y1bGxk+O02XqI9m3J1NC+wyiHfFGquJsvqrwn6QKbkJstjbOV0EbxYmhnSKPRmU3UyUfVnSCbLZWLpapld+FQYRVEH0+pPwkyVu/7LcO+oP9fdM0jG7Xag3aRrvdH7T2Djrdg/2uaf5CsxHIhRgGu3E2xXaHu0ns5J8eGn603htvERxQrbxvN4O3twDtZomJLC3BGCz5wFIP7GOapnjT65mQS4cVbUEczHmuLRR3ouWUgjrbs4J5KJb6LmSgAgwECW5RcqpVmZx6cjqqoTKVG20hx/OzEHA7VC7k1RnIyWIMRLmQtbWQ08UsMokL8aPdhTKRNTKRswXjEGUj62cj58txI1YUpFCmskam8mFJ7kTZyvrZysUTTmQ+++TlJQaYP2+TRHUiAchOvCU1sbkDJXxJbI4xv4pY4534ZE5XUBJbEXgUVEHIKiGqqiEqKiKmrcoWVEZUVkdI5Sk67SSslBDJNpVvq8nR8rJhct4X2HW5G5UVFPIqispKippqiuqKCnlVhbyyoqa6Ql5hUVNlUVNpUV1tUVVxIa26KOtSWH0hP8MmOccmPcs2RTVGVUVGTVVGVWVGRXVG9ek5eZVGbaVG5Vk48Xk4edVGXeWGNGhedHQq8rVI5DBlIccUxRyzFnRURdElMIKHEjk4UZ/yEcONSsiRhx3JAZ6aIlppxUdV1Yek8mOq6o+yjxRXgUgqQSqrQaorQuRVIRWVIfLqEHGFCF8lUh5quVpE6E3kJRmVZRlVpRkV5RmyEg1pmUZpXGs6hC2Ryirz6ArOiojT4NkDhWcVnlV49nXiWZWoU4B2SYBWeipdYVqFaRWmlYWzF393YamgZB53Fy710siWJET+bsVujcxKeC4+BmTdmCzi1sjirUWv4NrI9oFY39x4prjlqHjzRN01R0XOQu85KvJmvuio9s4iccFzbdV3a7qq70yn5V0FgbEuNHM3n7yyi4u40SdbhnxduroKclOvLUrettqXFsXRaiFmWMdLinD+AoLqK4oKY8jwurqE6GW7+Ky7HV8B7ild7igFPoYCPgr4KOCjyqAU8pkz8knCmgr6KOizQvcvLus0alsSajNacz2OKsrEb8h51E5N/t5oPyWBL0o8TJPBF3GXmsIX8Z+cw58qDc/jQ3keXnR1QhkrTpeGz9vKNLhxE3PxBSmJ8OVLp+JVReS65uIl4HPNMvHT3UX2KhPwuHiF6DTpd8E4y5B2Dol1f/0T6/7mHBbdELApPC1aizY7Cm0qtKnQ5qqjTXW5goKbC4ObtfetKsSpEOerR5wrcJSzOPnmc4Bz0THljuS3R4w9FVOeU0zZqEH5XYXyFcpXKH/dYsrqyiMF9hcdWxZfHK8Qv0L8KsasYsytp6DPfYU+FfpU6HPtYswKfir4uYRYs0KgCoGqmPOyYs5nCFGg+5SQ81bccz3+aSfKbTejtuhjmSPRA3MEx+AbREHUf+MgItMfVWY/MwXQTfQ9ggAyq9Wz35bS07dhOPYdIgh3mM5CglgCzNn6g6juSfgL4BYcgNDBkh8AT7iiPQODkYXwd2Jq+imB9dplhOvLh/N51M9xPZ9qt+h+9T9DiCbMukW7A074Ro46hPeFm5pTFXKvL4bqc2VmzrdYEIVqAk6mvECm2MskQuPLJmyXLNgWPHJEZV3iyauPiVZsweOJgGMIMIt02Zt6kWfdDv2HP7w+QQEmsG5tsqwQINDQ4lZ3xOJv1Yk/Xp14gyHWzmQc/Dvphp7nlvpLaeKHY+VGA+IYYQCvooaEpXFTaRyE2BOVGhJyz/RCZgvtZpETeUndaOZMoGZG0im/G0yIoMeSWck/IZyZ0xgimdq2a8fuNK/85xz+JVaS4swdvdpbvIThpy9/sum/yOhnmSU3tv8VOZcT1xTVi0ZzyFBzqHpV8x1vMi7eY8gvbOkDLzGDFrp1eyzs3ZYx7ZjhPHXGrY605j1N1VJXOU33ZCvc3jMWtgVZfBbkkdj9lmzHpgfQiYKD4rhmIp12s3QhXyogypPtfOVvE93cKlaFTBlTv6pqYMZI+i7C2hLtGoVOhAihsTdu5Jpbt5U2CYmpWbyUfZJI/LPO4YpZ1ZFbemcWS+9siIlvJfGhxyjSYTMtxTGOQRSA1V3vbjed8zozc0rTc1/zbYLVUfblWGa9pDO8evU9Lv7D237OBetGZue8zesG/6Hd5Dnc3QYt7m/DinR/nYyBBs3LZid/C99wl2+Yf0urw3/gLmXZt/j+Jn3Jie+nx3Ix+terkzgslcTLfkt/4lz7RfstQVradvIr8BGe0kOWCtX7/X4L7Hf3QLfVBrB9YLSbrWbX3LfedYDRbQ6ih2/TuFhz6/H/xs1ceXmnAAA= persistentvolumesusage.json.gz: - H4sIAAAAAAAC/+1bX2/bOBJ/96cQiF0gOXh3JSdu7AX2oc0ihwXaXm6b9h4uC4OWaJkXmtRSVP5skO9+Q1KSKYmqbaSbpq2BtohmhqPhzHDmx6lyPwgCNJtRnhUqRz8H//1jaCmS/FlQSRwa5lworKjgmngPJCAymisjAo8PRowkVOE5I0BdYJYTQ0w5Ub8lQOIFY5Yicba8EIIpmgE9NMQlTcip4EoKljfW08ZaRvmVY5gkCzB0CQR0FObI0sSNkTBWWluBGgvGcJY3bGswkg4nw5ywtSpXneFjRnF+KpiQxisPQ5c5xzJvazSMBOfL14SnSlsdhR0e6VumcC4KGesdoHMpVkQtSWH3XEstKGNabYf4T4kTSriq/F0zU0mTc+HZgPH7qEFiJCU8qTPAdUTKX+YXZeyVLBq2a4nr1M+ICymtWR7mCt/2MCj3MyRNl+odpJLPh8DPl+LGvzKHRf+hiYlKlW0OW0H+sx6l15gVJmparcNq+hMytxbqMG7KN0dtjpPsNVmbdy4oV2+E2aghNPMgI5AoXOHU6wiU6dU6Iwqtftzl+XMQIpUQSaR+54IJhVrcjGDlcR/KiaQk/9c1kZBsxLOjPMMx6TsUwNTRnjaJCsdXPm/mimQZSV6DU717UFimRDVPdftkG0Fym5mNHlzyIMiLVXBD1VIU6oByeDmPyTDg4P/D4ECJ7OogGgYHV8WcMKJm14IVKzIDMZXPYgybo+puNr9TJL+PWQEmyl8u0Xflj5doGPxPzIFSrtcEjlfEeEUL1g+aA6HNofJCdO1rYobpSkvZx0v0cHh4qG3+4ZGG42tMmT7ST2w5/EHDdjQWQq5MciFFtYUmo7pikLpEwnk8w7ESsn2gnCJ2Vut7D6U/eKdt7KqD9mJaF3qJGqyH4Xa58wV6/yldfyYJ2ej6Vy3XO0/NIqKWGgoIlngKjLb8DHqmrzpp3rslXSg/U5mehj4YL1lrg/e5rqxNwRrP3LdbDpZdcFExhey05KqjzNRdZl5NeUKvaVJAA+rtLpWswVZNy27xLc27ds2L+MoWQk+7W5WdRfumFRuks8q/qr+71i3SoEX/Fu7wLdmiLq/T0RyMbubAqSHMa6BmivQVNiiwm54WbniXWcARdujOjncqEF/GJtyjNvBsrYGGYxwvyQXkCxQ730mKNVB+BV07laLg3gNhZT7oXOlnd1MEyXSOD8bhMIhORsPgeDwMwh+nJ4ftxDVio6MTkBtNQS7UcpNpj5xWMz62f0HdIeotPFtC8zrkJThrslNcGLB234HAlT+iMOzC4IoZbn0U6zL5WqdY3lOYaqk3WF4RuRHdfvwacdQklZ3ClyU9kHeFs4zy9MJWuaiPt0X5KKuXrUiBEoEit6p7/q4rp+90rCvlEvN0C+WjLTsbJMGvkGPnFTJvZUL3QhALzkmsoO105C60RR7HZyJXC6prB0I+xhnczd/Rv4z6cfh9Swa6tXetoX90qXHVG7iAb1H5bQPvXndsxtqNobc/vfQwRb1wS5+Xt46jNlFeMXu1aB1Ufck2swATf11BjgDbRdEE/plMdQmJJp1KsyhY36VSv8XVZ9WNdJGbHnUUlcfdKOrHCBpOnmocw7uh2vleBEnZxbYltL3k5tZUIdznci+qb0UfNexzIG9z6/npkj8/j/1DV5tPei8YbbwX7HAR2+42gCZwbqYh+luh/UZsnkOXYkTHsqnftAO3SE7aRdJIbFckhbYT/bJjfaw60k4l0ix6W7a8apA3aK+r1/gHRCX1N0gSM2D2s38XN40Zcl30LsogNie3dWh/xflyLrBMAtCAmvzK28sXDqMMlQTpgRPJ/Rj5UWPk4/0YeT9G/rbHyI8YBVItlc8KKC5f5RjQTGDtJj/9CPax43tr1xc0tv8c2fLECROYwfHGjHlek2Nr7n50/CxGx1xw8qVPjp/xHvaD4/3g+JMMjsf7wfF+cLwfHH9Dg+OPTkCfGttunsk+4fXgW5rFPhos74exX+MwdlDuAOUAGVf4gz40xsro2JLVnX1VAuXWfg2scLoOlDmLkpP1Jxcozimq1SqyyhjsnKfdz5wH7Rg7Q8cGnq1im5AFLpgvvVyuDx6bj6Gb2KwH19fAwYGSDldk1TfbLixCfxZE3hlA6YOczkfVUYOaknanrkPkvH7Q2o7jMMxYBTtb23Bc6fphC4Bc+cqpZFDmoAgn5KWvR9aORGWhdnWtICLUs6bysmfJJheb183sJXabD0SHQfmSQ39IRptCUs4F3NDBGTB+z/9dWYWa3I75muYXLgNu9+cwoCdfWEVu/X3qLAh3zoK3VRfeJQ+4b9GnzoTtQMWDgyr2KfMkKXNeYzWLVk41VtslfWzQP3/u9APShx5E+nVk2KAaTNlfWzKTyrrf15dFcfNDVM1Dq3ugRSLrZRmNr8x/ZJWLS5fMKqDtYkQ0XqcfikLn4ch9iNaphMbOz5H7cBS6nPXUFo2cn6PENuM/qj3oq4GTaRvf4ip+4Sp23zI6dh/WgwN0krj2VrY03PeXMDdi9P7itERrFVZcn7HAHrLyl7sKMxZC02g6n47whEzC42iejF+c4MWUJHgejeLJcRhb4esaH4aDh/8D0iZvTe82AAA= + H4sIAAAAAAAC/+1bX2/bNhB/96cQiA1IBneV7LixB/QhTZFhQNtlS9o9LIVBS7TMRSZVisqfBvnuO5KSTEl0bCNZmnQG2iK6O1LHu+Pd7y7qTcfz0HhMWZrLDP3i/f25ayiCfMmpIBYNM8YllpQzRbwBEhATmkktAo+3WoxEVOJJQoA6xUlGNDFmRP4WAYnlSWIoAqezU84TSVOg+5o4oxE55EwKnmS19bS2NqHs3FJMkCkoOgMC6vsZMjR+qSW0lkZXoIY8SXCa1XSrMaIWJ8WMJIut7O00HycUZ4c84UJb5bZrMydYZM0dNSPC2ewdYbFUWgd+i0eWLZM447kI1QnQseBzImckN2eupKY0SdS2LeKvAkeUMFnau2LGgkbH3HEAbfdencQkERdYvQEF8/qbExITFlXRYRspZgfZaREXUuS1cymJi9jNCHMhjMoO5hxfLWFQ5mYIGs/kCYSZm53N+OUSDqz5i0baYWUgWmwJVyNx+Qx4YKxcO1Rta7HqpoagroRajMvizUGTY92DiqzUO+bgp/dcn1MT6o5KCcQQkzgmzjhL1WoVLLnaftDmucMTHBURQYR65zThEjW4KcHSYT6UEUFJ9vsFERCHxHGiLMUhWXZfgKmcPaoTJQ7PXdbMJElTEr0DozrPILGIiaxf+Oal14LkKtUH3Tljnpflc++SyhnP5Q5l8HIWkq7HwP673o7k6flO0PV2zvMJSYgcX/Akn5MxiMlsHGI4HJXX48m1JNlNmOSgonh9hn4ofjxDXe8fPgFKsV4RGJ4TbRUlWD0oDrg2g6QM3jWvCRNM50rKPJ6h293dXaXzi3sqji8wTdSNfmTN4Q/qNr0x5WKugwtJqjTUEdUWK5PXEQ4lF80LZeWwo2q/j1AVvBOlY3s7qDy6qqEDVGPddteLnWdo/cc0/ZEgZKXp3zRMbz3Vk4icKZTAk8iRYJTmR1BOXdlJ8U5mdCrdTKlLGvqkrWS09T5mKrPWBSuoc9MsOVi0cUfJ5KJVrcuKMpbXqX41ZRG9oFEOBWhpdSllNeyqa3aFr2jW1muSh+cmETrK3byoLMo2Dd8gFVXuVcura1UiNZB0H+EaX5E18vIiHPXFaEcO3BqSOBVUTB6/wRogtsPToA3nMoM3/BbdOvFGCeJ5HMK+ah3H0WpAOcThjJxCvECyc92kUGHoN1C1Y8Fz5rwQRuaTipXl7HaIIBFP8M7A73rBfq/r7Q26nv/zaH+3GbharNffB7neCOR8JTccLZFT2wz2zF/YbhctTTxrovbK5QU4q7NjnGuwdtNCwKU9At9vo+CS6a99Fas0+U6FWLYkMVVS77E4J2Ilur27w+g/QofRRP53AXEXqp7jNKUsPjWJNFjGWyNDFQnSJD1Pck+SK9m+4helXzfKHOXmArN4jc17axZPiLO3EMbHJfhvBFu75wg5YySUUNlacqdKI0cGSHkmp1SlJ4RcjCPOwIFf9fYD/8eGDAAC51pNv3OpNtV7aP/XKC4GI7Q7KnMpzMHQh5cHDiavFq5p86Kx6TeJ4jwx3UvjDqgWX08itP9VkuoDfAyCIfwzHKksFQxbyWyaJ8v6VvUWez+zXU/l0VG/tVGRUfRGy2GIup+HCiqxtqs2br0gKNvwuUDPZ0w3ZiWIfiqtV9V43anYtwD3urF6ecaensV+UtnmQVuP3srWY4Neb72GAw3h3ox89J92DyvhfwZVKiHKl/X9dTmwk+SwmSS1xHpJkis90esN82NZkTZKkXrRh6LklaPCTnNdtcY9gyqov0GQ6PG2m/0nv6xNsKukd1o4sT43rlz7FmezCcci8mAHVOeX1p69shiFqwRIdyxPbofY9xpi722H2Nsh9naI/fCDSKqksnEOeee7HELq+a855MMPgO/7ywOj1zP6pcG3iJZHDhhPj61XRszTmlsbdbeD6ycxuGackec+t37CZ9iOrbdj6wcZWw+2Y+vt2Ho7tt6OrR9ybH3n/PWx4fPqifAjdiD/p0nwvfH4dhT8PY6CO8UJUAaodI4/qUujtQz2DFlem1dFkG7Nl9ASxwtH6bsoGFl8U4LCjKJqW0nmaQInZ3H7E+9O08fWWLMGmUvfRmSK88QVXjbXhcD1h+B1+Fe1DkgVdO/EAFOLXyIIC7ZaXJ6WH67b+Ah9yYm41uDVBW+tL8uDGjUmzZJd+cp6fadxLstyOElKiFvHE7ZNbYOsAcZLo1kpDfIdZOOIHLiK5cKiRca295qDa6hjTWllx5JVJtavG5uGea1PYRvV47brFW/ddfuot8pHxVDC9iXcDu2I7I9STVTnts6jaG7hIgLMgS0GVOtTs5GdmR87LPyNw+JDWZ83CQzmWvTgobEW3Li18MY2ZB4lZI4rFGdwzKFCcZuEj3H6t4+d5VD1dglW/T4irFNOxcx/5tJj0goJVG0kv3wRlMPYskM0GGWxLKXhuf4tWrG4MMm4hOA2ekSDRfihwLce+vaDNVVBA+vnwH7o+zZnMTJGPevnIDLV+XN5BtU0WJG28i32xq/sje239Pbsh8VIAe1Htr6lLjXzfeW6V0YfTw8LHFeiyMUd88wlK/7LW65nUmgUjCajHh6Sob8XTKLBq308HZEIT4JeONzzQyN8USFHv3P7L0EqTdcFOAAA pod-total.json.gz: - H4sIAAAAAAAC/+2daW/bOBqAv/dXCJrsIsG6M5aPHAX6IclMFwXaTrbJdIFtg4CWaJsbWdRQVBJP4P3ty0MHSZGykx5JEPVDGpMUj5fv8fCVpdy+8Dz/4gIlaU4z/5X3+bwnSwj8M0cEKmUgSTAFFOGEF96yIlYYo4zyJuKTV5SKmkmOYvo2YZVBry6NAAUZzkkIWYV/QvAC0jnMM19pAxMwiXk9JTlUyucospSiECfHOMaEd0hmE7Dd73mDIGA/xuOeF+yoXSdgIQY+rNfi/d07jCGh2hToMhXtIpDNJxiQyC/qVuL/c/ZzJYQCI0SN2fqzBNK3EStJ8jiWJQSk8zOMY4pSVt4XhXw5xzihBMdcoFMQZ/J6pF0bo+RS2YUUJFC0lyIvBe6HOI5BmkG1J60iatTMCIpOcL2ZUsjGhl2zz4ORUnBTLqD4vOSfS+lUfYs1DKqP9azPqzICUwioslSl9C2FRGyPvfojvtYELKqyOb4+Q7ShOT4tCv3jnBCYUO8IJNE1iujc19ucor9Eu/muUlFoAsHXUgeKNdaCB+EcnqEFxHljMSHXyyMQXs4IzpPItjeYfAJxDu1VmWJarOynwcFBONpVFVUo/GC4xxR9cNDzRkz3+z/vH2hK/1M0GoEhKFW43oG1xuhHMEQLIHau2nN/islCbByT2gJeZJAgqFwzA/kM6kq1ADflKoO+qjwLlJQVajHfSlMifC/mzCHNcRy9AxOoG43Z4j0gl1DIj6tCQz2dqn9gqH4wWKP6QaPvOUSzOVX7ktYwrD8mTL2vQGyqi27qheTSFCWzM6mFga1cV5J6PYq7u+Iy9ij2KLyhimp4RZWyDmUljs4ISGZrOhvUnTWUjunCr0zvTjCTQqYrBFeHU+YqNLn7XESi9Xss/D8zjSSBIWUeTWtzxqdjSBSneryS+otgHP1uqRHWHIe6QOXOABbmlKJzbdkRnII8pmZncrFyieW/nlGPEl2ndIe1xWWepSCEr7ytFEe+0TBPkLDDozRTZ7fSZleoiq5YUjpXkBAZVG/1ayo7aspCX2HpqWSwJJcvZwTCxJinUPkI3jSXqugM3zetbtW7y7hLGMfMaTgHDpwDj6vN+YrhCYzcYw+cY+85xnaqmriw8nwNM6u9UIozOkV8dN83C98w6ihj3bj/N6WewOY1osx5ifAG7xlguL3QlAUWfh3fYd1fUGmy/odfDo0KXF3Q4kqyhq9gJeSSuVFoWnwc64w4ZHwYBPvsx/4BD5nBvhYyp3zkZnjhPav9yG4GLOgGB0OtAzWCNfZGECPrJ18kuqwpIDNIW0QJb1IxdpYvthEjJLjNnCEFbFrkglHnNSaXjNxDiK7gxWRJYXZBGefGt2GcZyzmvP7ibxW/fvF7lXN5/b8viqthNR7zNaKQ/f/FX33eKiPWqy3mF9jEue8839nRN62dCwqLyChITDetBcU3IKRCwJrF+jGcwSR6Uw2h98t0VAChf9imL5pXU8XOpvtGKqkWPXj56RxNabPCIMqPbCs8PPWOuMy9j3IHoiZFSjaqioU1q6a1r5qWqG03LcyPE/7rDa2q9DprDUs0/FDE+1CusaPfjn71grvh76jD3w5/O/zt8LfD3w5/vyP+UrYl2QLRjn8fjH/Pii2gzwOBHzLzHjhS7+NHlnr/pil3ECOQHZdHhDp2+xNAGlws7uG8g8mMCgH3tXJoa77uWMDdqXZ3gxf8k4AIQWHa/e+RZg7s+7xb07NwEPpQTFCz5DA7K+5Qmf4cXM0spaWyN2v4HavfFild2u/N/QcS3KyRDGh2JcnPLCX86HAqkcysK0KK0XvGGv9b6JXpUn3h+S0dWRlipZ5CYGbqMS+8LoYZtB5Y7ovwPPLYtDSFTBNZjJs1/UvKu+Rql2e6xZdnC70529QIEigC6DTG1F/jDmT8+r2AZGOdmWu+FiihILw0BcpCcJrC6J1kFX2iTy0L5k2W3jZrcx8cuFPMv71lo6xWztCvlXMJqwFiLRAou3t/IChSX9ZbrNX971vVqgERAdK0a0yofnSQdntRBgd2uEBXKMpBFcVXTdTgt93rCdyAG2T44kkeXkpVM5zHorBUvmLLFwiM1nbfVDmaz+eNKS7BTVvmotYdfrbVlYJnf2wAGePZEchgQ4ukA240t5y91XWszYc8pimWiv1caaGZlnPgwl6HCx0udLjwsFmDjhdqXihTBR0wdMDwWIBBSehoTkjN52gV3y6dM3Ckc/Yt6Ry7lF204ySeNuppIZ9Nv81rIaBWCnIKtBaqeb/IRCKbdGsJB847RULSBxaP2pxHOym5aamdmNZQUzs5uenJTVDrKMrtq9bQVBtRtd3IMbfERlduwnJQlpO0NqWtNuJaQ11t5NVCX+0E5qawtSTWSmN2InNS2Toyc9KZ9YbiXZI6bP48Cj8Epm2Iahvg2sbI1oZtDnRrv4vqwLg2lGvFOSMFVNwLKr8F5Z3I3TJvcTbhrg3wHJC3Eeg1/Yod+BzQ1wp+bfDnBsA1jrUJgs0lNIHQala1qqapRUVd5NVGXy0E5v5Wh4PENvmqwVNYgmpra0m4YzR76mpjSDNW31FaR2kdpT0aSqtyaR2mPQFMU76s05FaR2rPlNQaqe4f/xUp0/i+5pnkB8xgOu48BkGXwvxGKczhYA0dDzo67ui4o+OnkcOMCOaDd5D8lHKZ3q9y1zpS7ki5y2l20LZBTnMttQ07auuoraO2J5LT7LDtaeY2O3LryK3LcT5UjvM3Qjgg3ifF+aKYOXdD3Hfw2mFf9sWbKe/IzMI5XIBPkGRy/sG+LKbLGJavLJDXscBZa61/mU8gYR4fitdJyL4oXKQxE0Qy2+SlpzVg3doeRy7elOF4JLmstbF2+fJTNd1qV7XqRSUKJCu19StBVK/r/5lDshRKbX0lay3zQCudwRvjYXTl1anV8GZGWHvkJP6kvAHD+lyJepjY6AWyhazU74KjhMXlCB7a32tgF+SCbQayNC8FXET6u0hXjHQhHep2nt7+F08YM4QgukIsmjAU6HlFrzt28Q/Wib8ISqqeMCUXMs7+VU7D12sb8+Vl9sbF5soFKRV5Bs9kR9bHZOw77v/8D7UPkIv3XjQeO2LFFyHOhS4M+2aNdI5+0NdUYI0hckt/mS2ZoBcOY1RbWA1yE0VkFo0SVHhRffO/5suNTEsqmtzx252EpvjG8wb31Ptq7Htr/ndZ/F2M5BKlf5D4dJmEtkfHpAkFnQm1mZDDbh6dsbiPYStxDruTAX2jyKG/y+rH2c6dZPEs7cnYyB9oT2NXJBp/RQD6QfpcJy0cav3CdaDyMxjLlJc9W1dKZ9g3z1q1gHid62DqHs1y0HVshWszNh6qbWHB3DkWq3phO9RZfQQTQm+86GndPTUTrl4M2VnxQ5xnbOK/qw23WNXIremju2j66JmouPGXRkQ2r0pBVC8VxNcvK5sv3xsocyb1ZSliYEDqiwuZXZSTUbfXHyvpGVX3NRfsB7WeqzrvB+qHYV+tUR6sHii/B8XfVjkv18DT2U21c4+idryrdqyOMhipH5QXKexF6nzLuWji+wuLOwz+H2fHRdaoTGd9kASGkpn3i3dSQJ2fi/t5/h4I9vu7eyE8GA3BqA/gYDwaj3b3xmFwMJ2OQ9n4qspS9V+s/g+n66MejmcAAA== + H4sIAAAAAAAC/+2dW2/bOBaA3/srBE530WLdGcuXXAr0Ie1MFwXaTrfJdIFtg4CWaJsbWdRQVBJP4P3ty4suJEXKTnpJg6oPaUxSvByec/jx0GKuHwQBODvDaVawHDwNPp4OVApFfxaYIi0NpilhkGGSisRrnsQTE5wzUUR+CspUmTMrcMJepTwzHDSpMWQwJwWNEM8A7yhZIbZERQ60MiiFs0TkM1ogLX2JY0cqjkj6giSEigrpYgYfDQfBKAz5j+l0EISP9apTuJINHzVjCf4eHCWIMqMLbJ3JcjHMlzMCaQzKvI38/5T/3EihoBgzq7dgkSL2KuYpaZEkKoXCbHlCSMJwxtOHMlEM5wVJGSWJEOgcJrl6HhvPJjg912YhgymS5ZXIK4GDiCQJzHKk12RkxK2cBcXxO9JMphKyNWGX/PNooiVcVQMoP6/F50o6dd1yDKP6Y9Pr0zqNogxBpg1VS33FEJXT485+Ty4NAcusfEkuTzBraQ5gZSJ4UVCKUhY8h2l8iWO2BGaZY/yXLLfc0zJKTaDkUulAOcZG8DBaohO8QqRoDSYSevkcRucLSoo0ds0NoR9gUiB3Vq6ZFk/7aXR4GE32dEWVCj8a73NFHx0OggnX/eHPB4eG0v8UTyZwDCsVbmZgqzGCGEV4BeXM1XMO5oSu5MRxqa3QWY4oRtozC1gskKlUK3hVjTIc6sqzwmmVoSeLqbQlIuZiyR3SkiTxazhDptHYJd5Aeo6k/IQqtNTTq/qHluqHoy2qH7bqXiK8WDK9LmUN4+ZjytX7Aia2upimXkouy3C6OFFaGLrSTSVpxqO5uwsh44CRgKErpqlGUGZp49BG4qmMwnSxpbJRU1lL6bgu/Mr17h3hUshNhRDqcMxdhSF3IEQkS78h0v9z00hTFDHu0YwyJ6I7lkRJZq5XSn8xSuLfHTnSmpPIFKiaGciXOS3p1Bh2jOawSJhdmRqsGmL1b2Dl49TUKdNhPRQyzzMYoafBw4zEwCpYpFja4fMs13u3MXpXqoqpWEo6F4hStahem8/UdtSWhTnCylOpxZKeP1lQhFKrn1LlY3TVHqqmM2LejLzN4CbtrlGScKfhbTj0NjytJ+czmqco9rc98ra972nbq2rywdrztcys8UIZydkci9YBsBNfcuqo1rrp8G9aPkXtZ2Sa9xHpDd5wwPB7oTlfWMRzYoZNf8GUyYK3vxxZGaR+oMOV5C1fwVPoOXejyLb4JDEZccz5MAwP+I+DQ7FkhgfGkjkXLbeXF1GzXo+qZsQX3fBwbFSgr2CtuZHEyOspVqkpawbpArEOUaKrTLadF6tHmBMSesSdIYO8W/SMU+cloeec3COEL9DZbM1QfsY45ybXUVLkfM159gk8LH/9BAa1c3n2v0+aq+E5Afc1MpH//wlsPj6sVqynD7lf4B0XvvP08WNz0rq5oLSInMHUdtPGovgSRkwK2LBYkKAFSuOXdRNmvVxHJRCCoy59MbyaLnbe3ZdKSY3VQ6QfL/GctTMsonzPpyIg8+C5kHnwXs1A3KZIxUZ1srRm3bQOdNOSud2mRcR2Ajzb0aoqr7PVsGTBt+V6H6kx9vTb06+ZcDP8nfT42+Nvj789/vb42+PvV8RfxqckX2HW8++d8e9JOQXsx0Dgu4y8h57Q+/Q7C71/0ZA7TDDMX1RbhGbtBjNIW1wsz3Beo3TBpICHRjpyFd+2LRDu1DjdEAn/pDDGSJr28GuEmUP3PO819CwdhNkUF9QiPcpPyhMq25/Di4UjtVL2do44sfptlbG1+2zuP4iSdo5iQLsqRX52KhVbh2OFZHZeuaRYtee88L+lXtkuFUjP76jIyRAbfReCcluPReJl2cyoc8NyW4QXK49LSzPENZGvcYu2f8lElULtity0+GpvYRbnkxojiuQCOk8IA1vcgVq/fi8h2Rpn7uuvA0oYjM5tgfIlOMtQ/FqxitnR+xYFC2br4BEvcxscuNGaf33NW9lsvEu/kS4krC8QW4FAm93bA0EZ+nIesdbn39e6VUMqF0jbrgll5tZB2e1ZtTjwzQW+wHEB61V800YNcezedOAKXmHLF8+K6FypmuU8VqWlihE7vkBglXb7ptrRfDxtdXENr7oiF43uiL2tqRQi+uMCyIQsnsMctbRIOeBWccfeWx/H1njI99TFSrF/VFpoh+U8uLDf40KPCz0u3G3UoOeFhheqUEEPDD0wfC/AoAV0DCekx3OMjC8Xzhl5wjkHjnCOW8o+2vESTxf1dJDPrt/mdRBQJwV5BdoI1T4vspHIJd1GwqH3pEhK+tDhUdv96CYlPy11E9MWauomJz89+QlqG0X5fdUWmuoiqq6DHHtKXHTlJywPZXlJa1fa6iKuLdTVRV4d9NVNYH4K20pinTTmJjIvlW0jMy+dOQ8UbxLU4f0Xq/BdYNqOqLYDru2MbF3Y5kG37lNUD8Z1oVwnzlkhoPIsqPoWVPBOzZZ9xNmGuy7A80DeTqDX9itu4PNAXyf4dcGfHwC3ONY2CLaH0AZCp1k1qpplDhX1kVcXfXUQmP9bHR4S2+WrBvdhCLqtbSXhntHcoaudIc0afU9pPaX1lPbdUFodS+sx7R5gmvZlnZ7UelL7QUmtFer+9l+Rso3vc95JvsMIpufkMQz7EOYXCmGOR1voeNTTcU/HPR3fjxhmTIlovIfk+xTLDH5Vs9aTck/KfUyzh7YdYppbqW3cU1tPbT213ZOYZo9t9zO22ZNbT259jPOuYpy/USoA8TYhzgdlz4UbEr5D5I6Hqi5RTLsjM4+WaAU/IJqr/ocHKpmtE1RdWaCe4wtno7XgvJghyj0+ktdJqLoYWmUJF0S62OXS0wawrl2vI5c3ZXheSa5yXaxdXX6qh1srVQPiQpHgWPGw47JTjZa13OZuEN39gj8LRNdSu513szbCD43UBbqy3krX7lCtm7dDw8a7J8kH7SoM5wsm+q5ip5tkS6HpXwrHKV+gY3TkvuDAabxgxWcFO4pXAi6X/JtIV7Z0pjzroyK7/i+ZcXiIYHyB+bLCmWAQlLU+dot/tE385eqkKwzXdinj/F9VN4CZ2+qvSHMXLidXDUjLKHJ0oipyvi/jnnHw8z/0OmAhL8BovX/Ek88iUkhdGA/tHOUlQTg0VGCLRQqTf5KvuaBXHqvUSzgtcxdF5KaNU1y6U3PyP+dbjlxLaqx8DLq9haH41osHt9T7uu1ba/5XGfxNjOQcZ3/Q5HidRq53yJQJhb0JdZmQx26+O2Px78c2ckN2IwP6QiuHeanVt7OdG8nih7QnayK/oT1NfSvR9DMWoG+kz030wqPWD3w7K5CjRMW+3GG7Sjrjob3pagQk8nw7VH9rjh2vZyp8k7FzU10DC5fetnjWA9fuzukjuBAG09XAqO6+mXB9Q2RvxXexn3GJ/6Y23GFVE7+mT26i6ZMfRMWtPzkiw3p1LKK+XZBcPqltvrpAUAVPmscyzMGANg+XMjurOqNPL5hqcRpd9w0XDMJGz3WdB6H+YTzUc7Q3rEfa72H5R1ZOqzGIuHZb7fyt6BXv6RXrrYwm+gftRoX9WO9v1RdDfH8RedQA/jh5UYaPqrjWW0VgOF0EvwTvSqgDhTzYA/swPBju7UfocDKGkyFEo+lkOtnbn0bh4Xw+jVThizpcNXyw+T+Bl8Fjl2cAAA== proxy.json.gz: - H4sIAAAAAAAC/+1dW3PbthJ+96/gsD0zzoyao3uszuTBcU4ymSapm6jtQ5PRQCRE8ZgCGAC0pXp0fvtZgBeBJGjLcR1JMV4y4i4ui93F7oclHVwfOY47mYQkTgR3f3b++txKKQx/SUKGNRoihAokQkok8RpIQIxCLlQTeFyrZtgPBZpGGKgzFHGsiAHB4o0PJJJEUUphKJ6PKY1EGAO9rYjz0MdnlAhGI17qH5b6RiG50ARjeAaCzoHg9trcTWn0SrVQUqayAtWjUYRiXpKtxPBrnBgRHG2G0odLeyJvjsfhAtNEaDJu+DSi7AXyLgJGE1Ibf9PmDxQluJldFkHRf+iORl5/6LbKZBZM0XG396zldLqjltNvt5z205PRk2q7H/x+H/WQq1E/l+b1kUCcJsyTYrnnjC6wmOOElwZyZ5QtkFy6SyjBZV6AkgAXzlKQF2iZr7bTblfEWoQkZ1ZZfE6vTBoClphLF6CR/xZNcdl5TK3eIXaBlU4FS7DWaF2Wn4X+OVXuXmYof+yWSURgdokikxOUHVbTQxyHJBivYqWKJl7d9GV9QmuCFspIl1JzjqCOwEtRMbiTseVUJca6td3gDJFgi8G75cEb3Qu84CV42DkFzfG6K7hSi4r5jvpqfo8Sgj0Bm7TWbiwlMig+plzMwqXs7ZoYryDYfAz/VsMP2v+qtIHAYuyr6Dd2Vap6BxHldtPNYF+p3QOy15Uq0oW57/99amDSouOWOucQzmqeC0R2AS5q2KmzMIrOZPhR9peBpdeBuNI5gX9ORjKwdE5qgWUmBTJvQDmLPl46XBdCVGfUqw2k7/fGPaqSDYyZLEjdVAKxAIstrICXsZKJJ4vjJL72ooTDdn7+yf0x+/nJbTn/pVOgXCRT/FPM6HL1yV0/qVtlExAFpIUJxyzEvN4sjxevkCeUPrq1JhEOMPFfFePVBwFPVGnVPd3WA4oQyA3aCoXK2+7vcYVR5OnrqoUQ2yTNZhulEc7lENEizAFFlMdXoUPfUCfVDaVabLehqJTTfX7HvZRHrzttJ9XpfRYeIXFt+mnrL8EFFIWIn+UJvZJVpogZMxdkYj5/i0kg5ipO1ni4qdtWCTxU27VTI75myA8xEdVMfEta7JVTn/LhuueAIgJyysdVqKg3uQwaOF7CWCqYiQuJpYkTkgYOC4M5eJ/fJEsWiiRaqHKg05+hr0xTzT9q6wjUFA2V+/Bbd48MmtwwuWJcZVN3tgEctZRaSzpujMFdiECBGYvGsrf0i0QOP6jzzJ4I1vIxwyrEziJa2f4MxxgZ83caPH+9xAxcDhtWBLnLw01bI8t2ZSkh+ngXRiEhzMcx9t+mmbDO/6pkwpDAxzJlqIwx4SviTdKfLIFQOPETps5UkCcA3/h84sExQWydgFpOSGBFxMPP/wdt8wfITH8NFp+ffNP0JJf6j6eoir2lzK9SxFRzFsn7OA9nwszMctsHqXbnI9jB+VAVeNtEVw0BlAnDcUVt70me+0Lih5ehn6BSdjHnSXU4Lku2RMuQ1+WaJt5F6pOG2LPIdrlUTBVbZaDe0Ks51BXxSh33zUtYoeU2B5aNF9LY4H2RPMUZxZNMGrxAvHZk0mK/sVsa/Ns1urbeOx2LDmEJ+h6z2ORnp/812MSwE7aBJoZu90UmJvjxoMDkxuqIxSUHikvmIRcUcsxi8iVBRIQRPm4/HY1ad8YqafY5SLByfZ1Pv14fBmhJMctbsBHxVs5oJObOb5n5LIjZMYg5eAhzWADmqLIDzZE6o76B2KDClpn9gV6V3kwV8oyzzVd+H1RsyZeAbaYUMd+BEdwyPy9jzYcaI9sWDFofacZ5yJdTjwDRDWy1yVab9gnVDfeo2kSwuKLsQoI4CffAIQNbcPrW2O19agTnfGMEW3mylSdbeXpUOGVoK0+28vToMUpT5cn5WvDyD1egnOnKOc6pLSfCtih1N2Bjy1O2PGXLU7Y8ZctTKex7ZstTtjy1T9Cvv/PyFEASMfEiuZnUX5nAI58oz7g/jGt5YEtJ6z59upOSVHe5vANwa+2j6no7Ul3vRtW9OATV9Xekuv6Nqjs7BNUNdqS6wY2qe7kfJ61fQJXO6fkb50Oqfls/foT1YyPr4ErId1uFrSKXjxMn9jhhjxP7dJw42d9KsgG23L9+3ARjQIuy7fmvH8flYrJktJyERbuoJsvJ12vn+hrmP4SKMgQ+UWAcW0q2pWRbSralZFtKTrHfyH5BYL8g2Bfc1+k+MuDXWMByMuj3+j8W+X098nuNLfCzwM8CPwv8LPCrAr+KfLbqZ6t+j/UjAkBjHuYcYJ10EsB3C7ygbDWZrgTm967ire23nk347J3SswVhOwZhys+/g5eOe74O+9qxgkA6FoFYBGIRSPY3p8c5DPHipCgo3fGDqBv/DsXikCYccnb+u5Nw6ckWiuy4HjSX2jv0mtAeL8KCkAoI6VoQYkGIBSEyz9BJQBlNhDSgLXs84GupQssWbzw2vPFApY89X8f3/DbqKFuBy705XqA/MOOplJ1+ShardCofsYv0HiHIVxsvVdGUESxrV8VYAi/iCJZLgvqtSEdVb9AARgm+5VcT+HiGksh0O4LONaFBdXdSObg0+FRxn4sG4zQujfMrnvRo6n5JMFtlJfg63NPuYOqUqAGuXqBS2EWb/qiyHE1hKIry64Aqy9BUqethC3Ca60rLVhB2IXf6+NR0dUmhSDfLr/pYC7BIaOiTa9nQ5TYVq+kmaRRV/yHGJKb+JCQz2nKy4Z6Yld+9TflZCtKNBC6uNMx/y+d3y9yaoJJmbpyZNl2Jxkg4HqcD6Yj0W9u73WTvcg5r2je3WToHNvcy9cP93yfrzTdM34f3HOVJKr14TqGXIgQX1yrRq586OUbKb0xKM8KmWxyCJtmmc6aSSY5vdaDiDjau5Xba2kNPf+gsNr8H2u+O/tBr65wNknO72u+On8bHz/kaJCLX/OrWWfSBh/rA+izdvv6wuWLLfebr8uaylNT3NyXptUHjsyxrFh/0S9dLaYk6xLvDXhd3hwMft7uj4Unf67en3Y43fTaceqg/G/XTxpdFam4frf8PZcHkVKRwAAA= + H4sIAAAAAAAC/+1d3W/bthZ/z18haLtACni9luy48YA+pOlaFGu7rPF2H9bCoCVa1iKLKkkl9gLvb7+H1IcpiYrtfsVu+BLY55DU4fn88cgBb48syx6PwzhJObN/tv760MkoFH9MQ4oVGopjwhEPSSyIt0ACYhQyLofA15Uchv2Qo0mEgTpFEcOSGMSYv/KBFKdRlFEoSmYjQiIeJkDvSuIs9PE5iTklEavMDytzozC+UgSjeAqCzoBg97rMzmjkRo6QUmayAtUjUYQSVpGtwvAbnATFOFovpS6XzUTeDI/COSYpV2Rc80lE6DPkXQWUpHFj/fWYP1GU4nZ2VQRJ/8EdDr3+wO5UyTSYoGO396RjOe6wY/W7Hav7+HT4qD7uB7/fRz1kK9QPlef6iCNGUuoJsewLSuaYz3DKKgvZU0LnSGzdjkmMq7wApQEunaUkz9Gi2K3T7dbEmodxwayz2Izc6DQELD4TLkAi/zWa4Krz6Ea9QfQKS51ymmJl0KoqPw39CyLdvcqQ/uhWSTHH9BpFQhPOvKqHCAdY2r6mCBSFQXzGRnm8CFnqxgyDGb+EsNgkajUmFFUnSRgHo2Uitd3Ga3pXVVIYHaO59INrYRyLE4vjBa/5lJWzxaMqjFVnu8UpioMtFneri7d6MDjac3DiCwLGYU1vs0W0SuYbIjUMoRbH2OOQBxrjRkIiTYAnhPFpuBCzbR3jBeSzy/AfufxJ9z+1MZC7tHMl/c6pUlVvIGltNt0UQlcGKMjeVCrPNma//e+ZhknKiVvqnEHGbAQHEOkVuKgmGUzDKDoXGU7aX+SungOpyzmFP6dDkbuc00bumgqB9DEunqKuly3nQhZ0hr3GQmpKaY0tWc9gzXQeN03FEQ0w38IKeJFImVg6P06TWy9KGWSMp+/tH/OP7+2O9TeZAOUqneCfEkoWy/f26lHTKuucy6HyjBmmIWbNYUVKeoE8LvXhNoZkmelFuV5zEfBEWbnts209oMyyTKOtkMtUZ/+R1BglFLitWwjRdV1ut1GW4WwGGS3CDIBKdX2ZOtSAOq0HlByxXUARIaf9dMdYKrLXTuEkJ73N0yPUxvU8Zf8VRAJFBbHzAjPUCtcEUW1xhGLPZq9xHPCZzJMNHm6bthVGCGW4Og3iS4r8EMe8Xuw3VN7eN6i86DpoyTBeSmkms44LNaeNE8YtnHqV1ycpHQfm/C/0pdHqlUkGFUdteVI6FtsYVyKdMs3DJeMmf7SzDRRpFNtGObITDI4UcxTogXAiZguPScXyJ02e3kfBWD6mWCbfaURqiYHiBCNtZc/S6m/XmIIzYs2OoKp5uC1o8jpYlRLyknelFRIKQJJg/3VWI5v8TyozFHF8LIqJrCVjtoy9cfaRppAkx35K5YEOKgggH5+NPTij8K1LU8cKY9hR7OGn/8LY4gvUrL9+hBMkPHxcxOaHR9+0jIlnf/FSVrO+kPlFhqwariN4l7NwyvXMvAa+E0awLsEq1ru6wNsWxHpCIJRrTk4y2MdFjQxjP7wO/RRVqpC+nspzelWyBVqErCnXJPWuMg/VZKJ5HvNCMXUMloN/zaz2xFdmL9l50G9hiRbbHGzWXkgSjfdF4kCpFU8wSfAMscbRSikE2mlZJeg26Mp+dzo+HcIW1BgzGOZnq78nGEYz7eAgzKbOiEEwh4hgZiHjBOrPfPwxRTEPI3zcfTwcdnZGNVll+g5gze1tIcxqdRjwJkM3r0Fnsbe0hkM+s37PjWngzj3DnYMHO4cFdY5qEajP2zn1FeQGmcT07HfkpvI6rZRnlAdf9SVWGZLPAQVNCKK+BSvYVX7RGJsNFEYeFhRGHynG+Zpv1B4A9jsx/SvTvzoM9DfYo/5VjPkNoVcC7AlYCP4YmBbW/WK8t5lJrIu1SUwvy/SyTC/rQeGZgellmV6WQTOf0MuyPhXmfNWeljVZWsfFmI4VYdPm2g0CmYaXaXiZhpdpeJmGVwYQn5iGl2l4HQZE7N97wwvACh97kQgz+W848JWNpWd8PtzreGBLQXMfP96DJpe7WOwA8Dr7qMjeXiiyd6cinx2CIvt7ocj+nYo8PwRFnuyFIk/uVOTz/Ti7/QqKtc4uXlnvMmOY3vUD7F1rWQfXvt5tF6aDXT2gnJoDijmgHMYB5XR/e9gaePP5nes2uANaFGMvfrsc3dXGFsM6Vkqj++hji4evVtbtLTz/EHrZkCB5iYVME9s0sU0T2zSxTRM7w4hD8ysH8yuHQ0CIjvvAIGJrS8zKQeLLXwxG/FIY8SU2ENFARAMRDUQ0ELEOEWvymT6i6SOaHzrUQSKgNg8zBvBPOAngwDmeE7ocT5Ycs8/uC67ML1XbkNsbqWcDz+4Znkk//w5ecO75Pswrzho2cQw2MdjEYJM7sYnsURUAxUvSsiW144+0dvhvG4NX2vDK+cUfVsqEXxvIcs8dpZnQ3qF3lfZ4Ewas1MCKa8CKASsGrNwJVgIyDgglKRcGNI2Tr/jKq9SyQSIPDYl8pebJnu/je37TdZTvwGbeDM/Rn5iyTEqnn5H5MnuUj+hVdp0U1Ku1l8psSmMsul/lWhzPkwi2GwfNy7GO6t6g4IsKsCuuj/DxFKWR7gYLlavDifIKrWpyKX3KFnfhWJcZllP4xeU7CtJTuCQprvxS06r9McV0mXfzm4hQuZPLqVADXL/tpjSQ8vij2r4UzaEoKq6Hqnq0qlNVIVvg10JpStmC/AtF1Mdnuntm1hrNC6261hxME2rmFFrWTNmkYvm4cZZOxdUxmktiOla+7iO9FdxNVsiLkmotcHqpavZ7IYhd5TYkFjT94NzG2ZYURsrwKFtIxajf2vDdNsNXq1pLdt5o8gLqfFmblybffIXQ+gdS34d7HBV1KbtyUAKWMuuWt12Rm5+cAhYVF1llRWA9LQkBENH15FwlZXtOxSb2ydp3bKerfOmpX5TzqX2ifHbUL72uylmDN9tVPjt+lgk/FHsQIFxxnI1PURceqAurT3H76pf1zWf2E1+Vt5Clor5/SJzd5jQ6zwtl+Z8EwvUyWipP9Pag52J3cOLjrjscnPa9fnfiOt7kyWDiof502M8GX5fVuHu0+j/y+qz8nnIAAA== scheduler.json.gz: - H4sIAAAAAAAC/+1dW3PbthJ+96/gsD0z9oyaI+riWJnJg+OcZDJNWjdR24c2o4FIiOIxRTAAaMv16Pz2swBvIAnakqL4EuHFI+4C4GJ3sfstSIM3B5ZlTyZBFCec2S+svz53UgrFX5KAYoWGoohwxAMSCeINkIAYBozLJnC5ks2wF3A0DTFQZyhkWBL9CPN3HpCiJAxTCkXxfExIyIMY6F1JnAcePiMRpyRklf5BpW8YRBeKYBTPQNA5EOx+l9kpjVzJFlLKVFaguiQMUcwqslUYXoMTowiH5VDqcGlP5M7xOFhgknBFxpJPQkJfIffCpySJGuOXbf5AYYLb2VURJP2H3mjkDo7tTpVM/Sk67PWfdyynN+pYg27H6j47GR3V2/3gDQaoj2yF+rlyXw9xxEhCXSGWfU7JAvM5TlhlIHtG6AKJqdsRiXCV56PEx4WzFOQFWuazdbrdmliLIMqZdRabkyudhoDF58IFSOi9R1NcdR5dqw+IXmCpU04TrDRaVeWngXdOpLtXGdIfe1VSxDG9RKHOCaoOq+ghjoPIH1/HUhVtvKbpq/qE1hFaSCNdCs1ZnFgcL3nN4FbGFreqMFad9QanKPLXGLxXHbzVvcALXoOHnRPQHGu6gi20KJkfiCfv75Iowi6HRdpoNxYSaRQfE8ZnwVL0tnWMNxBsPgX/yOGH3X/V2kBg0faV9Fu7SlV9gIhyt+lmsK7k6gHZm0rl6cTsX/59qmGSouOaOmcQzhqeC0R6AS6qWamzIAzPRPiR9heBpe9AXHFO4M/JSAQW56QRWGZCIP0CFHdRx0uH60GIckb9xkDqem9dozLZwJjJImqaiiPqY76GFfAyljKxZHGYxDdumDBYzi//tn/Mfv5td6z/kilQLpIp/olB1PeSUNBXR03LlEGRQ2qYMEwDzJrN8pjxBrlc6qTXaBJiH0fem2K85iDgjTK12qfrekERBplGYwGXudv+Pa4xilx9U7cSomXibLdTGuVsBlEtxAyQRHV8GT7URXVSX1SyxXqLigg57Zcbrqc8gm20pGSnX7IQCcmr7KfMvwIZUBggdpYn9VpmmSKqzV6Qjdn8PY58PpexssHDbd3WSuKBXLJOg/iWIi/AEa9n4ztSY7+a/qQPNz0HFOFHp2ycwUWRimsRAF36LZHETShN5dJ0g9TS0k0YSM+hgT8H3/NaJMlikY4Dff4MPGmXegKS64ajtnAofedOFCKCJtPcWzKusjs76wCORkptJB07xuAqEUe+HovGorfwiUQMP2zy9F4IpvIwxTLEzkJSW/oUxxhp83caOH+9xBTcDWtmBLnLxW3LIst2VSkh8rgXWiEhzMcx9t6nmbDJ3yqZUMTxYZEtJrgH2SC9gjg48RIqKyrIEIBuPDZxoUjgG6WfjhVEMKfIxS//B+3zC8hLfw0Xn4+OrOm1dZiNUra915x1k09otbJubnIR4AK0sUE+62yt9WkQeUbdVqaGW1T+amcqV5wchT6hAZ8v9l3/pU6sQie3GONsZ8a4FPDYBJ7SEqlCbtH9662wdC05iem8Scu7RmYTvE/zYMb1zAyEfyo95iOYdCtEXocrhHLN3oqEIpMcpItAcRl4CarAYD2glzt5VcmWaBmwplzTxL1IE6gGJy0ySCIUUy8Esx0ITa92WFZgK7k3qZ/CNVqus7tSOiiJNY4Zii0nrXiCSfxXiDX2dxSYqu2W4tRug67Md6Pg8BSmoK4xU0S9sAamiDJF1N4XUfOAcQI5ZjH5kqCIByE+7D4bjTrWFtVVmoA2QznbgJyOFeKj77jE2sQmrbXXHhrjmxVgmxhkvcpsD61zP+XZJqZao27bQzs9weItBLtG7rU1GvG59Vtmd1PNPXA19+RruadVyR3UlqAesmbUd7DwZaTTsz+Sq8r7RIU842z1Vd/iKdbkayjypgRRz4IR7Co/f/A4P1YY2bKg0PpAMc63fKVoD0rb4TalrS7+rVPbakvYryxutbH4m1a3tz9dN+Xt15W3gwd/RghIhE/cUCwm+fIlXLKJ9Izd7Mp3XLCnoPWePSuw3n3iud5y+Q0f9t2b+voPpL7+rep79VTUN3gg9Q1uVd/ZU1Hf8IHUN7xVfY+k4PoZ1Gmdnr+zPqYmMM/MzDMz88xsrwqLY1NYmMLiMRUWJ4/3uZkGtOxmf7kVxVigS9Hj/NdP4+p+s2B0rISGD7HPLG4uN5nh/qvVzt+w3znOgfDHC4zz3mwtm61ls7VstpbN1rJEgM/NW1PmranHgv6cnoF/Tfj39j8G/W2P/t5iA/4M+DPgz4A/A/7q4O/EbP+Z7T/zXsELcUgJcTFjgOyEkwDEW+AFodeT6TXHbEf/53e/OK18A/TxY7QPUtkGiD0wEJPOfp9YRsv6ejz2yOdhHkJWUcjIoBCDQgwKkUfGcXyYQxE3Top9pR2+G5VuJBks0oZFzs5/txIm3NnAkT2DI93vAYuYt6G2BiI1+QwSMUhkX5GITyY+oSThwoC7+XdZAzlaH1EVmjaY46GfRc2F9p7+Fsgjn8f3/GTqIJuBLYLiAv2BKUuldAYpmV+nt/IQvUiP34ecVXqpjKg0wgI3FmNxvIhDmG7kNz8mcFD3BgVkVCBcfpqvh2coCXUHCqtcHSKUnxyoBpcWnyqOQVegnMIlcf5lBDWa2l8STK+z/fgm5FM+XeBUqD6unzte2EW5/UFtOorCUBjmp+jXpqGoUtXDGgA115WSrSDsQv708KnuxO9CkXaWY9WxFmCRQNMn17Kmy10qlrebpFFUHOLdclx3x8rGPtJboneXJbJ8pFoM/F2qm/2WC2NXuQ2pBU3fOLNzOi2FkTA8TgdSIep9G7/bZvxqQmtbRHeZPUc5W9t9N5tNK/2BlU/XSw7yzJR+pEVCliLuFp8gIFc/OTkwyr8ukKaBslscACSiZedMJZMc1KroxB6WLmQ7XeWir1445TE79lD57agX/a7KKeGb3VN+O14aFD/ncxAwXPGfO++iDnysDqzepTdQL8rPUdjPPVXeXJaK+v4hUXq8/vgsS5W142KymGcnsoK3e/h4eoz608HU82bOoPd82kfDIUJO33Hc42GauO3LIid3D1b/B+dQ2oPUZwAA + H4sIAAAAAAAC/+1d3W/bOBJ/z18haPeABPD2LH+k8QJ9SNNrUVy7m228ew+7hUFLtKyLLKoklTgb+P72G1JflEQ5duomccOXIJrhx3BmOPMbSpZuDyzLnkyCKE44s3+2/vzcSSkUf0kCihUaiiLCEQ9IJIi3QAJiGDAum8DlSjbDXsDRNMRAnaGQYUn0I8zfe0CKkjBMKRTF8zEhIQ9ioHclcR54+IxEnJKQVfoHlb5hEF0qglE8A0HnQLD7XWanNHItW0gpU1mB6pIwRDGryFZheA1OjCIclkOpw6U9kTvH42CBScIVGUs+CQl9jdxLn5IkaoxftvkDhQluZ1dFkPQfeqOROzi2O1Uy9afosNd/2bGc3qhjDbodq/viZHRUb/eDNxigPrIV6ufKvB7iiJGEukIs+5ySBeZznLDKQPaM0AUSS7cjEuEqz0eJjwtnKcgLtMxX63S7NbEWQZQz6yw2J9c6DQGLz4ULkND7gKa46jy6Vh8RvcRSp5wmWGm0qspPA++cSHevMqQ/9qqkiGN6hUKhCWdR1UOIfSxtX1MECgM/OmXjbL8IWerGDPw5v4BtcZeo1T2hqDqOg8gf38RS2228pndVJYXWEVpIP7gSxrE4sThe8ppPWRlbTFVhrDqbDU5R5G8weK86eKsHg6O9ASc+J2Ac1vQ2W+xWyfxIpIZhq0URdjnEgUa7sZBIs8FjwvgsWIreto7xFuLZRfC3HH7Y/UetDcQubV9JX9tVquojBK27TTeDrSs3KMjeVCpPF2b/8s9TDZMUHTfUOYOI2dgcQKSX4KKaYDALwvBMRDhpfxG7+g6ELucE/pyMROxyThqxayYE0u9xMYs6XjpcD6KgM+o3BlJDSuvekvkMxkwWUdNUHFEf8w2sgJexlIkli8MkvnXDhEHEePWX/WP27192x/ovmQLlMpninxgkFi8JBX111LRMGXc5ZJ8JwzTArNksD0tvkculTnqNJml0eluM1xwEvFFmb/t0Uy8oIi3TaCzgMtzZv8c1RgEHbutWQrTMze12SqOczSCqhZgBWKmOL8OHuqlO6ptKtthsUxEhp/1qy/2UR7CttpTs9EsWIiE/lv2U9VdQCSQWxM5y3FBLXlNEtQkSEj6bf8CRz+cyVjZ4uK3bRjghkFvWaRDfUeQFOOL1hH9H9u0/QPZFV35LlHETSlOZNd0g7bR0E8bTc+qJXh+ndBzo85/AkzarJye5pzhqC5XSr9gGyKJo1GBcZzM7m4CRRrptJCQ7xuBGEUe+HgrHorfwl0QMP2zy9B4KpvIwxTL8zkJSCwsUxxhpc3saVH+9whRcEWtWBHnNxW1bJsuEVSkhKrmXWiEhBcQx9j6kWbLJv1eioYjjwyKTTHAPMkV6BTFy4iVUFnSQPQD5eGziQo3Ct0pNHSuIYE2Ri1/9D9rnF5Cz/vwRqkiYfpLvzc9HR9b0xjrMxix7Pmh2u82Xt1pZt7e5CHAButki83XubYNpEHlG+TXlZ0pZY4DXOzOAsgFQ6BMa8PnCWEO1Rqkhq9DQGtOc7cw0VwJkmxDVZpdUPWss8eZe+LyW1MRy3qYlYyMjCt7FPJhxPTMD9hel/3wC9d4L5ddhDqFccyQkIcwkB/4iiFwFXoIq0FpfJMgDyKpkS7QMWFOuaeJepolXg68WGZQRiqkXl9mphqZXO5wrMJk8UtUv4QYtNzmxKR2UxBrHDMVJmVY8wST+a8QaZ0YKvNV2S/Ftt0FX1rtVqNiHJah7zBRmP1sDU5iZwswUZmuAzzxgnED+WUy+JCjiQYgPuy9Go451j4otTU7b4aGvh0MdK8RH33HZto2FWuu5Z2+ab1bUbWOezaq9Z2+rhyn5tjHcBrXgs7faHhaEIeg4cm+s0YjPrd8yLzAV4iNXiHtfH+5XdXhQ24J6qJtR38PGl3FPz/5EriuPVhXyjLPdV32gqdiTb6BwnBJEPQtGsKv8/Abp/FhhZNuCQusDxTjf8umqZ1AuD59IuaytivetXl7/fIApmL+uYB48+p1MwCh84oZim8knVOGSTaRn7OaOQMcFewpa78WLFkz4kLivt1x+w1uSD6bM/pNQZn+tMl/vizIHT0KZg7XKPNsXZQ6fhDKHa5X5REq4f4NyrdPz99an1CDmzp65s2fu7D2rUuXYlCqmVNmPUuXk6d7b04Cb3Zxst6IdC3Qpepz/ejFed9ItmnWshIaPccItJpfH2zD/arXz3yDsHA9BmOQFFvpgDrXNobY51DaH2uZQWyLFl+YZMPMM2D7gRKdngGITKL77l8GJu8KJ77CBiQYmGphoYKKBiXWYeGIOFM2Bonn2YS1OjClxMWOAAIWTABRc4AWhN5PpDcdsR7+DfFgEVz6/+vTR20epbAPRHhmiSWd/SJSjZX09Unvi6zA3PKv4ZGTwicEnBp+sxSfyqCoHKW6cFCdTO3xeS3cUZTBLG2Y5O//dSphwbgNbnhls6X4PmMU8oXVvwFKTzyAWg1gMYqkjFp9MfEJJwoUBd/PjYQNGWm9/FZo2aOSx73PNhfb2/xDlia/je77rdZCtwBZBcYH+wJSlUjqDlMxv0qk8RC/TLzFAziq9VEZUGmGBKIuxOF7EISw38pvflTioe4OCMSrgLn/rsodnKAl1L35WuTqsKL8+UQ0uhU/Z4jXy1kWK5xR+/t56Be0pXBLnX8tQw6r9JcH0Jjvab6JC5XMWToXq4/qL4gsDKdMf1NalaA6FYf5lhapHqzpVFbIBhs2VpqQtiL+QSD18qntFe6nRLNmqYy3ANIGmT65lTZe7VCynm6ThVLx1veX96h0rG/tIb4neXZbIEpNqMXB8qW72Wy6MXeU2pBY0fePMzumyFEbC8DgdSMWqD238bpvxq5mtJULfafYc7uze7oXZK+hvpX/t5/66w0Gei9Iv9EiQUkTa4uMQ5PonJ4dC+Xcf0sBfdosDAEG07JyppDiWU/GIPSx9xXa6ykVfvVDqUnuo/O+oF/2uyikBm91T/ne8NPp9ztcggLfiKHfOog58rA6sztIbqBflh0Lsl54qby5LRX1/kyj98MH4LEuOtRfkZMHNTmQ1b/fw8fQY9aeDqefNnEHv5bSPhkOEnL7juMfDNFXbV0UW7h6s/g/BmvPv0WkAAA== workload-total.json.gz: - H4sIAAAAAAAC/+2dX3PbNhLA3/0pOJxMx75TW1GybDczfXDS5i4zvbtc7LYzTTwaiIQkjCmCBUHLqkf32Q8A/wEkQFGOLNsxXhJzlwKBxe7ipwUk3R04jjseoyhOaeK+dj5d9TIJgX+miEBJBqIIU0ARjrjwjomYMEQJ5beIKyeXCs0kRSF9HzGl16ukAaAgwSnxIVO4HwheQDqHaeJK98AITEKupySFknyOAo0U+Th6i0NMeINkNgGH/Z4z8Dz2z2jUc7wjuekILMSDz6uxON845yEkVOkCXcXivgAk8wkGJHBz3Vr8f8X+XQujwADRWm/dWQTp+4BJojQMMwkB8fwS45CimMn7QsiH8xZHlOCQG3QKwiR7PVJeG6LoWpqFGERQ3J+ZvDC46+MwBHEC5ZYURdDQzAgKPuBqMjMj1yZsya4Hx5LgthhAfr3i14V1yrbFGAblZdXrq1JGYAwBlYYqSd9TSMT06NUf8VIxsFAlc7y8RLThOS7Nhe7blBAYUecNiIIlCujcVe+5QH+J++YnkiL3BIKXmQ/kYywND0IEEuGAwpKVCSZASJSucH/6BUYzKszcV+QwaczQxmhxpygMFUtzwT8ICBAbqDxT5tn+oTbb3mDDbHv62R6WlyGcwShQn8TMNIvOk0t9aIObWX3w3HWz+dJngp8XMV3pVX9AgpuaBbjVPGOBIo2UoNmcXmjzDXczjZTd+7twKdUruQoTPoRyNDXVTzDxm+1Rlp5CTcduQJjCwq0a88ByhcaLuHSZ982Tpde1iGS2uGCRqkS8y0fzAaOI/gsLcwiBK0W2D3UeHUPmtREFs2ZGinlr3EVT/vyRKm92nxktgASK/D4NcWVAQwZJIEEw+c8NJMzjYW2Iiam/SWPgCQX+daM3CYVxDINfmE0bOgrIDNJEWg/lFZGvFrexGAaf9zHrm3+YpItDxFIdPGTLGAWsVTJmy8cSk2u2BPsQ3cDxZEVhMhYeceeHKesB+fGz+yr/87Pb46uaGNiP/2Py8uqzu/70ipkUEuY1r1+xxRyHKU+qV0efo785OHIOy3t7MQ6YdEZwGo9DOKWHvAshBkGv+GPMU+GRU75kXCrYa19fpxPI/xjjJRvDawJDMIHb9rfnFG0KTXEhK0QveHP8fzbCI2eycg5571n3pdjiqRCThXAPlt0XcJz5hXpLYZ13wKeCIDxFnSWyd2U7d3cOe5KzXqutEDgVy5F7rsq5rwgvK4Xr/K/KIemczcsch0HNUXmX37GEX/duLr+YoyltKmqr3EfmVQ6eOm+4+zgfM2cKpJWt5JE7ObsBIlChnt+yJDaop6JxsT6iKEA3KEhZ1mqkpeIegUFVB27BLaqtR5PUv84iqJZEF3nyaUxigXT1pJtlaWP6/CTPU5GZS9lVYwgrcAtbArtytTdxzcVEHDQ6yBV49gYIYFOdLlunGrdnC1VfdbBqLapcrPf0u3ggm/nlAZUiaCGqY0tUlqgsUe2OqJiNomSBqEUqi1RfjlSXuTdRS1WWqp46VUn1QQWg5PKgothdddAzlAdHmvKg3somJDRhYRsatuBh1wq1BhNbUdFoTx0yarFRZ9vKvp4kU20j7HyiScDNbrTipBkpW7FyA1q246UZMc2YuQE1zbi5ATk3YGc7erbhpxFBm3OpRVEzjhqQ1IilHdC0DU83IGobpragajuumpF1I7a2oqseX80IuwljjShbT3QGpGWBZ4uED0O0Ham2A9luQbdthGugXHmBV2i3jXjbqLeVfFX6PWcRxEJ6U0HRiL9tCGzA4E4o3EyReiQ2YHErGrfjsRmRWzDZjMp6XJaRuTnUJjprk4mZT1sZtY1TW1jVxKtGZm2M65kO4UA3Zab3DJZmdYXQzjh7anHW4qzF2a8TZ22F1vLs4/CsrpprkdYirUVaGWkbmy37P8VZhO9OT3E+6vFZz7D/f/bEzs+WFnf++T5hS8ZqH+dn6wc4voYDtAP9fP+wzXmP+ibcpgMfdc1eT3zUdVsf+dh4ekMojMc3FH+W318Mup3eqKa0+Z6CQWwEfWUT+Cs7wiGNvnjDoBjUnol90Qc42IOe9vGNvFStJYbneFyDj7jzYY3aGCpat8cxnvAh16+AepqnXA3YIw3Ico/lHss9ti5qwWcn4FPUNC35WPKxB1GdgaHO5nk7PYmq24V/IUdRB4P2vXu1oa6b97pdhy679zrto27f6/T33r/vtAUv46F5D75x2kSLit224GtT3AEbX+I+fM1KOrx86G34LmU2NhK+IFvefHL78E32fGa78MXue3GO1PmQ+doL2HyvUWy3rXfNOJtEu4NN9fj5b6rHL+ec6AthTe1B0Y2wObSwaWHTwuZTh82ytmlp09LmA9OmdMrTAqcFzhcKnE/gFGc9+HZzdnPvFWXTlvaxrSjvqKI83AD5Iwv5FvIt5D+3inJAMO+GZX3L+vuqLDs/ZT5ngd8Cv60wW/bsUGHeBJ8nFj4tfFr4fHYVZkuflj73X2m2AGoB1FacH6vi/DMhnHPvU3A+yHvO0xDPHVw77Gdt8duk3+dK/DlcgN8gSbL+e2eZmK7C/BfFyHX2OgYAlde6PNUTtl4xTy6fRuEiDpkholmXH1yrQFFhfApvhccEcArSsIYFIrxkre4tQ/HDa3LxW+9q5e+rSawvaXFc/HqcnHXdP1NIVsKptT8HV9ncU6QzyJ3W1f9sW/n4en1e+WBZ+Fs+/towJFOut/zxutxW8iclUMRW6QCeh7oPchkMuWCTgTS3FwbO1/1trCueNM4S6mFJFiia4p6TN3ekt/tgk93z1Uh2EObdwrjJf4vnu6q20VEu09+cz2o2EkmRJvAya0j7KTj9VLvf/V1uA6QU6z5VyMRjH6fCCYb9uibLiq7XV+Z+QwRym3+brJihF4YolO/QRmIXD2ShjCKUp0911r/koO+6V1HqkdueHRSPr30M554OXz773i7/IIPfJkiuUfwrCS9Wka/7ZGgWQt5zCKHa1O0xhAxx8yDB8pDvydbVe6+tYmlHq0fx7HvH0mPYxobaXkMtgHGIV4v6N4PKyFje8NzCz1gSWddqIo8RnMKBnmJg3s9quw7bvg3btrAdmdhy9AVIuSfPr4qpBv8/MNVG3ASGWRVev4FQWGfYb3znZWkgrjPVmMxP0303sn4qTJPR+VFtA/Pmxmcx1YGuPqNNJswIvdGipzT33Fbeou5so/hRShM6828bwy1RdWz29ONtPP34hbj4QVGJXWc1RSRmKa8mTrOtDjfCy2/LmHeFm3OZq7wsRuxtMqlenNtsXHRGnl53JFVaZd9XUrDrVX4u+7zryRfDvqyRvkFkIP3tBdncXxVj4DtTTbczP0Vu+ERuWH7K4Fi+kL756DSQ+1v0RTHfX1hserq/Xr7NC8BFZfrfWT0CRTPne+d3+b2Zm4pzBu7p4GwyPT31fc87OQkG06E3HE6mg9HZ8cnZ6YnvZzfflFXn/sH6/+UubNfagwAA + H4sIAAAAAAAC/+2dX3PbuBHA3/0pOJxMx26VVpQs25eZe3CSS5uZ610a++5mmng0EAlJqCmCB4KWdR71sxcA/wEkQEmuLEsxXmxxQYHAYnfx4wIUH44cxx0OURSnNHHfOF9uOpmEwN9TRKAkA1GEKaAIR1z4wERMGKKE8lPEkZNLRckoRSH9GLFCr1NJA0BBglPiQ1bgfiJ4BukUpokrnQMjMAp5OSUplORTFGikyMfROxxiwiskkxE47nacnuexP4NBx/FO5KojMBMXvqz64vzJuQwhoUoT6CIW5wUgmY4wIIGbly3F/xv2dymUAgNEa611JxGkHwMmidIwzCQExNNrjEOKYibvCiHvzjscUYJDrtAxCJPs+0j5boiiW2kUYhBBcX6m8kLhro/DEMQJlGtSCoJGyYSg4BOuBjNTcm3A5uy4dyoJ7osO5McLflxop6xb9KFXHlatvillBMYQUKmrkvQjhUQMj774M54rChZFyRTPrxFtWI5Lc6H7LiUERtR5C6JgjgI6ddVzrtAf4rzpmVSQWwLB88wG8j6WigchAokwQKHJSgUjICRKU7g9/QijCRVq7ipymDRGaKW3uGMUhoqmueDvBASIdVQeKfNof1cbba+3YrQ9/Wj3y8MQTmAUqFdiappEl8m13rXB3aTeeW662XjpI8EPs5gu9EX/hgQ3S2bgXnONGYo0UoImU3qljTfczDRSdu5vwqRUq+RFmPAulL2pFb2Hid+sj7LwFGoadgfCFBZm1RgHFis0VsSl87xtniy9rXkk08UV81TF413em08YRfSfWKhDCFzJs32os+gYMquNKJg0I1LMa+MmmvLrD1R5s/lMaQEkUMT3cYgrBRoiSAIJgsnPd5Awi4e1Liam9iaNjicU+LeN1iQUxjEMfmQ6bZRRQCaQJtJ8KM+IfLa4j0U3+LgPWdv84ySdHSMW6uAxm8YoYLWSIZs+5pjcsinYh+gODkcLCpOhsIiH/+DR919dHwR3iFXy1e04fpiyNhEmfZV/ZFI+z4mufv9fJi+PvrrLL6+YkiFhdvTmFZvecZjyMHtz8jX6s4Mj57g8txPjgEknBKfxMIRjeswbFWIQdIoPQx4cT5zyK8OygH33zW06gvzDEM9Zr94QGIIRDB82a2/HKeoUJcWBXCBawavj/1kPT5zRwjnmrWfNl7yNB0dMZsJgWLyfwWFmKeophXY+AJ8KpvCU4iy0fSjreXhw2JWc5VKthcCxmKDcS1XOrUfYXSlc5p8qE6VTNi5THAY10+VN/sCmgLq9c/nVFI1ps6A2731mdubgsfOWG5TzOTOvQJrrSkJ5kOMdIAIe6hEvC2u9enAaFjMmigJ0h4KUxbFGoCrOEWBUNeAe3KPaDDVK/dvMp2phdZaHo8YgFpBXD8NZ3DYG1C/yOBWxupTdNLqwAPewxdUrU3sb10xM+EGjgbwAT94CgXCq0WUzV+P0bOrqqgZWzU6ViXX2v4lHsppfHmIpghbGOrWMZRnLMtb2GIvpKEpmiFrIspD1FJB1ndsXtZxlOWvfOUvKISpIJacQlYLtZRA9QwpxoEkh6rVsgkQTKLbBYgswrpvF1oBjKzwa9amDSC1I6nRb6deTZKpuhJ7PNAG42YxWwDRDZitoroDNduA0Q6cZPFfApxlAV0DoChBth9E2IDVCaXMstXBqBlQDpBpBdQ1YbQPWFdDaBq4t8NoOsGaIXQmyrTCrB1oz1K4CWyPc1gOdAXKZ49lE4q4Yd03OXYN1N+DdNuY1cK885Sv828bAbRzcysIqD18yn2JOvirpaATiNig2gPFacNwMmnpINoByKyy3A7MZmlvA2QzPeoCWIbrZ1SZMa8OLmVhbqbWNXFvo1USwRopt9OtAu3CkGzLTXYTlW12ydG3APbeAawHXAu63Cbg2i2sJd18IV5fxtZBrIddCrgy5jQWZ3e8GLdx3q7tBn3UbrmfYNXCxZ/twS407//iYsCljsYt9uPVtH9/CRtyefry/22SXSH2hbtU2kXrJTveJ1Ms23iiycs+HKDBu+lDsWb7j6K2356Ma0uZdBsPaCPrKQvE3tvFD6n1xC6Eo1O6ttds+lJsEdqH93vSRp7O1DHGImzx4j9fe4lHrQ8XvdhPHHm+W/QY4qLlb1gBCUocsCVkSsiRkc6cWhZ4IhYq8p2Uhy0J2Q6vTM+TiPG+rO1p1a/cvZEtrr9e+4q9WtO6Sv25lYp01f13psy7668ofveq/1sK9DIzmlfvGHhUtPK63cF8b4jVA8iWu3te0pAPOp168XycVx3rCJ2RLoAewet+k0QNbuy/W7Iv9qM6nzPpewJJ9jWvXW7DX9LPJuFtYio8Pfyk+fjn7TV8IfWo3nK7Ez77FT4ufFj/3HT/L/KflT8ufO+dPabeoRVCLoC8UQfdgN2jd+bazB3TnWWfTQvipzTpvKevcX4H9A4v9Fvst9h9a1jkgmDfD0r+l/+fLPjvvMyu0twD2FsBmoS2NrpGFXoWjZxZHLY5aHD24LLTlUcuj+5CNtkhqkdRmpZ8rK/0DIZx8H5OUPspbzsMQjx28tN/N6uKnSe8US/wpnIFfIUmy9nsXmZguwvwtaOQ2+x5DgspqXR7qCZvBmCWXV6NwFodMEdFknZfEVeioUD+F98JiAjgGaVgDBeFecqnuJqJ4WZycIC9MzX3PON+5ykBf83I46TZAKsVx8eo7Ofy6v6eQLIR1a99lVynfU6QTyK3X1b9zrrx8PZmvPLsW/porovZEgqTT5YZv3suVJj96gSI2XQfwMtQ9K6Z3XnfGRgVpTi8UnAPAJtoVVxpmkfW4RAwUjXHORlz2mlEdha9ZxwjyE8YEJSad6Mekt2pM8ilLtiLmAkLxyb+KtrlqaaMTXKY/OR/xrJdSQZrA66wi7UN4ejNw//oXuQ6QUqx7qJGJhz5OhYH0u/WSLHS6XlexixVumul+wRQ9M7iqfIbWXdexTubvKEJ5jFUtYru7ipndlCx64rYHFcU/ak8BPdI9yms/2kF2pI5NHOkWxb+Q8GoR+bqHVzM38w7BzWqDuUM3M/jWkzjUU97cLaubuI28a0uzT3HtR3vXc+jGutpOXS2AcYgXs/oPnMrsWZ5waO5nzK0sa8mV53BOYUD76JiP09q23bZr3bbNbQcm/hz8H9i5I8uvsrIG+z8yJVncBIZZgl+/NlFop99t/FBnqSBeZkpWma+m+4ln/VCYBmPtS7V1zJsar8WKjnSJHm0wYUroDGYdpbpDm3mLBLb14mdJbejUv6kPt3jVqdnSTzex9NMXYuJHRUp3mSUnkRilPC05ztZM3AjPX5c+7woz5zJX+VqM2I0zqb6c62xYNEYeXncgpWxl21dCsOtVdi7bvOvJB/2uXCL9pElP+uwF2djfFH3gS1xNszNfRa74TK5YvkrvVD6QfpzpPJDbW7RFUd8fWKynur9cv8szyUWK+6csQ4GiifM35zf53sxNxRYG97x3MRqfn/u+552dBb1x3+v3R+Pe4OL07OL8zPezk+/K9HX3aPk/kZkRRteEAAA= diff --git a/charts/kubezero-metrics/templates/rules/etcd-mixin.yaml b/charts/kubezero-metrics/templates/rules/etcd-mixin.yaml index dc9e87c6..be7404ee 100644 --- a/charts/kubezero-metrics/templates/rules/etcd-mixin.yaml +++ b/charts/kubezero-metrics/templates/rules/etcd-mixin.yaml @@ -49,9 +49,9 @@ spec: severity: warning - alert: etcdGRPCRequestsSlow annotations: - description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.' summary: etcd grpc requests are slow - expr: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) without(grpc_type)) + expr: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) > 0.15 diff --git a/charts/kubezero-metrics/templates/rules/kubernetes.yaml b/charts/kubezero-metrics/templates/rules/kubernetes.yaml index 953ff5dd..94225207 100644 --- a/charts/kubezero-metrics/templates/rules/kubernetes.yaml +++ b/charts/kubezero-metrics/templates/rules/kubernetes.yaml @@ -11,14 +11,10 @@ spec: rules: - alert: KubePodCrashLooping annotations: - description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is restarting {{`{{`}} printf "%.2f" $value {{`}}`}} times / 10 minutes. + description: 'Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is in waiting state (reason: "CrashLoopBackOff").' runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodcrashlooping summary: Pod is crash looping. - expr: 'increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[10m]) > 0 - - and - - kube_pod_container_status_waiting{job="kube-state-metrics"} == 1 + expr: 'max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="kube-state-metrics"}[5m]) >= 1 ' for: 15m @@ -89,7 +85,7 @@ spec: severity: warning - alert: KubeContainerWaiting annotations: - description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + description: pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting summary: Pod container waiting longer than 1 hour expr: 'sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0 @@ -177,7 +173,7 @@ spec: severity: warning - alert: KubeMemoryOvercommit annotations: - description: Cluster has overcommitted memory resource requests for Pods by {{`{{`}} $value {{`}}`}} bytes and cannot tolerate node failure. + description: Cluster has overcommitted memory resource requests for Pods by {{`{{`}} $value | humanize {{`}}`}} bytes and cannot tolerate node failure. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit summary: Cluster has overcommitted memory resource requests. expr: 'sum(namespace_memory:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0 @@ -195,7 +191,7 @@ spec: description: Cluster has overcommitted CPU resource requests for Namespaces. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit summary: Cluster has overcommitted CPU resource requests. - expr: "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"cpu\"})\n /\nsum(kube_node_status_allocatable{resource=\"cpu\"})\n > 1.5\n" + expr: "sum(min without(resource) (kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=~\"(cpu|requests.cpu)\"}))\n /\nsum(kube_node_status_allocatable{resource=\"cpu\", job=\"kube-state-metrics\"})\n > 1.5\n" for: 5m labels: severity: warning @@ -204,7 +200,7 @@ spec: description: Cluster has overcommitted memory resource requests for Namespaces. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit summary: Cluster has overcommitted memory resource requests. - expr: "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"memory\"})\n /\nsum(kube_node_status_allocatable{resource=\"memory\",job=\"kube-state-metrics\"})\n > 1.5\n" + expr: "sum(min without(resource) (kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=~\"(memory|requests.memory)\"}))\n /\nsum(kube_node_status_allocatable{resource=\"memory\", job=\"kube-state-metrics\"})\n > 1.5\n" for: 5m labels: severity: warning @@ -251,7 +247,7 @@ spec: description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup summary: PersistentVolume is filling up. - expr: "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\n" + expr: "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n" for: 1m labels: severity: critical @@ -260,7 +256,7 @@ spec: description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup summary: PersistentVolume is filling up. - expr: "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\n" + expr: "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n" for: 1h labels: severity: warning @@ -371,7 +367,7 @@ spec: rules: - alert: KubeClientCertificateExpiration annotations: - description: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days. + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration summary: Client certificate is about to expire. expr: 'apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 @@ -381,7 +377,7 @@ spec: severity: warning - alert: KubeClientCertificateExpiration annotations: - description: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours. + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration summary: Client certificate is about to expire. expr: 'apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 @@ -389,21 +385,21 @@ spec: ' labels: severity: critical - - alert: AggregatedAPIErrors + - alert: KubeAggregatedAPIErrors annotations: - description: An aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. - runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/aggregatedapierrors - summary: An aggregated API has reported errors. + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors + summary: Kubernetes aggregated API has reported errors. expr: 'sum by(name, namespace)(increase(aggregator_unavailable_apiservice_total[10m])) > 4 ' labels: severity: warning - - alert: AggregatedAPIDown + - alert: KubeAggregatedAPIDown annotations: - description: An aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. - runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/aggregatedapidown - summary: An aggregated API is down. + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown + summary: Kubernetes aggregated API is down. expr: '(1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85 ' @@ -423,9 +419,9 @@ spec: severity: critical - alert: KubeAPITerminatedRequests annotations: - description: The apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + description: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests - summary: The apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + summary: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. expr: 'sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20 ' @@ -597,33 +593,46 @@ spec: for: 15m labels: severity: critical + - name: kubernetes-system-kube-proxy + rules: + - alert: KubeProxyDown + annotations: + description: KubeProxy has disappeared from Prometheus target discovery. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeproxydown + summary: Target disappeared from Prometheus target discovery. + expr: 'absent(up{job="kube-proxy"} == 1) + + ' + for: 15m + labels: + severity: critical - name: kube-apiserver-burnrate.rules rules: - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[1d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[1d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n" labels: verb: read record: apiserver_request:burnrate1d - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[1h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[1h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n" labels: verb: read record: apiserver_request:burnrate1h - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[2h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[2h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[2h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[2h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[2h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[2h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n" labels: verb: read record: apiserver_request:burnrate2h - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[30m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n" labels: verb: read record: apiserver_request:burnrate30m - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[3d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[3d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[3d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[3d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[3d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[3d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n" labels: verb: read record: apiserver_request:burnrate3d - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[5m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[5m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[5m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[5m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[5m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[5m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n" labels: verb: read record: apiserver_request:burnrate5m - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[6h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[6h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[6h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[6h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[6h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[6h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n" labels: verb: read record: apiserver_request:burnrate6h @@ -724,12 +733,12 @@ spec: ' record: cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d - - expr: "1 - (\n (\n # write too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n -\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n ) +\n (\n # read too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"\ + - expr: "1 - (\n (\n # write too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n -\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n ) +\n (\n # read too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"\ })\n )\n ) +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d)\n" labels: verb: all record: apiserver_request:availability30d - - expr: "1 - (\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n # too slow\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"})\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n" + - expr: "1 - (\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n # too slow\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"})\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n" labels: verb: read record: apiserver_request:availability30d @@ -777,19 +786,19 @@ spec: record: node_namespace_pod_container:container_memory_cache - expr: "container_memory_swap{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n max by(namespace, pod, node) (kube_pod_info{node!=\"\"})\n)\n" record: node_namespace_pod_container:container_memory_swap - - expr: "kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" + - expr: "kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests - expr: "sum by (namespace, cluster) (\n sum by (namespace, pod, cluster) (\n max by (namespace, pod, container, cluster) (\n kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"}\n ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n" record: namespace_memory:kube_pod_container_resource_requests:sum - - expr: "kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" + - expr: "kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests - expr: "sum by (namespace, cluster) (\n sum by (namespace, pod, cluster) (\n max by (namespace, pod, container, cluster) (\n kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"}\n ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n" record: namespace_cpu:kube_pod_container_resource_requests:sum - - expr: "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" + - expr: "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits - expr: "sum by (namespace, cluster) (\n sum by (namespace, pod, cluster) (\n max by (namespace, pod, container, cluster) (\n kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"}\n ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n" record: namespace_memory:kube_pod_container_resource_limits:sum - - expr: "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n )\n" + - expr: "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n )\n" record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits - expr: "sum by (namespace, cluster) (\n sum by (namespace, pod, cluster) (\n max by (namespace, pod, container, cluster) (\n kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"}\n ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n" record: namespace_cpu:kube_pod_container_resource_limits:sum diff --git a/charts/kubezero-metrics/templates/rules/node-exporter.yaml b/charts/kubezero-metrics/templates/rules/node-exporter.yaml index 89d2ca3e..a844027d 100644 --- a/charts/kubezero-metrics/templates/rules/node-exporter.yaml +++ b/charts/kubezero-metrics/templates/rules/node-exporter.yaml @@ -190,11 +190,11 @@ spec: rules: - expr: "count without (cpu, mode) (\n node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}\n)\n" record: instance:node_num_cpu:sum - - expr: "1 - avg without (cpu, mode) (\n rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\"}[5m])\n)\n" + - expr: "1 - avg without (cpu) (\n sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\"}[5m]))\n)\n" record: instance:node_cpu_utilisation:rate5m - expr: "(\n node_load1{job=\"node-exporter\"}\n/\n instance:node_num_cpu:sum{job=\"node-exporter\"}\n)\n" record: instance:node_load1_per_cpu:ratio - - expr: "1 - (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"}\n +\n node_memory_Cached_bytes{job=\"node-exporter\"}\n +\n node_memory_MemFree_bytes{job=\"node-exporter\"}\n +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n" + - expr: "1 - (\n (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"}\n +\n node_memory_Cached_bytes{job=\"node-exporter\"}\n +\n node_memory_MemFree_bytes{job=\"node-exporter\"}\n +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n )\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n" record: instance:node_memory_utilisation:ratio - expr: 'rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) diff --git a/charts/kubezero-metrics/templates/rules/prometheus-operator.yaml b/charts/kubezero-metrics/templates/rules/prometheus-operator.yaml index f8d3f776..b5e57f80 100644 --- a/charts/kubezero-metrics/templates/rules/prometheus-operator.yaml +++ b/charts/kubezero-metrics/templates/rules/prometheus-operator.yaml @@ -86,4 +86,19 @@ spec: for: 5m labels: severity: warning + - name: config-reloaders + rules: + - alert: ConfigReloaderSidecarErrors + annotations: + description: 'Errors encountered while the {{`{{`}}$labels.pod{{`}}`}} config-reloader sidecar attempts to sync config in {{`{{`}}$labels.namespace{{`}}`}} namespace. + + As a result, configuration for service running in {{`{{`}}$labels.pod{{`}}`}} may be stale and cannot be updated anymore.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/configreloadersidecarerrors + summary: config-reloader sidecar has not had a successful reload for 10m + expr: 'max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0 + + ' + for: 10m + labels: + severity: warning diff --git a/charts/kubezero-metrics/update.sh b/charts/kubezero-metrics/update.sh index f3bc613e..6552d173 100755 --- a/charts/kubezero-metrics/update.sh +++ b/charts/kubezero-metrics/update.sh @@ -9,6 +9,7 @@ helm pull prometheus-community/kube-prometheus-stack --untar --untardir charts - rm -rf charts/prometheus-pushgateway helm pull prometheus-community/prometheus-pushgateway --untar --untardir charts --version $PG_VER +# workaround for https://github.com/prometheus-community/helm-charts/issues/1500 patch -p0 -i zdt.patch --no-backup-if-mismatch patch -p0 -i zdt-pushgateway.patch --no-backup-if-mismatch diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 64532c49..a3dfe366 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -160,7 +160,12 @@ kube-prometheus-stack: searchNamespace: ALL provider: foldersFromFilesStructure: true + script: "/opt/script.sh" # For the gunzip script + extraMounts: + - name: script-volume + mountPath: /opt/script.sh + subPath: script.sh extraContainerVolumes: - name: script-volume configMap: diff --git a/charts/kubezero-metrics/zdt.patch b/charts/kubezero-metrics/zdt.patch index 68c06715..19a403fd 100644 --- a/charts/kubezero-metrics/zdt.patch +++ b/charts/kubezero-metrics/zdt.patch @@ -1,20 +1,11 @@ -diff -tubr charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl charts/kube-prometheus-stack.zdt/charts/grafana/templates/_pod.tpl ---- charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl 2021-06-26 19:54:57.000000000 +0200 -+++ charts/kube-prometheus-stack.zdt/charts/grafana/templates/_pod.tpl 2021-07-01 13:31:20.750609396 +0200 -@@ -198,11 +198,16 @@ - - name: FOLDER_ANNOTATION - value: "{{ .Values.sidecar.dashboards.folderAnnotation }}" - {{- end }} -+ - name: SCRIPT -+ value: /opt/script.sh - resources: - {{ toYaml .Values.sidecar.resources | indent 6 }} - volumeMounts: - - name: sc-dashboard-volume - mountPath: {{ .Values.sidecar.dashboards.folder | quote }} -+ - name: script-volume -+ mountPath: /opt/script.sh -+ subPath: script.sh - {{- end}} - - name: {{ .Chart.Name }} - {{- if .Values.image.sha }} +diff -tuNr charts/kube-prometheus-stack.orig/crds/crd-prometheuses.yaml charts/kube-prometheus-stack/crds/crd-prometheuses.yaml +--- charts/kube-prometheus-stack.orig/crds/crd-prometheuses.yaml 2021-12-09 14:58:13.123561137 +0100 ++++ charts/kube-prometheus-stack/crds/crd-prometheuses.yaml 2021-12-09 14:58:50.570226461 +0100 +@@ -6,6 +6,7 @@ + metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 ++ argocd.argoproj.io/sync-options: Replace=true + creationTimestamp: null + name: prometheuses.monitoring.coreos.com + spec: diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index 06430640..386dbab0 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero description: KubeZero - Root App of Apps chart type: application -version: 1.21.7-8 +version: 1.21.7-10 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 6c9bb1f6..6405e4db 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -42,7 +42,7 @@ istio-ingress: metrics: enabled: false namespace: monitoring - targetRevision: 0.5.4 + targetRevision: 0.6.1 istio: grafana: {} prometheus: {} diff --git a/containers/admin/v1.21/kubezero_121.sh b/containers/admin/v1.21/kubezero_121.sh index 49270d8d..649901ca 100755 --- a/containers/admin/v1.21/kubezero_121.sh +++ b/containers/admin/v1.21/kubezero_121.sh @@ -17,3 +17,7 @@ kubectl delete daemonSet efs-csi-node -n kube-system # Remove calico Servicemonitor in case still around # kubectl delete servicemonitor calico-node -n kube-system + +# Upgrade Prometheus stack, requires state metrics to be removed first +kubectl delete deployment metrics-kube-state-metrics -n monitoring +kubectl delete deployment metrics-prometheus-adapter -n monitoring