From c9d3c3b27eff0785eef35497b552b694a93ef138 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 28 Jan 2022 17:19:41 +0100 Subject: [PATCH] feat: Version bump kube-prometheus-stack for 1.21 --- charts/kubezero-ci/Chart.yaml | 4 +- charts/kubezero-ci/README.md | 13 +- .../gitea/istio-authorization-policy.yaml | 7 +- .../jenkins/istio-authorization-policy.yaml | 7 +- charts/kubezero-ci/values.yaml | 3 +- charts/kubezero-metrics/Chart.yaml | 8 +- charts/kubezero-metrics/README.md | 40 +- .../charts/kube-prometheus-stack/Chart.yaml | 14 +- .../charts/kube-prometheus-stack/README.md | 61 ++- .../charts/grafana/Chart.yaml | 4 +- .../charts/grafana/README.md | 10 +- .../charts/grafana/templates/_pod.tpl | 116 +++--- .../templates/dashboards-json-configmap.yaml | 3 - .../charts/grafana/templates/deployment.yaml | 2 +- .../grafana/templates/extra-manifests.yaml | 4 + .../charts/grafana/values.yaml | 27 +- .../charts/kube-state-metrics/Chart.yaml | 6 +- .../charts/kube-state-metrics/OWNERS | 6 - .../kube-state-metrics/templates/_helpers.tpl | 5 +- .../kube-state-metrics/templates/role.yaml | 11 +- .../templates/rolebinding.yaml | 2 +- .../templates/servicemonitor.yaml | 6 +- .../charts/kube-state-metrics/values.yaml | 7 +- .../prometheus-node-exporter/Chart.yaml | 4 +- .../templates/_helpers.tpl | 7 + .../templates/daemonset.yaml | 2 +- .../templates/monitor.yaml | 35 -- .../templates/psp.yaml | 10 +- .../templates/servicemonitor.yaml | 48 +++ .../prometheus-node-exporter/values.yaml | 12 +- .../crds/crd-alertmanagerconfigs.yaml | 197 ++++++++- .../crds/crd-alertmanagers.yaml | 365 ++++++++++++----- .../crds/crd-podmonitors.yaml | 2 +- .../crds/crd-probes.yaml | 2 +- .../crds/crd-prometheuses.yaml | 378 +++++++++++++----- .../crds/crd-prometheusrules.yaml | 4 +- .../crds/crd-servicemonitors.yaml | 2 +- .../crds/crd-thanosrulers.yaml | 363 ++++++++++++----- .../templates/_helpers.tpl | 36 ++ .../templates/alertmanager/alertmanager.yaml | 14 +- .../alertmanager/podDisruptionBudget.yaml | 2 +- .../templates/alertmanager/psp.yaml | 10 +- .../kube-controller-manager/service.yaml | 6 +- .../servicemonitor.yaml | 6 +- .../exporters/kube-scheduler/service.yaml | 6 +- .../kube-scheduler/servicemonitor.yaml | 8 +- .../kube-state-metrics/serviceMonitor.yaml | 63 --- .../node-exporter/servicemonitor.yaml | 40 -- .../admission-webhooks/job-patch/psp.yaml | 10 +- .../prometheus-operator/certmanager.yaml | 6 +- .../prometheus-operator/deployment.yaml | 2 +- .../templates/prometheus-operator/psp.yaml | 10 +- .../templates/prometheus/_rules.tpl | 12 +- .../templates/prometheus/prometheus.yaml | 1 - .../templates/prometheus/psp.yaml | 10 +- .../templates/prometheus/service.yaml | 1 + .../charts/kube-prometheus-stack/values.yaml | 326 ++++++++------- .../charts/prometheus-pushgateway/Chart.yaml | 4 +- .../charts/prometheus-pushgateway/values.yaml | 2 +- charts/kubezero-metrics/configs/build.sh | 3 +- .../configs/dashboards/apiserver.json | 93 ++--- .../configs/dashboards/cluster-total.json | 2 +- .../dashboards/controller-manager.json | 57 +-- .../dashboards/k8s-resources-cluster.json | 132 ++---- .../dashboards/k8s-resources-namespace.json | 107 ++--- .../dashboards/k8s-resources-node.json | 60 +-- .../configs/dashboards/k8s-resources-pod.json | 108 ++--- .../dashboards/k8s-resources-workload.json | 134 +++---- .../k8s-resources-workloads-namespace.json | 134 +++---- .../configs/dashboards/kubelet.json | 52 +-- .../configs/dashboards/namespace-by-pod.json | 2 +- .../dashboards/namespace-by-workload.json | 2 +- .../dashboards/persistentvolumesusage.json | 22 +- .../configs/dashboards/pod-total.json | 2 +- .../configs/dashboards/proxy.json | 72 ++-- .../configs/dashboards/scheduler.json | 68 ++-- .../configs/dashboards/workload-total.json | 28 +- .../kubezero-metrics/configs/jsonnetfile.json | 2 +- .../configs/jsonnetfile.lock.json | 48 +-- .../configs/rules/alertmanager-prometheusRule | 3 +- .../configs/rules/etcd-mixin-prometheusRule | 10 +- .../rules/kube-state-metrics-prometheusRule | 2 +- .../configs/rules/kubernetes-prometheusRule | 100 ++--- .../rules/node-exporter-prometheusRule | 10 +- .../rules/prometheus-operator-prometheusRule | 20 +- .../configs/rules/prometheus-prometheusRule | 3 +- .../templates/grafana-dashboards-k8s.yaml | 34 +- .../templates/istio-authorization-policy.yaml | 6 +- .../templates/rules/etcd-mixin.yaml | 10 +- .../templates/rules/kubernetes.yaml | 97 ++--- .../templates/rules/node-exporter.yaml | 8 +- .../templates/rules/prometheus-operator.yaml | 15 - charts/kubezero-metrics/values.yaml | 55 ++- 93 files changed, 2110 insertions(+), 1763 deletions(-) create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/extra-manifests.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/OWNERS delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/servicemonitor.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml diff --git a/charts/kubezero-ci/Chart.yaml b/charts/kubezero-ci/Chart.yaml index 88d3f9a..0a495fc 100644 --- a/charts/kubezero-ci/Chart.yaml +++ b/charts/kubezero-ci/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-ci description: KubeZero umbrella chart for all things CI type: application -version: 0.4.20 +version: 0.4.23 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -26,7 +26,7 @@ dependencies: repository: https://dl.gitea.io/charts/ condition: gitea.enabled - name: jenkins - version: 3.10.3 + version: 3.11.3 repository: https://charts.jenkins.io condition: jenkins.enabled - name: trivy diff --git a/charts/kubezero-ci/README.md b/charts/kubezero-ci/README.md index 258a727..0c51e8b 100644 --- a/charts/kubezero-ci/README.md +++ b/charts/kubezero-ci/README.md @@ -1,6 +1,6 @@ # kubezero-ci -![Version: 0.4.20](https://img.shields.io/badge/Version-0.4.20-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.4.23](https://img.shields.io/badge/Version-0.4.23-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero umbrella chart for all things CI @@ -20,7 +20,7 @@ Kubernetes: `>= 1.20.0` |------------|------|---------| | https://aquasecurity.github.io/helm-charts/ | trivy | 0.4.9 | | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.5 | -| https://charts.jenkins.io | jenkins | 3.10.3 | +| https://charts.jenkins.io | jenkins | 3.11.3 | | https://dl.gitea.io/charts/ | gitea | 5.0.0 | | https://gocd.github.io/helm-chart | gocd | 1.39.4 | @@ -29,15 +29,15 @@ Kubernetes: `>= 1.20.0` - memory request 1.25GB - dark theme - trivy scanner incl. HTML reporting and publisher - + # goCD - + # Gitea ## OpenSSH 8.8 RSA disabled - https://github.com/go-gitea/gitea/issues/17798 -## Resources +## Resources ## Values @@ -83,7 +83,7 @@ Kubernetes: `>= 1.20.0` | jenkins.agent.resources.requests.cpu | string | `"512m"` | | | jenkins.agent.resources.requests.memory | string | `"512Mi"` | | | jenkins.agent.showRawYaml | bool | `false` | | -| jenkins.agent.tag | string | `"v0.2.4-2"` | | +| jenkins.agent.tag | string | `"v0.2.4-5"` | | | jenkins.agent.yamlMergeStrategy | string | `"merge"` | | | jenkins.agent.yamlTemplate | string | `"apiVersion: v1\nkind: Pod\nspec:\n serviceAccountName: jenkins-podman-aws\n containers:\n - name: jnlp\n resources:\n limits:\n github.com/fuse: 1\n volumeMounts:\n - name: aws-token\n mountPath: \"/var/run/secrets/sts.amazonaws.com/serviceaccount/\"\n readOnly: true\n volumes:\n - name: aws-token\n projected:\n sources:\n - serviceAccountToken:\n path: token\n expirationSeconds: 86400\n audience: \"sts.amazonaws.com\""` | | | jenkins.controller.JCasC.configScripts.zdt-settings | string | `"jenkins:\n noUsageStatistics: true\n disabledAdministrativeMonitors:\n - \"jenkins.security.ResourceDomainRecommendation\"\nunclassified:\n buildDiscarders:\n configuredBuildDiscarders:\n - \"jobBuildDiscarder\"\n - defaultBuildDiscarder:\n discarder:\n logRotator:\n artifactDaysToKeepStr: \"32\"\n artifactNumToKeepStr: \"10\"\n daysToKeepStr: \"100\"\n numToKeepStr: \"10\"\n"` | | @@ -102,6 +102,7 @@ Kubernetes: `>= 1.20.0` | jenkins.controller.installPlugins[6] | string | `"htmlpublisher:1.28"` | | | jenkins.controller.installPlugins[7] | string | `"build-discarder:60.v1747b0eb632a"` | | | jenkins.controller.javaOpts | string | `"-XX:+UseStringDeduplication -Dhudson.model.DirectoryBrowserSupport.CSP=\"sandbox allow-popups; default-src 'none'; img-src 'self' cdn.zero-downtime.net; style-src 'unsafe-inline';\""` | | +| jenkins.controller.jenkinsOpts | string | `"--sessionTimeout=180 --sessionEviction=3600"` | | | jenkins.controller.prometheus.enabled | bool | `false` | | | jenkins.controller.resources.limits.cpu | string | `"2000m"` | | | jenkins.controller.resources.limits.memory | string | `"4096Mi"` | | diff --git a/charts/kubezero-ci/templates/gitea/istio-authorization-policy.yaml b/charts/kubezero-ci/templates/gitea/istio-authorization-policy.yaml index 3f47bac..2e183eb 100644 --- a/charts/kubezero-ci/templates/gitea/istio-authorization-policy.yaml +++ b/charts/kubezero-ci/templates/gitea/istio-authorization-policy.yaml @@ -16,7 +16,8 @@ spec: - source: notIpBlocks: {{- toYaml .Values.gitea.istio.ipBlocks | nindent 8 }} - when: - - key: connection.sni - values: ["{{ .Values.gitea.istio.url }}"] + to: + - operation: + hosts: ["{{ .Values.gitea.istio.url }}"] + ports: [ "443", "22" ] {{- end }} diff --git a/charts/kubezero-ci/templates/jenkins/istio-authorization-policy.yaml b/charts/kubezero-ci/templates/jenkins/istio-authorization-policy.yaml index a8f8538..3d808ce 100644 --- a/charts/kubezero-ci/templates/jenkins/istio-authorization-policy.yaml +++ b/charts/kubezero-ci/templates/jenkins/istio-authorization-policy.yaml @@ -16,7 +16,8 @@ spec: - source: notIpBlocks: {{- toYaml .Values.jenkins.istio.ipBlocks | nindent 8 }} - when: - - key: connection.sni - values: ["{{ .Values.jenkins.istio.url }}"] + to: + - operation: + hosts: ["{{ .Values.jenkins.istio.url }}"] + ports: [ "443" ] {{- end }} diff --git a/charts/kubezero-ci/values.yaml b/charts/kubezero-ci/values.yaml index 040ce09..fcb556e 100644 --- a/charts/kubezero-ci/values.yaml +++ b/charts/kubezero-ci/values.yaml @@ -77,6 +77,7 @@ jenkins: enableRawHtmlMarkupFormatter: true # javaOpts: "-Xms512m -Xmx512m" javaOpts: "-XX:+UseStringDeduplication -Dhudson.model.DirectoryBrowserSupport.CSP=\"sandbox allow-popups; default-src 'none'; img-src 'self' cdn.zero-downtime.net; style-src 'unsafe-inline';\"" + jenkinsOpts: "--sessionTimeout=180 --sessionEviction=3600" resources: requests: @@ -129,7 +130,7 @@ jenkins: # Preconfigure agents to use zdt podman requires fuse/overlayfs agent: image: public.ecr.aws/zero-downtime/jenkins-podman - tag: v0.2.4-2 + tag: v0.2.4-5 resources: requests: cpu: "512m" diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index 34856e2..25d4455 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-metrics description: KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all Kubernetes integrations. type: application -version: 0.6.2 +version: 0.7.4 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -18,15 +18,15 @@ dependencies: version: ">= 0.1.4" repository: https://cdn.zero-downtime.net/charts/ - name: kube-prometheus-stack - version: 23.1.5 + version: 30.2.0 # Switch back to upstream once all alerts are fixed eg. etcd gpcr # repository: https://prometheus-community.github.io/helm-charts - name: prometheus-adapter - version: 3.0.0 + version: 3.0.1 repository: https://prometheus-community.github.io/helm-charts condition: prometheus-adapter.enabled - name: prometheus-pushgateway - version: 1.13.0 + version: 1.14.0 # Switch back to upstream once namespaces are supported # repository: https://prometheus-community.github.io/helm-charts condition: prometheus-pushgateway.enabled diff --git a/charts/kubezero-metrics/README.md b/charts/kubezero-metrics/README.md index 0163436..dd2551e 100644 --- a/charts/kubezero-metrics/README.md +++ b/charts/kubezero-metrics/README.md @@ -1,6 +1,6 @@ # kubezero-metrics -![Version: 0.5.4](https://img.shields.io/badge/Version-0.5.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.7.0](https://img.shields.io/badge/Version-0.7.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all Kubernetes integrations. @@ -10,18 +10,18 @@ KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all | Name | Email | Url | | ---- | ------ | --- | -| Quarky9 | | | +| Stefan Reimer | stefan@zero-downtime.net | | ## Requirements -Kubernetes: `>= 1.18.0` +Kubernetes: `>= 1.20.0` | Repository | Name | Version | |------------|------|---------| -| | kube-prometheus-stack | 18.1.0 | -| | prometheus-pushgateway | 1.10.1 | +| | kube-prometheus-stack | 30.2.0 | +| | prometheus-pushgateway | 1.14.0 | | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.4 | -| https://prometheus-community.github.io/helm-charts | prometheus-adapter | 2.17 | +| https://prometheus-community.github.io/helm-charts | prometheus-adapter | 3.0.1 | ## Values @@ -84,7 +84,6 @@ Kubernetes: `>= 1.18.0` | kube-prometheus-stack.alertmanager.enabled | bool | `false` | | | kube-prometheus-stack.coreDns.enabled | bool | `true` | | | kube-prometheus-stack.defaultRules.create | bool | `false` | | -| kube-prometheus-stack.global.rbac.pspEnabled | bool | `false` | | | kube-prometheus-stack.grafana."grafana.ini"."auth.anonymous".enabled | bool | `true` | | | kube-prometheus-stack.grafana."grafana.ini".alerting.enabled | bool | `false` | | | kube-prometheus-stack.grafana."grafana.ini".analytics.check_for_updates | bool | `false` | | @@ -102,13 +101,15 @@ Kubernetes: `>= 1.18.0` | kube-prometheus-stack.grafana.extraContainerVolumes[0].name | string | `"script-volume"` | | | kube-prometheus-stack.grafana.initChownData.enabled | bool | `false` | | | kube-prometheus-stack.grafana.plugins[0] | string | `"grafana-piechart-panel"` | | -| kube-prometheus-stack.grafana.rbac.pspEnabled | bool | `false` | | | kube-prometheus-stack.grafana.service.portName | string | `"http-grafana"` | | +| kube-prometheus-stack.grafana.sidecar.dashboards.extraMounts[0].mountPath | string | `"/opt/script.sh"` | | +| kube-prometheus-stack.grafana.sidecar.dashboards.extraMounts[0].name | string | `"script-volume"` | | +| kube-prometheus-stack.grafana.sidecar.dashboards.extraMounts[0].subPath | string | `"script.sh"` | | | kube-prometheus-stack.grafana.sidecar.dashboards.provider.foldersFromFilesStructure | bool | `true` | | +| kube-prometheus-stack.grafana.sidecar.dashboards.script | string | `"/opt/script.sh"` | | | kube-prometheus-stack.grafana.sidecar.dashboards.searchNamespace | string | `"ALL"` | | | kube-prometheus-stack.grafana.testFramework.enabled | bool | `false` | | -| kube-prometheus-stack.kube-state-metrics.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | -| kube-prometheus-stack.kube-state-metrics.podSecurityPolicy.enabled | bool | `false` | | +| kube-prometheus-stack.kube-state-metrics.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | | | kube-prometheus-stack.kube-state-metrics.tolerations[0].effect | string | `"NoSchedule"` | | | kube-prometheus-stack.kube-state-metrics.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | kube-prometheus-stack.kubeApiServer.enabled | bool | `true` | | @@ -131,13 +132,12 @@ Kubernetes: `>= 1.18.0` | kube-prometheus-stack.kubelet.enabled | bool | `true` | | | kube-prometheus-stack.kubelet.serviceMonitor.cAdvisor | bool | `true` | | | kube-prometheus-stack.nodeExporter.enabled | bool | `true` | | -| kube-prometheus-stack.nodeExporter.serviceMonitor.relabelings[0].action | string | `"replace"` | | -| kube-prometheus-stack.nodeExporter.serviceMonitor.relabelings[0].regex | string | `"^(.*)$"` | | -| kube-prometheus-stack.nodeExporter.serviceMonitor.relabelings[0].replacement | string | `"$1"` | | -| kube-prometheus-stack.nodeExporter.serviceMonitor.relabelings[0].separator | string | `";"` | | -| kube-prometheus-stack.nodeExporter.serviceMonitor.relabelings[0].sourceLabels[0] | string | `"__meta_kubernetes_pod_node_name"` | | -| kube-prometheus-stack.nodeExporter.serviceMonitor.relabelings[0].targetLabel | string | `"node"` | | -| kube-prometheus-stack.prometheus-node-exporter.rbac.pspEnabled | bool | `false` | | +| kube-prometheus-stack.prometheus-node-exporter.prometheus.monitor.serviceMonitor.relabelings[0].action | string | `"replace"` | | +| kube-prometheus-stack.prometheus-node-exporter.prometheus.monitor.serviceMonitor.relabelings[0].regex | string | `"^(.*)$"` | | +| kube-prometheus-stack.prometheus-node-exporter.prometheus.monitor.serviceMonitor.relabelings[0].replacement | string | `"$1"` | | +| kube-prometheus-stack.prometheus-node-exporter.prometheus.monitor.serviceMonitor.relabelings[0].separator | string | `";"` | | +| kube-prometheus-stack.prometheus-node-exporter.prometheus.monitor.serviceMonitor.relabelings[0].sourceLabels[0] | string | `"__meta_kubernetes_pod_node_name"` | | +| kube-prometheus-stack.prometheus-node-exporter.prometheus.monitor.serviceMonitor.relabelings[0].targetLabel | string | `"node"` | | | kube-prometheus-stack.prometheus-node-exporter.resources.requests.cpu | string | `"20m"` | | | kube-prometheus-stack.prometheus-node-exporter.resources.requests.memory | string | `"16Mi"` | | | kube-prometheus-stack.prometheus.enabled | bool | `true` | | @@ -153,12 +153,12 @@ Kubernetes: `>= 1.18.0` | kube-prometheus-stack.prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.accessModes[0] | string | `"ReadWriteOnce"` | | | kube-prometheus-stack.prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage | string | `"16Gi"` | | | kube-prometheus-stack.prometheus.prometheusSpec.walCompression | bool | `true` | | -| kube-prometheus-stack.prometheusOperator.admissionWebhooks.patch.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| kube-prometheus-stack.prometheusOperator.admissionWebhooks.patch.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | | | kube-prometheus-stack.prometheusOperator.admissionWebhooks.patch.tolerations[0].effect | string | `"NoSchedule"` | | | kube-prometheus-stack.prometheusOperator.admissionWebhooks.patch.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | kube-prometheus-stack.prometheusOperator.enabled | bool | `true` | | | kube-prometheus-stack.prometheusOperator.logFormat | string | `"json"` | | -| kube-prometheus-stack.prometheusOperator.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| kube-prometheus-stack.prometheusOperator.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | | | kube-prometheus-stack.prometheusOperator.resources.limits.memory | string | `"64Mi"` | | | kube-prometheus-stack.prometheusOperator.resources.requests.cpu | string | `"20m"` | | | kube-prometheus-stack.prometheusOperator.resources.requests.memory | string | `"32Mi"` | | @@ -166,7 +166,7 @@ Kubernetes: `>= 1.18.0` | kube-prometheus-stack.prometheusOperator.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | prometheus-adapter.enabled | bool | `true` | | | prometheus-adapter.logLevel | int | `1` | | -| prometheus-adapter.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| prometheus-adapter.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | | | prometheus-adapter.prometheus.url | string | `"http://metrics-kube-prometheus-st-prometheus"` | | | prometheus-adapter.rules.default | bool | `false` | | | prometheus-adapter.rules.resource.cpu.containerLabel | string | `"container"` | | diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml index eba1fa7..bf7cb79 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml @@ -6,20 +6,20 @@ annotations: url: https://github.com/prometheus-operator/kube-prometheus artifacthub.io/operator: "true" apiVersion: v2 -appVersion: 0.52.0 +appVersion: 0.53.1 dependencies: - condition: kubeStateMetrics.enabled name: kube-state-metrics repository: https://prometheus-community.github.io/helm-charts - version: 4.1.* + version: 4.4.* - condition: nodeExporter.enabled name: prometheus-node-exporter repository: https://prometheus-community.github.io/helm-charts - version: 2.2.* + version: 2.5.* - condition: grafana.enabled name: grafana repository: https://grafana.github.io/helm-charts - version: 6.18.* + version: 6.21.* description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus @@ -32,7 +32,11 @@ keywords: - kube-prometheus kubeVersion: '>=1.16.0-0' maintainers: +- email: andrew@quadcorps.co.uk + name: andrewgkew - name: bismarck +- email: cedric@desaintmartin.fr + name: desaintmartin - email: gianrubio@gmail.com name: gianrubio - email: github.gkarthiks@gmail.com @@ -46,4 +50,4 @@ sources: - https://github.com/prometheus-community/helm-charts - https://github.com/prometheus-operator/kube-prometheus type: application -version: 23.1.5 +version: 30.2.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md index d745a3d..d365b1e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md @@ -83,6 +83,61 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. +### From 29.x to 30.x + +This version updates kube-state-metrics to 4.3.0 and uses the new option `kube-state-metrics.releaseLabel=true` which adds the "release" label to kube-state-metrics labels, making scraping of the metrics by kube-prometheus-stack work out of the box again, independent of the used kube-prometheus-stack release name. If you already set the "release" label via `kube-state-metrics.customLabels` you might have to remove that and use it via the new option. + +### From 28.x to 29.x + +This version makes scraping port for kube-controller-manager and kube-scheduler dynamic to reflect changes to default serving ports +for those components in Kubernetes versions v1.22 and v1.23 respectively. + +If you deploy on clusters using version v1.22+, kube-controller-manager will be scraped over HTTPS on port 10257. + +If you deploy on clusters running version v1.23+, kube-scheduler will be scraped over HTTPS on port 10259. + +### From 27.x to 28.x + +This version disables PodSecurityPolicies by default because they are deprecated in Kubernetes 1.21 and will be removed in Kubernetes 1.25. + +If you are using PodSecurityPolicies you can enable the previous behaviour by setting `kube-state-metrics.podSecurityPolicy.enabled`, `prometheus-node-exporter.rbac.pspEnabled`, `grafana.rbac.pspEnabled` and `global.rbac.pspEnabled` to `true`. + +### From 26.x to 27.x + +This version splits Node Exporter recording and altering rules in separate config values. +Instead of `defaultRules.rules.node` the 2 new variables `defaultRules.rules.nodeExporterAlerting` and `defaultRules.rules.nodeExporterRecording` are used. + +Also the following defaultRules.rules has been removed as they had no effect: `kubeApiserverError`, `kubePrometheusNodeAlerting`, `kubernetesAbsent`, `time`. + +The ability to set a rubookUrl via `defaultRules.rules.rubookUrl` was reintroduced. + +### From 25.x to 26.x + +This version enables the prometheus-node-exporter subchart servicemonitor by default again, by setting `prometheus-node-exporter.prometheus.monitor.enabled` to `true`. + +### From 24.x to 25.x + +This version upgrade to prometheus-operator v0.53.1. It removes support for setting a runbookUrl, since the upstream format for runbooks changed. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +### From 23.x to 24.x + +The custom `ServiceMonitor` for the _kube-state-metrics_ & _prometheus-node-exporter_ charts have been removed in favour of the built in sub-chart `ServiceMonitor`; for both sub-charts this means that `ServiceMonitor` customisations happen via the values passed to the chart. If you haven't directly customised this behaviour then there are no changes required to upgrade, but if you have please read the following. + +For _kube-state-metrics_ the `ServiceMonitor` customisation is now set via `kube-state-metrics.prometheus.monitor` and the `kubeStateMetrics.serviceMonitor.selfMonitor.enabled` value has moved to `kube-state-metrics.selfMonitor.enabled`. + +For _prometheus-node-exporter_ the `ServiceMonitor` customisation is now set via `prometheus-node-exporter.prometheus.monitor` and the `nodeExporter.jobLabel` values has moved to `prometheus-node-exporter.prometheus.monitor.jobLabel`. + ### From 22.x to 23.x Port names have been renamed for Istio's @@ -331,9 +386,9 @@ For more in-depth documentation of configuration options meanings, please see The prometheus operator does not support annotation-based discovery of services, using the `PodMonitor` or `ServiceMonitor` CRD in its place as they provide far more configuration options. For information on how to use PodMonitors/ServiceMonitors, please see the documentation on the `prometheus-operator/prometheus-operator` documentation here: -- [ServiceMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/getting-started.md#include-servicemonitors) -- [PodMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/getting-started.md#include-podmonitors) -- [Running Exporters](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/running-exporters.md) +- [ServiceMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-servicemonitors) +- [PodMonitors](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#include-podmonitors) +- [Running Exporters](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/running-exporters.md) By default, Prometheus discovers PodMonitors and ServiceMonitors within its namespace, that are labeled with the same release tag as the prometheus-operator release. Sometimes, you may need to discover custom PodMonitors/ServiceMonitors, for example used to scrape data from third-party applications. diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml index 4aeff42..ca3c38e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 8.3.1 +appVersion: 8.3.4 description: The leading tool for querying and visualizing time series and metrics. home: https://grafana.net icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png @@ -19,4 +19,4 @@ name: grafana sources: - https://github.com/grafana/grafana type: application -version: 6.18.2 +version: 6.21.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md index 96dc902..a32af05 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md @@ -138,7 +138,7 @@ This version requires Helm >= 3.1.0. | `podLabels` | Pod labels | `{}` | | `podPortName` | Name of the grafana port on the pod | `grafana` | | `sidecar.image.repository` | Sidecar image repository | `quay.io/kiwigrid/k8s-sidecar` | -| `sidecar.image.tag` | Sidecar image tag | `1.12.3` | +| `sidecar.image.tag` | Sidecar image tag | `1.15.1` | | `sidecar.image.sha` | Sidecar image sha (optional) | `""` | | `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | | `sidecar.resources` | Sidecar resources | `{}` | @@ -169,6 +169,8 @@ This version requires Helm >= 3.1.0. | `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `nil` | | `sidecar.datasources.searchNamespace` | Namespaces list. If specified, the sidecar will search for datasources config-maps inside these namespaces.Otherwise the namespace in which the sidecar is running will be used.It's also possible to specify ALL to search in all namespaces. | `nil` | | `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | +| `sidecar.datasources.reloadURL` | Full url of datasource configuration reload API endpoint, to invoke after a config-map change | `"http://localhost:3000/api/admin/provisioning/datasources/reload"` | +| `sidecar.datasources.skipReload` | Enabling this omits defining the REQ_URL and REQ_METHOD environment variables | `false` | | `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | | `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | | `sidecar.notifiers.searchNamespace` | Namespaces list. If specified, the sidecar will search for notifiers config-maps (or secrets) inside these namespaces.Otherwise the namespace in which the sidecar is running will be used.It's also possible to specify ALL to search in all namespaces. | `nil` | @@ -226,8 +228,9 @@ This version requires Helm >= 3.1.0. | `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | | `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | | `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | -| `imageRenderer.service.portName` | image-renderer service port name | `'http'` | +| `imageRenderer.service.portName` | image-renderer service port name | `http` | | `imageRenderer.service.port` | image-renderer service port used by both service and deployment | `8081` | +| `imageRenderer.grafanaProtocol` | Protocol to use for image renderer callback url | `http` | | `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | | `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` | | `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` | @@ -237,6 +240,7 @@ This version requires Helm >= 3.1.0. | `networkPolicy.enabled` | Enable creation of NetworkPolicy resources. | `false` | | `networkPolicy.allowExternal` | Don't require client label for connections | `true` | | `networkPolicy.explicitNamespacesSelector` | A Kubernetes LabelSelector to explicitly select namespaces from which traffic could be allowed | `{}` | +| `enableKubeBackwardCompatibility` | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option | `false` | @@ -525,7 +529,7 @@ This example uses a CSI driver e.g. retrieving secrets using [Azure Key Vault Pr ## Image Renderer Plug-In -This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/docs/remote_rendering_using_docker.md) +This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/README.md#run-in-docker) ```yaml imageRenderer: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl index f7cae31..4d53c17 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl @@ -16,7 +16,7 @@ hostAliases: {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{- end }} -{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.sidecar.datasources.enabled .Values.sidecar.notifiers.enabled .Values.extraInitContainers) }} +{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.sidecar.notifiers.enabled .Values.extraInitContainers) }} initContainers: {{- end }} {{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }} @@ -77,49 +77,6 @@ initContainers: readOnly: {{ .readOnly }} {{- end }} {{- end }} -{{- if .Values.sidecar.datasources.enabled }} - - name: {{ template "grafana.name" . }}-sc-datasources - {{- if .Values.sidecar.image.sha }} - image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" - {{- else }} - image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" - {{- end }} - imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} - env: - - name: METHOD - value: {{ .Values.sidecar.datasources.watchMethod }} - - name: LABEL - value: "{{ .Values.sidecar.datasources.label }}" - {{- if .Values.sidecar.datasources.labelValue }} - - name: LABEL_VALUE - value: {{ quote .Values.sidecar.datasources.labelValue }} - {{- end }} - - name: FOLDER - value: "/etc/grafana/provisioning/datasources" - - name: RESOURCE - value: {{ quote .Values.sidecar.datasources.resource }} - {{- if .Values.sidecar.enableUniqueFilenames }} - - name: UNIQUE_FILENAMES - value: "{{ .Values.sidecar.enableUniqueFilenames }}" - {{- end }} - {{- if .Values.sidecar.datasources.searchNamespace }} - - name: NAMESPACE - value: "{{ .Values.sidecar.datasources.searchNamespace | join "," }}" - {{- end }} - {{- if .Values.sidecar.skipTlsVerify }} - - name: SKIP_TLS_VERIFY - value: "{{ .Values.sidecar.skipTlsVerify }}" - {{- end }} - resources: -{{ toYaml .Values.sidecar.resources | indent 6 }} -{{- if .Values.sidecar.securityContext }} - securityContext: -{{- toYaml .Values.sidecar.securityContext | nindent 6 }} -{{- end }} - volumeMounts: - - name: sc-datasources-volume - mountPath: "/etc/grafana/provisioning/datasources" -{{- end}} {{- if .Values.sidecar.notifiers.enabled }} - name: {{ template "grafana.name" . }}-sc-notifiers {{- if .Values.sidecar.image.sha }} @@ -168,7 +125,9 @@ imagePullSecrets: - name: {{ . }} {{- end}} {{- end }} +{{- if not .Values.enableKubeBackwardCompatibility }} enableServiceLinks: {{ .Values.enableServiceLinks }} +{{- end }} containers: {{- if .Values.sidecar.dashboards.enabled }} - name: {{ template "grafana.name" . }}-sc-dashboard @@ -223,6 +182,69 @@ containers: {{- if .Values.sidecar.dashboards.extraMounts }} {{- toYaml .Values.sidecar.dashboards.extraMounts | trim | nindent 6}} {{- end }} +{{- end}} +{{- if .Values.sidecar.datasources.enabled }} + - name: {{ template "grafana.name" . }}-sc-datasources + {{- if .Values.sidecar.image.sha }} + image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + - name: METHOD + value: {{ .Values.sidecar.datasources.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + {{- if .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.datasources.labelValue }} + {{- end }} + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: {{ quote .Values.sidecar.datasources.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- if .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ .Values.sidecar.datasources.searchNamespace | join "," }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if not .Values.sidecar.datasources.skipReload }} + - name: REQ_URL + value: {{ .Values.sidecar.datasources.reloadURL }} + - name: REQ_METHOD + value: POST + {{- end }} + resources: +{{ toYaml .Values.sidecar.resources | indent 6 }} +{{- if .Values.sidecar.securityContext }} + securityContext: +{{- toYaml .Values.sidecar.securityContext | nindent 6 }} +{{- end }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" {{- end}} - name: {{ .Chart.Name }} {{- if .Values.image.sha }} @@ -373,12 +395,12 @@ containers: name: {{ .Values.smtp.existingSecret }} key: {{ .Values.smtp.passwordKey | default "password" }} {{- end }} - {{ if .Values.imageRenderer.enabled }} + {{- if .Values.imageRenderer.enabled }} - name: GF_RENDERING_SERVER_URL value: http://{{ template "grafana.fullname" . }}-image-renderer.{{ template "grafana.namespace" . }}:{{ .Values.imageRenderer.service.port }}/render - name: GF_RENDERING_CALLBACK_URL - value: http://{{ template "grafana.fullname" . }}.{{ template "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} - {{ end }} + value: {{ .Values.imageRenderer.grafanaProtocol }}://{{ template "grafana.fullname" . }}.{{ template "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} + {{- end }} - name: GF_PATHS_DATA value: {{ (get .Values "grafana.ini").paths.data }} - name: GF_PATHS_LOGS diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/dashboards-json-configmap.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/dashboards-json-configmap.yaml index 3019067..59e0be6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/dashboards-json-configmap.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/dashboards-json-configmap.yaml @@ -9,9 +9,6 @@ metadata: labels: {{- include "grafana.labels" $ | nindent 4 }} dashboard-provider: {{ $provider }} - {{- if $.Values.sidecar.dashboards.label }} - {{ $.Values.sidecar.dashboards.label }}: "1" - {{- end }} {{- if $dashboards }} data: {{- $dashboardFound := false }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/deployment.yaml index 1c9ae86..8dbe5e1 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/deployment.yaml @@ -14,7 +14,7 @@ metadata: {{ toYaml . | indent 4 }} {{- end }} spec: - {{- if not .Values.autoscaling.enabled }} + {{- if and (not .Values.autoscaling.enabled) (.Values.replicas) }} replicas: {{ .Values.replicas }} {{- end }} revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/extra-manifests.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/extra-manifests.yaml new file mode 100644 index 0000000..a9bb3b6 --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/extra-manifests.yaml @@ -0,0 +1,4 @@ +{{ range .Values.extraObjects }} +--- +{{ tpl (toYaml .) $ }} +{{ end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml index ad0129a..77acde3 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml @@ -73,7 +73,7 @@ livenessProbe: image: repository: grafana/grafana - tag: 8.3.1 + tag: 8.3.4 sha: "" pullPolicy: IfNotPresent @@ -618,7 +618,7 @@ smtp: sidecar: image: repository: quay.io/kiwigrid/k8s-sidecar - tag: 1.14.2 + tag: 1.15.1 sha: "" imagePullPolicy: IfNotPresent resources: {} @@ -685,9 +685,12 @@ sidecar: # It's also possible to specify ALL to search in all namespaces searchNamespace: null # Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. - watchMethod: LIST + watchMethod: WATCH # search in configmap, secret or both resource: both + # Endpoint to send request to reload datasources + reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload" + skipReload: false notifiers: enabled: false # label that the configmaps with notifiers are marked with @@ -726,6 +729,7 @@ imageRenderer: HTTP_HOST: "0.0.0.0" # RENDERING_ARGS: --no-sandbox,--disable-gpu,--window-size=1280x758 # RENDERING_MODE: clustered + # IGNORE_HTTPS_ERRORS: true # image-renderer deployment serviceAccount serviceAccountName: "" # image-renderer deployment securityContext @@ -742,6 +746,8 @@ imageRenderer: # image-renderer service port used by both service and deployment port: 8081 targetPort: 8081 + # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana + grafanaProtocol: http # In case a sub_path is used this needs to be added to the image renderer callback grafanaSubPath: "" # name of the image-renderer port on the pod @@ -786,3 +792,18 @@ networkPolicy: ## - {key: role, operator: In, values: [frontend]} ## explicitNamespacesSelector: {} + +# Enable backward compatibility of kubernetes where version below 1.13 doesn't have the enableServiceLinks option +enableKubeBackwardCompatibility: false + +# Create a dynamic manifests via values: +extraObjects: [] + # - apiVersion: "kubernetes-client.io/v1" + # kind: ExternalSecret + # metadata: + # name: grafana-secrets + # spec: + # backendType: gcpSecretsManager + # data: + # - key: grafana-admin-password + # name: adminPassword diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml index 113f997..219f7af 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 2.2.4 +appVersion: 2.3.0 description: Install kube-state-metrics to generate and expose cluster-level metrics home: https://github.com/kubernetes/kube-state-metrics/ keywords: @@ -12,8 +12,10 @@ maintainers: name: tariq1890 - email: manuel@rueg.eu name: mrueg +- email: davidcalvertfr@gmail.com + name: dotdc name: kube-state-metrics sources: - https://github.com/kubernetes/kube-state-metrics/ type: application -version: 4.1.1 +version: 4.4.1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/OWNERS b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/OWNERS deleted file mode 100644 index 206b4fe..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/OWNERS +++ /dev/null @@ -1,6 +0,0 @@ -approvers: -- tariq1890 -- mrueg -reviewers: -- tariq1890 -- mrueg diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/_helpers.tpl index 88f5c06..976b273 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/_helpers.tpl @@ -57,7 +57,7 @@ Create chart name and version as used by the chart label. Generate basic labels */}} {{- define "kube-state-metrics.labels" }} -helm.sh/chart: {{ include "kube-state-metrics.chart" . }} +helm.sh/chart: {{ template "kube-state-metrics.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/component: metrics app.kubernetes.io/part-of: {{ template "kube-state-metrics.name" . }} @@ -68,6 +68,9 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- if .Values.customLabels }} {{ toYaml .Values.customLabels }} {{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} {{- end }} {{/* diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml index 9ad54ae..4a382a2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml @@ -9,16 +9,7 @@ kind: ClusterRole {{- end }} metadata: labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" $ }} - app.kubernetes.io/instance: {{ $.Release.Name }} - app.kubernetes.io/component: metrics - app.kubernetes.io/managed-by: {{ $.Release.Service }} - app.kubernetes.io/version: "{{ $.Chart.Version }}" - app.kubernetes.io/part-of: {{ template "kube-state-metrics.name" $ }} - helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version }} - {{- if $.Values.customLabels }} - {{ toYaml $.Values.customLabels | nindent 4 }} - {{- end }} + {{- include "kube-state-metrics.labels" $ | indent 4 }} name: {{ template "kube-state-metrics.fullname" $ }} {{- if eq $.Values.rbac.useClusterRole false }} namespace: {{ . }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml index 606a79f..135094f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml @@ -5,7 +5,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: - {{- include "kube-state-metrics.labels" . | indent 4 }} + {{- include "kube-state-metrics.labels" $ | indent 4 }} name: {{ template "kube-state-metrics.fullname" $ }} namespace: {{ . }} roleRef: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml index 54d7631..55fd8fe 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml @@ -10,10 +10,14 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - jobLabel: app.kubernetes.io/name + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} selector: matchLabels: + {{- if .Values.prometheus.monitor.selectorOverride }} + {{ toYaml .Values.prometheus.monitor.selectorOverride | indent 6 }} + {{ else }} {{- include "kube-state-metrics.selectorLabels" . | indent 6 }} + {{- end }} endpoints: - port: http {{- if .Values.prometheus.monitor.interval }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml index da09c89..aced5cf 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml @@ -2,7 +2,7 @@ prometheusScrape: true image: repository: k8s.gcr.io/kube-state-metrics/kube-state-metrics - tag: v2.2.4 + tag: v2.3.0 pullPolicy: IfNotPresent imagePullSecrets: [] @@ -34,6 +34,9 @@ service: customLabels: {} # app: kube-state-metrics +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + hostNetwork: false rbac: @@ -65,9 +68,11 @@ prometheus: enabled: false additionalLabels: {} namespace: "" + jobLabel: "" interval: "" scrapeTimeout: "" proxyUrl: "" + selectorOverride: {} honorLabels: false metricRelabelings: [] relabelings: [] diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml index 1389d05..2612f8f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.2.2 +appVersion: 1.3.1 description: A Helm chart for prometheus node-exporter home: https://github.com/prometheus/node_exporter/ keywords: @@ -14,4 +14,4 @@ name: prometheus-node-exporter sources: - https://github.com/prometheus/node_exporter/ type: application -version: 2.2.2 +version: 2.5.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/_helpers.tpl index 2fa1313..b6cbc91 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/_helpers.tpl @@ -54,6 +54,13 @@ Create the name of the service account to use {{- end -}} {{- end -}} +{{/* +The image to use +*/}} +{{- define "prometheus-node-exporter.image" -}} +{{- printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} +{{- end }} + {{/* Allow the release namespace to be overridden for multi-namespace deployments in combined charts */}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml index cf58944..2ace6c5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -36,7 +36,7 @@ spec: {{- end }} containers: - name: node-exporter - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: {{ include "prometheus-node-exporter.image" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: - --path.procfs=/host/proc diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml deleted file mode 100644 index 01c2b76..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{- if .Values.prometheus.monitor.enabled }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ template "prometheus-node-exporter.fullname" . }} - namespace: {{ template "prometheus-node-exporter.monitor-namespace" . }} - labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} - {{- if .Values.prometheus.monitor.additionalLabels }} -{{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }} - {{- end }} -spec: - selector: - matchLabels: - app: {{ template "prometheus-node-exporter.name" . }} - release: {{ .Release.Name }} - endpoints: - - port: {{ .Values.service.portName }} - scheme: {{ $.Values.prometheus.monitor.scheme }} - {{- if $.Values.prometheus.monitor.bearerTokenFile }} - bearerTokenFile: {{ $.Values.prometheus.monitor.bearerTokenFile }} - {{- end }} - {{- if $.Values.prometheus.monitor.tlsConfig }} - tlsConfig: {{ toYaml $.Values.prometheus.monitor.tlsConfig | nindent 8 }} - {{- end }} - {{- if .Values.prometheus.monitor.proxyUrl }} - proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} - {{- end }} - {{- if .Values.prometheus.monitor.scrapeTimeout }} - scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} - {{- end }} -{{- if .Values.prometheus.monitor.relabelings }} - relabelings: -{{ toYaml .Values.prometheus.monitor.relabelings | indent 6 }} -{{- end }} -{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/psp.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/psp.yaml index ec1259e..ea8bcba 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/psp.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/psp.yaml @@ -12,12 +12,6 @@ metadata: {{- end}} spec: privileged: false - # Required to prevent escalations to root. - # allowPrivilegeEscalation: false - # This is redundant with non-root + disallow privilege escalation, - # but we can provide it for defense in depth. - #requiredDropCapabilities: - # - ALL # Allow core volume types. volumes: - 'configMap' @@ -42,13 +36,13 @@ spec: supplementalGroups: rule: 'MustRunAs' ranges: - # Forbid adding the root group. + # Allow adding the root group. - min: 0 max: 65535 fsGroup: rule: 'MustRunAs' ranges: - # Forbid adding the root group. + # Allow adding the root group. - min: 0 max: 65535 readOnlyRootFilesystem: false diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/servicemonitor.yaml new file mode 100644 index 0000000..aad25c4 --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/servicemonitor.yaml @@ -0,0 +1,48 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }} + namespace: {{ template "prometheus-node-exporter.monitor-namespace" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} + {{- if .Values.prometheus.monitor.additionalLabels }} + {{- toYaml .Values.prometheus.monitor.additionalLabels | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ default "app.kubernetes.io/name" .Values.prometheus.monitor.jobLabel }} + selector: + matchLabels: + {{- if .Values.prometheus.monitor.selectorOverride }} + {{ toYaml .Values.prometheus.monitor.selectorOverride | indent 6 }} + {{ else }} + app: {{ template "prometheus-node-exporter.name" . }} + release: {{ .Release.Name }} + {{- end }} + endpoints: + - port: {{ .Values.service.portName }} + scheme: {{ .Values.prometheus.monitor.scheme }} + {{- with .Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + {{- with .Values.prometheus.monitor.relabelings }} + relabelings: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.prometheus.monitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml index 0d2c9aa..adcf6b5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml @@ -3,7 +3,8 @@ # Declare variables to be passed into your templates. image: repository: quay.io/prometheus/node-exporter - tag: v1.2.2 + # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} + tag: "" pullPolicy: IfNotPresent service: @@ -21,6 +22,9 @@ prometheus: enabled: false additionalLabels: {} namespace: "" + + jobLabel: "" + scheme: http bearerTokenFile: tlsConfig: {} @@ -29,7 +33,13 @@ prometheus: ## proxyUrl: "" + ## Override serviceMonitor selector + ## + selectorOverride: {} + relabelings: [] + metricRelabelings: [] + interval: "" scrapeTimeout: 10s ## Customize the updateStrategy if set diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml index 3c2ca57..0db926a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -63,13 +63,24 @@ spec: items: description: Matcher defines how to match on alert's labels. properties: + matchType: + description: Match operation available with AlertManager + >= v0.22.0 and takes precedence over Regex (deprecated) + if non-empty. + enum: + - '!=' + - = + - =~ + - '!~' + type: string name: description: Label to match. minLength: 1 type: string regex: description: Whether to match on equality (false) or regular-expression - (true). + (true). Deprecated as of AlertManager >= v0.22.0 where + a user should use MatchType instead. type: boolean value: description: Label value to match. @@ -85,13 +96,24 @@ spec: items: description: Matcher defines how to match on alert's labels. properties: + matchType: + description: Match operation available with AlertManager + >= v0.22.0 and takes precedence over Regex (deprecated) + if non-empty. + enum: + - '!=' + - = + - =~ + - '!~' + type: string name: description: Label to match. minLength: 1 type: string regex: description: Whether to match on equality (false) or regular-expression - (true). + (true). Deprecated as of AlertManager >= v0.22.0 where + a user should use MatchType instead. type: boolean value: description: Label value to match. @@ -102,6 +124,88 @@ spec: type: array type: object type: array + muteTimeIntervals: + description: List of MuteTimeInterval specifying when the routes should + be muted. + items: + description: MuteTimeInterval specifies the periods in time when + notifications will be muted + properties: + name: + description: Name of the time interval + type: string + timeIntervals: + description: TimeIntervals is a list of TimeInterval + items: + description: TimeInterval describes intervals of time + properties: + daysOfMonth: + description: DaysOfMonth is a list of DayOfMonthRange + items: + description: DayOfMonthRange is an inclusive range of + days of the month beginning at 1 + properties: + end: + description: End of the inclusive range + maximum: 31 + minimum: -31 + type: integer + start: + description: Start of the inclusive range + maximum: 31 + minimum: -31 + type: integer + type: object + type: array + months: + description: Months is a list of MonthRange + items: + description: MonthRange is an inclusive range of months + of the year beginning in January Months can be specified + by name (e.g 'January') by numerical month (e.g '1') + or as an inclusive range (e.g 'January:March', '1:3', + '1:March') + pattern: ^((?i)january|february|march|april|may|june|july|august|september|october|november|december|[1-12])(?:((:((?i)january|february|march|april|may|june|july|august|september|october|november|december|[1-12]))$)|$) + type: string + type: array + times: + description: Times is a list of TimeRange + items: + description: TimeRange defines a start and end time + in 24hr format + properties: + endTime: + description: EndTime is the end time in 24hr format. + pattern: ^((([01][0-9])|(2[0-3])):[0-5][0-9])$|(^24:00$) + type: string + startTime: + description: StartTime is the start time in 24hr + format. + pattern: ^((([01][0-9])|(2[0-3])):[0-5][0-9])$|(^24:00$) + type: string + type: object + type: array + weekdays: + description: Weekdays is a list of WeekdayRange + items: + description: WeekdayRange is an inclusive range of days + of the week beginning on Sunday Days can be specified + by name (e.g 'Sunday') or as an inclusive range (e.g + 'Monday:Friday') + pattern: ^((?i)sun|mon|tues|wednes|thurs|fri|satur)day(?:((:(sun|mon|tues|wednes|thurs|fri|satur)day)$)|$) + type: string + type: array + years: + description: Years is a list of YearRange + items: + description: YearRange is an inclusive range of years + pattern: ^2\d{3}(?::2\d{3}|$) + type: string + type: array + type: object + type: array + type: object + type: array receivers: description: List of receivers. items: @@ -952,6 +1056,43 @@ spec: type: string type: object type: object + pagerDutyImageConfigs: + description: A list of image details to attach that provide + further detail about an incident. + items: + description: PagerDutyImageConfig attaches images to + an incident + properties: + alt: + description: Alt is the optional alternative text + for the image. + type: string + href: + description: Optional URL; makes the image a clickable + link. + type: string + src: + description: Src of the image being attached to + the incident + type: string + type: object + type: array + pagerDutyLinkConfigs: + description: A list of link details to attach that provide + further detail about an incident. + items: + description: PagerDutyLinkConfig attaches text links + to an incident + properties: + alt: + description: Text that describes the purpose of + the link, and can be used as the link's text. + type: string + href: + description: Href is the URL of the link to be attached + type: string + type: object + type: array routingKey: description: The secret's key that contains the PagerDuty integration key (when using Events API v2). Either this @@ -1020,6 +1161,7 @@ spec: description: How long your notification will continue to be retried for, unless the user acknowledges the notification. + pattern: ^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$ type: string html: description: Whether notification message is HTML or plain @@ -1272,6 +1414,7 @@ spec: description: How often the Pushover servers will send the same notification to the user. Must be at least 30 seconds. + pattern: ^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$ type: string sendResolved: description: Whether or not to notify about resolved alerts. @@ -2614,19 +2757,21 @@ spec: to true for the first-level route by the Prometheus operator. type: boolean groupBy: - description: List of labels to group by. + description: List of labels to group by. Labels must not be repeated + (unique list). Special label "..." (aggregate by all possible + labels), if provided, must be the only element in the list. items: type: string type: array groupInterval: - description: How long to wait before sending an updated notification. - Must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds - seconds minutes hours). + description: 'How long to wait before sending an updated notification. + Must match the regular expression`^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$` + Example: "5m"' type: string groupWait: - description: How long to wait before sending the initial notification. - Must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds - seconds minutes hours). + description: 'How long to wait before sending the initial notification. + Must match the regular expression`^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$` + Example: "30s"' type: string matchers: description: 'List of matchers that the alert’s labels should @@ -2636,13 +2781,24 @@ spec: items: description: Matcher defines how to match on alert's labels. properties: + matchType: + description: Match operation available with AlertManager + >= v0.22.0 and takes precedence over Regex (deprecated) + if non-empty. + enum: + - '!=' + - = + - =~ + - '!~' + type: string name: description: Label to match. minLength: 1 type: string regex: description: Whether to match on equality (false) or regular-expression - (true). + (true). Deprecated as of AlertManager >= v0.22.0 where + a user should use MatchType instead. type: boolean value: description: Label value to match. @@ -2651,14 +2807,27 @@ spec: - name type: object type: array + muteTimeIntervals: + description: 'Note: this comment applies to the field definition + above but appears below otherwise it gets included in the generated + manifest. CRD schema doesn''t support self-referential types + for now (see https://github.com/kubernetes/kubernetes/issues/62872). + We have to use an alternative type to circumvent the limitation. + The downside is that the Kube API can''t validate the data beyond + the fact that it is a valid JSON representation. MuteTimeIntervals + is a list of MuteTimeInterval names that will mute this route + when matched,' + items: + type: string + type: array receiver: description: Name of the receiver for this route. If not empty, it should be listed in the `receivers` field. type: string repeatInterval: - description: How long to wait before repeating the last notification. - Must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds - seconds minutes hours). + description: 'How long to wait before repeating the last notification. + Must match the regular expression`^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$` + Example: "4h"' type: string routes: description: Child routes. diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml index 3df6ea4..74a8caf 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -1225,8 +1225,7 @@ spec: info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should - be specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -1287,9 +1286,10 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving - a TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward compatibility. + There are no validation of this field and lifecycle + hooks will fail in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name to connect to, @@ -1312,18 +1312,16 @@ spec: is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container - crashes or exits. The reason for termination is passed - to the handler. The Pod''s termination grace period countdown - begins before the PreStop hooked is executed. Regardless - of the outcome of the handler, the container will eventually - terminate within the Pod''s termination grace period. - Other management of the container blocks until the hook - completes or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + crashes or exits. The Pod''s termination grace period + countdown begins before the PreStop hook is executed. + Regardless of the outcome of the handler, the container + will eventually terminate within the Pod''s termination + grace period (unless delayed by finalizers). Other management + of the container blocks until the hook completes or until + the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should - be specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -1384,9 +1382,10 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving - a TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward compatibility. + There are no validation of this field and lifecycle + hooks will fail in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name to connect to, @@ -1411,8 +1410,7 @@ spec: info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -1433,6 +1431,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -1496,9 +1513,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -1596,8 +1612,7 @@ spec: fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -1618,6 +1633,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -1681,9 +1715,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -1764,12 +1797,14 @@ spec: This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' + 2) has CAP_SYS_ADMIN Note that this field cannot be set + when spec.os.name is windows.' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by - the container runtime. + the container runtime. Note that this field cannot be + set when spec.os.name is windows. properties: add: description: Added capabilities @@ -1789,25 +1824,29 @@ spec: privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to - root on the host. Defaults to false. + root on the host. Defaults to false. Note that this field + cannot be set when spec.os.name is windows. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType - feature flag to be enabled. + feature flag to be enabled. Note that this field cannot + be set when spec.os.name is windows. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root - filesystem. Default is false. + filesystem. Default is false. Note that this field cannot + be set when spec.os.name is windows. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -1825,7 +1864,8 @@ spec: process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. + value specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -1834,7 +1874,8 @@ spec: random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. properties: level: description: Level is SELinux level label that applies @@ -1857,6 +1898,8 @@ spec: description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. + Note that this field cannot be set when spec.os.name is + windows. properties: localhostProfile: description: localhostProfile indicates a profile defined @@ -1882,6 +1925,8 @@ spec: containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is + linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission @@ -1927,8 +1972,7 @@ spec: This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -1949,6 +1993,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -2012,9 +2075,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -2419,8 +2481,7 @@ spec: info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should - be specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -2481,9 +2542,10 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving - a TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward compatibility. + There are no validation of this field and lifecycle + hooks will fail in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name to connect to, @@ -2506,18 +2568,16 @@ spec: is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container - crashes or exits. The reason for termination is passed - to the handler. The Pod''s termination grace period countdown - begins before the PreStop hooked is executed. Regardless - of the outcome of the handler, the container will eventually - terminate within the Pod''s termination grace period. - Other management of the container blocks until the hook - completes or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + crashes or exits. The Pod''s termination grace period + countdown begins before the PreStop hook is executed. + Regardless of the outcome of the handler, the container + will eventually terminate within the Pod''s termination + grace period (unless delayed by finalizers). Other management + of the container blocks until the hook completes or until + the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should - be specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -2578,9 +2638,10 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving - a TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward compatibility. + There are no validation of this field and lifecycle + hooks will fail in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name to connect to, @@ -2605,8 +2666,7 @@ spec: info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -2627,6 +2687,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -2690,9 +2769,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -2790,8 +2868,7 @@ spec: fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -2812,6 +2889,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -2875,9 +2971,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -2958,12 +3053,14 @@ spec: This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' + 2) has CAP_SYS_ADMIN Note that this field cannot be set + when spec.os.name is windows.' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by - the container runtime. + the container runtime. Note that this field cannot be + set when spec.os.name is windows. properties: add: description: Added capabilities @@ -2983,25 +3080,29 @@ spec: privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to - root on the host. Defaults to false. + root on the host. Defaults to false. Note that this field + cannot be set when spec.os.name is windows. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType - feature flag to be enabled. + feature flag to be enabled. Note that this field cannot + be set when spec.os.name is windows. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root - filesystem. Default is false. + filesystem. Default is false. Note that this field cannot + be set when spec.os.name is windows. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -3019,7 +3120,8 @@ spec: process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. + value specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -3028,7 +3130,8 @@ spec: random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. properties: level: description: Level is SELinux level label that applies @@ -3051,6 +3154,8 @@ spec: description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. + Note that this field cannot be set when spec.os.name is + windows. properties: localhostProfile: description: localhostProfile indicates a profile defined @@ -3076,6 +3181,8 @@ spec: containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is + linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission @@ -3121,8 +3228,7 @@ spec: This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -3143,6 +3249,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -3206,9 +3331,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -3486,7 +3610,8 @@ spec: set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of - any volume." + any volume. Note that this field cannot be set when spec.os.name + is windows." format: int64 type: integer fsGroupChangePolicy: @@ -3496,13 +3621,15 @@ spec: support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". - If not specified, "Always" is used.' + If not specified, "Always" is used. Note that this field cannot + be set when spec.os.name is windows.' type: string runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -3519,7 +3646,8 @@ spec: Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. + takes precedence for that container. Note that this field cannot + be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -3528,6 +3656,7 @@ spec: SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. properties: level: description: Level is SELinux level label that applies to @@ -3548,7 +3677,8 @@ spec: type: object seccompProfile: description: The seccomp options to use by the containers in this - pod. + pod. Note that this field cannot be set when spec.os.name is + windows. properties: localhostProfile: description: localhostProfile indicates a profile defined @@ -3570,7 +3700,8 @@ spec: supplementalGroups: description: A list of groups applied to the first process run in each container, in addition to the container's primary GID. If - unspecified, no groups will be added to any container. + unspecified, no groups will be added to any container. Note + that this field cannot be set when spec.os.name is windows. items: format: int64 type: integer @@ -3578,7 +3709,8 @@ spec: sysctls: description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) - might fail to launch. + might fail to launch. Note that this field cannot be set when + spec.os.name is windows. items: description: Sysctl defines a kernel parameter to be set properties: @@ -3597,7 +3729,8 @@ spec: description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. + the value specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission @@ -3792,7 +3925,11 @@ spec: type: object resources: description: 'Resources represents the minimum resources - the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity recorded + in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' properties: limits: additionalProperties: @@ -4007,7 +4144,11 @@ spec: type: object resources: description: 'Resources represents the minimum resources - the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify resource + requirements that are lower than previous value but + must still be higher than capacity recorded in the status + field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' properties: limits: additionalProperties: @@ -4103,6 +4244,27 @@ spec: items: type: string type: array + allocatedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: The storage resource within AllocatedResources + tracks the capacity allocated to a PVC. It may be larger + than the actual capacity when a volume expansion operation + is requested. For storage quota, the larger value from + allocatedResources and PVC.spec.resources is used. If + allocatedResources is not set, PVC.spec.resources alone + is used for quota calculation. If a volume expansion + capacity request is lowered, allocatedResources is only + lowered if there are no expansion operations in progress + and if the actual volume capacity is equal or lower + than the requested capacity. This is an alpha field + and requires enabling RecoverVolumeExpansionFailure + feature. + type: object capacity: additionalProperties: anyOf: @@ -4155,6 +4317,13 @@ spec: phase: description: Phase represents the current phase of PersistentVolumeClaim. type: string + resizeStatus: + description: ResizeStatus stores status of resize operation. + ResizeStatus is not set by default but when expansion + is complete resizeStatus is set to empty string by resize + controller or kubelet. This is an alpha field and requires + enabling RecoverVolumeExpansionFailure feature. + type: string type: object type: object type: object @@ -4287,7 +4456,7 @@ spec: tells the scheduler to schedule the pod in any location, but giving higher precedence to topologies that would help reduce the skew. A constraint is considered "Unsatisfiable" for - an incoming pod if and only if every possible node assigment + an incoming pod if and only if every possible node assignment for that pod would violate "MaxSkew" on some topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 @@ -4750,9 +4919,7 @@ spec: volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information. \n A pod can use both types of ephemeral volumes and persistent - volumes at the same time. \n This is a beta feature and only - available when the GenericEphemeralVolume feature gate is - enabled." + volumes at the same time." properties: volumeClaimTemplate: description: "Will be used to create a stand-alone PVC to @@ -4869,7 +5036,11 @@ spec: type: object resources: description: 'Resources represents the minimum resources - the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity recorded + in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' properties: limits: additionalProperties: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml index 55522bf..d4d4d26 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml index 82853fa..aa2f8d2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml index 7470214..d94310d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -1640,8 +1640,7 @@ spec: info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should - be specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -1702,9 +1701,10 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving - a TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward compatibility. + There are no validation of this field and lifecycle + hooks will fail in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name to connect to, @@ -1727,18 +1727,16 @@ spec: is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container - crashes or exits. The reason for termination is passed - to the handler. The Pod''s termination grace period countdown - begins before the PreStop hooked is executed. Regardless - of the outcome of the handler, the container will eventually - terminate within the Pod''s termination grace period. - Other management of the container blocks until the hook - completes or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + crashes or exits. The Pod''s termination grace period + countdown begins before the PreStop hook is executed. + Regardless of the outcome of the handler, the container + will eventually terminate within the Pod''s termination + grace period (unless delayed by finalizers). Other management + of the container blocks until the hook completes or until + the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should - be specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -1799,9 +1797,10 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving - a TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward compatibility. + There are no validation of this field and lifecycle + hooks will fail in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name to connect to, @@ -1826,8 +1825,7 @@ spec: info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -1848,6 +1846,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -1911,9 +1928,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -2011,8 +2027,7 @@ spec: fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -2033,6 +2048,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -2096,9 +2130,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -2179,12 +2212,14 @@ spec: This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' + 2) has CAP_SYS_ADMIN Note that this field cannot be set + when spec.os.name is windows.' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by - the container runtime. + the container runtime. Note that this field cannot be + set when spec.os.name is windows. properties: add: description: Added capabilities @@ -2204,25 +2239,29 @@ spec: privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to - root on the host. Defaults to false. + root on the host. Defaults to false. Note that this field + cannot be set when spec.os.name is windows. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType - feature flag to be enabled. + feature flag to be enabled. Note that this field cannot + be set when spec.os.name is windows. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root - filesystem. Default is false. + filesystem. Default is false. Note that this field cannot + be set when spec.os.name is windows. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -2240,7 +2279,8 @@ spec: process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. + value specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -2249,7 +2289,8 @@ spec: random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. properties: level: description: Level is SELinux level label that applies @@ -2272,6 +2313,8 @@ spec: description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. + Note that this field cannot be set when spec.os.name is + windows. properties: localhostProfile: description: localhostProfile indicates a profile defined @@ -2297,6 +2340,8 @@ spec: containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is + linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission @@ -2342,8 +2387,7 @@ spec: This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -2364,6 +2408,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -2427,9 +2490,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -2925,8 +2987,7 @@ spec: info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should - be specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -2987,9 +3048,10 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving - a TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward compatibility. + There are no validation of this field and lifecycle + hooks will fail in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name to connect to, @@ -3012,18 +3074,16 @@ spec: is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container - crashes or exits. The reason for termination is passed - to the handler. The Pod''s termination grace period countdown - begins before the PreStop hooked is executed. Regardless - of the outcome of the handler, the container will eventually - terminate within the Pod''s termination grace period. - Other management of the container blocks until the hook - completes or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + crashes or exits. The Pod''s termination grace period + countdown begins before the PreStop hook is executed. + Regardless of the outcome of the handler, the container + will eventually terminate within the Pod''s termination + grace period (unless delayed by finalizers). Other management + of the container blocks until the hook completes or until + the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should - be specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -3084,9 +3144,10 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving - a TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward compatibility. + There are no validation of this field and lifecycle + hooks will fail in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name to connect to, @@ -3111,8 +3172,7 @@ spec: info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -3133,6 +3193,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -3196,9 +3275,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -3296,8 +3374,7 @@ spec: fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -3318,6 +3395,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -3381,9 +3477,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -3464,12 +3559,14 @@ spec: This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' + 2) has CAP_SYS_ADMIN Note that this field cannot be set + when spec.os.name is windows.' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by - the container runtime. + the container runtime. Note that this field cannot be + set when spec.os.name is windows. properties: add: description: Added capabilities @@ -3489,25 +3586,29 @@ spec: privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to - root on the host. Defaults to false. + root on the host. Defaults to false. Note that this field + cannot be set when spec.os.name is windows. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType - feature flag to be enabled. + feature flag to be enabled. Note that this field cannot + be set when spec.os.name is windows. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root - filesystem. Default is false. + filesystem. Default is false. Note that this field cannot + be set when spec.os.name is windows. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -3525,7 +3626,8 @@ spec: process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. + value specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -3534,7 +3636,8 @@ spec: random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. properties: level: description: Level is SELinux level label that applies @@ -3557,6 +3660,8 @@ spec: description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. + Note that this field cannot be set when spec.os.name is + windows. properties: localhostProfile: description: localhostProfile indicates a profile defined @@ -3582,6 +3687,8 @@ spec: containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is + linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission @@ -3627,8 +3734,7 @@ spec: This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -3649,6 +3755,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -3712,9 +3837,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -4261,6 +4385,14 @@ spec: bearerTokenFile: description: File to read bearer token for remote read. type: string + headers: + additionalProperties: + type: string + description: Custom HTTP headers to be sent along with each + remote read request. Be aware that headers that are set by + Prometheus itself can't be overwritten. Only valid in Prometheus + versions 2.26.0 and newer. + type: object name: description: The name of the remote read queue, must be unique if specified. The name is used in metrics and logging in order @@ -4736,6 +4868,11 @@ spec: description: MinShards is the minimum number of shards, i.e. amount of concurrency. type: integer + retryOnRateLimit: + description: Retry upon receiving a 429 status code from + the remote-write storage. This is experimental feature + and might change in the future. + type: boolean type: object remoteTimeout: description: Timeout for requests to the remote write endpoint. @@ -5173,7 +5310,8 @@ spec: set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of - any volume." + any volume. Note that this field cannot be set when spec.os.name + is windows." format: int64 type: integer fsGroupChangePolicy: @@ -5183,13 +5321,15 @@ spec: support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". - If not specified, "Always" is used.' + If not specified, "Always" is used. Note that this field cannot + be set when spec.os.name is windows.' type: string runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -5206,7 +5346,8 @@ spec: Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. + takes precedence for that container. Note that this field cannot + be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -5215,6 +5356,7 @@ spec: SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. properties: level: description: Level is SELinux level label that applies to @@ -5235,7 +5377,8 @@ spec: type: object seccompProfile: description: The seccomp options to use by the containers in this - pod. + pod. Note that this field cannot be set when spec.os.name is + windows. properties: localhostProfile: description: localhostProfile indicates a profile defined @@ -5257,7 +5400,8 @@ spec: supplementalGroups: description: A list of groups applied to the first process run in each container, in addition to the container's primary GID. If - unspecified, no groups will be added to any container. + unspecified, no groups will be added to any container. Note + that this field cannot be set when spec.os.name is windows. items: format: int64 type: integer @@ -5265,7 +5409,8 @@ spec: sysctls: description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) - might fail to launch. + might fail to launch. Note that this field cannot be set when + spec.os.name is windows. items: description: Sysctl defines a kernel parameter to be set properties: @@ -5284,7 +5429,8 @@ spec: description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. + the value specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission @@ -5580,7 +5726,11 @@ spec: type: object resources: description: 'Resources represents the minimum resources - the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity recorded + in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' properties: limits: additionalProperties: @@ -5795,7 +5945,11 @@ spec: type: object resources: description: 'Resources represents the minimum resources - the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify resource + requirements that are lower than previous value but + must still be higher than capacity recorded in the status + field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' properties: limits: additionalProperties: @@ -5891,6 +6045,27 @@ spec: items: type: string type: array + allocatedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: The storage resource within AllocatedResources + tracks the capacity allocated to a PVC. It may be larger + than the actual capacity when a volume expansion operation + is requested. For storage quota, the larger value from + allocatedResources and PVC.spec.resources is used. If + allocatedResources is not set, PVC.spec.resources alone + is used for quota calculation. If a volume expansion + capacity request is lowered, allocatedResources is only + lowered if there are no expansion operations in progress + and if the actual volume capacity is equal or lower + than the requested capacity. This is an alpha field + and requires enabling RecoverVolumeExpansionFailure + feature. + type: object capacity: additionalProperties: anyOf: @@ -5943,6 +6118,13 @@ spec: phase: description: Phase represents the current phase of PersistentVolumeClaim. type: string + resizeStatus: + description: ResizeStatus stores status of resize operation. + ResizeStatus is not set by default but when expansion + is complete resizeStatus is set to empty string by resize + controller or kubelet. This is an alpha field and requires + enabling RecoverVolumeExpansionFailure feature. + type: string type: object type: object type: object @@ -6380,7 +6562,7 @@ spec: tells the scheduler to schedule the pod in any location, but giving higher precedence to topologies that would help reduce the skew. A constraint is considered "Unsatisfiable" for - an incoming pod if and only if every possible node assigment + an incoming pod if and only if every possible node assignment for that pod would violate "MaxSkew" on some topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 @@ -6843,9 +7025,7 @@ spec: volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information. \n A pod can use both types of ephemeral volumes and persistent - volumes at the same time. \n This is a beta feature and only - available when the GenericEphemeralVolume feature gate is - enabled." + volumes at the same time." properties: volumeClaimTemplate: description: "Will be used to create a stand-alone PVC to @@ -6962,7 +7142,11 @@ spec: type: object resources: description: 'Resources represents the minimum resources - the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity recorded + in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' properties: limits: additionalProperties: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml index a110d48..5b178a8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -46,7 +46,7 @@ spec: description: 'RuleGroup is a list of sequentially evaluated recording and alerting rules. Note: PartialResponseStrategy is only used by ThanosRuler and will be ignored by Prometheus instances. Valid - values for this field are ''warn'' or ''abort''. More info: https://github.com/thanos-io/thanos/blob/master/docs/components/rule.md#partial-response' + values for this field are ''warn'' or ''abort''. More info: https://github.com/thanos-io/thanos/blob/main/docs/components/rule.md#partial-response' properties: interval: type: string diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml index 02bf69a..880b36c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml index ff5c001..3aacac5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.52.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.53.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -1152,8 +1152,7 @@ spec: info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should - be specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -1214,9 +1213,10 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving - a TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward compatibility. + There are no validation of this field and lifecycle + hooks will fail in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name to connect to, @@ -1239,18 +1239,16 @@ spec: is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container - crashes or exits. The reason for termination is passed - to the handler. The Pod''s termination grace period countdown - begins before the PreStop hooked is executed. Regardless - of the outcome of the handler, the container will eventually - terminate within the Pod''s termination grace period. - Other management of the container blocks until the hook - completes or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + crashes or exits. The Pod''s termination grace period + countdown begins before the PreStop hook is executed. + Regardless of the outcome of the handler, the container + will eventually terminate within the Pod''s termination + grace period (unless delayed by finalizers). Other management + of the container blocks until the hook completes or until + the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should - be specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -1311,9 +1309,10 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving - a TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward compatibility. + There are no validation of this field and lifecycle + hooks will fail in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name to connect to, @@ -1338,8 +1337,7 @@ spec: info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -1360,6 +1358,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -1423,9 +1440,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -1523,8 +1539,7 @@ spec: fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -1545,6 +1560,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -1608,9 +1642,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -1691,12 +1724,14 @@ spec: This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' + 2) has CAP_SYS_ADMIN Note that this field cannot be set + when spec.os.name is windows.' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by - the container runtime. + the container runtime. Note that this field cannot be + set when spec.os.name is windows. properties: add: description: Added capabilities @@ -1716,25 +1751,29 @@ spec: privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to - root on the host. Defaults to false. + root on the host. Defaults to false. Note that this field + cannot be set when spec.os.name is windows. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType - feature flag to be enabled. + feature flag to be enabled. Note that this field cannot + be set when spec.os.name is windows. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root - filesystem. Default is false. + filesystem. Default is false. Note that this field cannot + be set when spec.os.name is windows. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -1752,7 +1791,8 @@ spec: process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. + value specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -1761,7 +1801,8 @@ spec: random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. properties: level: description: Level is SELinux level label that applies @@ -1784,6 +1825,8 @@ spec: description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. + Note that this field cannot be set when spec.os.name is + windows. properties: localhostProfile: description: localhostProfile indicates a profile defined @@ -1809,6 +1852,8 @@ spec: containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is + linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission @@ -1854,8 +1899,7 @@ spec: This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -1876,6 +1920,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -1939,9 +2002,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -2465,8 +2527,7 @@ spec: info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should - be specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -2527,9 +2588,10 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving - a TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward compatibility. + There are no validation of this field and lifecycle + hooks will fail in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name to connect to, @@ -2552,18 +2614,16 @@ spec: is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container - crashes or exits. The reason for termination is passed - to the handler. The Pod''s termination grace period countdown - begins before the PreStop hooked is executed. Regardless - of the outcome of the handler, the container will eventually - terminate within the Pod''s termination grace period. - Other management of the container blocks until the hook - completes or until the termination grace period is reached. - More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + crashes or exits. The Pod''s termination grace period + countdown begins before the PreStop hook is executed. + Regardless of the outcome of the handler, the container + will eventually terminate within the Pod''s termination + grace period (unless delayed by finalizers). Other management + of the container blocks until the hook completes or until + the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should - be specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -2624,9 +2684,10 @@ spec: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving - a TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: Deprecated. TCPSocket is NOT supported + as a LifecycleHandler and kept for the backward compatibility. + There are no validation of this field and lifecycle + hooks will fail in runtime when tcp handler is specified. properties: host: description: 'Optional: Host name to connect to, @@ -2651,8 +2712,7 @@ spec: info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -2673,6 +2733,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -2736,9 +2815,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -2836,8 +2914,7 @@ spec: fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -2858,6 +2935,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -2921,9 +3017,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -3004,12 +3099,14 @@ spec: This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' + 2) has CAP_SYS_ADMIN Note that this field cannot be set + when spec.os.name is windows.' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by - the container runtime. + the container runtime. Note that this field cannot be + set when spec.os.name is windows. properties: add: description: Added capabilities @@ -3029,25 +3126,29 @@ spec: privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to - root on the host. Defaults to false. + root on the host. Defaults to false. Note that this field + cannot be set when spec.os.name is windows. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType - feature flag to be enabled. + feature flag to be enabled. Note that this field cannot + be set when spec.os.name is windows. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root - filesystem. Default is false. + filesystem. Default is false. Note that this field cannot + be set when spec.os.name is windows. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -3065,7 +3166,8 @@ spec: process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. + value specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -3074,7 +3176,8 @@ spec: random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. properties: level: description: Level is SELinux level label that applies @@ -3097,6 +3200,8 @@ spec: description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container level, the container options override the pod options. + Note that this field cannot be set when spec.os.name is + windows. properties: localhostProfile: description: localhostProfile indicates a profile defined @@ -3122,6 +3227,8 @@ spec: containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is + linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission @@ -3167,8 +3274,7 @@ spec: This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be - specified. Exec specifies the action to take. + description: Exec specifies the action to take. properties: command: description: Command is the command line to execute @@ -3189,6 +3295,25 @@ spec: to 3. Minimum value is 1. format: int32 type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is an alpha field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number + must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to + place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior + is defined by gRPC." + type: string + required: + - port + type: object httpGet: description: HTTPGet specifies the http request to perform. properties: @@ -3252,9 +3377,8 @@ spec: format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a - TCP port. TCP hooks not yet supported TODO: implement - a realistic TCP lifecycle hook' + description: TCPSocket specifies an action involving a TCP + port. properties: host: description: 'Optional: Host name to connect to, defaults @@ -3688,7 +3812,8 @@ spec: set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of - any volume." + any volume. Note that this field cannot be set when spec.os.name + is windows." format: int64 type: integer fsGroupChangePolicy: @@ -3698,13 +3823,15 @@ spec: support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". - If not specified, "Always" is used.' + If not specified, "Always" is used. Note that this field cannot + be set when spec.os.name is windows.' type: string runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -3721,7 +3848,8 @@ spec: Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. + takes precedence for that container. Note that this field cannot + be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -3730,6 +3858,7 @@ spec: SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. properties: level: description: Level is SELinux level label that applies to @@ -3750,7 +3879,8 @@ spec: type: object seccompProfile: description: The seccomp options to use by the containers in this - pod. + pod. Note that this field cannot be set when spec.os.name is + windows. properties: localhostProfile: description: localhostProfile indicates a profile defined @@ -3772,7 +3902,8 @@ spec: supplementalGroups: description: A list of groups applied to the first process run in each container, in addition to the container's primary GID. If - unspecified, no groups will be added to any container. + unspecified, no groups will be added to any container. Note + that this field cannot be set when spec.os.name is windows. items: format: int64 type: integer @@ -3780,7 +3911,8 @@ spec: sysctls: description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) - might fail to launch. + might fail to launch. Note that this field cannot be set when + spec.os.name is windows. items: description: Sysctl defines a kernel parameter to be set properties: @@ -3799,7 +3931,8 @@ spec: description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, - the value specified in SecurityContext takes precedence. + the value specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission @@ -3986,7 +4119,11 @@ spec: type: object resources: description: 'Resources represents the minimum resources - the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity recorded + in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' properties: limits: additionalProperties: @@ -4201,7 +4338,11 @@ spec: type: object resources: description: 'Resources represents the minimum resources - the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify resource + requirements that are lower than previous value but + must still be higher than capacity recorded in the status + field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' properties: limits: additionalProperties: @@ -4297,6 +4438,27 @@ spec: items: type: string type: array + allocatedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: The storage resource within AllocatedResources + tracks the capacity allocated to a PVC. It may be larger + than the actual capacity when a volume expansion operation + is requested. For storage quota, the larger value from + allocatedResources and PVC.spec.resources is used. If + allocatedResources is not set, PVC.spec.resources alone + is used for quota calculation. If a volume expansion + capacity request is lowered, allocatedResources is only + lowered if there are no expansion operations in progress + and if the actual volume capacity is equal or lower + than the requested capacity. This is an alpha field + and requires enabling RecoverVolumeExpansionFailure + feature. + type: object capacity: additionalProperties: anyOf: @@ -4349,6 +4511,13 @@ spec: phase: description: Phase represents the current phase of PersistentVolumeClaim. type: string + resizeStatus: + description: ResizeStatus stores status of resize operation. + ResizeStatus is not set by default but when expansion + is complete resizeStatus is set to empty string by resize + controller or kubelet. This is an alpha field and requires + enabling RecoverVolumeExpansionFailure feature. + type: string type: object type: object type: object @@ -4912,9 +5081,7 @@ spec: volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information. \n A pod can use both types of ephemeral volumes and persistent - volumes at the same time. \n This is a beta feature and only - available when the GenericEphemeralVolume feature gate is - enabled." + volumes at the same time." properties: volumeClaimTemplate: description: "Will be used to create a stand-alone PVC to @@ -5031,7 +5198,11 @@ spec: type: object resources: description: 'Resources represents the minimum resources - the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity recorded + in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' properties: limits: additionalProperties: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl index 2c4bc64..e806178 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl @@ -163,4 +163,40 @@ Use the prometheus-node-exporter namespace override for multi-namespace deployme {{- else -}} {{- print "policy/v1beta1" -}} {{- end -}} + {{- end -}} + +{{/* Get value based on current Kubernetes version */}} +{{- define "kube-prometheus-stack.kubeVersionDefaultValue" -}} + {{- $values := index . 0 -}} + {{- $kubeVersion := index . 1 -}} + {{- $old := index . 2 -}} + {{- $new := index . 3 -}} + {{- $default := index . 4 -}} + {{- if kindIs "invalid" $default -}} + {{- if semverCompare $kubeVersion (include "kube-prometheus-stack.kubeVersion" $values) -}} + {{- print $new -}} + {{- else -}} + {{- print $old -}} + {{- end -}} + {{- else -}} + {{- print $default }} + {{- end -}} +{{- end -}} + +{{/* Get value for kube-controller-manager depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeControllerManager.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.22-0" $insecure $secure $userValue) -}} +{{- end -}} + +{{/* Get value for kube-scheduler depending on insecure scraping availability */}} +{{- define "kube-prometheus-stack.kubeScheduler.insecureScrape" -}} + {{- $values := index . 0 -}} + {{- $insecure := index . 1 -}} + {{- $secure := index . 2 -}} + {{- $userValue := index . 3 -}} + {{- include "kube-prometheus-stack.kubeVersionDefaultValue" (list $values ">= 1.23-0" $insecure $secure $userValue) -}} {{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml index 5ebc70e..b296acf 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml @@ -13,7 +13,15 @@ metadata: {{- end }} spec: {{- if .Values.alertmanager.alertmanagerSpec.image }} - image: {{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }} + {{- if and .Values.alertmanager.alertmanagerSpec.image.tag .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.sha }} + image: "{{ .Values.alertmanager.alertmanagerSpec.image.repository }}@sha256:{{ .Values.alertmanager.alertmanagerSpec.image.sha }}" + {{- else if .Values.alertmanager.alertmanagerSpec.image.tag }} + image: "{{ .Values.alertmanager.alertmanagerSpec.image.repository }}:{{ .Values.alertmanager.alertmanagerSpec.image.tag }}" + {{- else }} + image: "{{ .Values.alertmanager.alertmanagerSpec.image.repository }}" + {{- end }} version: {{ .Values.alertmanager.alertmanagerSpec.image.tag }} {{- if .Values.alertmanager.alertmanagerSpec.image.sha }} sha: {{ .Values.alertmanager.alertmanagerSpec.image.sha }} @@ -91,7 +99,7 @@ spec: - topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} labelSelector: matchExpressions: - - {key: app, operator: In, values: [alertmanager]} + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-alertmanager]} {{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} podAntiAffinity: @@ -101,7 +109,7 @@ spec: topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} labelSelector: matchExpressions: - - {key: app, operator: In, values: [alertmanager]} + - {key: app.kubernetes.io/name, operator: In, values: [alertmanager]} - {key: alertmanager, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-alertmanager]} {{- end }} {{- if .Values.alertmanager.alertmanagerSpec.tolerations }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/podDisruptionBudget.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/podDisruptionBudget.yaml index c8d977e..76880b9 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/podDisruptionBudget.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/podDisruptionBudget.yaml @@ -16,6 +16,6 @@ spec: {{- end }} selector: matchLabels: - app: alertmanager + app.kubernetes.io/name: alertmanager alertmanager: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/psp.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/psp.yaml index 6fa4450..b60fbbd 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/psp.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/psp.yaml @@ -12,12 +12,6 @@ metadata: {{ include "kube-prometheus-stack.labels" . | indent 4 }} spec: privileged: false - # Required to prevent escalations to root. - # allowPrivilegeEscalation: false - # This is redundant with non-root + disallow privilege escalation, - # but we can provide it for defense in depth. - #requiredDropCapabilities: - # - ALL # Allow core volume types. volumes: - 'configMap' @@ -38,13 +32,13 @@ spec: supplementalGroups: rule: 'MustRunAs' ranges: - # Forbid adding the root group. + # Allow adding the root group. - min: 0 max: 65535 fsGroup: rule: 'MustRunAs' ranges: - # Forbid adding the root group. + # Allow adding the root group. - min: 0 max: 65535 readOnlyRootFilesystem: false diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-controller-manager/service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-controller-manager/service.yaml index d55ca2a..197f0f4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-controller-manager/service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-controller-manager/service.yaml @@ -12,9 +12,11 @@ spec: clusterIP: None ports: - name: http-metrics - port: {{ .Values.kubeControllerManager.service.port }} + {{- $kubeControllerManagerDefaultInsecurePort := 10252 }} + {{- $kubeControllerManagerDefaultSecurePort := 10257 }} + port: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.port) }} protocol: TCP - targetPort: {{ .Values.kubeControllerManager.service.targetPort }} + targetPort: {{ include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . $kubeControllerManagerDefaultInsecurePort $kubeControllerManagerDefaultSecurePort .Values.kubeControllerManager.service.targetPort) }} {{- if .Values.kubeControllerManager.endpoints }}{{- else }} selector: {{- if .Values.kubeControllerManager.service.selector }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-controller-manager/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-controller-manager/servicemonitor.yaml index 809beb1..3b26ac7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-controller-manager/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-controller-manager/servicemonitor.yaml @@ -25,12 +25,12 @@ spec: {{- if .Values.kubeControllerManager.serviceMonitor.proxyUrl }} proxyUrl: {{ .Values.kubeControllerManager.serviceMonitor.proxyUrl}} {{- end }} - {{- if .Values.kubeControllerManager.serviceMonitor.https }} + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . false true .Values.kubeControllerManager.serviceMonitor.https )) "true" }} scheme: https tlsConfig: caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - {{- if .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify }} - insecureSkipVerify: {{ .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify }} + {{- if eq (include "kube-prometheus-stack.kubeControllerManager.insecureScrape" (list . nil true .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true {{- end }} {{- if .Values.kubeControllerManager.serviceMonitor.serverName }} serverName: {{ .Values.kubeControllerManager.serviceMonitor.serverName }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-scheduler/service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-scheduler/service.yaml index 7a9c53d..eef9df0 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-scheduler/service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-scheduler/service.yaml @@ -12,9 +12,11 @@ spec: clusterIP: None ports: - name: http-metrics - port: {{ .Values.kubeScheduler.service.port}} + {{- $kubeSchedulerDefaultInsecurePort := 10251 }} + {{- $kubeSchedulerDefaultSecurePort := 10259 }} + port: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.port) }} protocol: TCP - targetPort: {{ .Values.kubeScheduler.service.targetPort}} + targetPort: {{ include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . $kubeSchedulerDefaultInsecurePort $kubeSchedulerDefaultSecurePort .Values.kubeScheduler.service.targetPort) }} {{- if .Values.kubeScheduler.endpoints }}{{- else }} selector: {{- if .Values.kubeScheduler.service.selector }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-scheduler/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-scheduler/servicemonitor.yaml index e6b5440..fd4674c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-scheduler/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-scheduler/servicemonitor.yaml @@ -25,13 +25,13 @@ spec: {{- if .Values.kubeScheduler.serviceMonitor.proxyUrl }} proxyUrl: {{ .Values.kubeScheduler.serviceMonitor.proxyUrl}} {{- end }} - {{- if .Values.kubeScheduler.serviceMonitor.https }} + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . false true .Values.kubeScheduler.serviceMonitor.https )) "true" }} scheme: https tlsConfig: caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - {{- if .Values.kubeScheduler.serviceMonitor.insecureSkipVerify }} - insecureSkipVerify: {{ .Values.kubeScheduler.serviceMonitor.insecureSkipVerify }} - {{- end}} + {{- if eq (include "kube-prometheus-stack.kubeScheduler.insecureScrape" (list . nil true .Values.kubeScheduler.serviceMonitor.insecureSkipVerify)) "true" }} + insecureSkipVerify: true + {{- end }} {{- if .Values.kubeScheduler.serviceMonitor.serverName }} serverName: {{ .Values.kubeScheduler.serviceMonitor.serverName }} {{- end}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml deleted file mode 100644 index 8493d9d..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml +++ /dev/null @@ -1,63 +0,0 @@ -{{- if .Values.kubeStateMetrics.enabled }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ template "kube-prometheus-stack.fullname" . }}-kube-state-metrics - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }}-kube-state-metrics -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -spec: - jobLabel: app.kubernetes.io/name - endpoints: - - port: http - {{- if .Values.kubeStateMetrics.serviceMonitor.interval }} - interval: {{ .Values.kubeStateMetrics.serviceMonitor.interval }} - {{- end }} - {{- if .Values.kubeStateMetrics.serviceMonitor.scrapeTimeout }} - scrapeTimeout: {{ .Values.kubeStateMetrics.serviceMonitor.scrapeTimeout }} - {{- end }} - {{- if .Values.kubeStateMetrics.serviceMonitor.proxyUrl }} - proxyUrl: {{ .Values.kubeStateMetrics.serviceMonitor.proxyUrl}} - {{- end }} - honorLabels: {{ .Values.kubeStateMetrics.serviceMonitor.honorLabels }} -{{- if .Values.kubeStateMetrics.serviceMonitor.metricRelabelings }} - metricRelabelings: -{{ tpl (toYaml .Values.kubeStateMetrics.serviceMonitor.metricRelabelings | indent 4) . }} -{{- end }} -{{- if .Values.kubeStateMetrics.serviceMonitor.relabelings }} - relabelings: -{{ tpl (toYaml .Values.kubeStateMetrics.serviceMonitor.relabelings | indent 4) . }} -{{- end }} -{{- if .Values.kubeStateMetrics.serviceMonitor.selfMonitor.enabled }} - - port: metrics - {{- if .Values.kubeStateMetrics.serviceMonitor.interval }} - interval: {{ .Values.kubeStateMetrics.serviceMonitor.interval }} - {{- end }} - {{- if .Values.kubeStateMetrics.serviceMonitor.proxyUrl }} - proxyUrl: {{ .Values.kubeStateMetrics.serviceMonitor.proxyUrl}} - {{- end }} - honorLabels: {{ .Values.kubeStateMetrics.serviceMonitor.honorLabels }} -{{- if .Values.kubeStateMetrics.serviceMonitor.metricRelabelings }} - metricRelabelings: -{{ tpl (toYaml .Values.kubeStateMetrics.serviceMonitor.metricRelabelings | indent 4) . }} -{{- end }} -{{- if .Values.kubeStateMetrics.serviceMonitor.relabelings }} - relabelings: -{{ tpl (toYaml .Values.kubeStateMetrics.serviceMonitor.relabelings | indent 4) . }} -{{- end }} -{{- end }} - {{- if ne (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack-kube-state-metrics.namespace" .) }} - namespaceSelector: - matchNames: - - {{ printf "%s" (include "kube-prometheus-stack-kube-state-metrics.namespace" .) | quote }} - {{- end }} - selector: - matchLabels: -{{- if .Values.kubeStateMetrics.serviceMonitor.selectorOverride }} -{{ toYaml .Values.kubeStateMetrics.serviceMonitor.selectorOverride | indent 6 }} -{{ else }} - app.kubernetes.io/name: kube-state-metrics - app.kubernetes.io/instance: "{{ $.Release.Name }}" -{{- end }} -{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml deleted file mode 100644 index 02db6b2..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml +++ /dev/null @@ -1,40 +0,0 @@ -{{- if .Values.nodeExporter.enabled }} -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ template "kube-prometheus-stack.fullname" . }}-node-exporter - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }}-node-exporter -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -spec: - jobLabel: {{ .Values.nodeExporter.jobLabel }} - selector: - matchLabels: - app: prometheus-node-exporter - release: {{ $.Release.Name }} - {{- if ne (include "kube-prometheus-stack.namespace" .) (include "kube-prometheus-stack-prometheus-node-exporter.namespace" .) }} - namespaceSelector: - matchNames: - - {{ printf "%s" (include "kube-prometheus-stack-prometheus-node-exporter.namespace" .) | quote }} - {{- end }} - endpoints: - - port: {{ index .Values "prometheus-node-exporter" "service" "portName" }} - {{- if .Values.nodeExporter.serviceMonitor.interval }} - interval: {{ .Values.nodeExporter.serviceMonitor.interval }} - {{- end }} - {{- if .Values.nodeExporter.serviceMonitor.proxyUrl }} - proxyUrl: {{ .Values.nodeExporter.serviceMonitor.proxyUrl}} - {{- end }} - {{- if .Values.nodeExporter.serviceMonitor.scrapeTimeout }} - scrapeTimeout: {{ .Values.nodeExporter.serviceMonitor.scrapeTimeout }} - {{- end }} -{{- if .Values.nodeExporter.serviceMonitor.metricRelabelings }} - metricRelabelings: -{{ tpl (toYaml .Values.nodeExporter.serviceMonitor.metricRelabelings | indent 4) . }} -{{- end }} -{{- if .Values.nodeExporter.serviceMonitor.relabelings }} - relabelings: -{{ tpl (toYaml .Values.nodeExporter.serviceMonitor.relabelings | indent 4) . }} -{{- end }} -{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml index 19c43e5..9feff52 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -14,12 +14,6 @@ metadata: {{ include "kube-prometheus-stack.labels" . | indent 4 }} spec: privileged: false - # Required to prevent escalations to root. - # allowPrivilegeEscalation: false - # This is redundant with non-root + disallow privilege escalation, - # but we can provide it for defense in depth. - #requiredDropCapabilities: - # - ALL # Allow core volume types. volumes: - 'configMap' @@ -40,13 +34,13 @@ spec: supplementalGroups: rule: 'MustRunAs' ranges: - # Forbid adding the root group. + # Allow adding the root group. - min: 0 max: 65535 fsGroup: rule: 'MustRunAs' ranges: - # Forbid adding the root group. + # Allow adding the root group. - min: 0 max: 65535 readOnlyRootFilesystem: false diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/certmanager.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/certmanager.yaml index cfd5165..a1e06ae 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/certmanager.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/certmanager.yaml @@ -18,7 +18,7 @@ metadata: namespace: {{ template "kube-prometheus-stack.namespace" . }} spec: secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert - duration: 43800h0m0s # 5y + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.rootCert.duration | default "43800h0m0s" | quote }} issuerRef: name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer commonName: "ca.webhook.kube-prometheus-stack" @@ -35,7 +35,7 @@ spec: secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert {{- end }} --- -# generate a serving certificate for the apiservices to use +# generate a server certificate for the apiservices to use apiVersion: cert-manager.io/v1 kind: Certificate metadata: @@ -43,7 +43,7 @@ metadata: namespace: {{ template "kube-prometheus-stack.namespace" . }} spec: secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission - duration: 8760h0m0s # 1y + duration: {{ .Values.prometheusOperator.admissionWebhooks.certManager.admissionCert.duration | default "8760h0m0s" | quote }} issuerRef: {{- if .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef }} {{- toYaml .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef | nindent 4 }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml index 0ab8941..df6aee0 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml @@ -53,7 +53,7 @@ spec: - --deny-namespaces={{ .Values.prometheusOperator.denyNamespaces | join "," }} {{- end }} {{- with $.Values.prometheusOperator.namespaces }} - {{ $ns := .additional }} + {{ $ns := default (list nil) .additional }} {{- if .releaseNamespace }} {{- $ns = append $ns $namespace }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/psp.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/psp.yaml index 18d1d37..d9228f0 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/psp.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/psp.yaml @@ -12,12 +12,6 @@ metadata: {{ include "kube-prometheus-stack.labels" . | indent 4 }} spec: privileged: false - # Required to prevent escalations to root. - # allowPrivilegeEscalation: false - # This is redundant with non-root + disallow privilege escalation, - # but we can provide it for defense in depth. - #requiredDropCapabilities: - # - ALL # Allow core volume types. volumes: - 'configMap' @@ -38,13 +32,13 @@ spec: supplementalGroups: rule: 'MustRunAs' ranges: - # Forbid adding the root group. + # Allow adding the root group. - min: 0 max: 65535 fsGroup: rule: 'MustRunAs' ranges: - # Forbid adding the root group. + # Allow adding the root group. - min: 0 max: 65535 readOnlyRootFilesystem: false diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/_rules.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/_rules.tpl index 0e33d65..c774dab 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/_rules.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/_rules.tpl @@ -5,22 +5,22 @@ https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-promet {{- define "rules.names" }} rules: - "alertmanager.rules" + - "config-reloaders" + - "etcd" - "general.rules" - "k8s.rules" - - "kube-apiserver.rules" - "kube-apiserver-availability.rules" - - "kube-apiserver-error" - "kube-apiserver-slos" - "kube-prometheus-general.rules" - - "kube-prometheus-node-alerting.rules" - "kube-prometheus-node-recording.rules" - "kube-scheduler.rules" - "kube-state-metrics" - "kubelet.rules" - - "kubernetes-absent" + - "kubernetes-apps" - "kubernetes-resources" - "kubernetes-storage" - "kubernetes-system" + - "kubernetes-system-kube-proxy" - "kubernetes-system-apiserver" - "kubernetes-system-kubelet" - "kubernetes-system-controller-manager" @@ -29,10 +29,6 @@ rules: - "node-exporter" - "node.rules" - "node-network" - - "node-time" - "prometheus-operator" - - "prometheus.rules" - "prometheus" - - "kubernetes-apps" - - "etcd" {{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml index 7b5e12c..bfb5794 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml @@ -202,7 +202,6 @@ spec: {{- else if .Values.prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues }} ruleSelector: matchLabels: - app: {{ template "kube-prometheus-stack.name" . }} release: {{ $.Release.Name | quote }} {{ else }} ruleSelector: {} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/psp.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/psp.yaml index 08da5e1..eecf992 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/psp.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/psp.yaml @@ -12,12 +12,6 @@ metadata: {{ include "kube-prometheus-stack.labels" . | indent 4 }} spec: privileged: false - # Required to prevent escalations to root. - # allowPrivilegeEscalation: false - # This is redundant with non-root + disallow privilege escalation, - # but we can provide it for defense in depth. - #requiredDropCapabilities: - # - ALL # Allow core volume types. volumes: - 'configMap' @@ -41,13 +35,13 @@ spec: supplementalGroups: rule: 'MustRunAs' ranges: - # Forbid adding the root group. + # Allow adding the root group. - min: 0 max: 65535 fsGroup: rule: 'MustRunAs' ranges: - # Forbid adding the root group. + # Allow adding the root group. - min: 0 max: 65535 readOnlyRootFilesystem: false diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/service.yaml index c642006..f71362c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/service.yaml @@ -50,6 +50,7 @@ spec: {{- if .Values.prometheus.service.additionalPorts }} {{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} {{- end }} + publishNotReadyAddresses: {{ .Values.prometheus.service.publishNotReadyAddresses }} selector: app.kubernetes.io/name: prometheus prometheus: {{ template "kube-prometheus-stack.fullname" . }}-prometheus diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml index 8be9f18..4782a75 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml @@ -35,17 +35,16 @@ defaultRules: rules: alertmanager: true etcd: true + configReloaders: true general: true k8s: true kubeApiserver: true kubeApiserverAvailability: true - kubeApiserverError: true kubeApiserverSlos: true kubelet: true + kubeProxy: true kubePrometheusGeneral: true - kubePrometheusNodeAlerting: true kubePrometheusNodeRecording: true - kubernetesAbsent: true kubernetesApps: true kubernetesResources: true kubernetesStorage: true @@ -54,12 +53,11 @@ defaultRules: kubeStateMetrics: true network: true node: true + nodeExporterAlerting: true + nodeExporterRecording: true prometheus: true prometheusOperator: true - time: true - ## Runbook url prefix for default rules - runbookUrl: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md# ## Reduce app namespace alert scope appNamespacesTarget: ".*" @@ -71,6 +69,14 @@ defaultRules: ## Additional labels for PrometheusRule alerts additionalRuleLabels: {} + ## Prefix for runbook URLs. Use this to override the first part of the runbookURLs that is common to all rules. + runbookUrl: "https://runbooks.prometheus-operator.dev/runbooks" + + ## Disabled PrometheusRule alerts + disabled: {} + # KubeAPIDown: true + # NodeRAIDDegraded: true + ## Deprecated way to provide custom recording or alerting rules to be deployed into the cluster. ## # additionalPrometheusRules: [] @@ -95,7 +101,7 @@ additionalPrometheusRulesMap: {} global: rbac: create: true - pspEnabled: true + pspEnabled: false pspAnnotations: {} ## Specify pod annotations ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor @@ -361,13 +367,13 @@ alertmanager: scheme: "" ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. - ## Of type: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#tlsconfig + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig tlsConfig: {} bearerTokenFile: ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep @@ -375,7 +381,7 @@ alertmanager: # sourceLabels: [__name__] ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -386,7 +392,7 @@ alertmanager: # action: replace ## Settings affecting alertmanagerSpec - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#alertmanagerspec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerspec ## alertmanagerSpec: ## Standard object's metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata @@ -475,7 +481,7 @@ alertmanager: retention: 120h ## Storage is the definition of how storage will be used by the Alertmanager instances. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/storage.md + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md ## storage: {} # volumeClaimTemplate: @@ -644,6 +650,11 @@ grafana: adminPassword: prom-operator + rbac: + ## If true, Grafana PSPs will be created + ## + pspEnabled: false + ingress: ## If true, Grafana Ingress will be created ## @@ -759,7 +770,7 @@ grafana: ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep @@ -767,7 +778,7 @@ grafana: # sourceLabels: [__name__] ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -799,7 +810,7 @@ kubeApiServer: provider: kubernetes ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep @@ -807,7 +818,7 @@ kubeApiServer: # sourceLabels: [__name__] ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: @@ -855,7 +866,7 @@ kubelet: resourcePath: "/metrics/resource/v1alpha1" ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## cAdvisorMetricRelabelings: [] # - sourceLabels: [__name__, image] @@ -870,7 +881,7 @@ kubelet: # action: drop ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## probesMetricRelabelings: [] # - sourceLabels: [__name__, image] @@ -885,7 +896,7 @@ kubelet: # action: drop ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## ## metrics_path is required to match upstream rules and charts cAdvisorRelabelings: @@ -899,7 +910,7 @@ kubelet: # action: replace ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## probesRelabelings: - sourceLabels: [__metrics_path__] @@ -912,7 +923,7 @@ kubelet: # action: replace ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## resourceRelabelings: - sourceLabels: [__metrics_path__] @@ -925,7 +936,7 @@ kubelet: # action: replace ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - sourceLabels: [__name__, image] @@ -940,7 +951,7 @@ kubelet: # action: drop ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## ## metrics_path is required to match upstream rules and charts relabelings: @@ -969,8 +980,11 @@ kubeControllerManager: ## service: enabled: true - port: 10252 - targetPort: 10252 + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.22. + ## + port: null + targetPort: null # selector: # component: kube-controller-manager @@ -985,9 +999,10 @@ kubeControllerManager: proxyUrl: "" ## Enable scraping kube-controller-manager over https. - ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. ## - https: false + https: null # Skip TLS certificate validation when scraping insecureSkipVerify: null @@ -996,7 +1011,7 @@ kubeControllerManager: serverName: null ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep @@ -1004,7 +1019,7 @@ kubeControllerManager: # sourceLabels: [__name__] ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1033,7 +1048,7 @@ coreDns: proxyUrl: "" ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep @@ -1041,7 +1056,7 @@ coreDns: # sourceLabels: [__name__] ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1074,7 +1089,7 @@ kubeDns: proxyUrl: "" ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep @@ -1082,7 +1097,7 @@ kubeDns: # sourceLabels: [__name__] ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1093,7 +1108,7 @@ kubeDns: # action: replace ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## dnsmasqMetricRelabelings: [] # - action: keep @@ -1101,7 +1116,7 @@ kubeDns: # sourceLabels: [__name__] ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## dnsmasqRelabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1159,7 +1174,7 @@ kubeEtcd: keyFile: "" ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep @@ -1167,7 +1182,7 @@ kubeEtcd: # sourceLabels: [__name__] ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1194,8 +1209,11 @@ kubeScheduler: ## service: enabled: true - port: 10251 - targetPort: 10251 + ## If null or unset, the value is determined dynamically based on target Kubernetes version due to change + ## of default port in Kubernetes 1.23. + ## + port: null + targetPort: null # selector: # component: kube-scheduler @@ -1208,9 +1226,10 @@ kubeScheduler: ## proxyUrl: "" ## Enable scraping kube-scheduler over https. - ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks. + ## If null or unset, the value is determined dynamically based on target Kubernetes version. ## - https: false + https: null ## Skip TLS certificate validation when scraping insecureSkipVerify: null @@ -1219,7 +1238,7 @@ kubeScheduler: serverName: null ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep @@ -1227,7 +1246,7 @@ kubeScheduler: # sourceLabels: [__name__] ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## relabelings: [] # - sourceLabels: [__meta_kubernetes_pod_node_name] @@ -1273,7 +1292,7 @@ kubeProxy: https: false ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## metricRelabelings: [] # - action: keep @@ -1281,7 +1300,7 @@ kubeProxy: # sourceLabels: [__name__] ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig ## relabelings: [] # - action: keep @@ -1293,45 +1312,6 @@ kubeProxy: ## kubeStateMetrics: enabled: true - serviceMonitor: - ## Scrape interval. If not set, the Prometheus default scrape interval is used. - ## - interval: "" - ## Scrape Timeout. If not set, the Prometheus default scrape timeout is used. - ## - scrapeTimeout: "" - ## proxyUrl: URL of a proxy that should be used for scraping. - ## - proxyUrl: "" - ## Override serviceMonitor selector - ## - selectorOverride: {} - - ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig - ## - metricRelabelings: [] - # - action: keep - # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' - # sourceLabels: [__name__] - - ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig - ## - relabelings: [] - # - sourceLabels: [__meta_kubernetes_pod_node_name] - # separator: ; - # regex: ^(.*)$ - # targetLabel: nodename - # replacement: $1 - # action: replace - - # Keep labels from scraped data, overriding server-side labels - honorLabels: true - - # Enable self metrics configuration for Service Monitor - selfMonitor: - enabled: false ## Configuration for kube-state-metrics subchart ## @@ -1339,52 +1319,54 @@ kube-state-metrics: namespaceOverride: "" rbac: create: true - podSecurityPolicy: - enabled: true + releaseLabel: true + prometheus: + monitor: + enabled: true + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Scrape Timeout. If not set, the Prometheus default scrape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + # Keep labels from scraped data, overriding server-side labels + ## + honorLabels: true + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + selfMonitor: + enabled: false ## Deploy node exporter as a daemonset to all nodes ## nodeExporter: enabled: true - ## Use the value configured in prometheus-node-exporter.podLabels - ## - jobLabel: jobLabel - - serviceMonitor: - ## Scrape interval. If not set, the Prometheus default scrape interval is used. - ## - interval: "" - - ## proxyUrl: URL of a proxy that should be used for scraping. - ## - proxyUrl: "" - - ## How long until a scrape request times out. If not set, the Prometheus default scape timeout is used. - ## - scrapeTimeout: "" - - ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig - ## - metricRelabelings: [] - # - sourceLabels: [__name__] - # separator: ; - # regex: ^node_mountstats_nfs_(event|operations|transport)_.+ - # replacement: $1 - # action: drop - - ## RelabelConfigs to apply to samples before scraping - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig - ## - relabelings: [] - # - sourceLabels: [__meta_kubernetes_pod_node_name] - # separator: ; - # regex: ^(.*)$ - # targetLabel: nodename - # replacement: $1 - # action: replace - ## Configuration for prometheus-node-exporter subchart ## prometheus-node-exporter: @@ -1398,6 +1380,48 @@ prometheus-node-exporter: - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ service: portName: http-metrics + prometheus: + monitor: + enabled: true + + jobLabel: jobLabel + + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## How long until a scrape request times out. If not set, the Prometheus default scape timeout is used. + ## + scrapeTimeout: "" + + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + + ## MetricRelabelConfigs to apply to samples after scraping, but before ingestion. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + metricRelabelings: [] + # - sourceLabels: [__name__] + # separator: ; + # regex: ^node_mountstats_nfs_(event|operations|transport)_.+ + # replacement: $1 + # action: drop + + ## RelabelConfigs to apply to samples before scraping + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#relabelconfig + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + rbac: + ## If true, create PSPs for node-exporter + ## + pspEnabled: false ## Manages Prometheus and Alertmanager components ## @@ -1453,6 +1477,11 @@ prometheusOperator: # Use certmanager to generate webhook certs certManager: enabled: false + # self-signed root certificate + rootCert: + duration: "" # default to be 5y + admissionCert: + duration: "" # default to be 1y # issuerRef: # name: "issuer" # kind: "ClusterIssuer" @@ -1542,7 +1571,7 @@ prometheusOperator: # logLevel: error ## If true, the operator will create and maintain a service for scraping kubelets - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/helm/prometheus-operator/README.md + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/helm/prometheus-operator/README.md ## kubeletService: enabled: true @@ -1639,7 +1668,7 @@ prometheusOperator: ## image: repository: quay.io/prometheus-operator/prometheus-operator - tag: v0.52.0 + tag: v0.53.1 sha: "" pullPolicy: IfNotPresent @@ -1657,7 +1686,7 @@ prometheusOperator: # image to use for config and rule reloading image: repository: quay.io/prometheus-operator/prometheus-config-reloader - tag: v0.52.0 + tag: v0.53.1 sha: "" # resource config for prometheusConfigReloader @@ -1673,7 +1702,7 @@ prometheusOperator: ## thanosImage: repository: quay.io/thanos/thanos - tag: v0.23.1 + tag: v0.24.0 sha: "" ## Set a Field Selector to filter watched secrets @@ -1741,7 +1770,7 @@ prometheus: scheme: "" ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. - ## Of type: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#tlsconfig + ## Of type: https://github.com/coreos/prometheus-operator/blob/main/Documentation/api.md#tlsconfig tlsConfig: {} bearerTokenFile: @@ -1812,6 +1841,13 @@ prometheus: ## type: ClusterIP + ## Additional port to define in the Service + additionalPorts: [] + + ## Consider that all endpoints are considered "ready" even if the Pods themselves are not + ## Ref: https://kubernetes.io/docs/reference/kubernetes-api/service-resources/service-v1/#ServiceSpec + publishNotReadyAddresses: false + sessionAffinity: "" ## Configuration for creating a separate Service for each statefulset Prometheus replica @@ -1995,7 +2031,7 @@ prometheus: scheme: "" ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. - ## Of type: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#tlsconfig + ## Of type: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#tlsconfig tlsConfig: {} bearerTokenFile: @@ -2018,14 +2054,14 @@ prometheus: # action: replace ## Settings affecting prometheusSpec - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#prometheusspec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#prometheusspec ## prometheusSpec: ## If true, pass --storage.tsdb.max-block-duration=2h to prometheus. This is already done if using Thanos ## disableCompaction: false ## APIServerConfig - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#apiserverconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#apiserverconfig ## apiserverConfig: {} @@ -2054,7 +2090,7 @@ prometheus: enableAdminAPI: false ## WebTLSConfig defines the TLS parameters for HTTPS - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#webtlsconfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#webtlsconfig web: {} # EnableFeatures API enables access to Prometheus disabled features. @@ -2066,7 +2102,7 @@ prometheus: ## image: repository: quay.io/prometheus/prometheus - tag: v2.31.1 + tag: v2.32.1 sha: "" ## Tolerations for use with node taints @@ -2090,7 +2126,7 @@ prometheus: # app: prometheus ## Alertmanagers to which alerts will be sent - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#alertmanagerendpoints + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#alertmanagerendpoints ## ## Default configuration will connect to the alertmanager deployed as part of this release ## @@ -2146,13 +2182,13 @@ prometheus: configMaps: [] ## QuerySpec defines the query command line flags when starting Prometheus. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#queryspec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#queryspec ## query: {} ## Namespaces to be selected for PrometheusRules discovery. ## If nil, select own namespace. Namespaces to be selected for ServiceMonitor discovery. - ## See https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#namespaceselector for usage + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#namespaceselector for usage ## ruleNamespaceSelector: {} @@ -2220,7 +2256,7 @@ prometheus: # prometheus: somelabel ## Namespaces to be selected for PodMonitor discovery. - ## See https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#namespaceselector for usage + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#namespaceselector for usage ## podMonitorNamespaceSelector: {} @@ -2240,7 +2276,7 @@ prometheus: # prometheus: somelabel ## Namespaces to be selected for Probe discovery. - ## See https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#namespaceselector for usage + ## See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#namespaceselector for usage ## probeNamespaceSelector: {} @@ -2321,14 +2357,14 @@ prometheus: # - e2e-az2 ## The remote_read spec configuration for Prometheus. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#remotereadspec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotereadspec remoteRead: [] # - url: http://remote1/read ## additionalRemoteRead is appended to remoteRead additionalRemoteRead: [] ## The remote_write spec configuration for Prometheus. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#remotewritespec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#remotewritespec remoteWrite: [] # - url: http://remote1/push ## additionalRemoteWrite is appended to remoteWrite @@ -2344,7 +2380,7 @@ prometheus: # memory: 400Mi ## Prometheus StorageSpec for persistent data - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/storage.md + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md ## storageSpec: {} ## Using PersistentVolumeClaim @@ -2460,7 +2496,7 @@ prometheus: ## SecurityContext holds pod-level security attributes and common container settings. ## This defaults to non root user with uid 1000 and gid 2000. - ## https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md + ## https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md ## securityContext: runAsGroup: 2000 @@ -2475,7 +2511,7 @@ prometheus: ## Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. ## This section is experimental, it may change significantly without deprecation notice in any release. ## This is experimental and may change significantly without backward compatibility in any release. - ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#thanosspec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#thanosspec ## thanos: {} # secretProviderClass: @@ -2693,6 +2729,6 @@ prometheus: # matchNames: [] ## Endpoints of the selected pods to be monitored - ## https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#podmetricsendpoint + ## https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmetricsendpoint ## # podMetricsEndpoints: [] diff --git a/charts/kubezero-metrics/charts/prometheus-pushgateway/Chart.yaml b/charts/kubezero-metrics/charts/prometheus-pushgateway/Chart.yaml index 59e5947..17e55d0 100644 --- a/charts/kubezero-metrics/charts/prometheus-pushgateway/Chart.yaml +++ b/charts/kubezero-metrics/charts/prometheus-pushgateway/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: 1.4.1 +appVersion: 1.4.2 description: A Helm chart for prometheus pushgateway home: https://github.com/prometheus/pushgateway keywords: @@ -13,4 +13,4 @@ maintainers: name: prometheus-pushgateway sources: - https://github.com/prometheus/pushgateway -version: 1.13.0 +version: 1.14.0 diff --git a/charts/kubezero-metrics/charts/prometheus-pushgateway/values.yaml b/charts/kubezero-metrics/charts/prometheus-pushgateway/values.yaml index fbb4904..9b730d9 100644 --- a/charts/kubezero-metrics/charts/prometheus-pushgateway/values.yaml +++ b/charts/kubezero-metrics/charts/prometheus-pushgateway/values.yaml @@ -10,7 +10,7 @@ fullnameOverride: "" image: repository: prom/pushgateway - tag: v1.4.1 + tag: v1.4.2 pullPolicy: IfNotPresent # Optional pod imagePullSecrets diff --git a/charts/kubezero-metrics/configs/build.sh b/charts/kubezero-metrics/configs/build.sh index 3869153..c9c5eb8 100755 --- a/charts/kubezero-metrics/configs/build.sh +++ b/charts/kubezero-metrics/configs/build.sh @@ -9,7 +9,8 @@ which jb > /dev/null || { echo "Required jb ( json-bundler ) not found!"; exit 1 if [ -r jsonnetfile.lock.json ]; then jb update else - jb install github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus@main + #jb install github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus@main + jb install github.com/prometheus-operator/kube-prometheus/jsonnet/kube-prometheus@release-0.9 fi rm -rf dashboards && mkdir -p dashboards diff --git a/charts/kubezero-metrics/configs/dashboards/apiserver.json b/charts/kubezero-metrics/configs/dashboards/apiserver.json index bd6c74b..1c940dc 100644 --- a/charts/kubezero-metrics/configs/dashboards/apiserver.json +++ b/charts/kubezero-metrics/configs/dashboards/apiserver.json @@ -56,11 +56,7 @@ }, "gridPos": { }, "id": 3, - "interval": "1m", - "legend": { - "alignAsTable": true, - "rightSide": true - }, + "interval": null, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -132,14 +128,13 @@ "fillGradient": 0, "gridPos": { }, "id": 4, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -241,11 +236,7 @@ }, "gridPos": { }, "id": 5, - "interval": "1m", - "legend": { - "alignAsTable": true, - "rightSide": true - }, + "interval": null, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -316,14 +307,13 @@ "fillGradient": 0, "gridPos": { }, "id": 6, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -416,14 +406,13 @@ "fillGradient": 0, "gridPos": { }, "id": 7, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -499,14 +488,13 @@ "fillGradient": 0, "gridPos": { }, "id": 8, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -606,11 +594,7 @@ }, "gridPos": { }, "id": 9, - "interval": "1m", - "legend": { - "alignAsTable": true, - "rightSide": true - }, + "interval": null, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -681,14 +665,13 @@ "fillGradient": 0, "gridPos": { }, "id": 10, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -781,14 +764,13 @@ "fillGradient": 0, "gridPos": { }, "id": 11, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -864,14 +846,13 @@ "fillGradient": 0, "gridPos": { }, "id": 12, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -959,14 +940,13 @@ "fillGradient": 0, "gridPos": { }, "id": 13, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": false, "sideWidth": null, "total": false, @@ -988,7 +968,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1041,14 +1021,13 @@ "fillGradient": 0, "gridPos": { }, "id": 14, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": false, "sideWidth": null, "total": false, @@ -1070,7 +1049,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1123,7 +1102,6 @@ "fillGradient": 0, "gridPos": { }, "id": 15, - "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -1152,7 +1130,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1218,14 +1196,13 @@ "fillGradient": 0, "gridPos": { }, "id": 16, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -1300,14 +1277,13 @@ "fillGradient": 0, "gridPos": { }, "id": 17, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -1329,7 +1305,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])", + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1382,14 +1358,13 @@ "fillGradient": 0, "gridPos": { }, "id": 18, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -1477,7 +1452,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": null, "name": "datasource", "options": [ ], "query": "prometheus", @@ -1495,7 +1470,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(up{job=\"apiserver\"}, cluster)", + "query": "label_values(apiserver_request_total, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -1515,7 +1490,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "query": "label_values(apiserver_request_total{job=\"apiserver\", cluster=\"$cluster\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/cluster-total.json b/charts/kubezero-metrics/configs/dashboards/cluster-total.json index 9cab13f..86e147d 100644 --- a/charts/kubezero-metrics/configs/dashboards/cluster-total.json +++ b/charts/kubezero-metrics/configs/dashboards/cluster-total.json @@ -1619,7 +1619,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": null, "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/configs/dashboards/controller-manager.json b/charts/kubezero-metrics/configs/dashboards/controller-manager.json index 56bffc9..373d6ab 100644 --- a/charts/kubezero-metrics/configs/dashboards/controller-manager.json +++ b/charts/kubezero-metrics/configs/dashboards/controller-manager.json @@ -36,11 +36,7 @@ }, "gridPos": { }, "id": 2, - "interval": "1m", - "legend": { - "alignAsTable": true, - "rightSide": true - }, + "interval": null, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -110,7 +106,6 @@ "fillGradient": 0, "gridPos": { }, "id": 3, - "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -139,7 +134,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} {{name}}", @@ -205,7 +200,6 @@ "fillGradient": 0, "gridPos": { }, "id": 4, - "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -234,7 +228,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} {{name}}", @@ -300,7 +294,6 @@ "fillGradient": 0, "gridPos": { }, "id": 5, - "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -329,7 +322,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} {{name}}", @@ -395,14 +388,13 @@ "fillGradient": 0, "gridPos": { }, "id": 6, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -424,28 +416,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"2..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"3..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"4..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"5..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -498,14 +490,13 @@ "fillGradient": 0, "gridPos": { }, "id": 7, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -527,7 +518,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -593,7 +584,6 @@ "fillGradient": 0, "gridPos": { }, "id": 8, - "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -622,7 +612,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -688,14 +678,13 @@ "fillGradient": 0, "gridPos": { }, "id": 9, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -770,14 +759,13 @@ "fillGradient": 0, "gridPos": { }, "id": 10, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -799,7 +787,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -852,14 +840,13 @@ "fillGradient": 0, "gridPos": { }, "id": 11, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -947,7 +934,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": null, "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/configs/dashboards/k8s-resources-cluster.json b/charts/kubezero-metrics/configs/dashboards/k8s-resources-cluster.json index f84c0ea..d08fa6c 100644 --- a/charts/kubezero-metrics/configs/dashboards/k8s-resources-cluster.json +++ b/charts/kubezero-metrics/configs/dashboards/k8s-resources-cluster.json @@ -24,12 +24,10 @@ "id": 1, "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -49,7 +47,7 @@ "steppedLine": false, "targets": [ { - "expr": "1 - sum(avg by (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", cluster=\"$cluster\"}[$__rate_interval])))", + "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[$__rate_interval]))", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -101,14 +99,11 @@ "fill": 1, "format": "percentunit", "id": 2, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -128,7 +123,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"cpu\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -180,14 +175,11 @@ "fill": 1, "format": "percentunit", "id": 3, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -207,7 +199,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"cpu\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -259,14 +251,11 @@ "fill": 1, "format": "percentunit", "id": 4, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -286,7 +275,7 @@ "steppedLine": false, "targets": [ { - "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})", + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -338,14 +327,11 @@ "fill": 1, "format": "percentunit", "id": 5, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -365,7 +351,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"memory\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -417,14 +403,11 @@ "fill": 1, "format": "percentunit", "id": 6, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -444,7 +427,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"memory\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -507,14 +490,11 @@ "datasource": "$datasource", "fill": 10, "id": 7, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -598,14 +578,11 @@ "datasource": "$datasource", "fill": 1, "id": 8, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -764,7 +741,7 @@ ], "targets": [ { - "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(kube_pod_owner{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -884,14 +861,11 @@ "datasource": "$datasource", "fill": 10, "id": 9, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -911,7 +885,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -975,14 +949,11 @@ "datasource": "$datasource", "fill": 1, "id": 10, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1141,7 +1112,7 @@ ], "targets": [ { - "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(kube_pod_owner{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1159,7 +1130,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1177,7 +1148,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1195,7 +1166,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1263,12 +1234,10 @@ "id": 11, "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1412,7 +1381,7 @@ ], "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1421,7 +1390,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1430,7 +1399,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1439,7 +1408,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1448,7 +1417,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1457,7 +1426,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1523,14 +1492,11 @@ "datasource": "$datasource", "fill": 10, "id": 12, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1550,7 +1516,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1602,14 +1568,11 @@ "datasource": "$datasource", "fill": 10, "id": 13, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1629,7 +1592,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1693,14 +1656,11 @@ "datasource": "$datasource", "fill": 10, "id": 14, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1720,7 +1680,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1772,14 +1732,11 @@ "datasource": "$datasource", "fill": 10, "id": 15, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1799,7 +1756,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1863,14 +1820,11 @@ "datasource": "$datasource", "fill": 10, "id": 16, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1890,7 +1844,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1942,14 +1896,11 @@ "datasource": "$datasource", "fill": 10, "id": 17, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1969,7 +1920,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2033,14 +1984,11 @@ "datasource": "$datasource", "fill": 10, "id": 18, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -2060,7 +2008,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2112,14 +2060,11 @@ "datasource": "$datasource", "fill": 10, "id": 19, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -2139,7 +2084,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2204,14 +2149,11 @@ "decimals": -1, "fill": 10, "id": 20, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -2231,7 +2173,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])))", + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\"}[5m])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2283,14 +2225,11 @@ "datasource": "$datasource", "fill": 10, "id": 21, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -2310,7 +2249,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2374,14 +2313,11 @@ "datasource": "$datasource", "fill": 1, "id": 22, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -2529,7 +2465,7 @@ ], "targets": [ { - "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "sum by(namespace) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2538,7 +2474,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "sum by(namespace) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2547,7 +2483,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "sum by(namespace) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2556,7 +2492,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2565,7 +2501,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2574,7 +2510,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, diff --git a/charts/kubezero-metrics/configs/dashboards/k8s-resources-namespace.json b/charts/kubezero-metrics/configs/dashboards/k8s-resources-namespace.json index db4905f..a5f8e4e 100644 --- a/charts/kubezero-metrics/configs/dashboards/k8s-resources-namespace.json +++ b/charts/kubezero-metrics/configs/dashboards/k8s-resources-namespace.json @@ -22,14 +22,11 @@ "fill": 1, "format": "percentunit", "id": 1, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -49,7 +46,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -101,14 +98,11 @@ "fill": 1, "format": "percentunit", "id": 2, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -128,7 +122,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -180,14 +174,11 @@ "fill": 1, "format": "percentunit", "id": 3, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -207,7 +198,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -259,14 +250,11 @@ "fill": 1, "format": "percentunit", "id": 4, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -286,7 +274,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -349,14 +337,11 @@ "datasource": "$datasource", "fill": 10, "id": 5, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -479,14 +464,11 @@ "datasource": "$datasource", "fill": 1, "id": 6, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -717,14 +699,11 @@ "datasource": "$datasource", "fill": 10, "id": 7, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -767,7 +746,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -847,14 +826,11 @@ "datasource": "$datasource", "fill": 1, "id": 8, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1028,7 +1004,7 @@ ], "targets": [ { - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1046,7 +1022,7 @@ "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1064,7 +1040,7 @@ "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1073,7 +1049,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1082,7 +1058,7 @@ "step": 10 }, { - "expr": "sum(container_memory_cache{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1091,7 +1067,7 @@ "step": 10 }, { - "expr": "sum(container_memory_swap{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1159,12 +1135,10 @@ "id": 9, "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1308,7 +1282,7 @@ ], "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1317,7 +1291,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1326,7 +1300,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1335,7 +1309,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1344,7 +1318,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1353,7 +1327,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1419,14 +1393,11 @@ "datasource": "$datasource", "fill": 10, "id": 10, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1498,14 +1469,11 @@ "datasource": "$datasource", "fill": 10, "id": 11, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1589,14 +1557,11 @@ "datasource": "$datasource", "fill": 10, "id": 12, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1668,14 +1633,11 @@ "datasource": "$datasource", "fill": 10, "id": 13, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1759,14 +1721,11 @@ "datasource": "$datasource", "fill": 10, "id": 14, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1838,14 +1797,11 @@ "datasource": "$datasource", "fill": 10, "id": 15, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1930,14 +1886,11 @@ "decimals": -1, "fill": 10, "id": 16, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1957,7 +1910,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])))", + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -2009,14 +1962,11 @@ "datasource": "$datasource", "fill": 10, "id": 17, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -2036,7 +1986,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -2100,14 +2050,11 @@ "datasource": "$datasource", "fill": 1, "id": 18, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -2255,7 +2202,7 @@ ], "targets": [ { - "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2264,7 +2211,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2273,7 +2220,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2282,7 +2229,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2291,7 +2238,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2300,7 +2247,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2389,7 +2336,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "query": "label_values(kube_pod_info, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -2412,7 +2359,7 @@ "multi": false, "name": "namespace", "options": [ ], - "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/k8s-resources-node.json b/charts/kubezero-metrics/configs/dashboards/k8s-resources-node.json index 7cc7790..c226330 100644 --- a/charts/kubezero-metrics/configs/dashboards/k8s-resources-node.json +++ b/charts/kubezero-metrics/configs/dashboards/k8s-resources-node.json @@ -21,14 +21,11 @@ "datasource": "$datasource", "fill": 10, "id": 1, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -41,32 +38,12 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "max capacity", - "color": "#F2495C", - "dashes": true, - "fill": 0, - "hiddenSeries": true, - "hideTooltip": true, - "legend": true, - "linewidth": 2, - "stack": false - } - ], + "seriesOverrides": [ ], "spaceLength": 10, "span": 12, "stack": true, "steppedLine": false, "targets": [ - { - "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max capacity", - "legendLink": null, - "step": 10 - }, { "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "time_series", @@ -132,14 +109,11 @@ "datasource": "$datasource", "fill": 1, "id": 2, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -370,14 +344,11 @@ "datasource": "$datasource", "fill": 10, "id": 3, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -390,32 +361,12 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "max capacity", - "color": "#F2495C", - "dashes": true, - "fill": 0, - "hiddenSeries": true, - "hideTooltip": true, - "legend": true, - "linewidth": 2, - "stack": false - } - ], + "seriesOverrides": [ ], "spaceLength": 10, "span": 12, "stack": true, "steppedLine": false, "targets": [ - { - "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "max capacity", - "legendLink": null, - "step": 10 - }, { "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", "format": "time_series", @@ -481,14 +432,11 @@ "datasource": "$datasource", "fill": 1, "id": 4, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -814,7 +762,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "query": "label_values(kube_pod_info, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -837,7 +785,7 @@ "multi": true, "name": "node", "options": [ ], - "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, node)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/k8s-resources-pod.json b/charts/kubezero-metrics/configs/dashboards/k8s-resources-pod.json index 9db65a2..798184a 100644 --- a/charts/kubezero-metrics/configs/dashboards/k8s-resources-pod.json +++ b/charts/kubezero-metrics/configs/dashboards/k8s-resources-pod.json @@ -21,14 +21,11 @@ "datasource": "$datasource", "fill": 10, "id": 1, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -75,7 +72,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "requests", @@ -83,7 +80,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "limits", @@ -147,14 +144,11 @@ "datasource": "$datasource", "fill": 10, "id": 2, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -174,7 +168,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{job=\"cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[5m])) by (container) /sum(increase(container_cpu_cfs_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[5m])) by (container)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -247,14 +241,11 @@ "datasource": "$datasource", "fill": 1, "id": 3, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -485,14 +476,11 @@ "datasource": "$datasource", "fill": 10, "id": 4, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -533,7 +521,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -541,7 +529,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "requests", @@ -549,7 +537,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "limits", @@ -613,14 +601,11 @@ "datasource": "$datasource", "fill": 1, "id": 5, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -794,7 +779,7 @@ ], "targets": [ { - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -812,7 +797,7 @@ "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -830,7 +815,7 @@ "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -839,7 +824,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -848,7 +833,7 @@ "step": 10 }, { - "expr": "sum(container_memory_cache{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -857,7 +842,7 @@ "step": 10 }, { - "expr": "sum(container_memory_swap{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -925,12 +910,10 @@ "id": 6, "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -950,7 +933,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1004,12 +987,10 @@ "id": 7, "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1029,7 +1010,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1095,12 +1076,10 @@ "id": 8, "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1120,7 +1099,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1174,12 +1153,10 @@ "id": 9, "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1199,7 +1176,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1265,12 +1242,10 @@ "id": 10, "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1290,7 +1265,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1344,12 +1319,10 @@ "id": 11, "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1369,7 +1342,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1434,14 +1407,11 @@ "decimals": -1, "fill": 10, "id": 12, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1461,7 +1431,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[5m])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Reads", @@ -1469,7 +1439,7 @@ "step": 10 }, { - "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[5m])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Writes", @@ -1521,14 +1491,11 @@ "datasource": "$datasource", "fill": 10, "id": 13, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1548,7 +1515,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Reads", @@ -1556,7 +1523,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Writes", @@ -1621,14 +1588,11 @@ "decimals": -1, "fill": 10, "id": 14, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1648,7 +1612,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceil(sum by(container) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))", + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -1700,14 +1664,11 @@ "datasource": "$datasource", "fill": 10, "id": 15, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1727,7 +1688,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -1791,14 +1752,11 @@ "datasource": "$datasource", "fill": 1, "id": 16, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1946,7 +1904,7 @@ ], "targets": [ { - "expr": "sum by(container) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "expr": "sum by(container) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1955,7 +1913,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "expr": "sum by(container) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1964,7 +1922,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "expr": "sum by(container) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1973,7 +1931,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1982,7 +1940,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1991,7 +1949,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2080,7 +2038,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "query": "label_values(kube_pod_info, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -2103,7 +2061,7 @@ "multi": false, "name": "namespace", "options": [ ], - "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "sort": 1, @@ -2126,7 +2084,7 @@ "multi": false, "name": "pod", "options": [ ], - "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/k8s-resources-workload.json b/charts/kubezero-metrics/configs/dashboards/k8s-resources-workload.json index bfc912f..7334989 100644 --- a/charts/kubezero-metrics/configs/dashboards/k8s-resources-workload.json +++ b/charts/kubezero-metrics/configs/dashboards/k8s-resources-workload.json @@ -21,14 +21,11 @@ "datasource": "$datasource", "fill": 10, "id": 1, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -112,14 +109,11 @@ "datasource": "$datasource", "fill": 1, "id": 2, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -257,7 +251,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -266,7 +260,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -275,7 +269,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -284,7 +278,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -350,14 +344,11 @@ "datasource": "$datasource", "fill": 10, "id": 3, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -441,14 +432,11 @@ "datasource": "$datasource", "fill": 1, "id": 4, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -586,7 +574,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -595,7 +583,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -604,7 +592,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -613,7 +601,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -681,12 +669,10 @@ "id": 5, "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -830,7 +816,7 @@ ], "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -839,7 +825,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -848,7 +834,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -857,7 +843,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -866,7 +852,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -875,7 +861,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -941,14 +927,11 @@ "datasource": "$datasource", "fill": 10, "id": 6, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -968,7 +951,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1020,14 +1003,11 @@ "datasource": "$datasource", "fill": 10, "id": 7, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1047,7 +1027,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1111,14 +1091,11 @@ "datasource": "$datasource", "fill": 10, "id": 8, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1138,7 +1115,7 @@ "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1190,14 +1167,11 @@ "datasource": "$datasource", "fill": 10, "id": 9, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1217,7 +1191,7 @@ "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1281,14 +1255,11 @@ "datasource": "$datasource", "fill": 10, "id": 10, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1308,7 +1279,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1360,14 +1331,11 @@ "datasource": "$datasource", "fill": 10, "id": 11, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1387,7 +1355,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1451,14 +1419,11 @@ "datasource": "$datasource", "fill": 10, "id": 12, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1478,7 +1443,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1530,14 +1495,11 @@ "datasource": "$datasource", "fill": 10, "id": 13, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1557,7 +1519,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1644,7 +1606,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "query": "label_values(kube_pod_info, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -1667,30 +1629,7 @@ "multi": false, "name": "namespace", "options": [ ], - "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "", - "value": "" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "type", - "options": [ ], - "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)", + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "sort": 1, @@ -1713,7 +1652,30 @@ "multi": false, "name": "workload", "options": [ ], - "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}, workload)", + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/k8s-resources-workloads-namespace.json b/charts/kubezero-metrics/configs/dashboards/k8s-resources-workloads-namespace.json index 3e995a5..9654b4f 100644 --- a/charts/kubezero-metrics/configs/dashboards/k8s-resources-workloads-namespace.json +++ b/charts/kubezero-metrics/configs/dashboards/k8s-resources-workloads-namespace.json @@ -21,14 +21,11 @@ "datasource": "$datasource", "fill": 10, "id": 1, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -151,14 +148,11 @@ "datasource": "$datasource", "fill": 1, "id": 2, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -335,7 +329,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -344,7 +338,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -353,7 +347,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -362,7 +356,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -428,14 +422,11 @@ "datasource": "$datasource", "fill": 10, "id": 3, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -478,7 +469,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(\n container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}} - {{workload_type}}", @@ -558,14 +549,11 @@ "datasource": "$datasource", "fill": 1, "id": 4, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -733,7 +721,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -742,7 +730,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -751,7 +739,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -760,7 +748,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -769,7 +757,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -837,12 +825,10 @@ "id": 5, "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1001,7 +987,7 @@ ], "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1010,7 +996,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1019,7 +1005,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1028,7 +1014,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1037,7 +1023,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1046,7 +1032,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1112,14 +1098,11 @@ "datasource": "$datasource", "fill": 10, "id": 6, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1139,7 +1122,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1191,14 +1174,11 @@ "datasource": "$datasource", "fill": 10, "id": 7, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1218,7 +1198,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1282,14 +1262,11 @@ "datasource": "$datasource", "fill": 10, "id": 8, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1309,7 +1286,7 @@ "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1361,14 +1338,11 @@ "datasource": "$datasource", "fill": 10, "id": 9, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1388,7 +1362,7 @@ "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1452,14 +1426,11 @@ "datasource": "$datasource", "fill": 10, "id": 10, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1479,7 +1450,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1531,14 +1502,11 @@ "datasource": "$datasource", "fill": 10, "id": 11, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1558,7 +1526,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1622,14 +1590,11 @@ "datasource": "$datasource", "fill": 10, "id": 12, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1649,7 +1614,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1701,14 +1666,11 @@ "datasource": "$datasource", "fill": 10, "id": 13, - "interval": "1m", "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -1728,7 +1690,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1815,30 +1777,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "", - "value": "" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "namespace", - "options": [ ], - "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "query": "label_values(kube_pod_info, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -1875,6 +1814,29 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false } ] }, diff --git a/charts/kubezero-metrics/configs/dashboards/kubelet.json b/charts/kubezero-metrics/configs/dashboards/kubelet.json index e1a47c2..a5f43f8 100644 --- a/charts/kubezero-metrics/configs/dashboards/kubelet.json +++ b/charts/kubezero-metrics/configs/dashboards/kubelet.json @@ -154,7 +154,7 @@ "refId": "A" } ], - "title": "Running Containers", + "title": "Running Container", "transparent": false, "type": "stat" }, @@ -294,7 +294,7 @@ "pluginVersion": "7", "targets": [ { - "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval]))", + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -347,7 +347,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)", + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (operation_type, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -432,7 +432,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -517,7 +517,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -602,14 +602,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} pod", "refId": "A" }, { - "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} worker", @@ -694,14 +694,14 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} pod", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} worker", @@ -788,7 +788,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", @@ -875,7 +875,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", @@ -962,7 +962,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))", + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", @@ -1047,7 +1047,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{operation_type}}", @@ -1132,7 +1132,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -1218,7 +1218,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1303,7 +1303,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1388,7 +1388,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1473,28 +1473,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"2..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"3..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"4..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"5..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -1579,7 +1579,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{verb}} {{url}}", @@ -1749,7 +1749,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1893,7 +1893,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": null, "name": "datasource", "options": [ ], "query": "prometheus", @@ -1927,11 +1927,11 @@ "datasource": "$datasource", "hide": 0, "includeAll": true, - "label": "Data Source", + "label": null, "multi": false, "name": "instance", "options": [ ], - "query": "label_values(up{job=\"kubelet\",cluster=\"$cluster\"}, instance)", + "query": "label_values(kubelet_runtime_operations_total{cluster=\"$cluster\", job=\"kubelet\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/namespace-by-pod.json b/charts/kubezero-metrics/configs/dashboards/namespace-by-pod.json index 1d2e1f1..cbe880c 100644 --- a/charts/kubezero-metrics/configs/dashboards/namespace-by-pod.json +++ b/charts/kubezero-metrics/configs/dashboards/namespace-by-pod.json @@ -1137,7 +1137,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": null, "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/configs/dashboards/namespace-by-workload.json b/charts/kubezero-metrics/configs/dashboards/namespace-by-workload.json index 67f4851..c05bda2 100644 --- a/charts/kubezero-metrics/configs/dashboards/namespace-by-workload.json +++ b/charts/kubezero-metrics/configs/dashboards/namespace-by-workload.json @@ -1349,7 +1349,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": null, "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/configs/dashboards/persistentvolumesusage.json b/charts/kubezero-metrics/configs/dashboards/persistentvolumesusage.json index cf722b5..e9ab9c2 100644 --- a/charts/kubezero-metrics/configs/dashboards/persistentvolumesusage.json +++ b/charts/kubezero-metrics/configs/dashboards/persistentvolumesusage.json @@ -26,14 +26,13 @@ "fillGradient": 0, "gridPos": { }, "id": 2, - "interval": "1m", "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -125,11 +124,7 @@ }, "gridPos": { }, "id": 3, - "interval": "1m", - "legend": { - "alignAsTable": true, - "rightSide": true - }, + "interval": null, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -212,14 +207,13 @@ "fillGradient": 0, "gridPos": { }, "id": 4, - "interval": "1m", "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -311,11 +305,7 @@ }, "gridPos": { }, "id": 5, - "interval": "1m", - "legend": { - "alignAsTable": true, - "rightSide": true - }, + "interval": null, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -398,7 +388,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": null, "name": "datasource", "options": [ ], "query": "prometheus", @@ -416,7 +406,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"kubelet\"}, cluster)", + "query": "label_values(kubelet_volume_stats_capacity_bytes, cluster)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/pod-total.json b/charts/kubezero-metrics/configs/dashboards/pod-total.json index e5c0b6e..e854c41 100644 --- a/charts/kubezero-metrics/configs/dashboards/pod-total.json +++ b/charts/kubezero-metrics/configs/dashboards/pod-total.json @@ -903,7 +903,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": null, "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/configs/dashboards/proxy.json b/charts/kubezero-metrics/configs/dashboards/proxy.json index d7ad8ac..8cf589a 100644 --- a/charts/kubezero-metrics/configs/dashboards/proxy.json +++ b/charts/kubezero-metrics/configs/dashboards/proxy.json @@ -36,11 +36,7 @@ }, "gridPos": { }, "id": 2, - "interval": "1m", - "legend": { - "alignAsTable": true, - "rightSide": true - }, + "interval": null, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -110,14 +106,13 @@ "fillGradient": 0, "gridPos": { }, "id": 3, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -139,7 +134,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval]))", + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "rate", @@ -192,7 +187,6 @@ "fillGradient": 0, "gridPos": { }, "id": 4, - "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -221,7 +215,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -287,14 +281,13 @@ "fillGradient": 0, "gridPos": { }, "id": 5, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -316,7 +309,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval]))", + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "rate", @@ -369,7 +362,6 @@ "fillGradient": 0, "gridPos": { }, "id": 6, - "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -398,7 +390,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -464,14 +456,13 @@ "fillGradient": 0, "gridPos": { }, "id": 7, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -493,28 +484,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"2..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"3..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"4..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"5..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -567,14 +558,13 @@ "fillGradient": 0, "gridPos": { }, "id": 8, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -596,7 +586,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\",verb=\"POST\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -662,7 +652,6 @@ "fillGradient": 0, "gridPos": { }, "id": 9, - "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -691,7 +680,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -757,14 +746,13 @@ "fillGradient": 0, "gridPos": { }, "id": 10, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -839,14 +827,13 @@ "fillGradient": 0, "gridPos": { }, "id": 11, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -868,7 +855,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\"}[$__rate_interval])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -921,14 +908,13 @@ "fillGradient": 0, "gridPos": { }, "id": 12, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -1016,7 +1002,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": null, "name": "datasource", "options": [ ], "query": "prometheus", @@ -1034,7 +1020,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(up{job=\"kube-proxy\"}, cluster)", + "query": "label_values(kube_pod_info, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -1054,7 +1040,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(up{job=\"kube-proxy\", cluster=\"$cluster\", job=\"kube-proxy\"}, instance)", + "query": "label_values(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/scheduler.json b/charts/kubezero-metrics/configs/dashboards/scheduler.json index b480c68..de71562 100644 --- a/charts/kubezero-metrics/configs/dashboards/scheduler.json +++ b/charts/kubezero-metrics/configs/dashboards/scheduler.json @@ -36,11 +36,7 @@ }, "gridPos": { }, "id": 2, - "interval": "1m", - "legend": { - "alignAsTable": true, - "rightSide": true - }, + "interval": null, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -110,7 +106,6 @@ "fillGradient": 0, "gridPos": { }, "id": 3, - "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -139,28 +134,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} e2e", "refId": "A" }, { - "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} binding", "refId": "B" }, { - "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} scheduling algorithm", "refId": "C" }, { - "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} volume", @@ -213,7 +208,6 @@ "fillGradient": 0, "gridPos": { }, "id": 4, - "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -242,28 +236,28 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} e2e", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} binding", "refId": "B" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} scheduling algorithm", "refId": "C" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} volume", @@ -329,14 +323,13 @@ "fillGradient": 0, "gridPos": { }, "id": 5, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -358,28 +351,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -432,14 +425,13 @@ "fillGradient": 0, "gridPos": { }, "id": 6, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -461,7 +453,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -527,7 +519,6 @@ "fillGradient": 0, "gridPos": { }, "id": 7, - "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -556,7 +547,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -622,14 +613,13 @@ "fillGradient": 0, "gridPos": { }, "id": 8, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -704,14 +694,13 @@ "fillGradient": 0, "gridPos": { }, "id": 9, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -733,7 +722,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -786,14 +775,13 @@ "fillGradient": 0, "gridPos": { }, "id": 10, - "interval": "1m", "legend": { - "alignAsTable": true, + "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, + "rightSide": false, "show": true, "sideWidth": null, "total": false, @@ -881,7 +869,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": null, "name": "datasource", "options": [ ], "query": "prometheus", @@ -919,7 +907,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(up{job=\"kube-scheduler\", cluster=\"$cluster\"}, instance)", + "query": "label_values(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-scheduler\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/configs/dashboards/workload-total.json b/charts/kubezero-metrics/configs/dashboards/workload-total.json index 9cf5daf..f1ce702 100644 --- a/charts/kubezero-metrics/configs/dashboards/workload-total.json +++ b/charts/kubezero-metrics/configs/dashboards/workload-total.json @@ -89,7 +89,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ pod }}", @@ -184,7 +184,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ pod }}", @@ -290,7 +290,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ pod }}", @@ -385,7 +385,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ pod }}", @@ -506,7 +506,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -597,7 +597,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -699,7 +699,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -790,7 +790,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -901,7 +901,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -992,7 +992,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -1061,7 +1061,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": null, "name": "datasource", "options": [ ], "query": "prometheus", @@ -1079,7 +1079,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)", + "query": "label_values(kube_pod_info, cluster)", "refresh": 2, "regex": "", "sort": 0, @@ -1099,14 +1099,14 @@ "value": "kube-system" }, "datasource": "$datasource", - "definition": "label_values(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\"}, namespace)", + "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", "hide": 0, "includeAll": true, "label": null, "multi": false, "name": "namespace", "options": [ ], - "query": "label_values(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\"}, namespace)", + "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "skipUrlSync": false, diff --git a/charts/kubezero-metrics/configs/jsonnetfile.json b/charts/kubezero-metrics/configs/jsonnetfile.json index 97f3b96..1619309 100644 --- a/charts/kubezero-metrics/configs/jsonnetfile.json +++ b/charts/kubezero-metrics/configs/jsonnetfile.json @@ -8,7 +8,7 @@ "subdir": "jsonnet/kube-prometheus" } }, - "version": "main" + "version": "release-0.9" } ], "legacyImports": true diff --git a/charts/kubezero-metrics/configs/jsonnetfile.lock.json b/charts/kubezero-metrics/configs/jsonnetfile.lock.json index ecbfbee..2edbb7a 100644 --- a/charts/kubezero-metrics/configs/jsonnetfile.lock.json +++ b/charts/kubezero-metrics/configs/jsonnetfile.lock.json @@ -8,8 +8,8 @@ "subdir": "grafana" } }, - "version": "199e363523104ff8b3a12483a4e3eca86372b078", - "sum": "/jDHzVAjHB4AOLkJHw1GyATX5ogZ1iMdcJXZAgaG3+g=" + "version": "90f38916f1f8a310a715d18e36f787f84df4ddf5", + "sum": "0kZ1pnuIirDtbg6F9at5+NQOwKNONIGEPq0eECzvRkI=" }, { "source": { @@ -18,8 +18,8 @@ "subdir": "contrib/mixin" } }, - "version": "29292aa7bdafaf65cb5e054591fe0ff07b36f5ee", - "sum": "cdKL5kPYfpWSpTCu4qctmh+gWQqL+4YWom6rw9qLYJU=" + "version": "a62444690780fd1eb72615b0cd6d804ea54cc771", + "sum": "W/Azptf1PoqjyMwJON96UY69MFugDA4IAYiKURscryc=" }, { "source": { @@ -38,7 +38,7 @@ "subdir": "grafana-builder" } }, - "version": "e2cfc18b7be54acc6978b7dab2c61ec5e37385a8", + "version": "5a128df878434da37969b811e99bb9cd0a3779e3", "sum": "0KkygBQd/AFzUvVzezE4qF/uDYgrwUXVpZfINBti0oc=" }, { @@ -48,8 +48,8 @@ "subdir": "" } }, - "version": "9821d07e94e9a9916575a234fb699ae3331fa939", - "sum": "xubNXyvDwUw9GZzi9BRb6ob3bYzfoMr5F5zCVn2d7ag=" + "version": "fb9d8ed4bc4a3d6efac525f72e8a0d2c583a0fe2", + "sum": "xjKkdp+5fkekCNBUIgZCHTRmVdUEmQNFKslrL2Ho8gs=" }, { "source": { @@ -58,7 +58,7 @@ "subdir": "lib/promgrafonnet" } }, - "version": "9821d07e94e9a9916575a234fb699ae3331fa939", + "version": "c76b9378b86d28bd617d94a57c72b4770efed510", "sum": "zv7hXGui6BfHzE9wPatHI/AGZa4A2WKo6pq7ZdqBsps=" }, { @@ -68,8 +68,8 @@ "subdir": "jsonnet/kube-state-metrics" } }, - "version": "b761b5382bdd85d7af915516f48cba1c46859c1d", - "sum": "U1wzIpTAtOvC1yj43Y8PfvT0JfvnAcMfNH12Wi+ab0Y=" + "version": "f170cc73f11c1580d7f38af746be0f2fa79c6a1e", + "sum": "S5qI+PJUdNeYOv76jH5nxwYS9N6U7CRxvyuB1wI4cTE=" }, { "source": { @@ -78,7 +78,7 @@ "subdir": "jsonnet/kube-state-metrics-mixin" } }, - "version": "b761b5382bdd85d7af915516f48cba1c46859c1d", + "version": "f170cc73f11c1580d7f38af746be0f2fa79c6a1e", "sum": "u8gaydJoxEjzizQ8jY8xSjYgWooPmxw+wIWdDxifMAk=" }, { @@ -88,8 +88,8 @@ "subdir": "jsonnet/kube-prometheus" } }, - "version": "9b532dac0c8b00e984d2f77b2f2f0ef0dae6e751", - "sum": "HCUXyGj9sJWtBV10TBxC+Q0RFz5VFiMM/y5u6/3t4tw=" + "version": "c8db3c85db6e13b94726b7c5d630a6ebe7038cf9", + "sum": "7DfVEg2kDzIQPgxTThnjQ4S2ZksrF0hkNLysXP6Wv24=" }, { "source": { @@ -98,8 +98,8 @@ "subdir": "jsonnet/mixin" } }, - "version": "2200303e6a9629b3ccc0e9d466438dc079572aa3", - "sum": "qZ4WgiweaE6eeKtFK60QUjLO8sf2L9Q8fgafWvDcyfY=", + "version": "83fe36566f4e0894eb5ffcd2638a0f039a17bdeb", + "sum": "6reUygVmQrLEWQzTKcH8ceDbvM+2ztK3z2VBR2K2l+U=", "name": "prometheus-operator-mixin" }, { @@ -109,8 +109,8 @@ "subdir": "jsonnet/prometheus-operator" } }, - "version": "2200303e6a9629b3ccc0e9d466438dc079572aa3", - "sum": "9R1mw4Tz0/1V1QWkJMzqE4+iXXONEfYVikW8Mj5AOcA=" + "version": "83fe36566f4e0894eb5ffcd2638a0f039a17bdeb", + "sum": "J1G++A8hrtr3+OZQMmcNeb1w/C30bXqqwpwHL/Xhsd4=" }, { "source": { @@ -119,7 +119,7 @@ "subdir": "doc/alertmanager-mixin" } }, - "version": "e2a10119aaf7777fa523d216e05897c5b719134c", + "version": "b408b522bc653d014e53035e59fa394cc1edd762", "sum": "pep+dHzfIjh2SU5pEkwilMCAT/NoL6YYflV4x8cr7vU=", "name": "alertmanager" }, @@ -130,8 +130,8 @@ "subdir": "docs/node-mixin" } }, - "version": "2028460438d502be6137ac0738068538cd8fafe1", - "sum": "MlWDAKGZ+JArozRKdKEvewHeWn8j2DNBzesJfLVd0dk=" + "version": "832909dd257eb368cf83363ffcae3ab84cb4bcb1", + "sum": "MmxGhE2PJ1a52mk2x7vDpMT2at4Jglbud/rK74CB5i0=" }, { "source": { @@ -140,8 +140,8 @@ "subdir": "documentation/prometheus-mixin" } }, - "version": "a14960eae102fb6e83817f7faf15695110ff433b", - "sum": "ZjQoYhvgKwJNkg+h+m9lW3SYjnjv5Yx5btEipLhru88=", + "version": "751ca03faddc9c64089c41d0da370a3a0b477742", + "sum": "AS8WYFi/z10BZSF6DFkKBscjB32XDMM7iIso7CO/FyI=", "name": "prometheus" }, { @@ -151,8 +151,8 @@ "subdir": "mixin" } }, - "version": "4ed5206d1893975f6c45045d680f8c8a7dd2a25e", - "sum": "1Y1cPIeoPg2nCAEhKPCt8bAGuwuOP2eZ3kVF432mlMA=", + "version": "ff363498fc95cfe17de894d7237bcf38bdd0bc36", + "sum": "cajthvLKDjYgYHCKQU2g/pTMRkxcbuJEvTnCyJOihl8=", "name": "thanos-mixin" } ], diff --git a/charts/kubezero-metrics/configs/rules/alertmanager-prometheusRule b/charts/kubezero-metrics/configs/rules/alertmanager-prometheusRule index a18dcfd..75e44f6 100644 --- a/charts/kubezero-metrics/configs/rules/alertmanager-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/alertmanager-prometheusRule @@ -4,10 +4,9 @@ "metadata": { "labels": { "app.kubernetes.io/component": "alert-router", - "app.kubernetes.io/instance": "main", "app.kubernetes.io/name": "alertmanager", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "0.23.0", + "app.kubernetes.io/version": "0.22.2", "prometheus": "k8s", "role": "alert-rules" }, diff --git a/charts/kubezero-metrics/configs/rules/etcd-mixin-prometheusRule b/charts/kubezero-metrics/configs/rules/etcd-mixin-prometheusRule index 2aa76d4..02e0ba7 100644 --- a/charts/kubezero-metrics/configs/rules/etcd-mixin-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/etcd-mixin-prometheusRule @@ -64,10 +64,10 @@ { "alert": "etcdGRPCRequestsSlow", "annotations": { - "description": "etcd cluster \"{{ $labels.job }}\": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method }} method.", + "description": "etcd cluster \"{{ $labels.job }}\": gRPC requests to {{ $labels.grpc_method }} are taking {{ $value }}s on etcd instance {{ $labels.instance }}.", "summary": "etcd grpc requests are slow" }, - "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~\".*etcd.*\", grpc_method!=\"Defragment\", grpc_type=\"unary\"}[5m])) without(grpc_type))\n> 0.15\n", + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~\".*etcd.*\", grpc_type=\"unary\"}[5m])) without(grpc_type))\n> 0.15\n", "for": "10m", "labels": { "severity": "critical" @@ -112,7 +112,7 @@ { "alert": "etcdHighFsyncDurations", "annotations": { - "description": "etcd cluster \"{{ $labels.job }}\": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}." + "message": "etcd cluster \"{{ $labels.job }}\": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}." }, "expr": "histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~\".*etcd.*\"}[5m]))\n> 1\n", "for": "10m", @@ -135,7 +135,7 @@ { "alert": "etcdBackendQuotaLowSpace", "annotations": { - "description": "etcd cluster \"{{ $labels.job }}\": database size exceeds the defined quota on etcd instance {{ $labels.instance }}, please defrag or increase the quota as the writes to etcd will be disabled when it is full." + "message": "etcd cluster \"{{ $labels.job }}\": database size exceeds the defined quota on etcd instance {{ $labels.instance }}, please defrag or increase the quota as the writes to etcd will be disabled when it is full." }, "expr": "(etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100 > 95\n", "for": "10m", @@ -146,7 +146,7 @@ { "alert": "etcdExcessiveDatabaseGrowth", "annotations": { - "description": "etcd cluster \"{{ $labels.job }}\": Observed surge in etcd writes leading to 50% increase in database size over the past four hours on etcd instance {{ $labels.instance }}, please check as it might be disruptive." + "message": "etcd cluster \"{{ $labels.job }}\": Observed surge in etcd writes leading to 50% increase in database size over the past four hours on etcd instance {{ $labels.instance }}, please check as it might be disruptive." }, "expr": "increase(((etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100)[240m:1m]) > 50\n", "for": "10m", diff --git a/charts/kubezero-metrics/configs/rules/kube-state-metrics-prometheusRule b/charts/kubezero-metrics/configs/rules/kube-state-metrics-prometheusRule index a2d279e..210c4a3 100644 --- a/charts/kubezero-metrics/configs/rules/kube-state-metrics-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/kube-state-metrics-prometheusRule @@ -6,7 +6,7 @@ "app.kubernetes.io/component": "exporter", "app.kubernetes.io/name": "kube-state-metrics", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "2.3.0", + "app.kubernetes.io/version": "2.1.1", "prometheus": "k8s", "role": "alert-rules" }, diff --git a/charts/kubezero-metrics/configs/rules/kubernetes-prometheusRule b/charts/kubezero-metrics/configs/rules/kubernetes-prometheusRule index 04e6b56..abbe47b 100644 --- a/charts/kubezero-metrics/configs/rules/kubernetes-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/kubernetes-prometheusRule @@ -123,7 +123,7 @@ { "alert": "KubeContainerWaiting", "annotations": { - "description": "pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour.", + "description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} has been in waiting state for longer than 1 hour.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting", "summary": "Pod container waiting longer than 1 hour" }, @@ -232,7 +232,7 @@ { "alert": "KubeMemoryOvercommit", "annotations": { - "description": "Cluster has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.", + "description": "Cluster has overcommitted memory resource requests for Pods by {{ $value }} bytes and cannot tolerate node failure.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit", "summary": "Cluster has overcommitted memory resource requests." }, @@ -249,7 +249,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit", "summary": "Cluster has overcommitted CPU resource requests." }, - "expr": "sum(min without(resource) (kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=~\"(cpu|requests.cpu)\"}))\n /\nsum(kube_node_status_allocatable{resource=\"cpu\", job=\"kube-state-metrics\"})\n > 1.5\n", + "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"cpu\"})\n /\nsum(kube_node_status_allocatable{resource=\"cpu\"})\n > 1.5\n", "for": "5m", "labels": { "severity": "warning" @@ -262,7 +262,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit", "summary": "Cluster has overcommitted memory resource requests." }, - "expr": "sum(min without(resource) (kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=~\"(memory|requests.memory)\"}))\n /\nsum(kube_node_status_allocatable{resource=\"memory\", job=\"kube-state-metrics\"})\n > 1.5\n", + "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"memory\"})\n /\nsum(kube_node_status_allocatable{resource=\"memory\",job=\"kube-state-metrics\"})\n > 1.5\n", "for": "5m", "labels": { "severity": "warning" @@ -332,7 +332,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup", "summary": "PersistentVolume is filling up." }, - "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n", + "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\n", "for": "1m", "labels": { "severity": "critical" @@ -345,7 +345,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup", "summary": "PersistentVolume is filling up." }, - "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n", + "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\n", "for": "1h", "labels": { "severity": "warning" @@ -468,7 +468,7 @@ { "alert": "KubeClientCertificateExpiration", "annotations": { - "description": "A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days.", + "description": "A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration", "summary": "Client certificate is about to expire." }, @@ -480,7 +480,7 @@ { "alert": "KubeClientCertificateExpiration", "annotations": { - "description": "A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours.", + "description": "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration", "summary": "Client certificate is about to expire." }, @@ -490,11 +490,11 @@ } }, { - "alert": "KubeAggregatedAPIErrors", + "alert": "AggregatedAPIErrors", "annotations": { - "description": "Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.", - "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors", - "summary": "Kubernetes aggregated API has reported errors." + "description": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/aggregatedapierrors", + "summary": "An aggregated API has reported errors." }, "expr": "sum by(name, namespace)(increase(aggregator_unavailable_apiservice_total[10m])) > 4\n", "labels": { @@ -502,11 +502,11 @@ } }, { - "alert": "KubeAggregatedAPIDown", + "alert": "AggregatedAPIDown", "annotations": { - "description": "Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.", - "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown", - "summary": "Kubernetes aggregated API is down." + "description": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/aggregatedapidown", + "summary": "An aggregated API is down." }, "expr": "(1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85\n", "for": "5m", @@ -530,9 +530,9 @@ { "alert": "KubeAPITerminatedRequests", "annotations": { - "description": "The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.", + "description": "The apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests", - "summary": "The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests." + "summary": "The apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests." }, "expr": "sum(rate(apiserver_request_terminations_total{job=\"apiserver\"}[10m])) / ( sum(rate(apiserver_request_total{job=\"apiserver\"}[10m])) + sum(rate(apiserver_request_terminations_total{job=\"apiserver\"}[10m])) ) > 0.20\n", "for": "5m", @@ -581,7 +581,7 @@ "expr": "count by(node) (\n (kube_pod_status_phase{job=\"kube-state-metrics\",phase=\"Running\"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job=\"kube-state-metrics\"})\n)\n/\nmax by(node) (\n kube_node_status_capacity{job=\"kube-state-metrics\",resource=\"pods\"} != 1\n) > 0.95\n", "for": "15m", "labels": { - "severity": "info" + "severity": "warning" } }, { @@ -748,71 +748,53 @@ } ] }, - { - "name": "kubernetes-system-kube-proxy", - "rules": [ - { - "alert": "KubeProxyDown", - "annotations": { - "description": "KubeProxy has disappeared from Prometheus target discovery.", - "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeproxydown", - "summary": "Target disappeared from Prometheus target discovery." - }, - "expr": "absent(up{job=\"kube-proxy\"} == 1)\n", - "for": "15m", - "labels": { - "severity": "critical" - } - } - ] - }, { "name": "kube-apiserver-burnrate.rules", "rules": [ { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[1d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[1d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate1d" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[1h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[1h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate1h" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[2h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[2h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[2h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[2h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[2h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[2h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate2h" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[30m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate30m" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[3d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[3d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[3d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[3d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[3d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[3d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate3d" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[5m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[5m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[5m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[5m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[5m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[5m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate5m" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[6h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[6h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[6h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[6h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[6h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[6h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n", "labels": { "verb": "read" }, @@ -934,37 +916,21 @@ "record": "code:apiserver_request_total:increase30d" }, { - "expr": "sum by (cluster, verb, scope) (increase(apiserver_request_duration_seconds_count[1h]))\n", - "record": "cluster_verb_scope:apiserver_request_duration_seconds_count:increase1h" - }, - { - "expr": "sum by (cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_duration_seconds_count:increase1h[30d]) * 24 * 30)\n", - "record": "cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d" - }, - { - "expr": "sum by (cluster, verb, scope, le) (increase(apiserver_request_duration_seconds_bucket[1h]))\n", - "record": "cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase1h" - }, - { - "expr": "sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase1h[30d]) * 24 * 30)\n", - "record": "cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d" - }, - { - "expr": "1 - (\n (\n # write too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n -\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n ) +\n (\n # read too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"})\n )\n ) +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d)\n", + "expr": "1 - (\n (\n # write too slow\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~\"POST|PUT|PATCH|DELETE\"}[30d]))\n -\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[30d]))\n ) +\n (\n # read too slow\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~\"LIST|GET\"}[30d]))\n -\n (\n (\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30d]))\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30d]))\n )\n ) +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d)\n", "labels": { "verb": "all" }, "record": "apiserver_request:availability30d" }, { - "expr": "1 - (\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n # too slow\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"})\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n", + "expr": "1 - (\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30d]))\n -\n (\n # too slow\n (\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30d]))\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30d]))\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n", "labels": { "verb": "read" }, "record": "apiserver_request:availability30d" }, { - "expr": "1 - (\n (\n # too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n -\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\"})\n", + "expr": "1 - (\n (\n # too slow\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~\"POST|PUT|PATCH|DELETE\"}[30d]))\n -\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[30d]))\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\"})\n", "labels": { "verb": "write" }, @@ -1026,7 +992,7 @@ "record": "node_namespace_pod_container:container_memory_swap" }, { - "expr": "kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", + "expr": "kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", "record": "cluster:namespace:pod_memory:active:kube_pod_container_resource_requests" }, { @@ -1034,7 +1000,7 @@ "record": "namespace_memory:kube_pod_container_resource_requests:sum" }, { - "expr": "kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", + "expr": "kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", "record": "cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests" }, { @@ -1042,7 +1008,7 @@ "record": "namespace_cpu:kube_pod_container_resource_requests:sum" }, { - "expr": "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", + "expr": "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", "record": "cluster:namespace:pod_memory:active:kube_pod_container_resource_limits" }, { @@ -1050,7 +1016,7 @@ "record": "namespace_memory:kube_pod_container_resource_limits:sum" }, { - "expr": "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n )\n", + "expr": "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n )\n", "record": "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" }, { diff --git a/charts/kubezero-metrics/configs/rules/node-exporter-prometheusRule b/charts/kubezero-metrics/configs/rules/node-exporter-prometheusRule index 332a296..799ebaa 100644 --- a/charts/kubezero-metrics/configs/rules/node-exporter-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/node-exporter-prometheusRule @@ -6,7 +6,7 @@ "app.kubernetes.io/component": "exporter", "app.kubernetes.io/name": "node-exporter", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "1.3.1", + "app.kubernetes.io/version": "1.2.2", "prometheus": "k8s", "role": "alert-rules" }, @@ -25,7 +25,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup", "summary": "Filesystem is predicted to run out of space within the next 24 hours." }, - "expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 20\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n", + "expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 40\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n", "for": "1h", "labels": { "severity": "warning" @@ -255,11 +255,11 @@ "name": "node-exporter.rules", "rules": [ { - "expr": "count without (cpu, mode) (\n node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}\n)\n", + "expr": "count without (cpu) (\n count without (mode) (\n node_cpu_seconds_total{job=\"node-exporter\"}\n )\n)\n", "record": "instance:node_num_cpu:sum" }, { - "expr": "1 - avg without (cpu) (\n sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\"}[5m]))\n)\n", + "expr": "1 - avg without (cpu, mode) (\n rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\"}[5m])\n)\n", "record": "instance:node_cpu_utilisation:rate5m" }, { @@ -267,7 +267,7 @@ "record": "instance:node_load1_per_cpu:ratio" }, { - "expr": "1 - (\n (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"}\n +\n node_memory_Cached_bytes{job=\"node-exporter\"}\n +\n node_memory_MemFree_bytes{job=\"node-exporter\"}\n +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n )\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n", + "expr": "1 - (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n", "record": "instance:node_memory_utilisation:ratio" }, { diff --git a/charts/kubezero-metrics/configs/rules/prometheus-operator-prometheusRule b/charts/kubezero-metrics/configs/rules/prometheus-operator-prometheusRule index ddc63b4..39c0894 100644 --- a/charts/kubezero-metrics/configs/rules/prometheus-operator-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/prometheus-operator-prometheusRule @@ -6,7 +6,7 @@ "app.kubernetes.io/component": "controller", "app.kubernetes.io/name": "prometheus-operator", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "0.52.1", + "app.kubernetes.io/version": "0.49.0", "prometheus": "k8s", "role": "alert-rules" }, @@ -110,24 +110,6 @@ } } ] - }, - { - "name": "config-reloaders", - "rules": [ - { - "alert": "ConfigReloaderSidecarErrors", - "annotations": { - "description": "Errors encountered while the {{$labels.pod}} config-reloader sidecar attempts to sync config in {{$labels.namespace}} namespace.\nAs a result, configuration for service running in {{$labels.pod}} may be stale and cannot be updated anymore.", - "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/configreloadersidecarerrors", - "summary": "config-reloader sidecar has not had a successful reload for 10m" - }, - "expr": "max_over_time(reloader_last_reload_successful{namespace=~\".+\"}[5m]) == 0\n", - "for": "10m", - "labels": { - "severity": "warning" - } - } - ] } ] } diff --git a/charts/kubezero-metrics/configs/rules/prometheus-prometheusRule b/charts/kubezero-metrics/configs/rules/prometheus-prometheusRule index 04587ae..404d4b4 100644 --- a/charts/kubezero-metrics/configs/rules/prometheus-prometheusRule +++ b/charts/kubezero-metrics/configs/rules/prometheus-prometheusRule @@ -4,10 +4,9 @@ "metadata": { "labels": { "app.kubernetes.io/component": "prometheus", - "app.kubernetes.io/instance": "k8s", "app.kubernetes.io/name": "prometheus", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "2.31.1", + "app.kubernetes.io/version": "2.29.1", "prometheus": "k8s", "role": "alert-rules" }, diff --git a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml index 84f5102..91600f8 100644 --- a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml +++ b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml @@ -16,36 +16,36 @@ binaryData: node.json.gz: H4sIAAAAAAAC/+192XLjyBHte39FmbbvtGa0cF8c4QeJlDy67kVuqWfCywQDAosSLBDgYJFa3bf97TerAJDYQRILATAfZlokQKAqazvnZFbWtzeENKZTSVmaht74C/k3fCbkG/8/XFGEBYVvG5Pb6c2nj+8v736+/HzbOHYuy8I9ldn1G01dUOORmvr64ozqoiYtDUlV2C3rC8brkj90JhiCrpqaSNfXlrL5ICnXM3Z9GfJQ6/oHu1iu1/IbvsP/fzu2qqTR301JoyGVct6/FBQo/erREn/pg2A+uMrjGOBv3q+fqaY79bLefBz+jgdNmAuKEHiL/+vVe/wXXG8anHZOB/GvC6+SJiwfQ1/l/nrjKoU1mxTZYEqwqcJe2TptnjZ3qJsuKQ8y1Q3BCL7yNuRasJarLiMoigr3wlXWZ6zXN2RJN1Y9aF0ouPKnPz0K+uPf6St7lnr/Xyoaf2k1e8PVu+Cee1OSjWv2tpbrW5cBww0D91BFuJfZdUMzqev7R2kW8q0kqspYlVWNPVB7uBfeNo9Ju9WC//V6x6R15H60Y57zdX3J/yHnMtUMTxHW7a0/3quCNmvY177zf397YzdVg84kw1faxoNCDT6KW8N+0/qGdbc7VZUNaQnfW1/yRlRMWbY+GVQT7Pmi1W91+81es9XvD/v8qiwpTyGjmdWdFZN+gV8rgkzYfa7JRnjgP/pt/Y1kyNYAkIxH8z44L3kfYGp8hns0jKX+l7OzB/6jU1FdnGlzTZDkzpk9mE9WhtLDO3KKkvpnhc2Kav2Kl9X++2xdxjPWMv4xwMdYiJFFVZaFpU5Zc80FWafrWT6pM0PDS7MbdT2mrH7sGxMv8LnddX3xxekj9mc2zJpOB/ROAu1+a70+rCqwNqNGl1QwXB3Na9x/mJL4RMY3n8kZeU8X8P+JpD8Fba2pL+HtKgriI72TFlQ1A29JNo9vmbww9VfC5ixK1DkRZJmXTFRhJSOG+sB+ra1/PJeoPBurylx68Np3RueCKRteq1sNyWcJ95fw9ULl00rDeIT3PKqy04V99rYeYeqGumDP8H6/EJZLmHF113TpnzTdrdY89n+tsnmh8ddG4IIBI4Zd+nB2HnLRap5W4MKzIJu8UqxBGp6r312ffvNVgnW8VrPp/VZS/CXmT/2FveK9bTv+Gs8tLmtGWly411XZNKivYg3doMugKYPGdLWpNfP3YOpvDdrHpAszf/N0NDgK2MxlHFZo39Xvx9u+st0ZwDvbI3hnk71zOIp957CX/o2scr2u9R9UMvZ9o8D7PJ9/i+zppiLxfrekMH4VY92HXLc1VIAUMMNZQPO3wAwVOft1fbNfJ2Hya0VMfs01GncWyPXUpy69eMb6UpOgNs5C23iEz19V+MLde2HSnJki/Rjyez7jyWKwcwId0I0PqvHBN968I4xPWbqHEaxbS3fWl1BTw1h6uXPG1DvGPXT/euS/672gPQHks6FJwIQWnfjFj7Fda7IGU663qh5L0C9L3ivfvn0rqqZi2P9XYHRPxaU51Sks+jN9agDUkr9JCszsikj/+p/Gn9gt/2kc/1e9Z5/gn/80vh+R+1fyFn53dHRETojw/PBWNxf8SzZfHL0FeESjHs7u+OsP0kymPxwnvejffwKCBM8C4gdYBEz/2xF75Y9s5juCFTDbungb2oawvmZj/dkuy5UgGnyse+b0hkwfqDK7UrUFX8x93Uejc4s1epcIPo9ybNF84x/+IUCLrbVsBQ6u/hYvLMf6T972CCxJ0D8ALz/QI8QDiAcQD9QcD3Q2wwMtIPwICPYPCNjazVdOWRVmvQ1WyzOS8aprreZem85Xq6cB6xU8G1pez2Z5zmYBvn3VyTuwGCxxC4Z/jsq+ErdwKcalGJfig1qK+xsuxSNcicu2ErdKsxSXZZ1t7X2h/aCy9VOQZOZEIp/O35MFXajaaznX0hkVpYXAu3czcpHFlavwlatZ8MrVrOLKNUpcudi0xAb2R7sEAYeftbT1cWkrherM1yZrupy+p4s7tjJN718NqieuT+SE+H59pVG64Y/ZIpnq5Wn5qTcCIttl8zje6lBqsF3A9ufOErapCVZqewo7HhXP7y/S4A62un9mcQR7Qxvs7eT2RVgiVUeqXmHA0yoY8LQrSdVb7Q3d6C0ENKUDNGyWToFo2M/TQJqtXx+OaYrh87+e35RhYf2kqga5usW1FddWFBNqLia0epu6pLu4uJZgcfXw1rkkU/0VxvBiyrXXzda44wVTxJcqrGnw3Rl8MdfZ9PYH+KTB1D/Xg8TW9SZd+kozfNH25LeQtdheA7NdjvnsciGITw8aWCYYh8+v/2LPJyGXvP0jeY7aZN6MnM4DttkaTHDcBRZY3FONud9Xrvc0yGINC3aZJ9fdC0at5tpTZLWopwiwSP+y4mnuSF5Yp50LTd98EDJNGIkTibHhJBI5ybf9mzD8k/xww0l+Pcc7g8zfhUPmfhuR3fkQkvv7mInN2cnEp2ViqITDsOCM7Sp1tLrnPEwTlIeEh7VjZgFo+Ql09hs2c+ne5mfXbqj2iTf1WkBnNuK3h8I0/unOgpceay5V3ZhLX7xbK+0vr2CpvIW5ll3sNf/sug4zXOA3/LvIn3B7vBeWMe0w1/iwCsOXEbi4YahB3Bs0pr6EXg3dxje6YEWRvdvdOi2Yg6CrQncdsUmoNfTMU3P2nuDgYU92P8d6TJvNZ6OO5wHu8Rno/3zzGzzHXPg3um66MucZbu0ajY1GWeKt3bTDbTB3HPbYO90762fYjk8+MCN7ML8a34PDiF10543gbcF68hudzcqiqWkrjI5Lv3vpX3nUW1F44JYDSGIuGbQrBQao6frv34PZbu7iMnZ75xwSWDw+CN+iPep6R3OmKCLilb3wV+aINdKjCh1hRcVhBYcSNhfmWGJjBf0eKOx2v0yAFu0CwcNn3yKRK3KIGPADb8UODl/EarybA4wgTEmHL5rxegNTbtI5L7KCF1wkO1SJoRPho+2gxIASA2KB3bFAflJ8RbaZeaDDoHk8ajbC9furW8KXBNQfDkp/SMIH5+8RHBQLDgL6QwQ4GPQQHJQSHAwQHFQDHOwWgN7Y0+LdCI8tx0UbF23fos1CI3HVLnjVbm+2aq+pP67auGrjqr3zqr1trHqJlm0eun5Y63Ypsun2orLpdtJk070QdEn0ZNP9QI3tc+oKsiToYweIuKrKs/PBL/94eX4x7Izcw40n7LlWXwQrJvqPw1HzqtkO3KHyFLrshtbVYDhuuW+4nlm1+GOz177oBS6RE/IrPB06JYF1nei8edgnmPgfYfGhSsSPH0yqW4UanY+7Q3cXa0jR75TWlQlWV9J+51curloXnj0nDUWy+s8fx63euDXwTEnq3HB+eNnuTroe+8CAtgKF/ng1vmxPLj3XuG7IL/aaw773h6Zum7TXGvXGl41A17oXND2kt+uP76jyYPD+2/R8T/UdBocLFLYj0zqtOqikzNVsgKE3bmoFWHbCixJfdLqeL/6mCTMWD+8Bu5EzwcA3E7TaCTNBPyykY0aVW0u3Dc0AMBisERoPFPOWA8bvg3Ku39l54f1rqPD8EPKtM2kGr1ibXgLf8k0v/m816eER1oYwedlZkX3p83W4+Vdpxrthu+dB2CzQ1zvV2d9OoFcFr/BYvZD3hm4c+O7GuFT3h82wL1/sUrXi4fAOADF014XADgFY7UuI3JBgbTYRHgKRPklbFbgTgfVlk7245/0+2NGgM8yoZs0sc1l1YQDLo/DRPZTC13q+jqyyudmLw3HYfrPVepGI+VfP5KtFxMMGlxft/mSLh/ECWmFuUc+014Atn/lZd2WM9z2xed7tDMbbPvGjJwm975H9Seuq307A5yINm/cB54lP/j7FQOaSzt5ZeN7bQTaGyk4WZQcWb5BJ+T/2igfIeZdsyrlnX2zHButG9qYM82DsYNQfGFD4obIGDQ6lbbJi5GBOCyAWatBd7BY2764sN96H5f73n8bpjwCEix3dO9nu0z/0CMtNMrFcorH+4GSVtz9YY9j+YE2Sq49Oh3Q+ar+v77SoR+k7a8jqtrL4ZWZ9Nf98/sWYLIiBVsa6SqsWeY5CWtArSzD06Q4L+ok+2FDW94PbR2kerVTwXP+MDLpkidXxVN/cnEHQ6CyENVjcoOmH+VNH3JCUmfQszUxBDvJh1/lw7kPYvghfJB8mvzfFJwvj+LjGKiOCO9h1LUn7OUso81nxkhAm+ip82Tq4vt2J6m/ePZP2Vmz79EDf1+oDNEsgY4LNARuBPm2xwIbvW1d9dwvab3d3qIrX6ImVCdxu1SXwdYiQHDZ4XoO9h/PwMALOvn9Hn1el9pzttrEad77kQiug/vbo0qtdXZjzueVagqut7mTk7hqNMXNduvmC/9rMlpNag65H9Rqri4VkGM71gBLFEr64qY3r0s+CNnuBscx2lmnmEp5BTsg5j25jW4yZ+9h4FAz4HyVPVFOoTAA8KoY0l+BWQWcbkO3fsa3tBnlRtSdJebCktqur86srj2CowAwqPVul6Q273YFHibsB/szlEd0ubq953vLfMI29A0o8XdX3snk1mnjesM4JEy4T3kKvtYhvs395MfBfmq7b6LI56LUnnhtYArWIx8Kl2NfCddezgxolv2FVr/ZVb9DzKJGfFdN5upcRl1dutLw0xNSFB3qQkmMgD9dOmuOwnppjx6M5Hp6G6G3qEouIwQgjv0Lnn4TXw6vpA8OW1BWZSCa6AHzqJD8Ra4GNVPYsZ9J2j+aTblaC5vkz+/M+WiKdXE7Or1IYqwzCYmaRcwVKWRG9OENlcF9ZiVl6I/evLSS56Y9/8rzZxq8lyMOb3JqeTENZ65L7MWjeukXSTJqhPLl7fy7CCME5P0N9cZ/pPPM2XjDl5tZa4T7lr/cWJUEFbGcFLCq0d3dxK0IPSqNtoWwVIlt9ouKzNY1MgdO33fj2OPQuWY2SYNg9M01drh/Uv5w0L5vhN9nPCWo1/BYgIq4CBZWT9U32c8bj8473ljtNUPTkqrlvi6ybdVNS5Vx3RdbOuiepeq67IuunbdAmmrtNmlbU3ajbbZ6H32Q/Z9QfdLzRgpq7TZoRhdb8bRKMOjS8bdKMaBNjkzYxPG0SVTkj0CbB2hmeNomqnhFoE2+UTPGSW4i6plCDSbE8oI8sqUb4Sj4HMojhfeyGzi5aW7dIrY2RocvF0niNuPYvqqnFS3QWvFkVOzwC0Pejvah3qNN5paez0x/5xHX645kvyp99yyARNxJ9EJiD5uSfqaLVvNXIRFVahwXYU9sUlhkKRbWXh81SQoYEBfw4zJsVsfWQfPs2o8+SSL9/z01iCpqIt+1CMspvI15Sl5FII8OTlopjkB/sZRdw23wOyzBSySyo5FLfB0fbkksulznS3bLzyD26mTtRQJhtryJ8qSLMOe6cWL5O+KMTHujgzkh/yO7nMExMGUL0blJMxsmtXrtGTumNEG9gNzk6rPcGhPeLU7M4UcQCQX/43w9WLrEfsjk+JPDU3N0B376t86Slwb37RHR8Fbnlqwg/zQsRXQaIzjmbKQtUl0f0awk8BN5yFgfs3BvxPctQkfvw282ojfi9kI34McpHiKcj9c71pN3rCTvYE3exx+9kT9jNnryjPWlXe+LO9rjd7b5BFcEH4jhBDC/YrBtGhqGGcYRG8IyzEPQfxwDiWEAUEwhYKYoRBM3pZwax7CByVK5HZttXjJewL78ET370ZDAKFjOOJKyHcycEPQTL6aML/lPOV4wh5MKaNIRctKblsAt8Yg65EMMeolfM+O300Vvq47fVx7EM7zrtWUj87RRGOKJJRwTxiCYfGxCQSBKSSESCtYnchb8JJ4njJTHcJJ6f2FdD0sMkspdYBhMZwplEZCLJjH/9TLNDL/t94hsSFA9JsbBhzA3hLCY8sMk6a+eEwIIjUl2nOqFfqGhy9CApzt4YDrADD4ugOlF0J2R1yKphst1rvkujbGFztrMcLP6BfSGTZbjhWYWSzH6xd7MzvFYVs3+AsjKzwz+zNFYf793qme4TztnqGxKSaHNP9m/ubBMv5G1wXlqfya/PPjJji+piKVMjrndf7t/cGeYNyNvW2u9gaGADz5JozSDwCLZtVI+x8NXeLZx1doacrXxrFddjabsKcXb+2/7tzMSIyliZFRZszHJMEx3mZIMtiDyBIlGBgAh8MnEcjC+PVCGaqSj2wimQZ0kzTKC0X2F1pQp8UpWFXwr1NtDPe28gLnZVpYH+xgrrbSCnAVbW57kO2YQvEF63QMvFNMf15s3h+uSldRH+izgfRqwfI9aX4fVnQM39CYSDHow4L0aEJ2Mjb0aQsod7NSI8G7HejTgPR7SXI0G3CXo7glUIej1CB1+cT8EbL+KSMYI3hfsY4vwh0T6RSH/DJpPJxvUJGnyjeoT+LNxnEh0REz8MX8N7WIQjJdaZ4n1R7EbgKJ9EdOIQfjk2eQi/IyaByPp6RBIR64b4RCL8nuhkIvxysQlFLF9ObFIRfktSYpHVTdPEu+ITjPBbopOIrC7HJRKxbopMJrK6zIAe+8eTSCP+J3E5RtY3RecZ4fcEc42E3gDFYw9aFU8RdB3Gjv1L/158dBvV020U+DILv1G7e5COow46jopwHAU73d48R3HbPEJBmXu7R+hS/OPpWRh/CUsmEo+dSuKZ2mNajd0TOfh+vF0GCV8GAAYmdvrlGhPtnntgq5LvLAI8Rk6cuykEDGhDZd47uAkGxj3/VzuxwsWF5VKWRMGaWop056Vqn9LYd11yn5UNlSwAsN5T44VSZSXGCMqMLB9fdTC57MBFYTbTuOOpUL/efvr3LiqkU7i1hTmteqIU+jasQ+ITo1tLaAnduvIoPFOwPJh9Tg1WK8IOmyI6YxD3psG51is1rFsW6owTskLdeztOa6Xp9qzMwWnFRXSZZ4m3GSMjRIfFUTRM6OVciZQMnagv3LVK3vLbdCJLT5RIrNLHhJFk7fWYQOMdFeoG3HmlKk3DOAPlhp3sN+MbgKwmeMv/FFV4pGIcWY1TqM8vo1xQ+zSuXWww74Wswqxj7TIgb+npwylMOtpsJulPyba9ytW2KVJM7dO2SUJKkU690qSt2usU75Ld1jsLC/Xdue3oMLwVwataB89NLa6t+85GF7ecYKMfb69+vGCKM68fL+p6GtcXevBK7MFzMpLAysC6dkxmq9DbnVxhoTt2Vqk8Nnx44H7v00vmeEEXynY7b1q9lC6U4eG4UMrk7ECXxtYujbCTaHrDQSu4XLmdH3dhua7WE2NcvqsM/B5Bo2Tu+CgqH1bOQqYr49MJ+WTVoFDJvbCkWUUa8s6uQjbieRl5kC/RFlKhfVCh8Kl52I3r9/5kXj7KxGVw0yBvT1i2E0khb386ahTHPbbmUBEm6MWZANnWjmyrZCFiHQwRyy1ELI7gBJOSbUF8Wr3+4QSPReYu88eIhT0Xg8Qqy6jKSVmyzF3meJSyyrBWLFSPTIu2M9spI0b3pU5DjI7uCnRXIICuB05ul8IP0B6k3UrRHu0VDrtOskA3AcLdvPdEfKLCLIVX4HjzN+kzYRr6pujjY7d/xX3SK7yHge/yCjHpFfbxVyleMUusxdWw2xmneQVNfEXALb51c7cSXmFvTU3zinZSNS7Ou53zVO0tdJLecdWd9EbDVN02yVT21ts0r2gnWcraJFvyV3SSuq21eTjN+E5qi1G/PRmmGhlikqHs7expXpFkKDtLb5pZKslQ/d64N7lI9YokQ12NRp3uZapXJPao8363n6rT0lZip+W75nM11PjyfJyquWknsS0u25N2dcMGWFz+VAM8pE+dfGezzZzdtnr21//BF+wp1kc93Au+gWLW3TZKAMq8ytE221O0ADffiyaxEIE922/L4IBfeaE3MuBFDQTI64/+g5mSpcdQ5oja4+7ao6TGuv2vPxI2E9lefz6qyu74RzUy4+BpSSW8CwcP1A32P3T919j1n1rT7LYPzcOPCWBQ7AySGLakhuZ7yUPszF/rzF/qzF/pzF/o3EXnrCJr3SY6OyXh2tPmWA9puzVFloh/DhPgq4WVI9z1xZJfgyqH3BC2CRLbouI8mqXF/8zSS3LtBRgalxAwniccpsViqnyC7kf92IDzi2Vi3E/l+He4GfpxZljEWWGvdUWKjhS9Suf+jVIy9FZ7gBQdKTqG3++RREnq1EbR7gM/dgbwpHAEH75qp8L39Y3F5xDekGTpK09riti9NNi91ey24gaKPe+ZihR3ZsifTWjcEgfvR9S8HbtPthxwvRwutcAJ8KGnynrO+QEqZ6cLOyMfqAH/Zw7x9XntzsDToJolPgu+FXUWfD+bs+Dx3BU8d2V9U/K5KzkdolKVo1Cy3v/STMNEB820TLTTx7NESs0325nxTdbx34WpMtsRzfKSyGKP23Dm/xKmx3eKtk7H/vIoiY8EMKiV4B7WGiZ5s3aWrVztp4RcG0TSCdxOCYXhLsFQZznb7ym7UxagjDOeqt067HNpaktVp/s7IOK8rNY/99nePn/ANj1Pac2NvLI/OwDC1IEkwt8Me6zNbSq8pYR7XZVNg7LrlLnfBO21vu4222y2Gc/ICmgVRNpFc2HKPCYEg1hxAz2GrCJzROaIzLHCzDE0zDgD6thD6nggrspqUcfNA0ZX68f0/NY+HXLn4FPrWe8k+F/UQZOBvdFBVmLNEc2KBah6Dsly2bSE7MzX5OuV3j5faAm8x6ZlsqyKAlvYVYUv/1aitb0RXlcHK61dne6/NqtjV3suZ3ZlieqEtbqwvZ3rQXAtixWXGA55LfJa5LXIa5HXIq+trUd0mJrWjlpIa5HWokc0hUeUZaWu2lmcnsKzM2fhn5N7dnjczKEwQE8kx7P67tNnIku6sTc2uCqvoKhKZY3NCs9wCfzzCthE575Qfii7dVR4mMmP4StRNmcARoixWM518lZ/BKscZcMYd3dFV7Hbn2/Q6Tfv8uMizFzFDn++cXe3jS1TQTdOnBCBEx418BZa4Ch8CMS0yaSe0QBkQg3BF/ON4gmKJzuLJwk1KVY8qewe2fO1AjAadbvN8zj1pDdsdZvjvagnE0kzXt1pTaLUlYvB5OL8IlbqsHOG+m55LzCkbQkYvfZFr5WjFuJXMiwBIQcl487mhFGm59eZuBRT79U95CeyKlPcraubwtW2Xxbcg/B5LZ/4U7iifLJxWEB6/aTTPEj9pEJbmDGkvKy+c55YhvGvEjp4V2XzR5VLuk1a5FdyTxklWeUFYncbKpm5990VLZSsyn23WJbZrFC8tWU537t/JVefby950L1BF0tVEzQr6ZLVDvc2lNqX7MExVAlNyssV0ktfBMngjFklMEQDvZR5whJ66rgWXHo9kpn8YCFh3BCPNBpjEDAGAWMQMAYBYxDSxSCM0nPoIcYgYAxC5ULrw7LcdFudkPw+q8D5W+a5/dl8oGzC1YMEyEKcjjPSdvRy3Gq5ftfBw+RFMh7JIzyKLNmzNg2uP96pTsNS16lSGwYsobyEPNIqGLTfZ33dXJJCFtb3vEUIP4F9tb/+mVq7vBdwD50dE92ErwHIyNI9kHdpj9vleacsoZF5uRwbB0YG/3hPjRdmVB2QtibIZKmpbBM81Y8BOZq6yxfOECWj8Pr+YkG8g78og2eWDLDIuaugsBFepZvFrNh5JtsWWRWfE6gVg7Jbw2qGM94CR07b2GE80GYbtUE9wkRuLYOw/mj7XzE+ZA/CVjhoasamJ6+6BBZR50FsikcUyzDoBINOMOjksINOMhDMWhh0gkEnuGtnB/nl9rNiJ5wrozqwKhwQnxtB46yHLTzHludIFBTmF/LkKFQVdy4H3dTo/kSXT1ap2HxTRuu6ihdu3wWbpjzmXWtafMuCXvs0DdyBh+ERGB6B4RHI+JDx1ZHxIZnLLvih1UxL5ga4gQCjH3ADwU5kzp7Jxo+m8lS1PdLusgMbecfsoAPFUxVDAiBv6uQeLj8xhvJs3UoAbgvr+O45W0z3RfXs0vNVp6KWt1bME2L9q0tfqdvWNqdmJt+bc921UFfUxjzMwR0EG96VeRxm3fch2CZBco3kGsk17j3AvQe49wD3HqT1pabn363eCHcfIP9G/r0D/76A9UUso6vPKpgvnpdtLLc4tXUCOLm37kreWH5eCwJiGQVDRJF/IP9A5x4699C5h+wi3rnXSk0u2iP07iG7qPaxYasctKGnfEFZFZ2BAm4t+sAh38k/GxXeossSFhe+p3Gb9Mru8q1JjqQQYZVqeaNdcDkfWQ2FKbMJg+YDmqjZG515nO1qTyHf9lz/M6id3oM+KuSIyBHRR4U+KvRRoY8qtY8qPY3sN9FHVWoWifv9ykrm/s5XklujnGmmXaWDxdD65MQecoOdkrtHSWdBcmzF09Y72PZG66DPi0uzcolloNjjm89s3x9wPPaXE+G5SuEzewUCIInwxSvjfLIqzPhWQaAM5iHQP7v3nTHroJcQGSAyQPQSopewjF7CcPIY6iUMvxW9hBl6Cdup6V03W3oXNjWhmxDdhPUkeGtPHVtbSsXuvEUD7vHzyivH3E1M6FwCtLZkT6aMMo73So01Kdkby1sX/ZP+PCupVVnRvFZlMZ7WtjKBiByuLGAQsfNuHJOSOaw2a9M/Crqdw1iY0WNybxrQCM7NMID5DaxJVjftbTvgut63prYsaZOwokV3dOFefab8I5/L2bUzlt/4TH/Vz54XZ4o2ZY7rDDPolphvr134Y+YDgUkZ3a57Jd0JVLUR0S2rR7obDaTcSLmRciPlrqpLNTXnHrSQcqNPFSl3SsrNJ8ByUpFAapcVC4G/SxAe+7NDdKB4JfRMu4vnEDpmV27O2vtC173o1l9dJGboDUVihsQMiRnumERfqI+XddLvmCzwMNiweYtlwrtcLPnAjbr+L6qpEZeR16ErtbK8DhYsKhqwqLT+VkJG4iqdZ+OI5WayjykVdOB5kp7EUvKldquStt9XLk7WVfbUVr4oxsrdp+paufv31Fauhzd0ZREk20i2MfQYN59uSbZx8yluPkVPaR6MvDPAzadIqJFQ70CoPyss06hR0gMRXaXzLIam63t7l6Rz/OQP/HhE/cUiKKpp8IhbgTwLUCrjlf1aozB7Kvvzq75nKV7zPzFjB3u/f8dLBra+tV3SFt2zCsyClBdOulpAFhyKLNi1t0dEfwULLQjbnFp736u7WwrKjNhmwy2pyAuRF6IbFt2wtXTDWreubgpXEtBXm6GvtpueGbYxuy1SQ6SGu1DDD1e3QA/1snJDKB5xiufJkcouWLRFt3dWMp4CbwbzO/snrU2s1v5Li9jwe+DdrF/U/ZwNsBD6sJCrIFfJiau88b0O5o8lFfxDMzAs4R9JmauN9Q32wNCgbm9creKUuCGqsiws9cAITYaBobit4VvBOVxrd11f+KAah2nt9htfn7HAV7u/kuUbS0GhsneAxNG4MmDkUkHhdictEh70DwYJI9QtF9Td/CAH1TTqe4CDJhj0LQfYzwt4rTFdPiwfJCURVP/7T9Mp++3UAcm/HRWJs/kmI4DVJ9bmKvhLhX4lWD2zSBdKqP2gx1TCgMwRZVuQ/bmRCevhLLE2qV0r5Ix8NA3kHRXIIMK76NuTI2gyGO5vfzoqcNMXMpHivSYHDbdDg5JS4+12G+E2wm2E22WC2/rLsgJwm5fSxor6KpAJhu1+sTYUqwpY2ypmwHyqaRwI0Gb+aUTbiLYRbVckRgn3ruDelSyjonDvSmFemU56r0wPd64giawcifzTnx5hhP+d8qBV9f6/VDT+0m+1hkE4sKKbNw9zwZRXIvhC+C/blqHMCHQ1+Mu6GCeL2yO/GQLVXZwyS5Ia6ElFuIS4IUpPs9I2ZrFuIihfRczqlJQFtHGjMoawjT0virDnzsYkUK9qt4+krNqHd/Vt22dcG6mBXLFa67jpaR8SQ/gC3OnEjYYkMWIebM9yxRxGVLqbotKoUmyoUuBOKvsW3EmFO6lKLVSE+rNTKxWd5gB3UqFUgTupdqb2qrqYPsH43wvJySwtINSCsFpQjUjKs30EnU7qu2Xq48f35O+8wuhMLQvT6XUGqZiOGHKaXfmpDtQaqc52VGfHjVi/8Bm7cvuwOhH7sEadKu/DglVe1CQOVQKrWMk2aXXTBo32mxUD2dWICkW3XTD28xdBkzh4+/E0JAZUZFOAFc0wal4121UL/uTAm6GZL1Oqw7/spOop2EnVACqLqjLLMZfBYyTnzAOQXzrVI7x6LITMrmHx+e0sg6vzuU6Nutn5DopCrKoxG99T44Wdt81O4Zad5HfMAwo2hhEL9SUiy4xXfO55qxFgKq9ld38PWHFhLrbp7BV3ufGOd/uqiJqqSF9hnE809hMko/uN7I2hl5FdsoJhvDlxZdw5V3eCFL6tLjVDGrWQISFDqp23wIXcZFVd8j+nsIowzGZU6NSgm0dYEk7sdNGsJoS9nwiz/5q6UV8nAcdoN+/ekfNgRRGclW7bVfGYBvdZITYrt3jdSQvNWv0hQjOEZiheZwkG9VdFnDJnpFmlsyOvdUv8JKz4j45uZahEIBqFZmQZe618vORti/yVvFL9mDThD0U92pdqPdfo7yZVxNepBVYXMCCmPJ6/QoZ/x9Voy/arCpF1heqbAyGgkt7yQYNIHJE4InFE4pVSSTvpVdIuQnGE4nVWSQ1JfMrft505QLu1SrwKH7CgGqvM3sI1DEEqLGQje6bBfqJwKgjA99xQF5JIOBZ8e3d+fWQHbNQc9r6XdBGBLsYDIMgtKIp8NfxurbgvNgqZ1FG5iPFueMR4Zw1L8eSOlLr6ICWW77cRyucA5XHjZKlB/lJTRX16bwUSlOrYOZh0RKrrFDC8HebwIkgGOw2bnTN+ffaR6dyiuljK1KDFY3rLcJqpKFCkkhpOUggroOUGMIRYM1Ucn68rjXp0ybLl9JrNw9tDCrVu4R7Smkrc5VKx0wLfTquHyBeRb1mQb/7JQFf4jQMGHvVBKyTFfvtmgTny/Xt9A329eI4inEM4h3AO4Vz9ZcxWWjjX6mJ4MMK5QxIy1xngYEJ+0qe81xSa/W2jrfk7SXi8Rvbe+0NAe+SKVRjhXmngXr+bCu5Z/feM9eCqQb5+FyEfQr4iFLwMMN8IMR/GoaZAdptvCXsvfDn98Sz62KAqnl5ri4HTZ0kzAAxMFzw95fT+1aB6NkCy2ORNazxl14hYNSK69JXyrFqsZoU6zR0TA8bkJxI6NoapY0M7lyUjlrA6btFnXOFZkGTubN/AwjmdH4Q92WfncaF23rw3l9DW2XbwSV0oqZWrGCnpfuO+Z1SM6HErrhl1vSwsE7kjcsfMU2Gnj/7oIHVEd8EeSGWY6DZojuKOtr2e6ESWFpIRl4vkqt0d9ca1Iai+aJWlVKl9gx/MxT3VGJZkjbefMG3LcAyarz5VKdNfTKevTfQ2q6PdVVhu73eB6iLixpgfjPlBEF9PB1B6FN/EGG5E8WVB8d1RvxMD489Of7S38oU6kqBOis6mWW5V+sDPIj/5Z6OKjiWOQXV23jI/mtLeLujk/9hDpFLm8HR885l8+0bEpUm+fycnzjEhRF9C1yV2hYlAbOhdKAMIawK75x1OE9y/OrZnDeHeQ2s8SjqBH9d+VyjhzW/au2B18snulWfkV8scSDXKQjW6w3481ah2+peIKiPPQJ5RBM/opPcWYMJD5BnV8BbcAe4QZgfvMDAsM1RI8T6X2WGP7KRNu+z7dhtUz4QJfb82WRntaqLrAF0H6Dqod+7HVUBe5ZI/9sKTP7b7I0z+mFEY1DAlr8Et08hqMmA1Gl1SIRSyVHAztajCyvvFmOovkiE+0n1K5duI4VapiVPqPTkcWOeqwUZ0nvZdM5eGXl8S4fSYW7vHkDMSUW1kEgdyrhFG/aOQX2ohPy3eHSDeRbyLeNeRe2VVmLUKTOLd3eCASWFGWovilW9mil75TNFbFLqXet0rSmiLVqwxxtVG47baxyqK2Ls0Kn47VYhKVVX8dr+NKj7i+fzl61ZaPN/uYQKoPBA9Rt/4gmrOTn8ca5IhiYL84+lZXFTNZbs76bY3jao53ioBVdKrr4bdzrheKaocVdnWJesQt/7tG2Gwygpahw+SMldrndR+LSuTCTUESUaEi+oyqsuIRsuWj3SYejtqH+EoCsy5ANXq7PZkS58uSyKt3y7D+mK0W2er4Lr1CP1CRZPFo9+/EiqIj8wICN0qs2VwLyAPtw0iHqxLjsG0cLDVayEcRDh4QHCQI0GwjqYuX6ertMnTe8nQS3VW+KVVRFdmZ0MlmqDM1AXMo3xTFfyAAihlk2NtQZ9tBsR05dk21h8e4rax/gghHUK6AiS+dnqJD/dMIaY7OInPOfdDXJp1SiF2B2+z8oXVF+QxAdNY1ZMdYWLqdtIA3Qo6tHe84aYfTAeGuh6CwLrrep20GLCPycAw6DC/oMNQitxpJ6QWTg4PHI6aV812bcIDHUjKMlXNd8pSldtJciogLQJmpQQaXNSkZXYy4pbHTrKSlMU6H7e0SsX311+xmk4iaoroep8qa6dziCprp4Mq63YAO5CcK3Tjb2AT33tJFyuXrasfnq2r0+xitq6sAgr6qQMKUHxG5lEw8xi0+/HMYx8boyJKOkjLkWq2hYrHYzy+LFRlCgvTcirC/A1drEIZfVno7aO0ZJul4E+dKoCB2QdWneKzRLhNCb1+KsiCtsjfqnZ6s+AsWqTFnWFOzlmli09L4bd9ffqyY9lssluksekjANhD684/Szo8K8H+k9ztz2Ssw7A8rMIxlr6sturzs6DNXgSN8gWKBRCa8Dc0ssSshgpQWRSgQS/Wv2oPxBgNyNW8VdOBBj10tKKjtZBou37qaLsmCh4oeBQqeLT6vR7KCEGsKgJeYSeIzuizJMJHU5uy7b40d7Qan9U7u42+1kTAsCpDLAypSgpZJ2spXm3wWZwxhJwtnodHfBeDVv98JtWAv4ndgsRqQcT/ZcH/rf5geIB7p6Ha6AJG6F9EjGV65N9G5I/IPwXyL/MppeoLBfRsLpfy61RVWCPWBUUDvHPXjoE+q4L13WBzw+pLrPoixEOIhxCv4lF+QWdOJSP6BhHnb476GNGXEcxtNdMmMB/hkUQIc+sHc115IXlM9GyqqwxbTAVRpEuDztgxmwosWKwJapLU2xE5x+ua1ThfpNWs5Fa1ICPiXszonQrMIkRFFTL7jN7p8RnuuEB8ll8Awiqs4EqQZDqLiym4andHvXFjt/NjrhWAMtIzjX3B1WjYae74gvPExw86F1f90Y6Pn1BefsGQlIf4OowvBpM0dUh8xXh41R63G1mGcATGUz6yswPFTWWDfJzHPNQAPq4NX6rIgrjGyjswI4UhafmMSIvfy7GbAWe0pH0xfnbKexPHbsac8wWnVGaMWgPz3oWxmwElpYQDOnqdr8sGC0d4+Mwai9yyxkDx4RDFh0xdaag+7OQgswdj5VxjwwjX2KBZZdeYk1YJplcrCpU6yezfCnMYgGRBgTboR0Sdk+uzj0Sjv8NkALOoqC6WMmUn/SxZJANPIElgSBB4SWjs6lyi8mysKnPpIYSf07lgykawL3KxQzf40vDte3AUhXH6wNBtqB7vR+QsVi4/YSulDrXfY45ZCNG/qKZmpVFp0sOjcRsZl3TQKasFmWrGnYOvkmqTJHLJ5oOk/EI13Z4TBqedU2+SjHyEsNBESbtJZGFRLu1mJyEd4icqzE5/DNmkA1VVdLbc83ahDwKDyyf/bKSOvIEydePLpM+EaWiZXCLZ5UW7P8mkML2kwtwnFeby/GLYGWVSmH5SYcSkwvQvJ83LZiaFGSQVZpZoGWt/VxaFGSYVhiYWxkqsk0VhRokduJVQmN6w2x0MsihMt5lYmHaSaS7Ou53zTHpwt5VYmk5Saa66k95omElp2omDO6mhmue95nkrk8J0EgvTTmqnq9ZFM5PB3U2cg+87SeOpOei1M5mDu4lzsJjUTKN+ezLMZHB3k+fgpGbqt7qTUSeTwiTOwWJSM43Ox93heSaFSZyDZ0nN1O+Ne5OLTAqTOAfPkprpajTqdC+zKEwvcQ6eJY6m8363n8nQ7iVOwbSVOLSvRpNMVqdeO30zjS/Px5l04F7iDEw7iX3msj3J1qtZaJjhTNKfphqAfn26UjOyjCsMaPHdDUICLenk+3dyQhgfcQkthfosfVZ60SSDFmWm9nZm+pWXbSM7VXyP+ASaglx/XOpkHFrZDfwYodIJOjK2dWSEzqmtYT/Oqyepy7jsUNdMXBVm5O3JETkjfMCRtz8dVW0zEVhhgJuJahqpGe0tUOfk/pVNw7wPz+H3RNV4Lzaowk7KXbsFXA4D9BCkC1VN6yLodNBHgD6CCvkIdg+WzV3fL0C1L0CLL0BhL0A3L0ANL0DjLkK5LkKPLkBlLkA7LkARLkDnLUC9LUCTLUBpLUA/LUAVLUDrLEDBPFRdcsop1R63OqdRLi0+uG/REthneay4g7DJ2HOSIeugan46+5VMBENAPbM0emanH5sV6CJWzXSJQRUXNDuDJuqZh6NnCs9UEx4oYYOUxzOvIp4lXTfpzKdlwqd7SmAFeKazU3L3KMF9iiib8GR+G3+MvmRJou9f+Tfr5ykE/jLhlcrMfy97oCSyNMBwYXFaVZ20U44zFFPLpAOUSd0r66q4IZfZ6cWosaLGihoraqyosaLGihoraqyosaLGWlqN1ZbwWDhKCpGQnJHiAkuzTM++pbKrTYUXQTL2HouaW7PlGeladFp9V7u9JLZbHVTkc1u8+RWqSu78MivKyfuUk7vd+CNE4w5OhRqe1kJM7obtwkMxue5isqXzyvzNLFTWowMbj4JBXqhGQxVmDJFNd5xsau23h9ovar+o/aIii4osKrKoyKIii4osKrKoyOajyEqqJey9UIZT6Sy9wtcoUiCt79k9jrL2D85kb6WvKK2VR1rrtTpxqpKihh2lulLXhN/NE/1riY/5Ca9yF4W0wxHS1rvMuQrMIiYtDXilovE8tZ5ktGtdjctvqKtlGFLZSSurtXDrOepqqKthTCUqeKjgoYKHCh4qeKjgoYJXl3yaFhsrU5ZIviXdKlYJEmmWzz48i2aygeqy1/x9sKKoYe5Vwxz20iTPPPuoVy0WsBefLhQlzPpImDcrWYJJmNQ6zcza7z0zNbbN++VREh+9J2xFhgSSt/eCMuPaDDENSZa+CuxFvtO3jk7JxLpdFwxTs25RRdHU4NGPLCkn26zORzW8hYiyqvPt7K1m88/8UdZjdGsvOhRxVTAmZgiy/HpKyIVpeO4N3iwpZClocDuVj4luQiUFnXw6v54QQdOEV51ruWyu0hRyezvRj61y2YLvTIWHKqoBz5vLMGhY/SSNqbx8oMCiSWRpIRk6bpFPFyeZWtDt9FHPRT0X9VxUWVFlRZUVVVZUWVFlRZUVVdZ84ySrFR5JTsj1x32qr/A/UdBmFbWdXfr6qrNsxza55QnwJipTEQKyHmq0+9Ro+73Yo31ssmYqUoxu2fgzk6wqFm3a7w1Rqj3EaFPVNNiyMPNImgKXIYm1Ylh/6wJLF0JeBJ0YwhNVTsm1Imp0AcOBzpjiuf6xRsmD9GwdiiQsl5q61CRYV4gOPVU0nBun69SgM+p5ErzulcwlRdIfTzGINU0Qaze15tlFzRM1T9Q8UfNEzRM1T9Q8UfNEzRM1T9Q8c9M8FfWlHEezX38kUJb6bve+tritoFDV1HHTdxnFuEF7VK6AybxFuEEHD+I5EBEOda1UwXzpha1mC5WtqotTTCz+yoyIItWWK0272QnJLlKAnBVRmF5SYXIRviIK008qTC4SWURhBkmFyUVMiyjMMKkwuchuEYUZJXbgPAS68MJ0m4mFyUXKiyhNK7E0uYh+EaVpJw7uPOTBiMJ0EguTh5AYUZjEOTgXyTGiMIlzcC7iZERhkufgPGTMiMIkzsG5CJ4RhUmcg3ORRiMKkzgH5yKihhemlzgH5yK3RhQmcQrORZiNKEw7fTPtIOFGFCZxBj4MsdeOdczpIJ584zQndtnJquwliHnNM+dAPkcWrcx4EEkJrj8u9ZCeQ87Cqo/K+z6V93Yrfud+kvRevVQFUOMBau9bae9vfK+D2WpJBf9EsJ4EbmGOZPkB2GTQWF+2B5oGNXvjaj2nvGzZt7IJ+EZCsu4eqkA3fGIkF57brvXPH0xpHbIzeuPrW5aM3B6sHtVYCgqVvQOu/E4JUVoIvNAd9FbkFYXbaqd0VnT3m0kWvREYKruBF6KcrIvTBBj5VH+FZy+mwrMgydP7V4PqSRTh2ILqf/jfD5qqGnP9h+9xCMHeccdH8m7nom5FIxaqqRi8+TiVOGfV4gM/iCeooUliOIPJkaT5DT/XKN2X3YMTTUZmv4JKZUPbdrSqLn2tn1UDcVheq46rTYavVq1H+CRJhNCxuwEDjt6nggQ4HQHuDNuxBJiPuBgGHHW9zNFnUGekwBh+hoQuOfwsPaPD6LPy8T1kdZVldfxPTjHqROoYuyCsaoRVRK/vZgwGickHVscgo0IcvFccPEpzCl0jrvOWHAyPegiGEQwjGE7ybrTTYuH2sGLeDfReIM4tGufOZ1NoUGlhLhIDm3bGsztEhb0XvhB1SRWOUvX9+BfANIIsq6Jg0FmBxknG8x83McxFDZD7xF5PwS4Yw7UP6L4dIC8zFkeEjQgb5ebMIXZrhHoz6s2IwzPVm2slNa+EOmINkUPQmnfJ94Nac/GA1WouRK11Rq3eR5/x02w/KjLX5P84HDWvmu3ogbP36PnAdIMAOEOJuZM63GL/aawvF0vjFdEx6tflwc2BNsodNrOj01WY1AtEzlmA49VatM/4ecskU2hyVdvafsdznYFL+MJYLDOxZ4Y7on3Wts/05RWtt3BuR4FLyqqHkTNyGag3spJ9RsD0B2mScjZE1rmptgt5aYTkECko+qU/aGH0y1YsZ9fd0Ot5oGp7ojvNqD3R7V33RPtwLBWp9EyngGXYdDKFqjTdiQeP42+XVXfGOT/S1QRFX0jGpg8P3O99esmYabnYWzMle+s3D2b7sw4/+tVmSZ1ms6bOC9aJ34XJFduxr9IdAXTHJonQ3FV8+mDrJ7cUfRAMmKlO/lndFFYKNV5U7Wnqn3QzzL20P0LiStH0yarfntJbOVYOrD51M/OdXcH68r0PVlOyms7nkkjuX8mN1ZjI9fbrgVrG5rKyRxw7J5O8PTkCgg50/e1PR2VOcIVErRZBVOUKQkoN41vddmm8MGH9x+WGCbtcHhpQm7NIkUMgh3BxCLDXPpFtSopgOSz0sjCFqhrTLn+yNWvGBy6D9UUugFwAuQBygdIp+t30VKCDVACpAFIBpALhVGCmqcsKUwFW/LIQgYqa0vEIJNmyZjRg4q8tkgAkAUgCkASUzyGQAQtoIQtAFoAsAFlAOAtgZ4QBMtOzPdOtYC6wrkRZGEGlzbriBRvZtWbsYBxeZ+QIyBGQIyBHKJ2joJeeIvSRIiBFQIqAFCGcIixMAD6ioBsVZgirOtQ36ZIfyL4PrTLiWMSxiGMRx5ZP684AyHYRyCKQRSCLQDYcyM6luVphDMuKXxZ9u6KmXCnbCbasmaZ95a8tsgBkAcgCkAWUTs3upycBPSQBSAKQBGyUhK43CDmCHdmCxRY0gA57wLjZn0MQxiVY5Q5HC78KVBcR8B4TfvaGo7jhUT+oHGqEURPzfyKmLkBZTw2q2x2MIkdQfZiguiqwdR16LEDhqVZBdfbWABKhG4DX/JU5HKQ6DqkwYtXSHZmVIuU8yrMIJasqzw7SI0lMUIhIEuVZjNGI3jkny3q1N81BBQ4IsAaqi3AV4SrCVYSrJVA+U+PVfutgzsVBUFgNn32/34nx2X+4AgCisKKLT0SWQs/7ENmM4T70OYiZrFHcrBistBDlfLqywBRamJk8EUkWCRs9TWQXsPhjT8PMNOUdprzGiujPtQnbvQI0bVUVAXVZYhX6g2Gas0kjx3cqxJr34aT9wQhjExCh5y8oD9MLyk1E6IjQDy4AgGM4QVuWEuF++0YsZZRwafT80022OLeM4I1V8jKkkgjeSqeGXpYekyHUQqiVuRiaHmsNh4i1EGv5Cn0gWMtxVC8Mc3r/apQbcV2wAtYXa72/+4wYax8Ya0ZFaQEDKmg2T+++D+9+KwAWdR3hF8Kvuipdo/ToC5UuRF8Hjr70JaWz8uOvW1bM+uKvYPUQgSECQwSGCKzUAlgGEGyEEAwh2GFDsNVOjd9NCsu0zItXZjBGyDV7xhxsudpYTHjhiVX4+gK1f/BavgvWEvFaufCaoio0OecNAjYEbAckmbWaqQHbqInbjTeb83C7cc3i1raP+m132vHZINnJ6QAsD2Nnsq7ODcC806Wmivs+yZPdmAk0Ht98Jt++iUuTI+Mbp2p7SivvmHhm9av6GdgeMPXdsnJrtSC5CQGoyDH2uG2l3Ryk2bZiEQ6qEzY07TSb9kRYxVyb7eYQ97MgZSlAY07PWTot5CzIWXCvTZk5AQMB+lT/3aT0ax1x661dsxqHMNjA9aNpEHVO/mFCEyJ6rQl6rUC6I0SpiFL3J6y3UoPU5ghBKoJUBKnljNEwl99UsBa83mBw1FwCFi1XhIbkhGRwvPkRCiuwziHIhBeafL7ZQ0ok23p20vk8Y1rYg43IU55iE43ePL7qkgiGYgPRslasvlydlKLuXsDOAjBRT8a8ohiYkiN+fuN7HcwbSyr4R2tk8t/G+g57cGhQuTeuZnGKzHIOysJSD4zSZPgcCoQbPlzC8W+76/rCh3057u203vg6jR0mMuisvlgKCpW9gwS5xTbcopM25WoHmUUWzALJAZIDD7zVVfGJwcXp3fhmKsiyKuYftZ2ZPr0qM8snxf4F2DsjrEqhcbF5UwWPLSXF1GnFbMnLzI4tEJeOFYk9L8qv7ERNuJxN2MrOZl3QRX5GtU/ZDc6qeRkcagPm/gwIiMCfqvZKoKTEEJcxVh4XYGVVWz4KSsV6r1VoJhxYfyTPA5MCTGm8VMyMxgvz9dmj/wWot/JARFmNHfmXVff7WQ1GoP6oLhyiuoC7XtA5V+4QsrQMuuUWQpBCI4VGCp01hf48uXl3fXdZOernKTfjI7OlLBl0FwqYN7OGolbRvG7TlpFZsyLmyqxzMWo4ezZnGbHnUjMVqD8yFWQqyFSQqZTL1ddOT1T6SFSQqCBRyY+o2Ah0CqBzViHMd+sqtkujzoeelBv96Zhqs+zwL9L9hPAP4V99herU+K/dxq3OiP8Q/+Uc67Wgi6JSpmcGAFdlZjFK45uVkgq4z3gEWJS07+G8GBm10naFOmxt14uawOr3lop8K32liK73i64xVz1ia5RWvdC6k15a7SG0RmiN0Do/aH316fxvlYtQWBcaAOCVJjyUMfSDF9Ly81fNtHZ0QjBWYQ623mPYx6fzXyvXV1dlBnN+El526al1iQFhfYucEbAIspWyJKlq9Qbt8mWpypLZRNS6g1mqtuJAu+6yd0Z/5bbZt6O22Xdxm31WBLEdTRApo0KBVXhjn0wLiWMWxLE2mb3ApDdU+6QG+yeephG3UA4H/fjjND6axmEcpaHA7xgJuV5efoH/Kx9FA76pdsJcpxaMm1CRQgvNiMq/2dNhGl4jQ9/ao5W3sOOqoCzMCwZ6shEr7oL6YLUTub4h1wpwOpaB+GOwzkjt9knthsNRGmrH+rAIXVqFtrUOz5CUKp6b0RqOmkj20OFVQDBZboRmiHwG+Qx60crNC6YACF8EbSYpD9WmBet6AJ4FiDdffa7vXgUXor0Kry6C2X2C2VEv1WkabM4Dyy30qnkqRj08T6Ou4NV1rnpgYiqVUN/NC9e2WvuP8CoLcEWp/WCk9sMW0MXFcnqtvNcfKq+eszoARCbvqa5Dj9PJy6MkPhKY6gn0Qcl4JZotq5+SD6pBre0JxqOkEztwgkiKKJvQaYggy3BF1al9aUbuX4kEtrpWLjWNrVp7luRZs0FnrX672ZWAhgu0GzSM3XCCYdDFkjWDoRIdfp+iBeGFiU1YcYfA9fj92hWAtKncW70XTrevkrSPnAc5T6Fifm6kp9dEzoOcBznP4XEeCwVWnfVYtQiDzwHaQ+4BYcwovARmD/go6ORReGaOBQYYT/QlrAZzSSTUeuTbe2HGrxDxkYpPurnQjwn7TuaNekyoIZ4e1dcJwaseQhUQQyOGRgyNGLpqfoNebhAa3QYIoRFC7wyhjzd/0a0yOyys/nnGoPrECRioOlpfVQQA+/pvB6Dv2anAbA09uSbGdtfEY209ODPXSPtnKbhQ+q8GbVnFQSFvQd5y2MH6uZGTVnOE7ATZSf3YSbUQfM3Edoay1oiSh6GIqinPiKIa5J6SGZUB0GtWwIqgEGG5lCVRYB2vBCD/g3oDY73irWFXItAYK2+HCmYnS7iHvEjGI7QMkSUwpxKWICfHrFmhTfBOAqtmOyriDRzZl1lJLBvyv7Lv1ONGhj33k/h8b87rMJm4a2L3Yfg8p5rjfttAFJgUNF/cKrOaWN1dk1Crs+DC+p5VzuqLHs1ybabqtjvtMm6myj0VAFQc876hV3TPXtF+bmnPRig8oPCAblF0i+YgqtyJTFS5pYXv/mHvMsJHYz6SC6sjT+T1sIC+v6ZFx/ZWHxa+aO3zWRF/SbHQ/Cm5W20NWm8M0v1PYkLBOgE0hRreyxIsfzP4paIAUuGDdr+qDWtvGE17aPDsvbL+FmX+2GBrrhvF3Qw8hJV+8d4K1w1BUtgXqiKzeFc+7heSYWySQq/iDl92lhU6fNHhiw5fpGQJlEwXNYlj0CCnKZc3uJ+fN7iLuduQtKG3uMTEhiUbfsc9dawGULuXYjHvY+RoyQMQ+2rKDieFwul8/5blryS/m9QEQDwngn1gDVHt2/ceMupprommLuvfVLyWjL/884POHKHvrm/vLj+sjz19UFQttl0uimsX4Aa3r8oniw9Vmzh6qmKZ/wT+Ozsf/91F+HiL3GtUeCIz9UVZX9F1Th8BbMOPjqHH6MYZqwDD4a6fZ3PwUyqab1cxO6q/XWCA6/Ve2cXNqU+s6ABJP+bzlGIu7qnGZih+sLLzG/cvPBwdJjONLGCYkKVGnyXV1IG1b8XZJ40MLc5jIfYXhREqb61ELPKWnj6cWhtvmXVXe3GPYt3U1dAt0BddBdWi/MeKoUKBIelZO4YHuaWPx+2yqCHUz/EbGuLT68YfWPVe+DJWFYX8eBrivRXZ5MRu/ONw1LxqhkVJWRNGs2L+3AACHcOAvWSOx/zO0i2WKq8qxPQMhlpdLjx2dvE6R43j5GO2oISlfJTga41c3t6dXwC3/vlywtjC+N3H28sT8uv59V3xx0i7m8rusXVpKGcAnpB3EpAv5nZlrcLnXh+vc7ehK7uQKChEN5c80P7tRAJUK4msHf/TOGn9p3FUb8frOCJCAFnMPiNq+6N0pyjHRH2U+nyC7gDP1kLv7d69t7kxpxEevIXOWyRecYGwUNqDi4Rlnq7bV0VU1Sew3JUgyXRWYzekv6rsWDAFHiHNmG+L2NfKklQo0D6fqPh8EK3DKmo5Krdpk4t9tAmbNg6iTfh50N42SUgHNa4+SYXqj3llkaJWwtGGwcFILw/a9TbMzfXWRtcbMkAM3811X+K5yLjlxyWtesihqyIhbiweAfcoPFOyEGaUCGQmAbw3rDg2id3DYhGZo4SHKl5+uLN9XHOYgvn33Kc1sb4twf7CG0HX69Fw7pqkbrlP418m/pZzYn4TWq4GDq6JZZq7lWmQQ5SdQ1TBe4WsIYY1vPG9DiaUJRX8Y3U9Tj9Q40XVnsgHazJvrO+wx4cGlXvjahanyCzCRhaWemCgJpOGUPjf8IE6jvrbrh13fsTP0X6n88bXaewEKoNVApXGUlCo7B0n6LDLim71utF0ayNa1d3vGXrb86YDcJwhN4rhRgHz5hPiBxOAsKRTNstSBl6nM1Pj6TIBCsM6PdPrEkX27duqjt+/M32dV5w41a3vGXEfoM3I5RcW/EY1p9p3fvyIMLl4mBwDke3Bh6o6quoYtJUZCGz1BrVBgQjkqrG/pDUKC/V1B0RZieB+jN1dctXujnrj/cRM7QuJ6qYoUr3uANSuZfE7Rgz6xYCpfGV43g1rbmynzs6+9Rb5K+/eNd4CEgb/EfmXZR9IqzvsptsIkse+99zzqkO1e7gNJF/F3T3u4/T2N/ZD2YzHZil2tdO0ZvCGLj7ShfAL1XSLH7T71tfGq/WWmaA9WXcCNFuPgcaTeU81hRowLlbPN+hiKQNAUR5WBmqw9FGukePS6ddA+psXk7FZnc9KIcADpndeKjoXTNmH5vhIdl8NY2sNvjAEOoSzern9Ana23nM5BIV702naJMx1eQHvl0J+Zs9Qjcnt9ObTx/eXdz9ffr51/3ANo92LQeN3k2p8fC3DWJ6rZVuebx/oF9+K29CfpOVnTWbhqCHlc3qPq1Z+x4irCQVZ/sU2us+crsZ1G38DxspaT1KkELqaedP9X/V+mzb7r/f2pJbir5lay4sV6GCyJ00lZa4eE3jYUS4taK/b7qfAyOXNpP/DKVvDezVQCfZd+M12/7Bq6bpg6vTOepB7Nt1Tp4kz/TcP2mU5ny08fJRnT/tZ1Y2/bNPXWKGz6mwb1Rj7X7oV6d/Cydfffvp/yvOC/rt5MvrtJ8X6J2KJirx9hzWrvW139D0mqS/OJP1pRp8l0cP63F3yTRQ6dNsuTOLaxnjbmc8NrUJHzAbv3HCtFE3dAHb3xv1axiW+W6BI4la04dDc4oEwvl9OWg4bAoJmf9fw/GwpAQ/S1j+2x+gq3M1t+0bPFRfRaro+dNwfWov13z3X3y33h07TfWXN2Rpt19+tmVXj35w6MG0g2Cei3+J+cN/9YPdb2l33h/WepcZg5i6vUxaP+b6qXL5q3AMU1m2IHEWbrxw9s2FyIbmh/XM2m8k3v9rg93kFkHv9N9//P4On2tnk2wUA apiserver.json.gz: - H4sIAAAAAAAC/+2dW2/buBKA3/MrCLULJAu39S1pXKDnIE2TboG0zTbu7kNbGLREW9zIkkpSdrxZn99+hpRkUxfHztaJ7YQLbBvxJooznPk0nKjXOwhZnQ71w0hw6xX6+r0SlzDyI6KMaGXY9wOBBQ18WXgNRVDoUS5UE7icqGbEoQJ3PQKlPexxogr7PhHvHSjyI8+LSxgO3XYQeIKGUF5VhS51yHHgCxZ4PNOfZvp61L/UJhZin6j2X9Wc4plBuQ0jEV9Oz2q7BF2cfUK7nLAhtQnyyJB4KOj+RWxBh2QPYd9BgXAJQwMiGLU5cigPPTwmUO4j4VIowdztBpg5CDOCegFD1Ic/B2pRsIfCiIUBJxw6eOPnViWdiIMF5kHEbKI9RVxDuM1oKPuvf5p9Rp3zYCZcVejCZb0yux7J66ZWcJVKL7key+vkcjIdWwlwOo41CBy5FtYAs0snGPmzSXCQJtTUZm0FFUqbrI+BgDWZNRXjUJULciUsVTiBP2OdYKQHyisnbzWq3IrLglGplngeDnlGXzMVTqEmp3D6cHFPbLukTQckiERO4unIAXuD7cs+CyK/MP6szR/Yi8j86uwUVPmTeqtlNw+sSraY9bt4t954WYGFbVVQs1pB1eeHrb18uydOs4kb2NJKv2fum1Fl65wFoIYuibiVbUVsOsBqiRq5ioy+/xaM0AD7YxQSGNEXKOghaXgIFxztdkHR4RI7SutHjArYANRHjSqo+JgjFw8J6hLiQz0fEQZbgEe2TTjvwYqPVa8e5gIRP4j67n+zc4x3hJxFcvPIpyLbpI+jPslsh1h58VUqmFq1mlvBAfXTynwVd4NRmTClKrtSWwPPOcNdkrV9Za0+wLYhSvyCRURrNMnOf7ahsxVqN2YlQ8FWsiH25ILUBtl18EifKDXNLQT2aN8/4u3E3Mu55PWO9l1xQR2yaKpZk64tdRhSv9+Od3ptXl1xI2RnCq19PFAqO5TCQSJAympU8s2GqWAzFZPKcoMz7PeXGLyeHXzuZgNFewv77TwA4fCitlnSsKjKD4k5BY/ng6cAk1Vo15YzKrFF4AlEj0ojblllFafgRC/o32r4/eovuTZgZkv7qvIbu6ql+gD2dbHoemBl5Bhy7sVFFfGDWR9fHJVUBtOOS6554n+a+UJ2CSpaYgx61POOpTFW8pdmtlEDK1s7hD8OW9LM1g4LZlbapzl7XN5FHy8erg4Gu9ZqFAbSTcrcvaVwDMaMBn5RVAKzPhFLSIFchWpOOKSSTgjrJJb6FR5i6uEu9agYN6rONdR1X38D6+B9syrI9iIOpgUKniY/frMmRUnNzLEAx9mBW1DCi81SM3WKbaHWqF5oElur0+l4xUFAOxWMWkfLasXU8vKSFUz55EhbB7QLK7GH/oNarefVal75xZR7r/PyBGRzFks0IR8O9s8DEeCc41KGRt9+h/kZqBbLbb9AztN6fcudl9q6W20+1eljYkzxsD/rpz1/BrXABWF+nMJQzs11MSt1pZYk5DPi94WrrGqhjszrtnL4iWwXEcaA0buRAxsRAb97pCeQFwSXIF2EAYoiBmak1ar+gvSthvoRBisqCOF5tKHKvOQeTJa+Y9ih8TtR9Rao0LwHVJDiLjeJdsRYPOeyWnCS82qoP6cmjyXlVrWsBvr8SR2lN3lXqva1wPMMu9JtvnBrS/vPS26uKkbJrWvLsFOBDgr+MyVf3C9/yQhlb6kxkRx+v1hXvk1AWA7QuPIWPS/I2SZGQoJLUSS2+Z/AfYAykpInAjdsk3n7NnHch9lCAW9ZpZMERxSGxDmLnXqx/tZ+EeAM/Yp2V+Ie0bN4w8N/e/fpKpUlig3Ryr1mTpZy6qcx2BUUQdZduLQnyisTd3siJ/smtpqr8Lb5rR4wUfISp7ZxJ3XA1HfokDoRzri4cmetIl7ZmV3hK8qL8+pG9mWseyU2Jg2dyEXK42DyHlLSa75Jm9olFcMrf4QxvlrmHWue51vyfTtWSfkCXPoMsjLov8G88Cqo+YHSbrEjKK3S1uVWb3wP6VH1/buTL53u2nLDnZS+B1ODE7Apqf4cjDLR3+l82slmzsbXplv87TSSCSNY2foUbd2DYkSQQesdTYwm2LexwT7saBG/s/cX7cq7k7YJ860uzLdvwnwmzGfCfOVhvoYJ8/1smE+a8McV5/ssnVYx2GdCfI8oxFdONnNxBtgDcQJ220EOuA2XoOn+4ki+wHbHyAbrfteBvAMTyDOBvHUE8m62P2qLy7u8qD9//oIWTY2duson+wdHB8031q24bTZ8Y9Hwp/Xj4+rxvx2+uWj4Rv1l823r3w6/v2j4k2r9tNm8BQMtDKc2SsOpBZ1eeTSVRwNpE3elUdyL/wIQiS3zqwKbdNR+fcWwIPuDpcDkXsOq19fKuKPJZPOjqgpuLs7eo2foc+LITDR1LdHUmT6Clod8a0OlW/Mc5XHQ7afSZeJtGqDKpFFGRMR84qARFckBNTTfv7ram0OoqwLUlwZQDaBux0lzY10nzSkbpTz0c3wku77+3zcL4HI+LaEXaLV33QwqS+e+bWSmjr4Nl62by9Z0ulu9AzTb/Ed5qHQWUxeXiX8yLthqAXAl0qBe/Os70mbKXEDFa/8oXsOoT4fEn1qwO8ayQ4NlBssMlt2IZQnGdH5EWG3dEiByojhLppNs+lcu5SIAlzmY9tq4Y7UtxqS3yXIbUFozKD2E4BU3CXwmgc8k8OUCiuq3crWI4vknINTzL/D/Ufv4t3/enpydtE9MOt/q0vlaJp3PpPOZdD6TzndH6XzKoD+ufL4/lQ8zCX0moa9AOkvizaak9+VGMnE6E6cz+X0mv+9h5vfdTComwc+7iXZMhp/J8DMZfg8ww+/2xLrWfL9azRCrIVZzsnwfCX8pMN1zxt9mcdpWnWXrrGZy/kzOn8n5ezQ5f9Jqqpy/OQB3zwmAtbrhNMNphtPuMQNwc07ithmbTA6gyQE0OYAmB/DmHMANZMeVE1zjURJcqR03CLfhCHewxlDbrgxf7Y4CdgnIBo4dOw6PA1vXfwXqy80p1Uk4oz7MzrdVQO1pejGX2r4+7XTk8J10+33f21MxtrRnBUmHfc9BsfTmkwm6vpb33wrIAwGh36WE0JEDvgVW1TDemhkvuN/Dv7sIim3uI2x7MGzlQNM0QGOAxgDNrYHGIaFwDctsJsu8lcIxILPuYJUrV2/LUWaTH8LATA5m9jcEZkq6bd3x2qLfWTQk8xMkkzsJvkeUKR6V7cp/JauCShgnIZ3caVsn9kz3xj4V5JE9A0C3AqAzWFbfHhsEMud15rzOnNc9XiI0/1aDiW5tCRM214WEIQvkl2NkSjrM3hedARkEbNzpjgXhJZh3O8pbF7htPq99UMtsGG3NjKbU/AFw2oY/hwlW5dDEfKXfoIlBk5vRRAWkUj6xw2gag5qXSvTTESiDK/Nw5fj8C4q4VGtDLOZgzRysPSZWMZ+uN6xiWOVmVukHnX7AgkhIAZqwyZ1xyLvpIhsQeWwgcldHXJv9HA/5mGsneQKL2y4Z4D8I4/Es48xkMNPj+FYOZpdqILDM/ZmWWpdRlzCfyNjXdCxBBqGH5a/+T3cdOEkuNN2eaYOGFxmuS7/y6pAejryyD83qtWWYaLkxYWjGZapTlvy8NbqIUU6rT7+nrYGeVhuozyDkzar1IyJsnITyi0AoLbg0yFlVhdI+yX/Aeiog7fY7uefSVg57XvrF96xG62uqL8gS+Joumua2wP6CY3TIUdmno2crmjhPfawBiIaW9ElXuaTLoiVWt+vE5nQ3CouOfjL16HvlQqgvEkLik3Rhgc6rlea/p/OwsrWFCcuy8saJiOMn0ioiTtrxQDqh3rfcq/PknnVqc4zzQomnpLNSkc9huFlC1MPQhJ3UA01iK0sHs6/Czz5VH4ye1VIASr9CH5v7WbeQAvqwWedkSaZxOJ1CrP2Zmli1qnbR0C+0F1FrX/u5pl80qnrNDNOsuvZzzYlt3vf0GSRuazqy8C76wAf6wPpd6k39YvbPFlgvHX2+6Vwyy/d3oF5arC/t48Qlpg756Pw9irUyrojU+7tVbRH7EOMaabUOnINez3YODmsvu13c6zWdbq0bNx5OnW91Z/J/pLorRNWkAAA= + H4sIAAAAAAAC/+2d62/bthbAv+evINQNSAa39StpXKD3Is2jK5C2WetuH9rCoCXa5iJLKkk58bLcv/0eUpJNSVTitE7sJBywNuJLFHkePx2eqBcbCDm9Hg2iWHDnJfryrZaUMPI9poxoZTgIQoEFDQNZeAFFUOhTLlQTuLxUzYhHBe77BEoH2OdEFQ4DIt56UBTEvp+UMByNumHoCxpBeV0VjqhH9sNAsNDnuf4019enwak2sQgHRLX/ouaUzAzKXRiJBHJ6TndE0KfjD2iTEzahLkE+mRAfhf2/iSvohGwhHHgoFCPC0JgIRl2OPMojH08JlAdIjCiUYD7qh5h5CDOCBiFDNIA/x2pRsI+imEUhJxw6+NNnTi2biIcF5mHMXKI9RVJDuMtoJPuvfppDRr2TcL65qnAEl83a/PpMXre1gvNs99LrqbxOLy9nY6sNnI3jjENProUzxuzUC8+C+SQ47CbUNOZtBRVKmpz3oYA1mTcV00iVC3IuHFV4CX8mMsHIAIRXTt5p1bmTlIVnRinxfRzxnLzmKrxSTUHg9OGSntgdkS4dkzAWhR3PRg7Za+yeDlkYB6Xx523+xH5MqqvzU1DlT5qdjtvecWr5Yjbs481m60UNFrZTQ+16DdWf7Xa2iu2eeO02bmFHK/2Wu29OlJ0TFoIYjkjMnXwr4tIxVkvUKlTk5P338AyNcTBFEYERA4HCAZKGh3DB0WYfBB0usaek/oxRAQpAA9Sqg4hPORrhCUF9QgKo52eEgQrw2HUJ5wNY8anqNcBcIBKE8XD03/wcE42Qs0hvHgdU5JsMcTwkOXVIhBefZxvTqNcLKzimQVZZrOKj8My0mVKUR1JaQ987xn2St32mVu9AbYjafsFiojW6zM9/rtD5CqWN+Z2hYCvZBPsmec3bW20doogGw26iho2qurKU5tcTWgd4rORpIlcOiRApla4Vm02yVc9VXNYWG5zhYLjA4M384JWaAFJwAMpwEsLK8bIoOHIVVeW71NaBOwrAjIM9KbXryhkZFh7MtBhQaWEdx1RxBB7uE/1HDb9d/7XQBmygsa8qv7KrWqp3YPyu37oBmAA5hpx7eVFF8mDO++d7hspw1nHBNU+dQ7tYyE5BRA2aOqC+vy8tpdp/aQNbDTCBjV34Y7cjbWBjt2QDpfGoUEB5F328ZLgmWNNGp1UaSNf3Sh1VrARjxuOgvFUCsyERC+wCOY/UnHBEJToQ1kvN6Es8wdTHfepTMW3VvQuo67/66mDf/+rUkOvHHPQeCn5Jf/zqXJZ3am4rBXi1HtyCEl5ultmQI+wKtUbNUhOfDEngHc3GKw8C0qlI0dlbVCpmZpEbVjCDhz1tHdAmrMQW+g/qdJ7V60XhFzMovSjuJ/CUd/2OpljCwf75sAW44FWUodHVb7c4A9ViMfUL5TydVzfUvMzW3Uj5VKf3qTHFk+G8n/b8OQ7CPsV8PyOVgg/qY2b0c47E12MSDMVIWdVSHanqtnQyid0RIowBQPdjDxQRAVz7ZCCQH4ansLsIA7HEDMxIp1P/FemqhoYxBisqCOFF7qDKvBQeTJa+YdijyQtL/QZ+PG8NEwUrCy/sxTDY493iq5neBPbUXOPGjCUTM9WCJ6yqoUFFDaPDESiAVzWX1HZKvCnWQKe/qKeko+gwlfYKXGW+lQTzaxVYWnluuLmqOEtv3ViEkEoMUPKSGXzioZnzI9lbykUsh98u15mVAXbLAyBWPmHghwULxEhEsBE4Esv+AZwEiBwxPBE4W5dUaWfqnnfzhQJedIyTBHcTRcQ7Tlx3uf7G3g8QDP2GNpfiBNHTRK3hv627dIjK3iTmZum+sbCXcupHCb6VBEHWfRrRgTBXpk71UE72dWIbl+FTi6oeMmF4j1Jq3MvcLA08OqFejHOOzOySVdApP7NzfE55eV792D1NZM9gY7LohVykIvSlbxuGXtUmbWaXVBjN/AhTfL7Im1SVf1vwlTcRSfkOanwGWRkOX2NeeuHTHIGxW+IJjFXautzove4hPaquvxvF0pnWmg13WvoWTA1O8cVQ/TE8ywVgZ/PppsqcD3HNVPxgFkyEEZx8fQawo51yUI5B6w1tG228bW3jbdjTgm7Hbz91a28OuzbStrxI27aNtNlI272MtLVspO1nI23Svj6uUNtH6VHK8TYbZXtEUTYzdlSyBoAB4gTstoc8cBsjgmb6xZF8u+xPkQvW/bZjaTs2lmZjabcXS7vayihFlnd53nz27DktGxQ3c4hPtnf2dtqvnRvR2Xz41nXDHzX39+v7Pzp8+7rhW80X7YPOjw6/fd3wh/XmUbt9A9K5NqLZMkY0SzK99IAmj8fS8m1K07eV/AW4kdjflyUC6Sl9fcmwINvjhfDjTiObFxfKhKPLy/UPbCqE+XT8Fj1FH1N3ZQOaKwlozuURpDzi9zZaeW+ewxyKvP/suUjIS8NQmTrJiIhZQDx0RkV6EgzNt8/Ptyo4dFkY+sJiqMXQdTrSba3qSDcjoIx6fo6CZNdX//vqAEJWMxF6jpZ71/Vgr2zu942/1Bmzpa9V09eKjlHrtwBg6/8oD5XBErbiMo9Oxvg6HcCqdDeon/yqirSZMrVOUdm/isowGtIJCWYW7Jbha9fCl4UvC1/yHDWBld73GCsFNWCPFydJJ71UtV+OKBchOMbxrNfaHYTdYxg6SJfb4tCKceghBKK4zYez+XA2H64QHFS/Z6pFB08+AIeefIb/97r7v/97cHh82D202XHLy47r2Ow4mx1ns+MeZ3acsraPKz3uL+VgbH6czY8rYciC7LEu2XKFkWyozIbKbLqcTZe7z+lyV/OIzZfzr2IamzBnE+ZswtwDTJi7OZeuNH2u0bBcarnUHuEuL38uw6I7TqBbLxq7V4fGOpHZFDqbQmdT6B5NCp20miqFrgLT7jifrtG0NGZpzNLY0hPq1ufs7D7DkU2psyl1NqXOptRdnVK3hoS4dE5rPWBOM1ZZUFtzUNtZYdhsU4aiNs9CdgpgBu4bex5PglQXf4fq68EZu0kEowHMLnBVcOyX7KKSzb5sj79tbakIWda2hqQjvuOQVnbzy0t0cSHvfy/gDbYE/SH3BO154DNgmyy7rZjdwrs9oLuNkNb6PsJ9D2UtHVTaFlQsqFhQqQAVj0RiZBllXRjlQG6HBZRVB5dGcvXuOaKs80NYSClAyvaPQIpBHRZhFEO3n0UU08HWrR55XfkbhJZPfo5PCmewdwgo5eOrTfnvKtWQgVxSfimcgPUS73OLRFNDPtmyWHMjrDmGrQvcqQUbe2pmT83sqdnj5Tz7TwbYWNRasV57VagXsVB+9EQmecPsA9Ebk3HIpr3+VBBuwLeb0duq8Gz9qeydWmZLYismMSXmD4DG1vw5bKCpACD2Y/EWQCyAJF/WEmQzoxA3imcRpKq0nR+IH1kMqcKQ/ZPPKOZSkC2J2MMue9j1mBjEfjPdMohlEOVnwt4wZGEs5AbaoMet0cab2SJb3HhsuHFbx1Dr/RwP+ShqI30Ch7sjMsZ/EsaTWSa5vmCmp8mtPMxO1UBgmYdzKXVO4z5hAZGRq9lYgowjH8tfhZ9pHThJLjTZnkuDxhc5esu+YOqRAY5900dU9VoTDDqjBDE041IhU7NPRGsUp9WG6msARWvqfI8Jm6bx9zLtScMt7XBeQqF0SIrfZJ7ti3b7jcLjaAuGfT/7wnjhMbSl1NdhATbN1krzVmB2wR96ZM/0NeTZQma/Wq2PNYYdoYY+2Sobuly3xOp2vcSKblZ8w2bmyrfM29C8bhtSZ6RvFwi7Wmv+RzYTJ19bmrIsMzdONzl5Jq0i5qSbDKSj6V3vfL1q5/PerEqDrtvzDHGWvemm7CQj0c2Tlh6GeGxk/ugysbl0PP/++fyj7OHZ00aGQ9n31hPjP+8WUQAhNu+cLkkvQ1mdSZztuew4jbp20dIvGuP5z9vazw39olXXa+bQ5jS1nxteYgq/Zc8g4VsTnGvvog+8ow+s36XZ1i/mH+h3Xnj6fLO55Jbvn1C9wjifu/upg8zc897JW5RIZVIRq3d2p94h7i7GDdLp7Hg7g4Hr7ew2XvT7eDBoe/1GP2k8mbni+sbl/wFvRsHeLqIAAA== cluster-total.json.gz: - H4sIAAAAAAAC/+1d62/bthb/nr9C0HqHFDeeLT/yKFAMeTYF1q63STfstoVBS7StRZY0ksqjQe7ffknqRUmkbCdO7MTchy4+lI/I8/yRPKRvNwzD7PddP4wINt8YX79vxRQE/4lcBAUa8P2AAOIGPiPeUhIlei4m7BH+yUiovGUQuR5579NGayunOoAAHETIhrTB/ISCCSRjGGFTeAb6YOCxdoIiKNDHriOhunbgHwZegBhDNBqAzdaW0bYs+k+vt2VYr0XWPpjwF+/nYzF+NvY9iEihC+Qm5M85AI8HAUCOmbTd8f9/p//ecaFAxyWl3pojH5L3DqX4kefFFATC8XkQeMQNKb3FiWw4h4FPUOAxgQ6Bh+Pvu4Xveq5/IWghBD7kz8ciTwVu2oHngRBDkVOhwam0jJDrfApyZcZCLinsin5udwXCdTqA5PMN+5xKJ+PNx9DOPua9/p7REAwhIMJQBep7AhFXj7z5c3BVEDBvwuPg6twlFcsxSUI0DyOEoE+MA+A7V65DxmbxmTP3B39uvC00JJaAgqvYBpIxZoIHngswN0AuyVwEA8Apha4we/oN+iPCxdwq0CGuaGiqt5hD1/MKkmaEdwg4Lh2oqCm1tvdK2rbaU7RtybXdyT56cAR9p/gmKqaRv4/P5a4NLkflwTPTjfUljwTHk5DcyJv+C1FQbZmAa8k7Jq4voSJ3NCZn0njDzExCpc/+yU2qaJWsKUBsCNloSk1HENtVfoSGJ0/SsUvgRTA1q4oeaKyQWBGjXiV9s0TqRckjqSzOqKcWPN5ko/kUuD75EHBxcIIpeLYNZRYdQmq1PgGjakQKGTdmohF7f69Ir3afCs2BCPL4PvSCXICKCIIhciH+/RIiavGwNESs6i+uDBwTYF9UeoMJDEPo/EZlWmkjAI0gwUI+FDMiyxbXIR8G03uf9s3exNFk06WhDm7SNEYA5Yr6NH1cBeiCpmAbupewP7ghEPe5RdzaXkR7gN5+M18lf34zt1hW4wN7+79v5i///mbefX1FRQkRtZY3r2gSD7yIBdPvr18bgxtjM3v+tZgcWfAI0IQLlMbDCezHkiw+kvI9ATbhOdcqNMeuf5Lxub3NXnZ3V+SE4JAHcXO/SGcS5rrJiHfJX7kayZiOahx4Tkm9rNsnNEyWbYLRz8bukFQbSrnhM9WFEQyNAyZ043OsAkfIB1kWvxVjAkA8wZajQuz67bID99Os4vqOe+k6EfX1ijOnz3DwkHfgGly7pSg+iOyL2O5KoWeSuGxFkSkQKoeqOLYpg85XUU9pPMto3ytDuAHXsMYdcnM7CEtm5oEB9CodZA3B6ABwmFM0vDi6Vx6Pw3uraGB5BM9NbGv1u7ghinn9YEiBUINDuhqHaByiccjicAiVkY8nLtFAZHlA5DzRAdFYRGORVcciNnXziV8jbAKv+TjOqTcUB8LVmzVNFUnK6A/2NeOnfQWzrHlehgf1DA/mZnhYz/BwboZH9QyP5mZ4XM/weG6GJ/UMT+Zm+K6e4bu5GZ7WMzydnWGWYxQc8/aanDIz3M5B1JAm7XT1cq/1L3N+uD11jddSLPL2KqivMNV4BNBnAGz8YNB6BmiGbRRwWRU6xULaKQQO/0axJc6SgrBoMCtJw4mRMs9MFZnMBexuvLqUxKd08jBps4leIpNKELfTWWAu3dSuYA6I/qL/NT58aBwdGaenbyaTN7iUUxxouxPAV+3bxZxClVfNzElDvsthHiFqpoYTXPlm5bEviEnVLDaEgOIb5MuHrEBaBXhCJ10OLL0t8l0+XvpdRKZ7cib0ymZBdTFoPVQhz62zqMOPJgPqYlJ1MAjzEGXIAPFa6eNgmfrYp7NaOqXWziGHdctWxto7x9Ey9ZHO3FOXMD6BeE68lqo4XqwqwnupQnCI9dbGySpoo+wYxhEK2GLmeqrk3Yo6yHpr5XSZWvkoX0ZYBdkXV5XvJXqnuTvYAbuDTnvb2QHbQ2vY6ra73db2dmd7197eAcPmRUQl6UMCcSPZhXD9USNbPWmEgYN/DdDovfPW+hnBIVPO204L/3wJUP7Y21f9vg2FXaKyrhXLNQ9VdnmqWd01eEGFInwjtbQzgwmQbJfKNm3atZs2996n2Xr++15LkOvB4uQKLkfaXBOxHj6NWNfOXI+eJgyk5hrGsGgNBHv8xPF1fSR7shyTdWIkvwYCfrck010fCZ/eo0jmwcUwZwQQcQM0ReGxAFRFCOmBmMIOn3geptCwuOMwqq3Sbcl5mJqZoKSaU1XRWVfVWVPZOeuRLEmFZ22Vp1Kesu1nacWnTLa5fC2BVpQNl/OOxK6r3aitBFVXg9ZWhE6pCq2vDFVXh6orRKdUiaorRadUi06pGK2vGq2rHFVWj1Z1Ka0iVRcVKAoLlMUFM1SV1lWWTqkuraswrakyrS9nUFebTq04rS1OkBcoqKtPp1WgKhc5yoHuOcweZyxInaEodc7C1LpFD0XuFfNvIQdPX9eS5+jaPF3M1ek25JQTNMrK1brqVUUF60xVrNXAIq9mVVS01la11le2qqtbaypc1VWu8kpXsdq1OtRq1avUBdWlpbXlpXUlpjVlpqpSU2W5aWVcz3QIGzKV1e4GaAwoI84CAnc1CNQgUIPAlwkCn36tW6PAh6BAZUGaBoIaCGogqFo4fforciq1pIu4ImepdxNZimPieyt2OVFevXvqYpo2bp7icqLyOf/VvxZgur7bcn0L47nvvQAl4mNcC1Am8sBSIi7yUoCHne9XnvRqP/ykFwV9PrQL551f/hn/4lk0fdXQizzhnyyJSnPsczzRT4pn9erP85fGkONbfWL/cbv4oNuDXiBOaCtwoWVpnKBxgsYJ+iqgpQOFdNVMIwWNFFbobp9lldV1FGV1VnuhdXWy3dHnvala3M2pKazrTNlTtToL3VSVNDz2nqqsQYKdHnVHdXGborLDeFVstZg90SrO0hujUjz22PuiK3ZWRW+MzrYxOtutEC9yP5SUL3OaZTdUMs4qBFzAPmf4/Pc5w/UpeFtjcNZtTQFnXQ3ONDjT4GxlwdkyzrtqdDYfOpt6UZQGaBqgvXiAtgKFaGXnW0z52ZMvWao2GXt6yXIxqLg3DRVva1SsUbFGxc9myfIJb9LQ4PhhS5fyGyI1QtYIWS9harAmA2t7U8DajgZrGqxpsPZ8ljA1WntOS5kasGnApgGbBmyLA2y7GrA9d8BW7/ExkDjwgPT2/2Lm+XMMiOFi4/zwk/EZpljh12qciOIfAhgTEuI3zSawbei4wP/FDiZNyFJ3iFwMcXNAY0YzwR6NGHI0vADjBkq5Y0yRA24A32k4Uei5Nk1+DfqgH1x50BnBCWWHm+aMiVmj1WeJVjlY9almGE6lnSD9czvsJxZ4BkdYCkyVMNRoGnKOv0dkfm4xqI2v5X16TJu+94VA2kJgwUYQEUYGnmdgdk0whiPu8BrfLhnfxqGO/x7Kc8e5Kz8UjXefEO/uabyr8W6Od2+MSUAjQ4CMs78+GihPTdNB78gl42jAIW+YeUaTYw6Kcmj2gahJ4S0VRtNqtTs/8b/p0yzBNbqtXau929nuaWS7fsj2+Jr+7/DT2Y2fYKEFwdt7A2YNcRcLcVksUcBcIcRokKtB7pqD3BWoUz1GiGHp+5SpbiQ9N5MfsGStnVbMiz2WxxkT22M4AX9AhOP+x+uuNITdxL1wALqIv0ezcG7FZv47mmb2NgInoUcF4Y8yr6TgCBPB9gv33Xj8F1Gr1zSAiASSG2EYuW8HEcewnVa5Jda8abXE+CWg3sJshcBrbp+9STGmXCY9Yg2yKc8sMw+Olkslu65Pk50D9z3ZVTdyPzAnEY27kseTOGXmiVJ8eRAySjnelEM39GIkJofKqXQ6rcpds5mAWJsq5KnfJrvJW64KlTJmflXdwKyx8l20aUMWHoS0bP4TQXSTCGGrN9kqsBOcrl2gjuB16devTHzhhl+QRwGXLVF0kkTF2nDqhNxp8H/SLpjF1jKE4DT5w1kWjqGR2BZheB7z4n0ql5qvkxe3H9mLZeKf14drvKqrtvTuPJbeXV8Tn2J8DhwCqmOFBaatUjOUJIrUiswjaqLGWWyjErMRLFhuOFIthlJLz7VpTdNmKlHh9fcKDoJMV9Qvk8npPNLlb+rHU4LNKLz9OxjQaa4NnEuX2jmd5m4ZCdfX93Wm2F1aj+su8YCm+8pGCprvYvjncsklwG8YT1pNP7hqZPnR5CmB0czC10KXztlQ/uVEJP3UccVQaPYEUCzmiQJcMa08J4j5wbTED52W2CLcQdYW/rac2Ma/p2NgCw3VEK1+i8h4W2QsvqXdFT8Idw7uOGJ/074UxPcj4ItE5pfzwwSrp5OIj9lv3BtN41AwaTPiy87mcLjd6YFWuzewhxDsDOyO43SHvdbu3l4LdOBe/PBlNj9obdz9Hx+Hf4PapwAA + H4sIAAAAAAAC/+1d62/bthb/nr9C0HqHFDeeLT/yKFAMeTYF1q63STfstoVBS7StRZY0ksqjQe7ffknqRUmkbCdO7MTchy4+lI/I8/yRPKRvNwzD7PddP4wINt8YX79vxRQE/4lcBAUa8P2AAOIGPiPeUhIlei4m7BH+yUiovGUQuR5579NGayunOoAAHETIhrTB/ISCCSRjGGFTeAb6YOCxdoIiKNDHriOhunbgHwZegBhDNBqAzdaW0bYs+k+vt2VYr0XWPpjwF+/nYzF+NvY9iEihC+Qm5M85AI8HAUCOmbTd8f9/p//ecaFAxyWl3pojH5L3DqX4kefFFATC8XkQeMQNKb3FiWw4h4FPUOAxgQ6Bh+Pvu4Xveq5/IWghBD7kz8ciTwVu2oHngRBDkVOhwam0jJDrfApyZcZCLinsin5udwXCdTqA5PMN+5xKJ+PNx9DOPua9/p7REAwhIMJQBep7AhFXj7z5c3BVEDBvwuPg6twlFcsxSUI0DyOEoE+MA+A7V65DxmbxmTP3B39uvC00JJaAgqvYBpIxZoIHngswN0AuyVwEA8Apha4we/oN+iPCxdwq0CGuaGiqt5hD1/MKkmaEdwg4Lh2oqCm1tvdK2rbaU7RtybXdyT56cAR9p/gmKqaRv4/P5a4NLkflwTPTjfUljwTHk5DcyJv+C1FQbZmAa8k7Jq4voSJ3NCZn0njDzExCpc/+yU2qaJWsKUBsCNloSk1HENtVfoSGJ0/SsUvgRTA1q4oeaKyQWBGjXiV9s0TqRckjqSzOqKcWPN5ko/kUuD75EHBxcIIpeLYNZRYdQmq1PgGjakQKGTdmohF7f69Ir3afCs2BCPL4PvSCXICKCIIhciH+/RIiavGwNESs6i+uDBwTYF9UeoMJDEPo/EZlWmkjAI0gwUI+FDMiyxbXIR8G03uf9s3exNFk06WhDm7SNEYA5Yr6NH1cBeiCpmAbupewP7ghEPe5RdzaXkR7gN5+M18lf34zt1hW4wN7+79v5i///mbefX1FRQkRtZY3r2gSD7yIBdPvr18bgxtjM3v+tZgcWfAI0IQLlMbDCezHkiw+kvI9ATbhOdcqNMeuf5Lxub3NXnZ3V+SE4JAHcXO/SGcS5rrJiHfJX7kayZiOahx4Tkm9rNsnNEyWbYLRz8bukFQbSrnhM9WFEQyNAyZ043OsAkfIB1kWvxVjAkA8wZajQuz67bID99Os4vqOe+k6EfX1ijOnz3DwkHfgGly7pSg+iOyL2O5KoWeSuGxFkSkQKoeqOLYpg85XUU9pPMto3ytDuAHXsMYdcnM7CEtm5oEB9CodZA3B6ABwmFM0vDi6Vx6Pw3uraGB5BM9NbGv1u7ghinn9YEiBUINDuhqHaByiccjicAiVkY8nLtFAZHlA5DzRAdFYRGORVcciNnXziV8jbAKv+TjOqTcUB8LVmzVNFUnK6A/2NeOnfQWzrHlehgf1DA/mZnhYz/BwboZH9QyP5mZ4XM/weG6GJ/UMT+Zm+K6e4bu5GZ7WMzydnWGWYxQc8/aanDIz3M5B1JAm7XT1cq/1L3N+uD11jddSLPL2KqivMNV4BNBnAGz8YNB6BmiGbRRwWRU6xULaKQQO/0axJc6SgrBoMCtJw4mRMs9MFZnMBexuvLqUxKd08jBps4leIpNKELfTWWAu3dSuYA6I/qL/NT58aBwdGaenbyaTN7iUUxxouxPAV+3bxZxClVfNzElDvsthHiFqpoYTXPlm5bEviEnVLDaEgOIb5MuHrEBaBXhCJ10OLL0t8l0+XvpdRKZ7cib0ymZBdTFoPVQhz62zqMOPJgPqYlJ1MAjzEGXIAPFa6eNgmfrYp7NaOqXWziGHdctWxto7x9Ey9ZHO3FOXMD6BeE68lqo4XqwqwnupQnCI9dbGySpoo+wYxhEK2GLmeqrk3Yo6yHpr5XSZWvkoX0ZYBdkXV5XvJXqnuTvYAbuDTnvb2QHbQ2vY6ra73db2dmd7197eAcPmRUQl6UMCcSPZhXD9USNbPWmEgYN/DdDovfPW+hnBIVPO204L/3wJUP7Y21f9vg2FXaKyrhXLNQ9VdnmqWd01eEGFInwjtbQzgwmQbJfKNm3atZs2996n2Xr++15LkOvB4uQKLkfaXBOxHj6NWNfOXI+eJgyk5hrGsGgNBHv8xPF1fSR7shyTdWIkvwYCfrck010fCZ/eo0jmwcUwZwQQcQM0ReGxAFRFCOmBmMIOn3geptCwuOMwqq3Sbcl5mJqZoKSaU1XRWVfVWVPZOeuRLEmFZ22Vp1Kesu1nacWnTLa5fC2BVpQNl/OOxK6r3aitBFVXg9ZWhE6pCq2vDFVXh6orRKdUiaorRadUi06pGK2vGq2rHFVWj1Z1Ka0iVRcVKAoLlMUFM1SV1lWWTqkuraswrakyrS9nUFebTq04rS1OkBcoqKtPp1WgKhc5yoHuOcweZyxInaEodc7C1LpFD0XuFfNvIQdPX9eS5+jaPF3M1ek25JQTNMrK1brqVUUF60xVrNXAIq9mVVS01la11le2qqtbaypc1VWu8kpXsdq1OtRq1avUBdWlpbXlpXUlpjVlpqpSU2W5aWVcz3QIGzKV1e4GaAwoI84CAnc1CNQgUIPAlwkCn36tW6PAh6BAZUGaBoIaCGogqFo4fforciq1pIu4ImepdxNZimPieyt2OVFevXvqYpo2bp7icqLyOf/VvxZgur7bcn0L47nvvQAl4mNcC1Am8sBSIi7yUoCHne9XnvRqP/ykFwV9PrQL551f/hn/4lk0fdXQizzhnyyJSnPsczzRT4pn9erP85fGkONbfWL/cbv4oNuDXiBOaCtwoWVpnKBxgsYJ+iqgpQOFdNVMIwWNFFbobp9lldV1FGV1VnuhdXWy3dHnvala3M2pKazrTNlTtToL3VSVNDz2nqqsQYKdHnVHdXGborLDeFVstZg90SrO0hujUjz22PuiK3ZWRW+MzrYxOtutEC9yP5SUL3OaZTdUMs4qBFzAPmf4/Pc5w/UpeFtjcNZtTQFnXQ3ONDjT4GxlwdkyzrtqdDYfOpt6UZQGaBqgvXiAtgKFaGXnW0z52ZMvWao2GXt6yXIxqLg3DRVva1SsUbFGxc9myfIJb9LQ4PhhS5fyGyI1QtYIWS9harAmA2t7U8DajgZrGqxpsPZ8ljA1WntOS5kasGnApgGbBmyLA2y7GrA9d8BW7/ExkDjwgPT2/2Lm+XMMiOFi4/zwk/EZpljh12qciOIfAhgTEuI3zSawbei4wP/FDiZNyFJ3iFwMcXNAY0YzwR6NGHI0vADjBkq5Y0yRA24A32k4Uei5Nk1+DfqgH1x50BnBCWWHm+aMiVmj1WeJVjlY9almGE6lnSD9czvsJxZ4BkdYCkyVMNRoGnKOv0dkfm4xqI2v5X16TJu+94VA2kJgwUYQEUYGnmdgdk0whiPu8BrfLhnfxqGO/x7Kc8e5Kz8UjXefEO/uabyr8W6Od2+MSUAjQ4CMs78+GihPTdNB78gl42jAIW+YeUaTYw6Kcmj2gahJ4S0VRtNqtTs/8b/p0yzBNbqtXau929nuaWS7fsj2+Jr+7/DT2Y2fYKEFwdt7A2YNcRcLcVksUcBcIcRokKtB7pqD3BWoUz1GiGHp+5SpbiQ9N5MfsGStnVbMiz2WxxkT22M4AX9AhOP+x+uuNITdxL1wALqIv0ezcG7FZv47mmb2NgInoUcF4Y8yr6TgCBPB9gv33Xj8F1Gr1zSAiASSG2EYuW8HEcewnVa5Jda8abXE+CWg3sJshcBrbp+9STGmXCY9Yg2yKc8sMw+Olkslu65Pk50D9z3ZVTdyPzAnEY27kseTOGXmiVJ8eRAySjnelEM39GIkJofKqXQ6rcpds5mAWJsq5KnfJrvJW64KlTJmflXdwKyx8l20aUMWHoS0bP4TQXSTCGGrN9kqsBOcrl2gjuB16devTHzhhl+QRwGXLVF0kkTF2nDqhNxp8H/SLpjF1jKE4DT5w1kWjqGR2BZheB7z4n0ql5qvkxe3H9mLZeKf14drvKqrtvTuPJbeXV8Tn2J8DhwCqmOFBaatUjOUJAqFFaXWIhiu3F6kygulBp4r0ZqmxFSQwuvvFRMEUa6oOyZz0nmky9/Uj2cCm1F4+3cwoLNbGziXLjVvOrvdMhKur+/rQ7GXtB7XS+IBTXeRjRQr38Woz+WSS/DeMJ6rmn5w1cjSoskzAaOZha+FLp2qofzLiUj6qb+KEdDsCVhYTA8FlGJaeSoQ04JpiR86LbFFuHqsLfxtObGNf0/HwNYXqpFZ/RaR8bbIWHxLuyt+EK4a3HHE/qZ9KYjvR8DXhswv54cJRE/nDh+zn7Y3msahYNJmxFebzeFwu9MDrXZvYA8h2BnYHcfpDnut3b29FujAvfjhy2xa0Nq4+z91+P/u0acAAA== controller-manager.json.gz: - H4sIAAAAAAAC/+1dW2/bNhR+z68QtA1IALeTb208oA9puhbFmi1tvO1hKwxaomUtkqiSVOIs8H77DqkbJdGOk7aJ3fAliM/h5fBcP9K0dL1nWfZkEsRJypn9k/XXx05GofhTGlCs0FAcE454QGJBvAYSEMOAcdkEPi5lM+wFHE1DDNQZChmWRD/G/K0HpDgNw4xCUTIfExLyIAG6I4nzwMPHJOaUhKzWP6j1DYP4XBGM4hkIOgeC3XeYndHIpWwhpcxkBapLwhAlrCZbjeG1OAmKcVgNpQ6X9UTuHI+DCJOUKzJWfBIS+hK55z4ladwav2rzBwpTvJpdF0HSv+uNRu7gmd2pk6k/Rfu9/vOO1e2NOtbA6VjO08PRQbPdd95ggPrIVqgfa/N6iCNGUuoKsexTSiLM5zhltYHsGaEREku3YxLjOs9HqY9LZynJEVoUq+06TkOsKIgLZpPF5uRSpyFg8blwARJ679AU151H1+oE0XMsdcppipVGy7r8NPBOiXT3OkP6Y69OijmmFygUmuhGdT2E2MfS9g1FoDDw4yM2zuNFyNI0ZuDP+RmExU2i1mNCUXWSBLE/vkqktlfx2t5VlxRaxyiSfnAhjGNxYnG84A2fsnK2mKrGWHY2G5yi2N9g8F598JUeDI72Cpz4lIBxWNvbbBGtknlCpIYh1OIYuxzyQKvdWEikCfCEMD4LFqK3rWO8hnx2Fvwrhx86PzTaQO7S9pX0tV2lqk4gad1suhmErgxQkL2tVJ4tzP71xyMNk5QdN9Q5g4zZCg4g0nNwUU0ymAVheCwynLS/yF39LqSu7iH8ORyJ3NU9bOWumRBIH+NiFnW8bLgeZMHuqN8aSE0pK2NL1jMYM43itqk4oj7mG1gBLxIpE0uj/TS5dsOUQcZ48bf9ff7v33bH+odMgXKeTvETNyuEIaZPIhQjXzRYHrRNVCVgDmVowjANMGs3K/LTa+RyqZxeq0mWpl6X47UHAbeUZdw+2tQdypTLNKoLuMx79u9Jg1HiguumuRCtivRqg2XpzmaQ3kLMALXUx5d5RI2uw2Z0yRabRRcRctovbhlYRSq7VWzJTr/muRIKZdVPWX8NnkCFQey4ABCNKjZFVFspofKz+Tsc+3wuk2aLh1d12wgwBDJ2uy3iG4q8AMe8WflvKMP9eyjD6MJfkW7clNJMZk03qD8rugnj6TnNiq9PWDoO9Pkz8KTNmlVKxhRHq3Km9Cu2AcQoG7UYl/nM3U1QSavutiqTnWBwo5gjX4+JE9Fb+Esqhh+2eXoPBVN5mGKZh2chaaQFihOMtEU+S6q/XWAKrog1K4IC5+JVIZOXxCaVw65AKyUUgyTB3rusXrb5dyo5FHG8f0no+acUp3iCPI9NpE/crRJ1rCCGFcQufvEfdCw+QIn663vYPcJkkyIUPx4cWNMraz8fvOrZsQTuu9eSdl2sdrm0rq8LQeQHIcty+cVrXsNRxIJeZ3is5WWCdzYPZlzPzIvln2BD670wonXkedYHUPWdqmczfRDKNXsumRomRUENYi+4CLwU1UqWvvjKHX5dsgVaBKwt1zR1zzN/1uStKE8RQjlN9JZvGzS9VqfJMtfJMwv9Eq7QYpMtUeWmJNG4Zyi2olrxBJP4LxFrbcqUsqHtltUNLUtZ8q32XjuyCjXa9prUMsb0OTynvoW4lwdYevYHclk7oyrlGeehVz8ZKgPyFSCiKUEUYhFkr/MLgDl/pjDyAKHQek+x0Nc8pnoEOHBgcKDBgTuBA3tbhAM9nPC5gYA7CwFfCfsZ/PfA+A+mo/w+sZPzFeDfNi/CoD+D/taiv6FBfwb9GfS3Dv3NA8YJVOVo8ilFMQ9CvO88HY06lgYW5uAwzTImwC8AgB6bZJX6oeFixwrxgcGMd8aM70DXsXtlUONDo8Zv4MzQnBgazLirmPHZlmBGLTTcNdC4/kaGQY2fhxoHD35kCNiET9xQhJm8HAwfiy+Q7wr8Oi6YUNB6T5+ugIH3CfJ6i8UtYFxni/TX3wr99dfq7+UW62+wFfobrNXf8Rbrb7gV+huu1d+r7diG/QJato5O31ofMhuY+xvm/sYjvr/xeK+tPjebD7P52I3Nx+H2nlhrUM09nVp3LFCq6Hr629l43SG2aNaxUho+xLG1mFweU8P8u3BKDfmSl9AoP6e2RiM+t97nHmCw0mM6tXZ2/cja3HEw59VfCjIemjsO5o6DueOw84jxzc8GMH4pwPgGG7xo8KLBiwYvGrzYxIsjc8RojhjN/Ya1gDGhxMWMARQUTgKYMMIRoVeT6RXH7I4AcNU91/tFctUt1e1HcSdS6QaqPTBUk07/DXwRuuXrMF+F1nFKQz4DVAxQMUClCVTk4VWBVtwkLc+qPuchPrf4RY4BL6vAy/Hp71bKhJMb/GJ+hv0If4b9iJFL1yAXg1wMclmLXHwy8QklKRcGNEcq9/XFWKlyA0seGyz5Wr/13e51fMvfh+3lK7CZO8cR+gNTlknZHWRkfpVN5SF6nr0CA4pX5aUytdIYi3OxciyOoySE5cZ++4Uee01vUMBGDeUVT7n28Ayloe5B2ypXBxrlaz/qyaX0KVs8v986y4Cdwi9eGKDAPoVLkuI1JWpatT+lmF7lh/5teKi8R6Rbo/q4+YT+0kDK9HuNdSmaQ2FYvNKi7tGqTlWFbABmC6UpZQvyL1RUDx/pno1faTSvuupYEZgm0PQptKzpcpOK5XSTLJ2Kx93f9GD7jpVPcqA3Se8mk+QVSjUdRIDUO3tfSGXXuS3xBU3fODd4tj6FkTI8zgZS0et9e4GzygvqJW5Fqr7R/gXu+RwHuNv7DqrLVN+GX+wV1Sl7WZKELWXuLd/TQS6fdAtwVLyCIysFVbckAFhEq865SsoTOxWh2MPKaeyuo3zoqx+ULas9VP7vqh/6jsqpIJzdU/7velk+/FisQUBxxWNunEUd+Jk6sDpLb6B+qN7ZYj/3VHkLWWrq+5fE2asnxsd5uSyP80o/tE4yP8wapHLHbz/vYQc7wymeDZ3RCA9nzmA0dYYzzx30RtgbZI0vygLt7C3/B8u4cmNlawAA + H4sIAAAAAAAC/+1d3W/bNhB/z18haBuQAG4nf7XxgD6k6VoUa7q09baHrTBoiZa1SKJKUomzwPvbd6S+KIlKXHdNnIYvhnXHj+Pd8e6nMy1d7VmWPZsFcZJyZv9k/fmxl1Eo/pQGFCs0FMeEIx6QWBCvgATEMGBcNoHLtWyGvYCjeYiBukAhw5Lox5i/9oAUp2GYUShKllNCQh4kQHckcRl4+JjEnJKQ1foHtb5hEJ8pglG8AEGXQLCHDrMzGrmQLaSUmaxAdUkYooTVZKsxvBYnQTEOq6HU4bKeyF3iaRBhknJFxopPQkKfI/fMpySNW+NXbX5HYYq72XURJP27wWTijp7YvTqZ+nO0Pxg+7Vn9waRnjZye5Tw+nBw0233njUZoiGyF+rE2r4c4YiSlrhDLPqUkwnyJU1YbyF4QGiGxdDsmMa7zfJT6uHSWkhyhVbHavuM0xIqCuGA2WWxJLnQaAhZfChcgofcGzXHdeXStThA9w1KnnKZYabSuy08D75RId68zpD8O6qSYY3qOQp0T1B1W0UOSBLE/vUykKrp4bdPX9QmtYxRJI50LzVmcWByveMPgVs4WU9UY695mg1MU+xsMPqgP3ule4AUvwMNOCWiOtV3BFlqUzBPiyfldEsfY5bBJW+2mQiKN4hPC+CJYid62jvESgs2H4B85/Nj5odEGAou2r6Rf21Wq6gQiys2mW8C+krsHZG8rlWcLs9/+eKRhkrLjhjpnEM5angtEegYuqtmpiyAMj0X4kfYXgWXYh7jSP4SPw4kILP3DVmBZCIH0G1DMoo6XDTeAENWfDFsDqfu9c4/KZANjplHcNhVH1Md8AyvgVSJlYmm0nyZXbpgy2M7P/rK/z7/+Zfesv8kcKGfpHD9ysywVYvooQjHyRYP1QdtEVXTkkCNmDNMAs3azIni8RC6Xyhm0moTYx7H3shyvPQi4pcyx9tGm7lDGQ6ZRXcBlErd/SxqMMmlfNc2FaJVBuw2WhTubQXgLMQNIUR9fxhF1dx02d5dssdnuIkJO+9lnbqwilH3W3pKd3uaxErJY1U9Zfw07oDBA7LjI7o0UM0dUm8YgLbPlGxz7fCmDZouHu7ptlM0DuXf7LeIrirwAx7yZlm/IkcN6HpQ+3PYcUIQfH7FpjhtFTm6EAnTud4QUN6U0k0vTDXJMRzdhID2HBv4SfM/rkCQPSjoO9Pkj8KRdmplI7huOuuKi9J0b4YiInkwzt2Rc5DP3N0Eerdzayj52gsFVYo58PShNRG/hE6kYftzm6b0QTOVhimWsXYSksfUpTjDSJvIscP56jim4G9asCJKYi7u2RZ72mlQOsFwrJQT8JMHemywntvlbpRWKON6/IPTsU4pTPEOex2bSJ7bLNj0riGEFsYuf/QsdiwtIQ3+Oo48HB9b80trPh6va9iyB5m41UV0V61uvraurQhB5IWRZr//3TNZwDbGglxnKavmV4H1YBguuZ+Yp8A+wmvVOmM068jzrPVhyq5zYDBiEcs1tjgwGsyJNBrEXnAdeimqJSJ9S5U11XbIVWgWsLdc8dc8yD9ZEqigPCkI5TUyW3wxoenUHxjK6yTKBfgmXaLXJjU7lpiTRuGco7v604gkm8Z8j1rrVUhKFtluWKbQsZcmfdUd1T1ah7ra9JrXcY/qonVNfw76XNSM9+z25qJWFSnmm+darF2PKDfkCcM6cIAp7EWSv8wvYuHyiMPINQqH1nmKhr1kZegDobmTQnUF3O4PuBjuE7jyc8KUBdvcI2L0QFjOo7o5RHUxH+W0iIucrgLpdXoTBdAbTXYvpxgbTGUxnMN0yYJxA5o1mn1IU8yDE+87jyaRnacBeDvnSLCoCxAJY57FZlo1vHwT2rBAfGCS4NRJ8A9aN3UuDBe8aC34D9T1T3TNI8L4iwSfbIEFdFNwECmoR3xdiQW1E/qpg8PqTDwYNfhkaHN15gQ8QCJ+5odhM8oQsXBY/4m4L6HoumFDQBo8fl/DuNsHbYLX6DHjW2yGNDe9IY8NrNfZ8hzU2uiONja7V2PEOa2x8RxobX6uxF7tx0/QL6NU6On1tvc+0bk5GmJMRD/hkxMM95vnU3CqYW4VdulU43N26sQat3FLtuGeBUkXX018/TOulZMHoWSkN76J4LCaXxWKY/z7UiiEO8hLy5NViazLhS+tdbnODgR5S7di574Vjc37AVI3/Lyh4aM4PmPMD5vzAvcCBr342MHB7GPgKGxRoUKBBgQYFGhTYRIETUxA0BUFzduAn8bQY4mLGAOIJJwGsF+GI0MvZ/JJjtiWw6zobert4rTrnuftY7UQq3QCyOwZk0um/gR8nd3wd5ufJOhppyGfgiIEjDxWOyNJTgUncJC0rTV/yKJpr/6tiQEkXKDk+/c1KmXBrg0vM344f4N+OHzAi6RtEYhCJQSQiz5CZTyhJuTCgKYjc1o9XpcoN+Hho4ONr/dd1t9fxLf9mtZevwGbuEkfod0xZJmV/lJH5ZTaVh+hZ9oYESF6Vl8rQSmMsqlrlWBxHSQjLjf32+x72mt6goI0aliues+zhBUpD3aOeVa4OGsq3QtSDS4dPlU+qVzCdwiVJ8fIKNZran1JML/NKfRv7KW+X6NeoPm4+Gr60izL9XmM5isJQGBYvOmgsQ1GlqocNkGqhKyVbQdiFROrhI91D2UtF2nmyVceKwCKBpk+hZU2Xm1Qsp5tlUVQ8Z/2mJ6r3rHySA71JBjeZJE9MqunA8aXe2btCKrvObYkvaPrGucGz9SmMlOFpNpAKWm/bC5wuL6hntq7ddJP9C7jzJQ6w3YP2q5NN34Zf7BVJKXuFjkQrZcgtXxBBLh71C0xUvPshywBVtyQANESrzrlKZgWeVYGJPa6cxu47ysVQvehH1fex8r2vXgwdlVMhN3ugfO97WTz8WKxBIHDFY26cRR34iTqwOstgpF5ULwuxn3qqvIUsNfX9Q+LsnQfT4zxLlrW60g+tk8wPswapvJ23nw6wg53xHC/GzmSCxwtnNJk744XnjgYT7I2yxudlXnb21v8BquF9oXtpAAA= k8s-resources-cluster.json.gz: - H4sIAAAAAAAC/+2daXPbNhrH3/tTcNnsjt1YtSRb8jGT2fERN5lJGjd20tmJMxyIhCTWFMmCoI+47mdfADwEXhKtw6Kkx68sgudzAD8CfwKPG4qiItt2KKKmY3vqkfLINrGNlulR9uvbd/bzaZvvhg2Too6F2VZKfCy29WxM3xtsi+1bVrCFILd/5TgWNV22vS429k0Dnzo2JY7FL9FFlhccb5n2jScuI34S3CXY67MN6m7dU4Ntzp3YQ9xWcHNsq+5YFnI9LJ9NFPSx2evzO1cb9bp7r8YFLrKxNTyTfDZRjiwTeaeO5RBhhadtubCDiJe+lCgwkNf/gO0e5TfdqGfKcNFhFHmOT3T+AOoFcQaY9rHvqYm9uqZl8dMmNzpkgMQDupgdb1PfNmnyONPIHGXaFJNbZAnDDJK7W7iHbSN2vWyRnn3sXaWcLu9x28t7PO4fnxB2bwWlA3RfVGLaBSWE+/WSBVL+rXh95y6/hLLgtgpOyiziD10klSXdz8JU7JU+uyi4M43A++kSKbLjzTxPLhzmjY+OeBKxQUGe8gMTJ+mW0L2oh3NDyOVnIcgwfX6ZVrYsP/KYVwxMMOHX7lpOKnI8TEzsfbrFhDBL59y/5yIdFwU8K+TeayY3UqTf5N6KR7HrYuMDs2FuOUWkh2kyZdNpK3bE9654noZSUzx/sMnCUuk8KJsDZuMtZZMgijdt9r+mu77mYd2xDU8TYfH4p9N5c63ywho7i0NYllyr2wo/8s0/1yyTLPy36dwhk/7NbhdZvFC3fPY/Yce9Cv+9Vp++vdI0fiEtyrTvW1tb6nb6VofZS80B1gJ7Z3czbWY2kT7ZgJay+Rzp1CFpk4d+7oqKWT1WE0VP0q+ka2mf172OZXCLq/v17YN6Mjj4HZ+zqkqq7hNll32zS/MLqahB1NOLL8oXarK2RTQ3qdPHjcZjOrURwUZBBnvMZ1kDBImt0QdXXNe0DfPWNHxWDxTmeLSvZ9o9CzPzp1LjHt2bXvbmOr5+EwRp+rF5bRYmObdOysmqjQY4/6jiqiyurkSznP8cD+gel8iZYSCyyxGaDUELdbCVe4O80OmdINH6NjJlQeWee1hQu9cz26UnTgbrdmUfIrcofI50Y5LKuo2cx1tpFGkCigCKrBeKcAzhNby4Sc4dRzd+B2uuY2gMPyhiFyIaa29F4rF//mK+p94RO+wxFy+2lB2BNuIsgmV4C+V7GrIsR0fixSyEGb5LjZfiGktlYuoeg5boUqyc3QzbkH8VAJYssHwOnaOcOoOBSQe8LgFwAXABcFl5cNkFcAFwAXApBhfLZC0iYEsVseWDcA1AC0ALQMtaQcseQAtAy3oO/BwJxBjggUMetI94cHyLTItHsdZ5oLgEp6SOv+IRFBxbMFIEPFKGRz4Ki8LQD8AIwMh6wUgLYARgZJ17UAKYqM7oT3A/QC7PIxcYAwKCAYJZS4JpA8EAwQDBVGUYCPhlEn6BwSCgF6AXmV42UgnJ0tfFKJ1M4db3LPmDbsvc4s/OXeIjqPiGrsI0TH6iFCfnO4yMoElMll2aP0R5vx3kTOiL53z81GytysdP9SyS7QOSvSyS1QHJAiRr5DNZxnjzQTIOTkMuS7DY0ZDK+EdOvsdsm/zUidOZZvJPlIoYjX8yFZ9+GpIai0pBWp7H53t8jK/79KQW7M6MeVNY+zN7C6dNQl6pOJjRZ07cA5WkLPGFLgAWANYKAZYcT4k8BLIq2R0mwOoAwAr6uqoEVs/v7PLog1WmjRB5yR/oKtt+BZmEh3zyP/ZX+/ixdnamvHt3NBgceTnc4yLK8sUuPmfUAPdNw8C2+qzWIL7dC8fIubbOK5gwPHJrbz2qgZK+mfRZDaybAyTqvWyLZwWgltuPxsuuBPyeWMi+yU+xaL+Yl9QzwuosxXDubIU6iptrA37IFyJqqx1j56CFWu1mff9AN7r77O8QI9RAhrG3q+tYb+CdmwOvFvUrerWYP/97i0htWI++eTX8/z+8KGLniJzFxvjoN680TceWpTVGxsdXXoUoPx3nxEgxmCZiyPYHHZaQmXIxiBEzw2Qh9odDbiwHQZzdFRsiGWzoYL/bqRuHjUOs1zFqddutvd1Gcw832vpeE7VSwRafdzFhd1LNsMt/a1p02DWLwm5EPE0TdyNDrYx3T6vr3UhFAQ6exsFn1Xew8m9w8TQufjs/F8tih8kdHYwngpOncfJ5dfM4HC6GLJ7Kwb9WOYt/i9hxOXy8zi+TI2PNLvZjFWqTikeXZMidX37emdyGHiWm3XueDYtHpyYZHYz1WSyuMRmlplKmH/8T3btz0FClBwbVUSqrsqOA2+WMqDu+TfnckZL2LXpT54Y9Slr4iGAxAjNqNDU6fFsZ2nXZ7HwyaztXbCi7AiY+nYuJZzr72HIZ9Gz5Yzb6IHmNvfh2sWkxXtS8XOY8X8OkWDUf/qougfTpd5/5tlrSJ4Jsj3sy348xxecUgTQKpFGrKY1K5SkIpMpIzw9BIQXS8/WTng/RKpwciXjRjEg6Mm5N1lIXdSyxrdHB/2IFoDSfJW5FEytxtFY273YcRUd6P2VSEJ6/PF2JOcOArtaUroK0BLR6jvY8TVuAViA+B/E5iM9BfA7icxCfg/h8vmEnv0iB6q2i+vPgpWoqB4MEveIS9Bn6GPSrK6xCT0xsBX6uqBB9BtkMWnTQooMWHbTooEUHLTpo0UGLPnMJ4+LGtldfej6j1Q9AfT7fKM3oatfSb28XngsgOV/GTACR+QurnuIOPmbX/HdHEJyD4BwkUdWTRGXHX0AbVUYb1QBtFGijQBtVZW3UaZAkrIbTsXmLlRNkG0E0wcDFFAMXc5QpnbjelL6+4pjH4B+cPRtnn1TR2Z+ZDxSnGyW2oVygAMXB1dWUCblTuzrKawrerrxgyJ15YitnxOEoAy6vqnpoHgkOXq+4lmhyr4O+BPQloC9ZIX2JmDJIGgiyMeXyBo0ELbkmZIfBFEOlR4aGofjPtfrL62v16RuLSX4hLepv+7619sKUUdanYaMK5p+rXmVc8LsBz4D956RuGRv+4ID5SmHKJoARAD34YU7SltKJAI6YQBGzqAn8wq7t3wJnVnEdU9BWgLYCtBWlcxYUFhmFRc7EfqlxbpBYwMx+LyWxaC9wYr/l6EmBSQBHy2FHKU5g9r+XxzEuNgAYWwCMba8EiuwCigCKAIpUc1gBWGQUi4xRRAKMAIxAz9DL9QxlsxB6g0oh2B4gGCDYWiEYn2wEeoOWn8COWfjx1SBOIy8OUSzxvfRRrHsGQgNCg+6i5WWVFrAKsAqwCnQXrTKsSF/sAK8Ar0CP0oJ6lMpmLHQ4PQ/i2gBxAHEw5lfRb1mA4kYKkMrMkAGg9vKg5gKoQcfS5EyyD0wCTAJMUtXvOwFKykDJuLmcgEuAS6AD6eU6kKK8TOci9BOVYrIDYDJgMmCyik/5AGg2SX9R7hR8wGfAZ9BvtFSMcgiMAowCjFL16ZAAUibsPwJOAU6BfqRq9SNlchL6k4Z7DacFrjXKUlyzDhQHFLdWFKdj09pkKMcgSl4SdDMFdl1PIxgZIwYAk4uMFixynoNvymsle607Yo4Usk96sS2gw8np8P2ni8vNzzwKXv8h/LMFMAhzYEK31RJ3WzVhkU0AnrXrtirLOmO+ppsH8czpkoA9U0zv1CeO3+tf+FTAT8A+gD7wQR70gy2qH+ySVUX8g7z3n6Dr6zlrqjdhwu8Xxr31WFM9aMdSkcJyjW3cSxmXnUDP1ppPsEb7vNZoH/babFVqsb5aA5Zln8kSjMLBeR1y4OEKrcU+tYdFCrNXR/B0xZdin8bT4bumG75rbsHCzNM4eY4rsJ9MvDCz5OKcngTwcXWWXJ+Jj3kaR7U2uHoqV59X0dWwzjqssw7rrK/OOusL08Gs5brq4609NyXQOq6jviYir3Vcon3BY9pruSp76dprXY3+dvUCvXLiDVjgvfwC7znjxpWQb8Dq7qBsBYFHrsBDqSlnJn8r7PjiKmP0Hhvhfaue3scD9BUTL7i3YHWrYLSXH2EgciNOxrKqN4xM9cZn7/A2ZhW5Gp+L4oFrsYe0e3GmqRa7KSmeh0EgqRwSQhGK70VMGLiLfCs1xC5SSC7N0xnzcWCcjNQ4lNQzRJFyGWhDpPIwn2XliFTquNym6SpX/cvH5EF8a5inMOH1Pq+skxHKtvbwfaqhiGsu6fIbqeeSLIcs62toiGQgj7NpgTHzrVhCRBNZWqraWUXOGmkDH1s5spGCjFYHzJVmzu6RV8Jm/zkuEVfSgip303cznPEUc8VWvs+a43wWNmuyb1mKCMd4v0e3oSZLM/fLt+XvHEZE8EBSge/hq+BEckUVVEa8cnoKUtEUlguTsBtQgWo7d7VGJHxkrXm4TU0c5pqsTSTDg0OTxKAlN09qS5qkq1GXfuzKPyTRlNqS/m/IP3brcom0KmFT+r9hBMH6PXoGrsmVYmHsVeQTt+UTy1dp7sk/pO9G9w35fqN7SZjvhyMENOqXq9Ow3ozpyhm4PsXK56jjVtlRTqXIVn0hPlNZ1XbQ7hoNo643mg3UbO/t7XXau220223VD9FBsPNtXGXXN57+DxMbrcusNwEA + H4sIAAAAAAAC/+1daW/bOBr+nl+h1XYXydaZ2E7sHECxaJJmWqCdZtq0g0VbCLRE29roGorK0SDz24eHDuqw49hOLNtvPsWkRFLvxUcvH5F3G5qmI8/zKaK274X6kXbHilihY4eU/fr2g/28b/DLsGVT1HMwK6UkwqJs4GH6zmIlXuQ4soSgYHjh+w61A1beFIVD28InvkeJ7/Au+sgJ5f2O7V2Gohvxk+A+weGQFei7zVCXZf61uEIMSw6OlZq+46AgxGpromKI7cGQj1xvNZvBjZ5WBMjDTtaS2pqoR46NwhPf8YmQwn1DrewhEha7EhUWCofvsTegfNCtZqkOj7qNotCPiMkfQD8nvovpEEehnruqbzsObzZf6BMXiQcMMLvfo5Fn0/x9tlW6y/YoJlfIEYJx85c7eIA9K1V9JpGrQdXgufQjQljPI2pddDOqxvZG1IRD/1oxLKWGMtt0RtzFHijKJKzU5bXHrExcVWxdVFzbllResUYxzLSYm/m5z4T5wbeE6niBhkLtJyZ+XqqxdtAAV1pAwFshyLIj3k2nXFdtOEzsFiaY8L77jl9QfIiJjcOPV5gQ5nEV4w8DZOJR9soquXra+UKKzMvKoYQUBwG23jMZVtZTRAaY5j2u6HXiQnwTiOdpadsas7lNgije9JiADTOIjBCbvmeFhrCDO5cVv/rOLNzB3/WGZjoRGwZhJS/if7/r999eGAZvw0iM/sfWlt4o9pr5EbVdbEjRlS+zPSYBYepl21T86gyZ1CdF6cUq64sQqb/Wc1X3yq+8luiQR0Hfsbjw9P1m46CZ1zMf8RkLGkrgzdV9Htp9Wl1JRQDXT86/aF+ozaK8CPyF5tPwfVf0UkSwNcqFfULLApA+atDbQPRre5Z9ZVsRc+mR7ppcG9rewMFM/AUrv0E3dlgeXC8yL6W9FR+bR57YX7l0CkrWPeTi6rtGR6U08ogJsvo5btENnsD8M0Nk3RFaNkEH9bBTOUBe6Q+OkZgHW6U6GYgrb5ORuFkqV544b6yN2j5EZVX8HMV5oeB1GxWPt9KgoA0TP0z8tZv4w8jd5EFYDJJP+keXUQ8bgW8ZbO6niHVEDDYlCt9g//zJdE/DI3bbXSUA2NJ2NN6maEUACT6JRKGBHMc3kXiLuUvaY7eyHhmYqG4KgEMZOHyKNaCd+K5rU5dHBAAQACAAQKw8gNgFAAEAYskBhGOzSQvgw8Lgw3shfwAPAB4APKwVeNgD8ADgoZbLDny6PxJTvYtdn9waH7D7+grZDp/rjd4txRPghcL9F9yC5L13AA4mAQcfhOxgXQKQASCD9UIGHUAGgAxqnlaQM/szL03ITiG98DgEAQsUgCQASawlkugCkgAksfxIYt5rFIAjpsERsFIBKAJQhIoiNgoOydw3wKjoTHHpO+b8Mo1XWf3Jv859O5IO6CJ2w/yXHalzvsXIkvNevu6z/VPUD7vSZ2JdPOabkXZnVb4ZaZah0T5AoxI0agI0ktCoVY2NSsJ7GmjEAUyGj3KY6ChDR/xzkChkss1/FMJRkmHzLz5GYaXerZbBr1nAzoNoRnrVWdre3V3a7/29PuJyJszLkQGayVsobRpwVLCDOX02wjVQSyAkvj0EDAQYaIUwkGpPOT8E8DNh5khgnwPAPpAWeiT2eXxeKKS3ziRhXLgOf6CL8hQjjR1nEOJ/7G/7w4ft01Pt7dsj1z0KK6BJgCgLNN7oNpM5cmhbFvb0RwXsdLjnvlXRt8ljQGwelQHWTIJEXjfTPquFTdtFIjSVJyVHYqnKbBSvuxD49NhB3mW1iyXXpZBGPyUsrGiWf+1p1NeCShnwW74Q8Xn9jrVz0EGdbru5f2Ba/X32d4gRaiHL2ts1TWy28M7lQbidZOfC7RQi/vcKke0s1L16kf3/b16VwNsE3IrC9O5XLwzDxI5jtMbax1ceQrR/vq6wkdHYMWdDXuT2mEOW6kVKPp3WpzOxP3xy6fgI7Ox6tCDyxoYO9vu9pnXYOsRmE6NOv9vZ222193Cra+61UadgbGm7izG743qaXfWLzaLNrj3K7MbY0yx2N9bUJtHuSX21m3ACQMGzKPi0/grW/gUqnkXFb55OxerS/fSKlqtyoORZlHxWXz+OF13Bi2dS8K919uLfEuy4HDpe55fJsbbmjdZjHaJJza1LEeTOL//ZmV6GISW2N3icDEcvIE2zgJdSmZhdYzKHZTix0eQTsI2K63P6OD7SpItxjckEZfqRRzf5DnfZUmfyNs6Fd5SX4hHBYiFknDST2xtaJtdlk/PxvOVcsxXlGoj45ElEPNcdm5ZLoKfLb7PJR6trrMU3i3WLhzm+yyXOszV0ilXT4a/6EjCQfo+YbuvFQCLIC7kmq/WYIvWKKmAoAUNpNRlKBT8FntIkJO1DICoBSbuWJO0M/cR73JCwemObhpZe+g9WAQzseeKfZI8cjnW1zesdXzOROSyIFAjZzw93xEZPAHfWFO5ItwSs8xhOdqEnwDpAygZSNpCygZQNpGwgZQMpO/+uA2ywmvKy5XvPTAoGanbNqdlz1DHwOleYnZ3bNgn0XFOC9hy8GTjawNEGjjZwtIGjDRxt4GgDR3sqat9zLSivPgF7TlvTAwd7njZZ4pKupZbeLNzygWZdf7sHGvUz04jSdByTa/WbHlCqgVINHKP6cYzKqyVANpqEbFQoi8O9OADS1YGIBEQkICLNSEQ6kV7AApSJ7SusHSPPktYEqwQzrBI8ISfoOAhn1PUFR2kMu4Oy56Ps4zoq+xPTgeb3E8e2tHMkkTSoup6cnGBmVSd+TUHbtWfnBHN3bO2U+BzKgMrrStV5CgcHrdecuDO91oHMAWQOIHOsEJlD7GmjrNp4mHKegUHkTG4Ijp/cA2fEMk5meH991395+V2//8YskDdrJMmxH1trzwcZJ2saT6HrKuzj5xR2YtiBxCrrJ+2ThZj22or7dJHGbUkgvn5Sf7NQI19bsZ/Vfte3ON38m1RdHc+gBLoC0BWArjCxzwJpoURaqNgNrrD2DMwE2A5OYSZ0F7gbXB0TELBz3HjK5zhaBmwZ9/z4iK/IAzpaADpqrAQ22AVsANhgmbDBQhPmAA7GgYMHeHyADgAdQO7k+XInZS+EfMlEmGgPMBFgorphIr6dBORLlg0SvWbGxjfZP0l0lmGj3FezRyl9FiATQCZIqCwveOgAeADwsEzgARIqK4AelC8xAEAAgICcy4JyLpN6LKRkHoequoCqAFUtI4VlMdx3gFVjSSyTbEUAyOn5kVMAyAlSL9ODhH0ACQASlpLLAiihvijhoV1sACgAUIAUy/OlWBK/LPoiZFImAkkHAJIAJC1zJmUxH1gDVpomo1K5GxgAJgBMkFlZKtBwCKABQMNSZ1YANSxVhgWAAwAHyLTUK9NS8knIuGRXZVuGbrcmhVXtJsAqgFV1g1XsDd7ZZNiK4Rz1QL7NAtLqhwbByErXrPIH+jW0yiP4vnXcH1vaS63c1jWxFfbx5I1tAQCbHoC9+3j+efMT1+LLP4T8twBvwXZ3kKpZ4lRNuwWYAjBFDVM1k8KJ3FdI8wEVUzYJyGKGrWCGxI8Gw/OICnwh4QWgC/g0CbI5i8rmfGahiH+a9O4jJHAec+hvG7bPLSOq9TjYV041BUUzd2CFewXhsgbMcmC7h4OCn+qg4Cx3sVWrE6O2W3A28FzOARMKrkpLgYZrdCDwzBoWLsze3kDTNT8PeBZNx6+DQfw6uAWng86i5NM6nu+tqLjiZR90XJ9zf+eiY+7GSdQGVc+k6rM6qhoO+4XDfuGw39U57PfJ2BxreZzvw9Kcms+yjgf2LgnVaB1P933iZdu1PMJ34uixqkJ9s3yG+uz8AjjRd/ITfSuWNmvBMIDjfIHfCByESg6Ctq2d2vytqBeJXh6gJGzE49ZDc4hd9BWTUI5NHtYiVzv5HRYil6Ix5lWDzDL1y4i9w3qYBWo9bYtiN3DYQ3qD1NPYi31IFXvOjEBZxs9xGSi+ETZh4T6KnMISs3AhtbaKbcrXQXHeUlNT0k8RRdpnSV9Q6mN/VskNSq0fcJkWQ67+Z4TJrfioq4oEweM+D9Z5C2WlA3xTmCjSyKV0v1F4LkVyyHG+xoLIG/JDMh0hzGopTsDzSCSthHYWyNm0bOHXTgVtYoRH6y5TpV1xeaKVeKJ/jEpET4YMuZtRcPd/v8cgg4msK5vNRwwypEhiq1pn7Yd0Fk9rqm6ZiwjFhL8nw9DztaXx8rLqi2OLkA+kVEQhvpANqYFKBiMenO6lK9pCcrET9iUq0D3/eruVcPPYbB6X6bnbApvNiSS7ORZJ+smuOj3pHWW/oFZT+bGr/mi52f8d5f+W+mO3qdYoh2y1lf9bljTWH8kzcNqoYgsP9qI23FUbVntp76k/lA/09i11vMlYcuL76QsCif7l4iSOmym68t0golj7lCQutR3tRLFsPRL8KJ2FtoNu32pZTbPVbqF2d29vr9fd7aLdfqd5iA7kxVdpyG5u3P8NqZWh+zMsAQA= k8s-resources-namespace.json.gz: - H4sIAAAAAAAC/+1dbVPbuBb+zq/w9fbegbtkSQIBykznToGy7Uy7ZYH2zp1tJ6PYSuLFsV1ZJlCG/371YjuyrbyTxCSHT0SyLfk850iPpEfW45ZhmMjzfIqo43uheWI8siSW6DohZb/++s5+Pu3yy7DtUNRyMUulJMIireNh+sFmKV7kujKFoKB74/sudQKWXhWJXcfGZ75Hie/yItrIDeX9ruPdhqIY8ZPgNsFhlyWY+9XQlGl+X1whqiUrx1It33VREGL1aSKji51Ol9fcrFWrwb2ZZgTIw+7gSerTRD5yHRSe+a5PhBWedtXMFiJhviiRYaOw+xF7HcorXasW8vCw2ygK/YhY/AXMS+L3MO3iKDQzV7Ud1+WPzSb6pIfECwaY3e/RyHNo9j7HLtzleBSTO+QKw/Syl7u4gz07hV61SMd7G97kQFevuOvoXo/jExHC6jYkt4fuh+U43pAcwnG9Zo6kr0rY9fv6HMqc2x3yUGaRaACRkpeFn7mpuCr/dJHRd2yJfj5H8ew0mcfJpc/Q+OSLNxEJBgqNn5j4WVhieFEHa10o4E8hyHYiXkyjmKf3PIaKjQkmvOy26+c8J8TEweHnO0wIs7Sm/mGALDzM4VkmR28/m0iRdautSkhxEGD7I7OhNp8i0sE0G7L5sBUX4vtAvE8Y9bY9Ztemh3pY1LQZ+HbTYi0PYoWQk/S/phVEzShktm2GmKXaYVO4yQl7RNMhiOJHy41YBcmbb+ar+N9v5q6RPpmnpz++mU87xp7By7+NWrlSm6xJE6HO/vnBvI2Gj3/7LXY/v7TCzENxhYU/cayQlzBdubtG8nSWwV6K18TczRto0GZQp8dfmaNcvMzxWG1E0BbDSGlDLpBFfW7weuES1oCL7sB8a2aynpRfWYeiXd7i+67NcTaPqrvH1axL8hpfsAZS6WQyedddp031mVS0W+bZ5RfjC3VYjyY6OWO7zR5nJGDs5IpLu67HfAODCLaHtCOhT2jRILJ5adKHQNTD8WznzrEj1hoNbWmSa0PH67iYO0e2evfo3gmLlWtF1q0MlbwZeJsaNzXcWjnQTe5K+ruGN6hpoynIgf49HtA9niByB47JiiO06JIuamFXW0Ge6XdOkeAAtUKe7GK0t8k+plpIV94467y7pX0JbVb8HvkuLReFW5rXW2tCVAdCBIQICFGZCJHr9BygQ6WgQxIKIENAhoAMbQAZ2gcyBGRo88jQgIP0cM8nD82+T25Z78I4AG22HihOyIiF7DuHdWQzUJC0jH+wHH6l02MoyR+lmCaSrw7UqEiNPgnLwGQR8CPgR5vNjw6AHwE/An5USn602FkjYEdTsyOYOwJuBNwoTU0Dk4VxgFE+qOLUD6wREEGkz77y+xlhVVqhmzgcs7KnNEjfY2TLDjebd+38FPndQxkzMRbTCKrqjXURVFWLhK8BhG+5hK+6noRvdAsnYoI/60fEzGxU0omFYltn8cjhl/5yUT943TgrXpGGg5ZlxL5ebJ27jm1j71pylyE3c8HmQMSpvSQNA32ugnSR3mRY7VRdRMGAknuMNN/F6+P96oaYbyg/nGToUatrxx6FMF6zderWg7HNipuH748l9BLui/R5j4+sxKcnc8iFzLS3Q1kJs76AcKrASU1tIRcROb5KRj0ilh6nHTNxhs0SGZu32U9lBJW0ab/NvwA/tV3Ht6trYGDZ5q3SvMNa3ZmNO9m4NscOnkkLwVuTUo5dxV4KGLbCsHWNhq2qP42KQxi1jl7IEIPWQxi0wirFSlYphgwVpl+mCOmDO9XI+abYi8lIwgOa8j/2V/n0qXJ+brx/f9LrnYQarhIgyuLFG/7MpBuWIz5ztnGqnmEMhqixj2gbcitphrIAzfrCNracHhKNX1034rzVB1CceSMGdacuGnddyp3Mc8JaLsP2+56pvfQLEW3VSGi+8ug1fnmrgWc4M8zA50W9FouFQr5Y+U077dnRvRo9hQMATwTwafkBNv4JEM8D8dniIFbFHLMD/XHUXCKAPBHI5+WNYwkvRPF8AL8rcxRf+vbLQHfYrP8Cwd2z9w4bxzV8cIgPcMM6aqGDKqoeHR4c7r9u4PZRq76/d3scVpKJxrAS+PZ/7hCpDIapb14N/v8Xz0rmNJMZTZE4mNIcTGiKDPbAN6+aTQuzAe0oVwt0MJahGSm5cykm3Pvt33uz2zCkxPE609lw+BTuGi4HifmSBQi/8hPu5ihp2HOvXHABoDTRSWqSE2H5IDphdXbu8MlEGxfW1uani7D56l09Fl0C+CPBPytdwMVK2LW1+DmE26ZC/858AUvHf/Ll8HItHRPkhRxJPY4pwdNkwdIyLC2v6dJyNk5haXkSQfQRrC2DIBoE0SCIBkH0agTRy9iLaYzZjAnS6AVJo59llymoo0eoo1dr4XURSCd7fvlEiLHd3/MNC1ldvAN66RUPakUHAIPaDR3UqmEJ49ppJNPHMKwFyTRIpsssmdY2bqDVKpdqWvKPuQAG4XTJhdPPiDGoLtdYOx1jDfLpUsunnyGaQUG99grq7ITP1fX1DoA9D9gXpQ7oGOaz4pweAD0l0L+/BKCv+ygAnOfC+X05cYadL8NhhZ0vsPPl5ex8KcE3mGELTEGWLbGAjRAL2wVTIrcfqc8HR1jJjpjJzQ6bYiD6YIPMM7kBCZ8f+Bdk5IulGFlIqjbZzL8vxcxhHwWbbOX35ssQG8KuOthVBwLEFyFAhI11UwsQX4MAEQSIIEAs9TdbZZAYV9jCbJhpnCLPlt4EK2flVCOeBuGcWN9witdzKIBdelni7GBfMQwMv50Etm1cIknDAeqSahPnhjqJawpol16hGDx7YBvnxOdUBiAvq2BxEQEOqJdcuTg76hsuczKobwS+HYLcCeROayJ3Et9SVBZrPEz52nOTyK5cLj7Lby/OvXjz9BdzSl5eM5l3+76z4WqnYeancbcK9l+s7Gmc+weS0gAAi5IbjQ0AQGDB2qNJQ8CWrB6AWJT6Z+JQACRmkQit6kPF8Qz3HxLNMp53C/IKkFeAvGLimAWdRUFnofmAcT4NlBbwBeMlKS0OV/jZ2GnmU5bG00r/idgVcbPRChP4rubyeRcXFwDrWgHr2l0LzlEDzgGcAzjHqEUEIB0rJh1jpI7AOoB1wFzP8uZ6ilEI8zsTca06cC3gWsC1RgoGgGyteoZnki0HQLmWT7kCoFww0TM7+dgH8gHkA8jHaLEcsI+SsI9xu+CAgAABgTmf5c35JHGZj0WY+ZmIfB0A+QLyBeRrIp08cLCSzgBpdykDEQMiBjNBL4qMNICMABkBMjLZXjFgI+WdEQJCAoQEZobKNTNUiEmYIRpcNfhCSqU2MV07BLoGdG2j6JqFHXebcTZGnOQBK9s57tYOmwQjO121y5/ToiVt03A241ejWGSfOIoo/NnL3AF+OAs//PD58nr7invDr/8VAO0AHYRPAcAM1UueoToCygOUZ9NmqMazncymtKVynsWWDMRnpl1xXeJHne5lRAX9kewHyA/sjIO5sFXNhV2zRoj1ecaHzzD9Nc0RU7VjIHzLJXybccaU7MdynsJijSUe5IzLHmAVW80nOLNqUWdWDeZtdkr17fJKDY6pepYv0guAdVNygHCJzqaaG2ERwmzsCEiX/GiqeZCOx5pBPNbcgYNq5gH5vIxHzSkQa2YSAOPyHEH1LBjzME5abYB6Lqgvygg1nDsF507BuVNrdO7UpGIYzeEWS16wWdOTp0YBkJEGAQILOnuqTCEwXiEGbrCgE7CmWygHGBZ0DNYE7SHgkMPh3UaEw6RKEnCJl3Qcl2ZxuxQaEziLCwS4oELRqlCMinHu8JFrKxKljBGlbMX1NkOri3voKyahrJv8fo1ckuZ32IjcioexqOoMPNO8jVpssI1ZC2+mz6K4F7jsJb1OGmmmyyql+PPACRQpRkbNQvG98Akbt1Hk5nQAIoTUXJ0cmi9W46ynpq5kniOKjGspYFHy43hW5S1Krh9wm+abXPNHhMmDmLLRyWB4u88b66yHstQOvs91FGnLpRS/lXsvxXLIdb/Ghsg68jibDjGm3ooTKH0SSytNO2vIWe9t47euRtsyJKLNHoPS0VyeoBLzgWkgESU1ZZO7HQUxAeFeW2E9PsUV9jbEsUJGJVLqsaNHrz4OvbiDU1FmwSIgCv9MKmRmcws152n6i2PfkK+mZEQhvpEPUpusF+Ew1QU7TMoUZ3YZ7inN9DFN7jNR2Ay6rMsZ4Ut6FvukfEVgPXxsK+kKn2TD7wizx01+W3JQ0/P7lVqiBWbcMU4zM7cFDmNgZHBzbJKU1qtkyGwoX6OrVZUf++oPRUdoNpT/a+qP/aqao5xvUFf+r9nS078n78AF6oobjS1FffCh+mC1lPqB+kPZTn1kq/VN6pIx309faMrMLzdncS+dcnm/F0QUG1fJioaxZ/yReKGxfeknaiwzEsJM87iBGof16tGxZbeP2N9rjFAN2fbBvmVhqyZDybxLmUJ16+n/ZJnATaMfAQA= + H4sIAAAAAAAC/+1da1PbuBr+zq/w8facIbuwJIEAZaZzpkDZdqbdskC7c2bbySi2knhwbFe2CZThvx9dbEe25dwvTvLyiUi2LL83PXr1yHre0TQdOY4boMByHV8/055pES20LT+gv/75Tn++7LHLsGkFqGVjWhqQEPOyjoODDyYtcULbFiUEed0717UDy6PlVV7YtUx84ToBcW32iDayfXG/bTn3Pn8M/0lwm2C/Swv0w6qvizK3z6/g3RKdo6WGa9vI87HcGq/oYqvTZT3Xa9Wq96gnFR5ysD1oSW6N1yPbQv6Fa7uES+FlT65sIeJnH8UrTOR3P2KnE7BO16q5Olx0W4B8NyQGewH9mrg9HHRx6Oupq9qWbbNm04Uu6SH+gh6m9ztB6FhB+j7LzN1l4w52zES9g7d+6Kg6yCQcEkJbL6jtoceiGsspqPG7bl8yHqkmoPZnF9z1gOxwIEWpLq0hakn8qmzrvKJvmUJB2RrJ+JJiZsrXruUEn1yTq4cVaMjXfmLipgUdaQB1sFLLHmuFINMK2WMa+Tq1cVCxm5hgwp7dtt2Mcn1MLOx/fsCEUK9S9N/3kIGLbJJWMvUcpgsDZNwru+IH2POw+ZHKUFkfINLBQdqrsp7FL8SPHn8fP+ztOlSuTQf1MO9p03PNpkGDA6IPIWfJf03DC5uhT2Xb9DEtNf0mN5Mz2kTTIijAz4Yd0g6SN9/0V9G/3/Q9LWmZlSc/vukvFe1AY8+/D1uZpzZp1OHeSP/5Qa0t8Cdse0+LW6AVtOPsafpeVggD1w2sHnstpsn8ZZZDFcI9L+8qvJp2hnrFFTIClwm1nruExlEelfW3eqrqRfqVNpqgywKva5tMl/pJde+0mjY71uMrGqekWJ+qu+1a7UBdGfAxQ7+4/qJ9CSw6sPCxRttt0+a0WOCVzOOSEeQ5G0QQwWZRhHFJkBeICCHN4Mnj/bAc03qwzJBGnMJoEl/rW07Hpt1DGSd8RI+Wn+9cKzTuhTtkxcACYxROmLQySteZKanvKg6aSWDkY7T6PZ7QIx7DOweGSR9HgrxJ2qiFbWUHWaXbOUd8KK7l6sQ4obxNDBTVXLn0xmnj3SvtSyirovfIDlsZL9xRvN5G45I64BLAJYBLJsQlttWzAJUsDZUIcQMmAUwCmGQLMMkhYBLAJKXEJAMo0MM9lzw1+y65pwMAHaaDZuspwBNjgqTFf9EadqXVozoRP5aWIhEvA3gkj0c+cclAogRACYCS7QYlRwBKAJQAKFlWfgQgycSQBLIkAEgAkCSliWNSN/YwyjpVVPqBBgHuROrqG7efItQkHbqL3DFNd0mc9D1GphhC03W31k9e3z0WPhPpYhIiTb2xKUSaah5lNQBl5VBWdTNR1vAgxM2WtfUjpGLW9pMJdz4cGcy42aW/XNWPXjcu8lckFqsEApE55gNo1zJN7NwKeFFwM+PSDfh1yksSK1bXSprOI5AUlJwoiucEKODBUPFdvT49rG6J+Aoh3Dh4v1ZXAv6cG2/YwmjrSdulj5sFko/E3ELdV0l7z8/0iS8vesGFVLT3hcCBSp+rcCLHSURtIBsRMc2JJybclyae1jAQTAsp4DbpT2mSE8e032dfDZ5YrqPj6gYIWMS8VYq3KOpOLdzxpp4ZdDCnhXkWTUo5veQ0d5hZwsxyg2aWsj0N80OYWA5P8PN55THMKyF7PyGanzx97wdP9kST27v8QCOMHQ+QxP/o3/6nT/uXl9r792e93pmvgBMeCmi4cYrbjEdKMSnTp5tKqkHAYBYZ2Ygy1hpxpEgraNoXNrFh9RCPT3XVpPBe7UBR5R2fd53baNR1CbzRLwkNLprp9h1deekXwnxWH6qar8x7tV/eKtRTDN5S6nPCXov6Qq6eL1om4+r02r0ZnmUBBY+l4PPyK1j7N6h4FhVfLE7FMg9hekV/HJbuAyWPpeTL8vqxUC948WwKfldmL752zfXQblFifoHKPTAPjhunNXx0jI9wwzhpoaMqqp4cHx0fvm7g9kmrfnhwf+rvx7lAf99zzf8+ILI/mEm+eTX4/z+sKk47xklHXjjIOg5yjryCNvjmVbNpYDrnHGZqnkqNZQgjJTcuSYQHv/96ML0M/YBYTmcyGRZnWTdwxYZ/RmQB9KlsTlwfRrCa9+ICI8YJEZ0lIjnjkvfCM9pn6wGfLYBXv0YyP1+EzFdv6hE9EZQ/VPkXpXO4qTijayTxS3C3bVX9O30NVnf/YivW5VrdJcjxmSbVekwAnqIKVn9h9XdDV3/Tfgqrv+PQik9g+RdoxUArBlox0IqLJ29z30aojdhHCHTiBdGJ57J5EhjFQxjFq5XwppCK462sLDOh7fYPXM1ARhdXgGO84lkmD/cwy9zSWabsljDRnIRmfArzTKAZA814RpqxMv4Av6lcTGMBEWZSMJCNS042nqOOgam4wXzjSNdAOS415XgO3gys441nHadzMje3txVQ9izKviq1Q0dqvsin3UDREyr6j3VQ9G0feaDnmfT8vpx6ht0ixWqF3SKwW2R9doss/Xu+sEkkR1wWkoetAgvbJ7IyIx/KVwe1r2SHyPhih00iW+hrsD1kHkon/qxqXiORXi1FpJyttD1C/WMpQvX7yNsemb7X14OjB7vDYHcY8PbWgrcHG8Qm5u29ThdFEZ61VevpwOkDTh9w+mb9dKjwAu0GG5jO97Rz5JjCmmAxqpwEv3PPn1HXdwyh0ak7KLv0TL/plX1DdaC57dixTe0aCRQNqi4p3W9mVcd+HYC2S0/68+bu2NolcRmUAZWXlQO4CAcHrZecDDi91recOaQFrua5pg8MImAQbQiDiH/ST1pZcXDAlnybRAzlYs1XfAJw0lXVf6gJstabcZLse2XLCURFwg6iQXRrpX2+TGnHpu0JuLKF4r5YiXFvr7wvV2nepkDjWyj2dys18+2V+1Xpv2EbZZ3/FLor42mlwFgAxgIwFsb2WaAu5KgLim/bZsqAoLDBH7ednKBwvMIvipYyDVH674muCD4NJ2bARxiXD43YmjwAoxUAo72NgAU1gAUAC9YJFqw2YQ64QI0LRpD4ABgAMICMyfIyJnkvhCzJWHCoDnAI4NA6ZklWtMIKeKggTzIO3x1Q0fJRkQeoCNIl0+ODQ8AHgA/WMl0CAKGUAGHULinACIARIHOyvMxJ7JdZX4T8yVj46AjwEeCjdc6frIi6CzBpsjyKcqMpYCXASpBPWSu80AC8AHhhrfMpABjWJa8CmAEwA+RXypVfyfkk5FkGVw2+Q7FfGxtRHQOiAkRVNkRFJ+/2LoVVFNuIIx92M/Cq7TcJRmayPJU9OUKJq4phVaP3vaL9puUf0ieWxBqew1MqANOmgWkfPl/f7t4wjf/2N1dJBVAZ7LGGXM4653JOAHkA8ihhLmc06EhtJVow9Jj3swB/TLVfqUvcsNO9DgOOQgQIAQwCe5YgM7SqzNAtDULs9PEPnyEZNMmpNLVTwF053LUdR8+IoSajaOoOtPAoI1zagJEPbC9wlM2ijrIZZDgqpfqk8X4NTq+Zy4equYJVySvQcImOrJlZw9yF6YQONF3yE2tm0XQ0HfSi6WAFzq+YRcmXZTyBSlKxYrIPOi7PyTRz0TFz4zhqg6pnUvVVGVUNx9HAcTRwHM0GHUezTGrIVh5BM0zA86fFbOOxMxvIbtrG42yWv168lafYjBGPtk7O7zbCnMtDf4ATccY/EUex/loKGgQchwNUTSBKKIkS2r52abGZWyvkTxnBm9iJ+q37Rhf30FdMfNE38dkOsSTL7jARueeNUa/qDCxTvw/pPNvBNILrSVsB7nk2fUmnk3iabtNOSfY8MAKJa5AiXAT4kduEidsotDPr4NyF5FoVcZYt1uK0pSampF+iAGm3gmMh1Uf+LDMwpFrXYzLNhlz9R4jJE09ZqJgaLO6zYJ22UFrawY+ZgSKJXNLjdzLvJUkO2fbXSBBpQx4l0wJhqqU4BhkllrQU2mkgp+O1id/aCm5HgUfrPapKS3F5rJUIAUyiEv6kpgi5u8xYmxT4NC2n7SbYoqJWVn2UsqLxTFYq9Q2uEf+v+Pl6ujbXUVamvjgyBfEmUkXo4zvRkByh1sI+qgu2jwQKzsdClFu3X6S925thOTvxePYiorfFhRnF7bYAkrrj9vdrMeeUAsCoTE/d5lkURpHBzZFIkq3sMqLRG9KXtGpV6ceh/KPWG/zfkP6vyT8Oq3KN9IXzuvR/zRT2+z1+B0aEloxj5FPkho/lhuWn1I/kH9Im1hNT7m/cl5T4frqcGKV/ubuIhtoEkLs9LwywdhOn5bUD7c/YCrXdazemFOkhJwDqpw3UOK5XT04Ns31C/15jhGrINI8ODQMbNeEg+kMy3Fd3Xv4P5hxjp0QWAQA= k8s-resources-node.json.gz: - H4sIAAAAAAAC/+1d/1PbNhT/nb/C07Y72MFIAmxd7vZDC6PdXbuxQnu3W3c5xVYSDdvyJJlAOfa370lyHDmWS0hoCUH9KX5P1pf3ed8kPdPrjSBAOE2ZxJKyVKBucA0kIMZUSHj66294vNlWzUhEJe7HBKiS50TThimRv0ZASfM4NhSOs9EZY7GkGdBbmjiiETlkqeQsVkMMcCzM+zFNz4UeRj9yMuBEjICA9loCGRob6xZ6WmZyQA1ZHONMELs3zRgROhypmaPOQSu7RCUjwymJpz3ZvWk+jikWhyxmXEvhZttm9jEXs0NpRoTF6DVJh1JNut2q8UjTaxILlvNQLQCdcJYQOSK5QJVWAxrH9W6pkne7Skol4RdYNUbtpNpJTIYkjUpg7fUO0+fibAZSu8XF0DV5Jf2cc5LKBm6CL5s4NG3gcIXaKaiJeypixMZujgTVjRs6BYnkUwAsXhVcUELdarZ3zRjTSGPbmuVYeluSlRWcMEDjDdMr0YQAi+Aj4awKS0YA/FTiIXEqSKZ64TiiuRrmoM5z6xWgEhFOuBp7EDNZHVMQTon4/YJwDpKu2sKsPUxtQvUFkAYhznBI5RXanm0WKqtRzb4+7uz/dHBYb1GaQh3BqZ63agzwGxFJT/W0m15WvmXqb5xNShNwcy2UOzWukDg8d6gQKJH1VFUDAYIiTV4BmMoI2p0qtRimpoNCkiwj0WuYpBNxifmQyDmwJJeZxkjkyeZ53ie9FHS0B+PKXPQm2F6HcQ4j8p8/oG+Knx/QdqCa/vwf0NQPRQAvrb0XtAuz/AO62apjPmA8wdoTS5rASAbFWrOJ7zrGodRa1GnA77js79PaaFqDwM6tuDSDKck0MFU8t+cXoJZdihOioe5lLOqFEN8woMS75a8eyKaXC7BxWD1QI9HT7qoLXfQox5LMJ++braB/FWzCKF9UzNfXMOLNzT1KuNFi5EjFfRZHDqeqlnUMEdIxluadjuhAuplShzZ0ePIueKdgqLpDWXqN69logzmJGoKKYFzWxWdiTU9eZXpEmkb0gkY5hKbGsDNpqzOm6swu8SUV9Xn18/Dc2Hpd6CgpQo4SyQxkSGmq+63mwFoGT50CupdwhS/nCSNTFYXhuHQoFO6TuEmXYjZ8gXWm167xTKrhfM3kGvWYYq34Tub/gItwm5hZxy1xaWOWWtoW5AoZwbOGU1B/BWehdwRu9ls2rmT95YTOCpOz9elTdmh4p/Sj5o9+MNZSQPE0k/16rt/xuf6XzfXb65nr31uSWp/LbVmqkFfxnTYcZ/UoZiyJTPOTP+Hfzps3O0dHwatX3STpCkfek2EJ9pI29zkJw2avge4UE8rpujOM6eao0BGnIw8nbqgK0KILjkhIE6ydX8e11zl3G1DBPNPbiRcxvq1dmTuhIw6eK4jYOEXOpu+49lWfhOa9st7g6+cOeJozwwp8aZ70wRZq/DylVtBeHN235F/wL1J4gJcB+MXqAxx86yFeBuLDzwdxEUf14+JAv6YJ9Xa8HMhHq2vHBl5vxcsB/MsqW/EJizy6C6GbuSS3Cpa74nhaItz9/rvdxWUoJKfp8G4ybD41XeTsfzWPrvXJhOPQGrac+uzBeXOzwJl2fQxOBvosCzk2H8tfFBSS6ZYC72qBZ3kX5kwvSFdfxVRA6E0uV+CHSUnXTtQv0DrdyQS7gYe6CerDlbOqWCeIayfoI29TTwPoX9AjuG79IwcFWK3rVo5ToZB041hmaA6Wv47117Freh1btVN/HTtP7eWev4/1tZe+9tLXXj547WVCEsavfPllw7bEiKc3ZvycpkMQgez1ryQR80q67OkraOnLMJfdF7zRcJg6iWBzvMtAi8IR2fJVmQ+8DdBG4bcBT3QbYJul3wncpTBz328EfGGmL8xc5cJMp3PzNQOrVZtp8o+lAPblmStennmPGPvarjWu0Cyw9kWaK12keQ/W7Os0175Os3rg8/b0dMuDvQzYxytt0AXMh/UzPQ/0HYF++RiAPh3jzOO8FM6vVhNnX1/v6+t9ff3nra9f8m5ywavJp1Vmb2TsK+0f5/V7s4p/sjjYg/4la+7nl7Yvu39yBvbkC/A/G+RcrH+icPzwYtb1SWsv6JcPL2gxxtnay/kVehwFev7bHf/tji/aexRFe3N9vrNRTBUJiGYJfk+4MNNp7xuyqgFSb0SYn5s/ui7xcKqMSOV4PCXFgaLpS5Iki2Fd6bD+N+M3ZnG3atUq1X6SXGo1iMgA5/FMoZS2Gpu74dAv/QlCVTlL7UFHWOLg1BT4WfzChO3yP4vLsslfwrf9K4KtIr/SR3yuMkHrL9e3K9QhuZyJCqWzsobfmFmXJTkcx+8LQVR19zaZNgjTLcU5KiEnkra8OfhuCNQReR47iv8ajBglACV1NJ+gUoT+u0CiR+oZL7uZZ9f/sD7kBkprd9Q3FmQHVsNpKCBb2A6K/rfc6HVuQ6+IaTbKYCwaIvHHZEKoyq3NXNHcjQvdMEuzGLkgZ6Yj20s9CoVp3ZPCVANrqS8qLVxYWaYf49B0wJwZ6I1JQddDXzYmkcz8lx46wSnd98AkjyDR8U578u0DpH4FDVVeyygkUHz6ciGS3iRltnMZdDDVC9RuWQ979oNVNI0OrN9t+2GvZXOmX2igjvW7HRmt/XuyBvVJjqUYt45id/yD3bE9Smfffphe/KAfI3u+k7lUxPeR6YJc9O7ssIi45Ve5LMlySYK3xSmKCHaD30ABg80TFgmjhijX9eeo02rh8Nkg6g/6/R/3+3s/4cHg2bN2+xnZj9phJzSNL8qA39q4+R94i1XE8mUAAA== + H4sIAAAAAAAC/+1cW2/bNhR+z6/QuA1IBnexnaTNDPShTZamQLOluRQY1sKgJdomIokaScVOA++3jxdZpiQqceJkcRz6SeKhePm+cw6PySNdr3kegHFMOOSYxAx0vGtRJApDzLi4+/ubuJ00ZDUUYA57IRKlnKZIlQ1ixD8GoiROw1CXUJgMzwgJOU5EeVMVDnGA9kjMKQllF30YMv18iOMLprpRtxT1KWJDUQC2mgzoMjJSNdSw9OBEqU/CECYMma0pwRDhwVCOHLR3mskY5IIExiictWS2puQwxJDtkZBQhcKkYQp7kLJyV0oQQDb8hOIBl4NuNSsyVPcYh4yk1JcTAMeURIgPUcpAoVYfh2G1WSzxbhWKQjRAcZCTN5vT5cDWvcQvpRTFvEYawXGdBMc1EjYkI0M1DAkX2hXWPHUJw3SGkSEr4i/0RNUqt64EIxwo+JtliaFaebFU1GOCY35EAgW+LPAg874jSorwJ0jwE3M4QFYOE9kKhQFOZTc7VZmdegF7gCiisu9+SHixT4YoRuzPS0SpsBnL+FkCfVSncUIo6Wm1i6Uc+hc28BhHSYKCTwJD61A5pAPEizZTthtVEY0TNR+WRuuxwLUbwwipkXYTEnR9YfpQdEI7+VXXT9JuygS2XYZEacC6Sk06ookuppCjaz9MxQDp26/gp+zyK2h4svm3/4oyefEVTDa83pW3LnrZAI3ysPqERlD5Ao4j2ZHEtlpNUIWoUMQD6HMi59GuVNEGdpC3d30tepxMQE1FgeiF4RSLdiJAV8wVBBPjrkg4H0qXSMLAogxyWgfCeVj6UrLTIe5zu5ArRw72js+9c0lDUQ157sCvy1YOKQrqXAChvAqftvEuv0pUjzgO8CUOUuESas19WlctJsWRjeEYs+q4eql/oVW1CjqIMlOXkJQoA1JT7U/VO7TcaanV0T6FKzhGc1jOTEVFd5RbFAr2UFinSyEZvIdqEWxVZNqHWx/TTrxZVc3ZjIvK2VjaSdhNTM+jvKSUrGytXJrblvDRCYJlw8lKPwpnoYIlu/iEjAoBUT6gs8zkTH26yQ617BR/V/Lha20tGRUvMw6qhkFtFwZVwqCWC4NuDIOqY7ktDmL8KpzHmSvTkRM6qy40WtnRLIT4S/xeHR292t/3Dg87UdRhltAkgVy4m7i+zelKKf5hBSgGd3Lb+XDtQcDUvxCa6YjV1/pTT1Ek6L4TDpCPI6j8kyUK02GVxYAy4ZkKWN+H8LZ6eXgD9qlwLl5ARjGwVj2n0mbBjdR8kdbr/fjOQk998FagL06jnrCFijyNsbGu3p/dE/SP8C+cOYIXIfj98hPs/ewoXoTivcejOFtH1e39if6EI+zseDGS95fXjjW9zooXI/j3ZbbiYxI4du/FbmJDbhksd8n5NCDc/PWXzftjyDjF8eBuGNZvbK7O7rI6FrPsK4u/nGpzobobcL9t52ofFPXVdhOw/Pmo2WluzI92hkwnB7yjAE/SjhgzvkSdi7RXIqErNElt6IgLHZKuHNTvHwPqJ1Nsb9NzVNdRvbd0VhWqAHHlgN53NvUyiP4dPIMT0c+pUIDlOhGlMGaSSTuPeYRmEbkTU3diuqInpkU7dSem82SObbkjU5c59pwzxyIUEXrVHRF6geOBiNl4t3fF0ZyhUsPLW/pB1HQZZIvGS0eKDn1+7K2PNonnQ3+INlxC2ROHR8ooXHj0QsMj0yxdhHSXnLJtFyC5nDKXU7ZgTpnV/7jjzuVKK9MhwkIEu8yyJc8se0COXVrKCieXZVy7/LKlzi97AGt2KWYrn2JW3JM5OT3dcGQvQvbBUht0RvNeddvNEX1Hoj88B6JPRzBxPC/E8+Fy8uxSg11qsEsNXurjw3ueHr6sDGGNsUsSfqKExkdT8RvzGh3p/2e68Pxou4zhF2dgLz53+NEop2z1A4WDp4dZpRCtPNAfnh5oNoLJyuN8CJ5HDp177cC9duDy6p5FXt1cbx6sZUMFTKxmEfyCKNPDaW3rYpkDJJ8IIL3QX7vlcDBTRiBjPBqjbENRt8VRlIRiXvGg+rHetTLvRjJaISGPo7FSgwD1YRqWEqWU1ZjSNYt+qW/6FpUz1x6wDzn0TnUOniHPTNjM0DOkJJl+gtj0r0D8VaRXaovPlslnfDK4VSgdoHFpVcidldH9WmleBnIwDL9kQBR19zZMa8C0ozhHsuIUacObC98tFuoAvQstyX81RgwiQSW2VJ+yki39d6FE9dTVXnY9/0OC4z5peFlzG3ay2reRlS1hJqnCNhQj7PO0f1CUVgYqy+yVM1XQMzEEKUNnuiHTKT0L/Wg+kH4U19FcPWQU+DC6YY03JzrgXA11WZuuW/rL6SqcyZ11X4eKAtDRq9b0ZQQR6GVloPBYgkW4RGcPZ5B0pwGyGbmAnZlagFbTuNkyb1rR7HrHuG6ZN1tNUzJ7ZQK0jetWoJX223QO8h0ZQy9u7cVs+LXZsNlLe9u8mR3zgDeBOd7pWArwfScq/Racn+1l62v++iCJkpQj7yTbM2HepveHUEBv/ZgETKshSFVCOGg3m9Df7Qe9fq/3Zru39Rvs93d3W61dtB20/LavK1/my3tzbfIf9DiwZVlfAAA= k8s-resources-pod.json.gz: - H4sIAAAAAAAC/+1dW3PbNhZ+969g2e6O3caxJN8Sz/QhtuImM8nEtZ10duqMBiIhCWuKZEHQl3i8v30BkKJAEtTVNknp5CUmQOH2nQN85+CAeNgwDBO5rscQI54bmEfGA0/iiQ4JGH/6+zt/fHwlXsM2YajrYJ7KaIhlWt/F7KPNU9zQcaIUivzBpec5jPg8vSETB8TGJ57LqOeIKnrICaLfO8S9DmQ18pHiHsXBgCeYu43AjNK8W/mGbFbUOJ5qeY6D/ACrpcmMASb9gWi52dpv+HdmkuEjFzvjktTSZD5yCApOPMejchQeX6mZXUSDbFUyw0bB4BN2+0w0utnI5eGinzEUeCG1RAfMM+oNMRvgMDBTb/WI4+SLJWK8m+kkl2F6g8TLZnOYLsTBfezaCbBqf/vuu+AyA6n6xk1f13gx+iGl2GUFuUN0V5RD3IIcKlC74GKib0ow8G71OYyLrlNQKB+RcAyAkpcGlwuhfCtbusy4JbbEtpHNUeQ2SRZacOZxND57sicywUCB8QNTLw2Ljzn4LkN9rBUQX5RCkU1CUc1+Pk8vVxwVG1NMRd09x2PpOgNMCQ6+3GBK+UindSGrD2OdEGVR/A8fSZaWz5EierK6n09be2/3T/JvxFLcyGWIWWE8U+SxVYVXn6vg08rlBgxZ1xrwM/BP6rZDhmRKp0/fvtlt1KLTylNaagMfWbhoEuOZQmebrXRqXE1OZQKGfR/bn3gjtQLKEO1jNoPo4TtfjnAQDjddrk0dFw2xbGnH9+yOxVcTxCuhR8lfHcsPO2HANaoTYJ5qBx05ORzxIjqEIoYfkjJ+vzJ/SR6uzFcGL1Ok8f/Ek+WEvCNUpMR/XpmPW0b33thMqtvSgO7RIZJrDyND0QqhbvnXRrP1KbKYFKNWgQicJuU9PCT1Pj6aBa/zQb9WluKMXGBfgjuXIqggXLki6TrsZgDo8BVbrmSd0STx8F+vy4dOvLrNBYXhbb66UWIFRUP7ypgVmFFdPImjzUG5creu3JdEongmrAYI0ZS10hAUzcoLA1A4M7KBoKOeY2vWetGrU07cNHXJvIsB6TF9JpOMyzw5+2p8FfNVepVmyerwkCVBiGK7gOsEHmX50YsoUIfd+7JG4trkhtghZ0yFbGj0riTy6ZbdoTsS5NvVDa3raE7PD7o5jJmQGJIMZKaQMv2vivlewumkZaLvwj26m4XdjCWUV0eZRqBQFztFsuR4/WMkDZBmLi9iwNqfRRQ4zwuUHs81O5TYCb2KRf2Ywj82sqmJbvEJ1scoqzhx6kc+V0hDVZ997t2mjNGkQZexyqnyNEkPo7wL8kPmDw4ibYmhABs0tkFbFbFBNT+LZFeXARZoxSzQmtkixLUo5lPmZtrusHrc2hhQj/F5w+7woSXeyP6IuZiF7BvCV+l5uFZSx088sdAw+fuXTkeYN52RCn7fytoqxs6UtleuxatgXc3IKycLn/Q1jLTZooQRi083RQ6HQpdBUZ7nS7KnIQ5y7hLzzuvWfi7zPqaCfHA4y521209CmS8jNSNuH3hzybw5XkJClzw98WwCda4HddboI/DnHH/O0+dd2MJ5WQLdBAI9kUDn2zKNQQfs3plrH+kyv5pFmoTHbOw//N/258/b7bbx4cPRcHgUaFiejxjXF7e4zNFyPCC2jV1zsf0fvYcuR8u0s7k1mobSAC3aYRtbZIjk5NfSEbxrvQLFmZfS1Dl20LT3Eg5ltimfuQzbu9X4dsWrX6mcqyZC801or/HzOw08xZ7VFHxuOOxyXcjlS8qRrNyLo3s+eWMTAJ4J4OPqA2z8CyBeBuKT54NYtSIWB/rTpL16AHkmkNvV1eMIXtDi5QB+X2ktHjnhAOOFMLaKx68KWlxxVJWB3Hn9687iYxgwmvVNTh3DYpdpqfFSy0SKzBcvJV0YGl8+t00L9vgWc/Xn66C4Jz1fpsZKWT5gJx62o2SYjiQafnjE20xu8NFMAVVrhcPxc+BQTZUwdgwQkoWE5KRyyhoH3q0VCm1QVRCRySLy3qxBFOafIRedau0mU+QGAkk9jgnZ1GRBlCZEaa7oVnNaT2GXeZYozT3YZoY4zXqcFEzUQMtL1vIcYe2GpL6nDMeEfYiHHr3v3Hr0mrh9bl+wTvee4UATF7vcWapctCwZcjWPHuC8oUiqznnDSCjgyGG5Rw7LQ2FVTh1+liMYhTUZm39dXGxBGHXJhq1cXMCwXVPDVlVIsG3niaDeB9MWIqghgrrKEdRT2AYE91QnlDpiIUvBDNHUFY+mfkKMIRRzhQOqY6whprrSMdVPoM0QVr3yYdVpCnYOFGxJsE8rrdAxzCfIGmAAeimg/6gD0Be3yAecl8L5QzVxhuMwcBymPFRX6DhMrTf21+tgTAQQHHsoguJZzsaUoB+T1GFipD0ISGXOxcwOBRyNqa+mzrWSPZHqwjmZ55IXGjyjhBg//W7EH/1MJ519adcQkdMXQcQSjhrAZEZM/ngRTIJb5AMkM0LywaxHwCOc8YMzfhAKWYtQSDjmN/8xvwOIhXzZWEi4jiGOhTwo8zYG8aEHhcC5mAmTmJuSFuZWZmQTF15qsAih+19iYRZeYsCzX/iwFq+xjIsLnpqpnUeoGcfItSM1g5Mp5dKxYx/OpZRCxl6tAic5BE4CnAQ4ScxJpLk/JAxISa1IyWUMG7ASYCXgIirdRZTXQvAPzcLF3gAXAy4GXCzjH+LNF2shkLG6eIj4aBpez4g9RbZxFgEIlKzsmyCBkoGjaGFy8hbICZATICdZRxGwk3qyk5HLiAFBAYICPqPSfUYjvczqIniOZiFn2TRgZ8DOwHWUkDObeqJyIGk1dyEZ7QhIYGrA1MCVVF+20gS2AmwF2EqRLwnoysr4lICxAGMB31K1fEs5nQQf0/it8femtpsz87kW8Dngc2vF5yxMnE1O6jhxkrzJ2MyQu17QoRjZEzYGsx9t0bK6ZUjdi5K5c9HZl7/9ahYcbimZGMy/WkD8JXtbf1L98cvZBbBm+NQDePrq7OnbBWYIzHDdPH3TSeGU44XPzkjWgBnORgrXCYhVYYaXA+qF/cFZyIAfwilP8KqW5FW94POOuN7i4xdj22gT8Q35bihq2TzzbJ4k5n3j30Y062yBs/WpnK17QKmBUq+ts1X5kHUpLtdJbM74zSjF+1gd5+PDQ9KfVdjYFz7ITWm//JZdxoBxgkcSPJI19EjChfRAn9bSIzkLc3p+d9ii/KnclgGJegp3naRSEZMCIgWuO3DdVc91l1xoCh67ScRSwyvh4/4vzCuba8EroyUuIylc13jiXmZweQFWfkJ9nJunZsJ5R0Q137dpTDVg984si1Jy3+5lfsFc7Npc5WZcfZmjFX9AbBu7C14PPHYPVet27+1m3a73flfN+58lwDrPHyA8L8LHFUZYqjA3PgHpp0D6pJpIx2aoH5uhWxW/273iILefD2RhmC4NscbJABjPifH7imMsA0ziWRugXgrq0ypCnfgiANuFsLWKx68KC3LFUVUGcuf1rzuLj6Hwrrn9+caw2FX/jJs/1dxcqcD1ye/M54lNnwxK2YFDVUfluBRUIMKsdoJyUqKgVHnbugLQtMucWQGbidi8X1u1qWscSgWE5tSseDTKSbRRaoy3wKsViSI+ISYg1QOaGBWaLAj5hViVdYpVmRafshG32wysAR6ib5gGUduiE1TR7rT4hY3otSyMa1V/LJnmddjF1MXizGxSFsND3+GddPuJppkOb5Qiz2MhUKIyUoEtDN9JmbBxD4VOJiRAqpCaqwvAFvvWOC2piSiZbcSQcRHFsij5sT6rkS5KrueLMc1OueY/Iab38vt5uogYMe+LyTotoTy1j+8yC0UycynVb2T6pYwccpxv8UCkBXnamBYMpn4UZwj6GY20MrXziZyv5TZ+52jCXAo02hxyKInm9REqMTuYBxJZUyeacjdDP6YjQmq3+YrP8DbvDSVWwKlEQkS29Oi1pqEXL3AqylxZJETBn6MGmencXMtFmv7lWDairikZYYAvo4LUKasWAtN4ZoFJeOPCIiMkpZMU0xEyEwYdf8CXnAmypOe0j8r3Z0HGVkXGuB2ynHTxArj10vPmlqfizxk/SitpNWRsY0S3HiNyQeSwx7SiF9k5puvdbjdHUencPonTzNTPfMJZPh3/OB6SxHRUCbe5r1yv02woD7vqgxK2au4rfzfVh92GmqNc+NxS/m7akaR/H/VBHJhQhGlqLWrBB2rBai2tPfVB+brroa22d9SW1PD98GQIo/n18iRmgom96A39kGHjHEc6GRg7xlmsFmYoQ3/Ng/03Tbx3gPfwvnXYRXsN1Dg82DvYfbuPe4fd1m708k1CQBsbj/8Hik7lsdX9AAA= + H4sIAAAAAAAC/+1daXPbNhP+7l/Bsn07dhvHkmw5iWf6IZbjJjPJxLWddN6JMxqYhCSOeRUEfcTj/vYC4CGQBHUfpLz5EhOgcOyzCzxYLMDHLU3Tket6FFHLcwP9SHtkSSzRtgLKnr59Z49PL/hr2LQourYxS6UkxCKt72L6wWQpbmjbUQpB/uDS82xq+Sy9IRIHlok7nkuJZ/MqesgOot/blnsTiGrEI8E9goMBS9D3G4EepXl34g3RrKhxLNXwbBv5AZZLExkDbPUHvOV6q93w7/U0w0cutoclyaWJfGRbKOh4tkeEFJ5eyJnXiAT5qkSGiYLBR+z2KW90s1HIw2U/oyjwQmLwDuhnxHMwHeAw0DNv9SzbLhZrcXk3M0k27mPXTMEb9um2r6qeyy8kBLu0JNdB92U5lluSEwy8O0k1pBzKtMsu+dUtssOhjKS8rPyZnoi38qWLjDvLFOJv5HMk1UqTuaKeeZZLP3mmED5P0FCg/cDEy4rfxwwfl6I+VmLo81IIMq2QV9Mu5qmhZ2I3McGE192zPZqtM8DEwsHnW0wIs5msuuZVdqi2vCyC/2GSpFkVSmzFE9X9fNo6eNPuFN+IFa1RyOCGOzTmIray7qlzJXxahdyAIuNGAX4O/lHdti3HGtPp0zev9xu16LT0lNXawEcGLhtnWCY3ymYrmxpXUzCZgGLfx+ZH1kilglJE+phOoHr43hcSDkJn22XW1HWRg0VLu75ndg024CNWCTlK/+oaftgNA2ZR3QCzVDPoisHhiBXRtQii+DEt448r/Zf04Up/obEyeRr7jz8Zdsg6QnhK/OeV/rSjXT9o22l1OwrQPeIgMT1Qy+Gt4OZWfI1ZLyZsbDpFBhVq1CpRgdO0vMfHtN6nJ73kdSb0G2m2zOkF9gW4UxmCDMKVy5NuwuscAF02qYrJppsMEo8q8b3QJhV+Uh5LYogywV+5O1fuKqVdPtpVQ9DRsFR7MZeNrjMLuXSEowPO/DzbVMzZvFenjCMp6hJ5FwOrR9WZVPBVvXP2RfvCx53sbEvTUf4xT2YQwWYZ0/EILUovojJd+uCLGi3XtG4tM2TMp5TVJO8Kzpxt2T26t4Jiu65D4yYam4tC152Y0XCR5CDTuZapf1XO21JuJhYB6i48oPtJWMpQQ1l1hCoUCl1ju0yXbK9/jATXbxbyIqqq/FnEVYvzu9TjqUaANXZCbWJRP8bwiK18ampbbBD1McobTpz6gY0VYk2ozj737jLrvrRBl7HJyfo0yg6jvAvrh8gfHEbWEkMBy714udda2HJPYeeR9qkyYLG36sVezWi/5RoEs1FtO0vxjR4j9gPiUWbaZpeJ1vISqj85wU9K/IklljL+b23n+06e9mt7Y9q28hZtwkJkQuo2WnnEsjyxRoNY1DLYcFG2Ni9dXZfleb7gU4q5WYw9fNx42WoXMh9itsWEw4jkpN1eCCu9jMzEcvtATddMTeMpIHStxXO7JrDTerBThT0CRS1Q1CJD3YcNiQJHbQJHHclRi20ZR1ID+mBPtStyWZxwImXHQ8L0f/Zv99On3ZMT7f37I8c5ChREzEeUjTlueZnJjDmwTBO7+my7GWo/VYE5KQdcIxkpsgDN2mETG5aDxPjUUnGwG7UBxZmXYjVxbKNx76U0Rz8hbHDRTO9O4eHkr34h3Gb1kdB85dar/fxWAU+5fzEDnxs618wWCvmCFaST6+zono/epgOAJwL4uPoAa/8DiOeBuLM8iGWiPzvQH0ftPAPIE4F8Ul07juAFK54P4HeVtuLETwYYz4SxUS6/KlhxxVGVBLn38re92WUYUJJ3H46VYblXc63RP/PES0wX/SMiWRXudrY2Ldknm80bX6yD4J5wTumKVcr8oSmx2I5SMR0JNPzwiLXZusVHSw8Pqh0Ox8vAoZomoe1poCQzKUmncsa6gBCz2qFwAqYKKjJaRd7pNYhF/CtkqlOtDV+C3IAjqcYxJZuKLIhVhFjFDd0NztopbARPEqt4ADvBzyVacQVH01JNVVKHZ3lwrXYiqe+xtiGndrDjkYfunUduLLfPlgC0e/1A8bxHfAoxpZbDjDp6gONsPGm1x9kimOFE2/JPtK1P0ptyqO2TkGAUL6Rt/31xsQMhxGteMYopAVaMz3TFKBskLBqniR5uw5oRoochenjO6OExhAACW6oTRhwRhblghkjiikcSLxBjCEPc4GDiGGuIJ650PPECrBlCijc+pDhLwc6Bgs0J9mmlDTqGuYOMAQag5wL6zzoAfXGHfMB5LpzfVxNnOAoCR0HWh+oGHQWp0Y758zoCEsEBAf5lUCzlFMjSrWGU8o+MIAd1qMx5j8mhgCMfdbHLqWapBRkqnPZYlnaQYGH6oP30hxbfHplNOvt8UkP5n65E/gZ3sAACSgT+XAkCwR3yAQAlAO/1esQJwpkzOHMGEYS1iCCEY2fTHzs7zCbFQzwvrOnoEF5YCC+EC/Tj8MLDdd6fz+8NkNiWiylfibI1nYHZci9aisbX1M/Cvv5NF3bfful2eWXdxDKSS+pZ9ooPFrEa13Ex/aJp1XmEkXaMXDMyKjh9sV7udOzD2Yu1MKcXm0AgXgGBAAKxKQRCLK0diwKDqDCDuIxBAgoBFAKcL2t3vhStEDwvkxCn10CcgDhtmueFNZ9PZcCcqul7YdLUvJ4W+2BM7SyCC/jTur+hB/wJXDAzM4k3wCSASWycCwaoRB2oROKMocAmgE2AN2bt3pjELvO2CD6ZSZhUPg2oFFCpzXHKmMTjlQOjqpVzRjuJYANaBbQKnDT1pRZNoBZALTbWSwPcoqbeGqAXQC/Aa1Mtr03BJsF7M3xreCPQbnNi8tUCggUEq2oEi63z7W3Gshi3EdRG286xrV7QJRiZ6R5Y/nYMJc2amGW1HcarVsqnznlnVv+lnknkfEcsKeK77oL+W/Sm/rz1w+ezCyCmcLYePF919nztA/kC8lVB79Z43pU5BrYEUvAMyNdkvGuTBL0p5OtyQLywPzgLKVAwOF8HvsE1+QYv2LjDr9H/8Fnb1U4sflf1dchr2T7zTJbEx3XtVy0adXbAZbgol+EBsFZgrVV2GUo34a7AcZglVNrv2gp8aOt0oS3rS+5r9KRtiyXA7/mZAkgd+NXAr1ZDvxp8/hkYSlX9apOQk0U7fSanKMutGXjKIpxOgq1EZAW4CjigwAFVPQdU+vk/8DuN4m4K6nYI1K1A3ZrPgrpFs1AOaGYOLPEgJ1xWgFEc856mpoK56MeECxb7No4MBvTBnmTeSD8geVmc02b7DqT0qUd1mcmkPLBME7szfu9y6CSp1udqd5t1+17t22p+0FQArPJ/AcLTInxcYYSFCbP1HyC9CKQ71UQ6Xin68UoRPjA+F8gnywOZrx3nhljhBwCMp8T4XcUxFpEM8agNUM8F9WkVoe6Uf+0esJ0AW6NcflWYkCuOqiTIvZe/7c0uQ+4Ac/vTybDcm77E/ZX17G9U4Nuib/XlBDGPFvqyw1+qLvXjtUh9A+Ocqg50Z41Ar3PntgKiP1nnyPbMZf9uY9W+qqESFQD9tOqfKe9EO4HacJcWPlYOgZ8QTlG/cIpxIRRbcbv1wBhgB33FJIjaFh1ViXZn+S9MRG5EYcyq+kPN1G9CtvZ3MT+cmJZFsePbrJNuP7U03WaNkvR5qARS2EEm9oLie6ETJu6h0M5tiQsTknNVYbh83xZnNTVVJf0EUaRdROEWUn5sz3IwhpTr+Vym+SFX/yfE5EFct6UK2uDjPh+ssxrKUvv4PjdRpCOXVP1Wrl+S5JBtf40FkVXkcTItEaZaihPEpSSSloZ2NpCz2dvEb21FmEeJResOg9JSvJ6gEvOBaSARNXWjIXebK2uXMYWu5fa8lGnsqMFqjQMrns9kUJltCESCv5L69WxuoaE8Tf1yrApRT6SMMMCXUUHyCFUL/WgsWT9SYrgYDVFeOfkk3TkJmrMpmsMgX6LOlF9T+iQWMJuhR1sJL3qKWIAlRBvP/71oQaK73t1uM4lwZguJOE3P/My3GB0nwx/HIknvbpWZsd6WPkjRbEgP+/KDdB+y3pb+bsoP+w05R/r4aEv6u2lG2vw96QMPvpcUZmwtcsGHcsFyLa0D+UG6tfGVKbc3aUtGfD88EWunf7nsxJQtXdh5jh9SrJ3jyO4CbU87i1VfD0UYqX7Yft3EB4f4ALeNV9fooIEarw4PDvfftHHv1XVrP3r5NmWKja2n/wCbu7mVvPQAAA== k8s-resources-workload.json.gz: - H4sIAAAAAAAC/+1d+2/bOBL+PX+FTtdbJIvkYufhNAGKQ5M02AXau2ya9nDYFAZt0bYukqglqTwaeP/25UMPSqITO2li2Zn+Uosj8zXfDD8OJ/TdiuO4KIoIR9wnEXMPnDtRJAoDn3Hx9Ps38Thel69hz+eoF2BRymmCVdkwwvxXT5RESRDoEori0TkhAfdjUd5ShSPfw0ck4pQEsokBCpj+fuBHl0w1ox4pHlDMRqLA3W4xV5eRa/WG6pbunCjtkyBAMcNmbUowwv5wJHvubu224hs3F8QowkFRk1mbkqPAR+yIBISqWRivm8IeoqzalBJ4iI0+4mjIZafbrZoMT/oaR4wktC8H4J5SEmI+wglzS28N/CCoV+vL+W6XiyKO6RWSL7vtsFxJgIc48nLFmuMdRu/ZeUWl5htXQ1vn5ewnlOKIT5CG6GaSxI8mSKjU2mcBE3tX2Ihc2yVcQDeYUKmYkaRQgCErK1eAUL1VrV0Jrn1P6bZVlRi4zYulFZwSoY1PRI1EFTiIOd8xJWW1xFgoP+JoiK0AiWUtFHl+IpvZrcvsuBJa8TDFVLY9CAgvt8kw9TH7zxWmVMy0pf8sRn08Cc5CKLXX3iqXctS/tE0e4ziOsfdRzKG1qxzRIeZlg6wapXoR38RqPCwJVy8iWRSJ2e1GKMSqv92YeN2+8C5INEUP8k/dfpx0EyZmuMuwKPVYV4HlQFTU9Sni+K4fJKKb9N2F+yb9eOGuO3nNsjx/uHDHsvWfHRKt5oXrou013akhJUncDfCAr14TehkQ5K072acuv43xWlFzNxeICg4uk54eBbmWI6A4QD0czNi7ojEpyD6b5aoTUij/V8NZc3q3zqoeg7tenfgBoSFSrpT7oZxEiZ76a5nzOUF9TqSmtmqvaB90ktd3dyfaHI/dCS8KzFwaa0rZEwhYKWyWBGPjqQxpPpIrCgk8C9zlsE6E77W0pWSfR/6A24VcOU336PSL80VCrGxoPF//7qp+DFHsTXBXjFBenz7txZTuZIt+5PlXvpcIpzfRoWXvqrW43LMbdOOzer96Sf9SG2N90t0wdWZySioqcyUA7d+a7LJzt6zIhX0It+gGT+EbCoiK5ii3AEpa0iQsBWR4iBSHaNdkehGzfk2vYq06NIsRl8G53thB2E1Mj6O6aFasbKVamtuWWIVijKqGk5b+KpyF4pp28Rm5LvHJvEPnqcmZeLrPDrXss/9dyUcdbS2pKl4njayzyC1gkS/LItvAIu9lkfW+PEQjGb8NplkplF3KAZ3XVzFtSbjgJ/8T/zY+fdo4PnZ++eUgDA+YhffEiAt7iSbXmS3DYvfr4cidaU3Iu2tnGJnzIjTFiNWR9zM3VFbQYwfs4b4fIuX8LBRPczaLAaXCc8X3DwP00Hs5d3KPqfBcjidYsWt99QtVvupe1XyV1uv8/b1FPZOZYUl9URL2hC3U5EnkG4v247V7hv8Q/oUzUPBTFHzYfAU7/wAVP0XFR8+n4nQdVY+PV/RHP/TBjp+m5OPm2rFWL1jx0xT8oclWfEq8xdBufQ/z7Mrd9DY7u2/beKeDd/Buf6+HdlqotdfZ6Wzv7+LBXm9re/PyLdsQulRbUrYRE+9fV4huFNvUd2+Kzz9JURbjzCKcqrAIcRYBTiUQFb570+32sdjQ3ge12KbGJriRhoPLmMLNf/68+fg5ZJz60XC2OZwcwoWTgsU5KVCBIMsZgdjhq1CP1XE94gih3gbFAxU6dC17vQmnBuuz4imf/wI7mbsTHzTNv/s/6Ym5k69uiFFzvBFiYQ59Jmd4Vk1ltQuBACkga77IOnw+ZIGnegY8bYLlguUqyz2az5oQqC0j4GppcXUMK8Iyrghgt0tutx/cBcgs+i0RltyszCKKIiY1addjvv+3iCDzCDKPljTzqGynkHk0TQL7NqQeQQL7a01gLxhniENCbxWB86OhoPC827vlmM1M1/Iq/yZEssAPhWr0AySuQ+J6Zdn6pHAHueuNYJDK4oFBvlIGaTVFIJFTpK/vAIeE9HVIX29y+vpEngF5c83JYNf840kKhiT2hiex/0AdQwbsEuexp7qGVPZGp7L/AGuGbHbIZodsdshmh2z2hclmh2MDyGJf9Cx2jVwA19ImsoOTWr4EdjBayGEHaEEaO6wHkL4OJgsZ7LOkGEESOySxQwrSQqQgQR77zClIu5CCBClIkILU6Bs0tZE4Z7iP/SvsHKLI02iCk89m5iMdxuyJuj6XFE/sgUDZjU9Meryyz4QOHDLIDNtzTpGm4aDqhmYnPVnVmV1z0Hbjc5TiH27YzjElksqAypuasvQcBg5a/xFaP2mi1iFBDRLUIEFt0RPUVuWhlLpqarU4i4owl+csXarXcH3Sp6+mSs+l+si78hmhs59GjX8XqJTtdbOA27e1i0ieRDm1oyjbMdR8TqH+nPIYqjiEeo1ZaveCiafsANC0nGg6fFE0Za4p1jwT4LRscDqaj3MCPC0pno7n6p48vQ0GWC0brD7M100BrpYUVyeNv+M0Pa37t4ZmE2+jglQxSBWDVLGpbRZyxmo5Y5a7TzuQNPaySWNw92maNNaZ09WnECKeEw2F+0/t1PP+ZEC4BPXlaaXMAwNSOQdSub4MlGoPKBVQKqBUcFAKnGpunOqBpHsgVUCqIFL3cpG6uhVCdG4aKvkWqCRQyddFJQUcIToHTLIxTPK9sCNhvs5RhsaCUsq5PRXKzf84CDgmcEwI3C0s29oHtgVsC9gWBO6AbjWabhl/mAuMCxgXRPXmFNV72FYh3DcTAa2WAQMFBgrZePBXkUBBXzQfb5q7vIBrvjzXjIFrQnTv8eSqDeQKyBWQK7gjANhVA9jVQ9dnAsECggXBvJcL5mV2WbVFiNxNRS63gFwCuQRyCRcGAcdsbgTPen0zEE0gmhDJWyiytQ1kC8gWkC24Rg/YVrMjekC4gHBBZK9Zkb2aTU6I8K2kvXZZf4RD9BVTpnvW3tHF8rca5Tc8RC9VZWJZGxaoVD/2TcUahdVPsui6OA7jQAwxGuZWJugE4waWCwgYTKvEVjm+UYjw8AAlQWWZV+ZjSm3kV/6KIy7jNAeSeyz4qfNZE1RDntqySV8NKYml3qr+1f0jwfRWGaCN5sr7UaVnLuNTlA7xTeVC1dxrGc2vVMZlzBwKgq/pRJRh/NCcTphM+yxOweSzmTbctnDSYin38PvAQl0n2LMbClX6ltczraTkYBaVqJa62t2uJvE9P1E/zinTml17Ww9pL128TC0LY1EqYr9lHXLL0lrPZZn95RQbemiGIGH4XFdkOqyFAEzrmQGTs8ZHQ0Yx1oLNSswkrBuPxIJzD5bs7HtscFrA2LJgTM3ZY+H1nPukcWU/BJBbFshlam0k7CbvwgvJckBxJdsGjDXt9ZV2UsI70NttNyLXG+1sByp2xWmZW/pa7AvWTosvp1OSx2LMbaC7a5zbt1vGw7b5YARJ3V3jc9t82G6ZEuPini3jc9vTBvEtG4OMkRhwe7AVs+KOWbHZytaO+WBsXPY8s79ZX0rT952oEJz75fwo3aPkt4eTME7Ezugs+9FCZ9P5r2k9bqKizS5qd3bQ3qC1vb0/2N/Hb/f7GO32ezt7eL+HO+09/fJVvj9qrYz/AtFMtIzM4AAA + H4sIAAAAAAAC/+1da2/bOBb9nl+h1XQXycCZ2Hk4TYBi0CQNZoB2N5OmXSwmhUFbtC1EEjUUlUcD728fPvSgJDqxkziW3NsvtUiZpHjPvTy8OqHv1yzLRkFAGGIuCSL70LrnRbzQcyPGr/78xi8nLXEbdlyG+h7mpYzGWJaNAsx+d3hJEHueKqEoHF8Q4jE35OVtWTh2HXxMAkaJJ7oYIi9S3/fc4CqS3chLiocUR2NeYO+0I1uVkRt5hxyWGhwvHRDPQ2GE9dZkxRi7o7EYub291w5v7awiRAH28pb01mQ98lwUHROPUDkLk5Ze2Uc0KnclKxwUjT/iYMTEoDvtSh2e9jWGIhLTgXgA+4wSH7MxjiO7cNfQ9bxqs66Y706hyMMjHDiZ8fJnuh6ZuhfzF1OKAzal1ke302rcYEpNNCY3GjS0GsbR5U351jXy4nyOtLri/HOcyLvKrcuKG9eR098u12jQyooFUM+IG7BPxJGTLwosFFnfMSXF6Q8xt0/A0AgbbRiKVihy3Fh0s1etM5ueT7uDKaai76FHWLHPCFMXR/+5xpRynzGMPwrRAE9DHK8U5ulsF0sZGlyZJi9iOAyx85HPoXGoDNERZkWfKfuNvBHfhvJ5othfvwxEUcBntxcgH8vx9kLi9AY8ACDeFT3MPvUGYdyLIz7DvQjzUifqSbAc8oZ6LkUM3w+8mA+Tvru03yQfL+2WlbUsyrOLS3siev/ZIsF6VtjifW+oQY0oicOeh4ds/YbQK48gp2Wln3rsLsQbecu9rII3cHgV99VTkBvxBBR7qI+9OUeXdyYq0s96uRyEqBT/y8fZsPp31rp6BrtVnvghoT6S0Y65vphEgZ7qbRyMmHJXO0UDRoSltiu3qBBymrV3f8/7nEzsKTdyzFxpYb8YCTisJDYLFRPtqghpNhZBn3iOAe7isU55eDT0Jes+j90hM1cyuVTZx2dfrC8CYkVHY9kSdV+OY4hiZ1qQI5RVp09FMWk70aMbOO6168Q86E0NaOm9crksjuwW3bpRdVz9eHClnLE66bafBDMxJSWT2QKA5m9ND9lZWJbrv/kR7tAtniE25BDl3VFmAJTwpGlY8sjoCMllvlOpU6uU8WtqmWpXoZk/cRGcrdo+hNnF1HOUF82Sl62VSzPf4qtQiFHZcZLS33mwkHTQXH1ObgqULxvQReJyOp4e8kNV99n9LuvHXeUtiSl+TKZXJXrbQPQqRK8DRO9Bolcdy2NML2J33izBXLqOeKCL6kKjwI5zCvE//m/z06fNkxPrt98Off8wMlCTEDEeboLpbaYrJd9DOjiw5wrb2XDNJCCNL4QmGDHG2kEaKYoGeuoDO3jg+kjGJwMLU7TK4EBJ5YWk5Eceeuy+jN7YJ5QHF8vhxNU23vqFCp+1HzTNV+G91k/vDeaZTt4K5gtiv899oVIfB662rj7duuf4Lx5fWAQGfo6Bj+pvYOufYOLnmPh4cSZO1lF5+XRDf3R9F/z4eUY+qa8fK/OCFz/PwB/q7MVnxGmGdat7mIUbd8vZ6u697eDdLt7Fe4P9Ptpto/Z+d7e7c7CHh/v97Z2tq7fRJrel3DVGmyFxfr1GdDPfSb57k3/+l6hK05BpElIW5lnIPAcpK3iD7970egPM95wPQS00mbEOYaTm4NKmcOuXn7eePocRo24wmm8Op2dZIZnfnGS+fM9qSOPzHb7M5RgD1xOy/NU+KB7K7J5t2OtNSey35sVTNv85dtJwxz8omj+3NdIWeAUHIqBnueg5Whx6IBotAE9b4J0/kHceLye2e3LrB9hpNHZOILKvYmQH31wB3/xgN0CM80fMvbVeYhyKgkhY0mzHbD9uqAKxDoh1VlSsU/RTEOvMIsveAbUOyLJrLMvOiZ+PfULvJMdygxFn0qzXv2N4fgaYNfkPXiUKXJ+bRl2AHBvk2KWV5ZPEHSiya0HypMcDyftBSZ7RFYHnzSDK3gWaB6JsEGU/U5Q9lQqAGqw+umxFEZ5lYJBm11ya/YI2Bl3nCquzE1uDQLvWAu0X8GbQaINGGzTaoNEGjXZjNNqQ2QdtdhO02QqdAKCVlWdDIGqmLBscE5TZAB8QZ0NcXyVRNrgl6LJnV+WANBuk2aDaaYRqB9TZc6t2iiKRNMKLtjq+DYoeUPSAoue5xywqL7DO8QC719g6QoGj0AQvEusp7zkKo2fa+kIwNL5NAWPXXufzdGOfcxtYZJg6tmOdIcWiwdQ1Ffs829SpXzOwdu0lP+GLO7Z1QomgMmDyuiqAFuHgYPWXsPppHa0Oei/Qe4Heq+l6r3Xx3kiecbSevy4KMBOvSXpUreHqZZw6E2nek5D+5BgUrffS7Ni3jctAvDayKu+NTO+MlvPK6P8zvjPK3xj9iBKvB6HDEi4A2FkF7By9KnbSsBMqDgngaTZ4jpcTeAA9K4Gek6WGHkdtXwFEzQbRh+WGIEDRSqDotPaHYSZv0P6tgFjHM5FAfQXqK1BfzeyzIMOqyLAMh2R2QWtV0VrBIZmJ1qq7pDMyIbP6KrwQjsU0c8GHFXNwNubr8zwhlgKWtwSW11oFjrMPHAc4TqM4DrwCBJKzYJLziFQcWA6wHMhlvV4uq+qFkL+ahdu9BW4H3K523I7jDfJXQO2WRO3ec6/hzmodp9jLOZ6Y2zNu3OxvTID0AemD1FZj6c8B0B+gP42iP5DaAv5TA/6j/cElUCCgQJD3WlLe63FfhYTYXIywVAaUEChhUxRd8GdHwAkXpuma5dAkIH+vT/5CIH+Q/3o62+kA2wG200xtF9AdoDuLpjuPHRwIjAcYD6S7Xi/dlfpl2RchtzUT29sGtgdsr9G5LTjPAkjfa+e4jCfJAvMD5ge5rkaxnx1gP8B+mp3rAvoD9GcJOS9gQMCAIPdVr9xXxSen5MDWklHb0WCMffQV00iNrLOrisXvuIlvOIheycb4IjbKUWmLiE75ioTlzzWothj2Q48/YjDKvIyTh4hpWM4hoBGnAn1k+FYiwsFDFHulRV26j15rYqPiF95wEacZkOwTThitz4oxavWJL+t8UqslobBbOb7af8WY3kkHNPFOcU6jiMxFfPLSEb4tHeyYRS2t+7XSc2kzhzzvazIRRRg/NqdTJtM8izNQ63SmtbDNgzRfyh383jMQ1Sn+bPvclK7h9tQqCTmYxySyp54Kt+sZ/XCDIWlZSXMbZmNtP2asZK3Sjcp9Q1ok+iPt3y7WVgYqysw3J1BQT6JVxBG+UA3p8akR+GgvGB8ZSXwZhBh56UQjpoCcVUFOunF4MnAWueGZ5BsbQNyqIE7OWR3Rpm2vC7vrSWl7vRpQXEup/ERRV1daJyGtQ7VltgNys9lJd5F8Z5uU2YWvhS5n3jT/cjIlWT5F38rZe9rb6U5bu9jRL7Rf67b3tM8d/WKnrddoR5xsa587jnKIb+kziDyHBrlHe9Eb7uoN671s7+oX2uZj39HHm46lMH3fiUya2V8ujpN9RnYSMfHDmO9uztMfJbO2rP/q8dqOZQrXRp3uLtoftnd2DoYHB/jtwQCjvUF/dx8f9HG3s69uvs72OO21yd/0NFeRF9oAAA== k8s-resources-workloads-namespace.json.gz: - H4sIAAAAAAAC/+1d62/bOBL/nr9Cp+0dkr14YzuvJkBxaJwGu0C7103SHg6bwqAl2tZGFlWKyqOB728/PiSZsqj4kcSWnemXRhyJpDgP/mY4Iz9sWJaNgoAwxDwSRPax9cCbeKPvRYxf/fmNXw63xW3Y9Rjq+Ji3Mhpj2dYLMPvN5S1B7PuqhaKwf0mIz7yQt9dlY99zcYsEjBJfDNFFfqSe973gOpLDyEuKuxRHfd5g79YjW7WRW3mHnJaaHG91iO+jMMJ6b5LQx16vL2ZuN/fr4Z2dEUIUYH/Uk96bpCPfQ1GL+ITKVRhu68QOotH4UJLgoqj/EQc9JibdqBdouOwxhiISU0e8gP2ZkgFmfRxHdu6uruf7xW49sd6NfFPAML1B4ma7Mch34uMeDtyMsfr79oL30eUYS/U7bnqmyYvVjynFASuhDtBdGcULSihUcO2Ci4l5KlGf3JopjIuuX9IpX5F4xACNlmcuF0J513jvknDruZK39XGKJrdZs9CCz4Rz4xORbyIbLBRZPzAlebaEmDM/YKiHjQISil4ocr1YDLNfpJnlinPFxRRTMXbXJyw/ZoSph6N/32BK+UrndWFcH0Y6Ifr6HvNltmoWxd/5irK8nKYKSeSwP5019472W8U7MnUocnEk6/UCgdsOFwcXcuplDwv7MrI5xlsyNTBTNU43C9SIIefaIEZjgjTNAvrewJuwfGdHb3frr2T5tKu8JkUhcnCZYeVEYUcazXxrMkxBjSOGwxC7H/kkjUrDEO1hNoU64LtQ8iiKB5tXgWUFXMvbARpgOdt2SNy2w3c5xAeix9lfbSeM23HENb0dYd7qRm1ptI55N22PIoYfHD/mk6Tvruw3yZ9X9raV9Szas4sre3gV/GyRYDNr2uYjb4kJ9SiJw7aPu2zzltBrnyB320r/arP7EG+Nem1nBP748XXcUW9AbsXsKfZRB/szzmxsMEEV/8spb1mde6t0WldBUeK7hA6Q3MqZNxCLJ2W4cFu6+Z0hh0kdapZI71nW38NDOvpwyJVydCknMxzaJT1wEbrWwM6YlONQiupMBiITKQf5iG5KLnAIJKGBtBkzcyBZ+D6iLr9M++Itqe3+hcsj58jWIhd88v6xBgusbPsyl7dsd5l7cUvtM+sLoE5814CCxNudcUhrGEvSLvpel5mJTGJRu/X5i/VFWMw8fmHZHvUwDg8RxW4JCowIZcVVVOBQ6rsY0Qtc78ZzY44lS3Fieq90cfIzu0N3XlScVyd2rtXOUlx0e5BgRLEkYyyzhcCZnypHwhnalT6b+RXu0d00uG8kqXw4ygwCJTaHMlnySe8ESdesUaAp38D4mHIOighGe+OZNH+JL2FWMfUeE1DQxnhrplvcgIcYjStO0vobtxnShTeTz8ltzk3PJnSZqJwuT4/poaJdeD8kvX+gtCVhxev0zovOeROc88U65431dM6fzSUqzmWSTxSxe3+mCMFlcRdTmoRHMOW//F/t06fa6an166/Hg8FxZMAqIWJcX4LyPtNtWHm29nz++HkcBF7Qsz4Tt8wbT8TEaMud1BLleTTvO7vY8QZI2r+6ybm+NutQQryU/uuJjybdl8En+5Ry42W53Nezjbd+odJcPcqdr0KBrZ/eGzhUDg5zHAziQYerQ4EeB562b8/HYDOEXDZ3m6vG3ZPqcvf88VgkMHgqBreqz2Dr78Dip7D49OVYnAAleTk/oz8+FhQHJk/F5A/V1WPFXtDipzH4rMpa/J8kfr4aLC47w3pBDu+4O6hxsIcOu/Xd3aPu0RF+e+RgtO909g7xUQcfNA53rt9GtTScHNXSI4l/3SBaG0Uk3r0Z/f0PQUrD12nwWjaOotej2LUkpL2+e9NuO9j3ZaOMaCcN7eajwnhbyugqWJtUDK1LMRiYm3nMTe4orJpsrjhntcXc+eXnnfnXMGLUC3qzrWH5ic2sp9wOiQO2uZzj4scPix89WpMhU8OhWsSQDIoajf8cZ27FMSjuyiC7bQiJPPHwEhIOFpxwUAEpOnkpKcpWfiQxKexopwf0D3+RDl9NcWuNvzPDtQHm1siJxJrPyiPtjFwdjoM8LUGeWmCVVkCKdkBLX7WWni7D6qu8IZCmtZOmD2Dz18Xmg46uqY6e2SuQjfmHyDCtVjYmRUEkOGnmYxZEMZAgWxOyNdc0WzOvp5CtWcjWNNRS7kK65mLTNaGWEmopoZYSail1h9KyRj7PAA8IvZduhBf0uOvI2p17hlPnx0Hujcdx5DwuTzbI3zhJNHgDbjbUxVDMo8QBgtpKqK2cprZSiS6UV75ceeVyV3hdKiw/yVWEIstKuO1ycwO3/ZW67UZVBM+94LkXHfc9cNwX67hDnSXUWc7mq0Od5ZrXWZYCSci4rk6ppQKYT2IwVFtWvNryGXkMpVpP43KlCy4TXkPNZaVrLp9Bm6HsEsouoewSyi6h7HIVzA2UXULZJZRdVrPsEnIToAxzsQU5L13glR2iQ8b/EkSqBYYKKjNBcaE4swLFmSBQyxSoD7ATQL0mqC2UbL5MeidUbULVJqR/rkT6JxRuzpz+uQ/pn5D+CemfVU7/bCklsc6xg70bbJ2gwFXSBKeE1cwFPQmjJ/L6UkA87vsAsyufFzo/s885DyzSTRXbtT4jBcOB1dVMDw2fzOpUrxlwu/ppos+u2NYpJQLKAMsrmjH6IgoOXK96DuncXIfc0dvAYsQKjZVpq5VDas4ZhPxRyB+F/NHXlj+6KQ4n5bd1N0enkgFmQkDaVKE6df6rvsX75FPg4Z/cIInx2mkI9tuWOpW0CseSpkPJSpxJ5k8kt1bgDPLZs10fFRyWYEOQnNWXnJOFSk5qckLlUYDorLLotJZjdEB21kB2TpdqdlwVyAARWmUR+rBc8wMytAYyVP2fW0jOUX9XYljFbzRCEh8k8UES39Q6C9l8hWw+w88wHEA632LT+eqQzqfS+Q6W9O17CNVOAznf/e/K/uWfzwhCX/CD96v/YfDHMzXh6+CLR5YiSQ9w5RJw5fY6oKpDQFWAqgBVwTkmwKplwqoJRRGAqwBXQbxucfG6ohZCjG4aNPkW0CSgydeFJrk4QowOwGSVwOR7rkpcg61WKpAjVClWOU2nP86KuABrAtaEGN7Koq4jQF2AugB1QQwPYNdKwC6tkBqQFyAviPItKco3pcJCDHAmNDreBnAU4Cgk6kGBI+DRRafqTfMNNsCci8ecIWBOiPbNj68agK8AXwG+gq8AAMCqBsCa9OVTwFiAsSCut7i4XqqX47oI8bup8GUT8CXgS8CX8KUggJmVjuMZP74NWBOwJsTzVgpv7QLeArwFeAs+qweAq/JxPcBcgLkgvlet+F5BJ0vifBvJrO3I6eMB+opppGbW2FPN4sc2xRMuoteyM76z9UZSKX+lnfJtCsvf1FF9MTwIff6KQS/TMo4oIqbJ8kgENLCVA6wM30mJcHEXxf7YTi/VR6ea8K/4GU6cl9NMkOxTDlGtC4VRNXqiyzqC1agkFHwbt6/29xjTe6mAJqQrPqMqLHNePnlrD9+NfXc1s1ra8Btj76WtHPL9r8lC5MV40pqWLKZ5FacA8+lKa2abG2m+m7v4vW9AryX6bA84Kz3D7SlXEnwwC0vkSG1lbjfjMIFKQmprHNsxXONvQz0n4nAhQ01bZu41J3Ev2bx0LnNlkSyK/kgnZOephZmLNvPNiWyoV9MIcYQvVUe6wVoJgam/sMBkwHFukclAqxd0ySPSY4bcQw3IvkKpQjEjBu6I5rZDYilvu/VxitqY7UY9JzETt4nQJ/cDcUfZTpHdMLfU8t3GC7xk787LyWJ9nuGY07O1BN0a+9Gs2dSqQss1iypee+EX6l/cB45hYRJFrVdDUTdSjDpUmMyTXEvQWFf5gnZAbmuN1D2ypbKKNjv3WOhxSElHDycLlsUKdB/F3teOlnUNtnf1Cy2IZ+9rfzf0i926TtE+O9PU/m64Sp2/pe8gvHhNFCeOond8oHesj9Lc0y80VH3o6vNN55Jbvh9EhojsL5etBEBnX8AmgzDmsP08/RVEa8f6PZVjazOttYmUuNqxDI3a6O1ht1N3jxpH2KljtN892N/bbTT3cOPA2WuifXXzTYbk6xvD/wPkH1LMs/YAAA== + H4sIAAAAAAAC/+1d62/bOBL/nr9Cp+0dkr2ksZ1XE6A4NE6DXaDdy6ZpD4dNYdASbQuRRZWi8mjg+9uPD0mmLCp+JLGlZPqlESmR1Lz0m+EMfb9mWTYKAsIQ80gQ2UfWPW/ijb4XMX7113d+OdoUt2HXY6jrY97KaIxlWz/A7HeXtwSx76sWisLBBSE+80Le3pCNA8/FbRIwSnwxRQ/5kXre94KrSE4jLynuURwNeIO904hs1UZu5B1yWWpxvNUhvo/CCOujyY4B9voDsXK7tdcIb+2sI0QB9scj6aPJfuR7KGoTn1BJhdGm3tlFNJqcSna4KBp8wkGfiUU3G4U+XPYYQxGJqSNewD6jZIjZAMeRnbur5/l+cVhP0LuZa/JxHwduxrzxO133TdML+sWU4oCV9A7RbVmPF5T0RANyo4mG1sO4dPklT10jPx7TSOvL05/LibxrcnTZceO5kvyNyR5NtLJmIahnxAvYZ+JK4osGC0XWT0xJnvwh5vwJGOpjIw9DMQpFrheLafaKfWbWc7K7mGIq5u75hOXnjDD1cPTva0wp15m8uE6K7FhsxVg/Yk5ma8ui+AenKMuLUqozRE77y2lr93CvXbwjk9giF8fi2Ch0cPV2cfBFLr3sYWECxmbBeEsmxeZejdOtQm/EkHNlEKMJQZqFgL439KaQ7/Tw3U7jlZBPu8prUhQiB5fZPt4pDEWzlW9NpimoccRwGGL3E1+kUWkYon3MZlAHfBtKHkXxcP0ysKyAa3knQEMsV9sJidtx+IcI8YnoUfZXxwnjThxxTe9EmLe6UUcarSM+TMejiOF7x4/5Iun7S/tN8uelvWllI4v27OLSHl0Gv1okWM+aNvnMG2JBfUrisOPjHlu/IfTKJ8jdtNK/OuwuxBvjUTtZB3/86CruqjcgN2L1FPuoi/05VzYxmegV/8slb1jdO6t0WZdBUeJ7hA6R/NoybyiIJ2W4cBs3hphyU3+KHCZ1qFUivafZePf36eyjEVfK8aVczGhkl4zARehKwyMTUo5DKapzGYhMpBzkI7ouucBRivx6S5sxNwcSwg8QdfllOhZvSW33Wy6PnCMbyyT49O/HCyCwsu2rJG/Z12Vh4pbaZzYQWJr4rgEFibc75ajTMJfs+zLweszcyaQHYLfPvlpfhcXM4xeWfaPuJ+Ehotgtw46EsiIVFTiU+i5m9ALXu/bcmGPJUpyY3iu9kPzKbtGtFxXX1Y2dK/VlKRLdHiYYUZBkgmW2EDjzU+VIOEO70q0yv8Idup0F940llU9HmUGgxMehTJZ80j9G0ntqFvoU+Dc+ptB/EcFobzyX5q/wJcwqpt5jCgpam2zNdIsb8BCjScVJWn/nNkN62ebuc3KT86SzBV0kKqfL00N6qPq+eD9l/2BfaUvCitfpQBf95xb4zwW3oPky/ecn81qKa5nmtkTszp/Lib8ofmiUsOMxkvgv/7f1+fPWyYn1229Hw+FRZIATIWLc3ATlY6ZfSuV82ou5zOdxEHhB3zojbpnDnIiJ0dw6qbHI82jRd3ax4w2RNFENk/97ZdahpPNCupjHPpp2X4Zw7BPK7YvlcnfMNt76lQq1tR/kzjehwNYvHwwcKsdvOQ4G8bDL1aHQHwee9mldjMFmlLdq7rbqxt3j6nL3/OFwITB4Jga3q89g6+/A4sew+OT5WJwAJXm5OKM/PRS3BibPxOSP1dVjxV7Q4scx+LTKWvyfJMRdDxaXbTM9I4e33W3U3N9FB73Gzs5h7/AQvzt0MNpzursH+LCL95sH21fvoq004httpbsG/7pGdGscNHj/Zvz3P0RXGmFO48uycRxgHoeXZUc66vs3nY6DfV82yqBz0tBpPSiMN6WMroK1ScXQuhCTgblZxNzkdquqyeaKc1Yj5vbbX7cXp2HEqBf056Nh+abKvBvRDokDtr6aHd2H93Mf3P2SiV6Gfa+IIRn1NBr/BbbFinNQ3JNxcNsQEnnk/iLkBCw5J6ACUnT8XFKUUX4sMSns6KR76HPzQduqVnvUIDMrkJk2WJ4aSNE2aOKL18STVVhvlaIDElNLifkItvul2G7Qwxrr4aldgwTGP0VSZrUSGCkKIsFJMx+zoIahCxIcIcHxhSY45vUUEhwLCY6GCsEdyHAsZDhChSBUCEKFIFQI6n6dZY1djyEeEnonkb4X9LkHxzrdO4bn90GyIf/Gu0SDN+RGQl2MxKwlHgnUB0J94Cz1gUpQoUTw+UoEV0vhl1Il+FlSEQoFK+FHy08Z+NGv1I82qiK40gVXuuhJ74InXXAQoFYQagXnc56hVvCF1wqWYj3IGq5OuaDCgI9iMFQMVrxi8Al5DOVGj+NypYsGE15D3WCl6wafQJuhdBBKB6F0EEoHoXSwDuYGSgehdBBKB6tZOgjJAlA4WPXCwWznGvLeVyA2bTA9UDkIqgjFgyA0UD8I9rvO1YOgiKtUxMoXECbBdaghhBpCyH2sRe4jlBHOnfuYT7VLLbwYqzm0IS8S8iIhL/KReZFtpQXWOXawd42tYxS4Sppg+6yaSZLHYfRIXl8IhMbdE2B25RMmF2f2OeeBRXqpYrvWGVIoGlhdzbzJ8NGsTvWaAbernz/55IptnVAioAywvKKplM+i4MD1qidXLsx1SKq8CSxGrNBYslWv5EpzMh0kVkJiJSRWvrbEynWxfygPY10fbxwGmAkB6VCF6tSGrDq8dd4jW//i5keM3knjpd831BaiVdhDNO0gVmIDMb99uFGDDcMnT/p8UExYggRBTuomJ8dLlZPUnITKWwBBqY+gtFdjUEBSaicpJys1Ka4KQIDA1EdgPq7WtIDE1E5iqn+efrK3+YcSuiqe+Qd5cZAXB3lxM+ssJMgVEuQM5+zvQxZcIQuuAVlwKgtuf0WHm0OEs4gB3//v0n77zydEhc94onn9T35+OJ0Rjn9ePtQTmWwA9FYA9DZfAsw5AJgDMKdWMOe17tABzlkmzpmSyg9AB4AORLSWF9EqaiFEsWaBd+8A3gG8qxy84/IGUSxAd6tDdx+44nB9tdqp+I1hnqBympV9lNUCAfgD8AdRrtrCoEOAQQCDagWDIMoFOKhCOEgrkAUoBFAI4mArioPNqLAQJZsLHk60AT4EfFiXZK9XV1UEAHGp6V6zHHYFIHD5IDAEEAjxsMUBTxMADwCeeqZ9AeIBxLMExDPtzEcAPQB6IPK1vMhXqpeTuggRrpkAXwsAHwC+Wke4Xt3hFoD7qhDpMp4DDOAPwB9EvGoFgHYAAAEAqnfECxAQIKDVRL4ABAEIgghYtSJgBZ0siYStJau2I2eAh+gbppFaWXNXNYsf4hNPuIheycH4d6w/lkpbGHXKP0pY/t6GGovhYejzVwz6mZZx/BAxTZbHIqBhpxyCZPhWSoSLeyj2J77rUn30XhMgFT/Rh/NymgmSfcIxo/VFgUatP9FlHVJqvSQUfJu0r/aPGNM7qYAm6CmOcxSWOS+fvLWPbyfOf8ysljb92sR7aZRDvv8tIURejKfRtISYZirOgK5TSmtmmxtp/jV38QffgFVL9NkeclZ6httTriT4YB6WyJk6ytyuZwjEC3pk00qG2zAzqzWNWcm3Smcq1w3JkejPdH4731tYqGgz35yIgnoTrSOO8IUaSLdP88kHihkx0Fk0dxwSS8nZaUz2KItpNxs53k/V39And0NxR5kKZzcsLH/cDHiBlxjVPM+XC0ZHk7+5bj9sj55DSyZ+6WQ+FakQueZRxSsv/Er9L3eBYyBMoqiNGirqigz5c4toJhRPY8qNcjjSBPFlmPi1FHaOFMzyJDETgNVT7p0dkJutZurx2NLMizY791jocZRIxw8nJMncf93tsPe0/VTd9ts7+oX20+D2nvZ3U7/Yaeg92nkdLe3vpqvk93v6DsIx14Rj6iz6wPv6wPosrV39QgPKB66+3nQtOfL9JDLGY3+9aCeYODtclwzDmCPx8/RHz6xt649UCq31tAQjUgJpxzL8aKN3B71uwz1sHmKngdFeb39vd6fZ2sXNfWe3hfbUzdcZOG+sjf4P5J8blcXvAAA= kubelet.json.gz: - H4sIAAAAAAAC/+1dW2/bOBZ+z68QhHloAU8R23ESLzAPbdIWi+lMs0mnC2xbCIxE25rIokpRSbxB9rfvIakLZUlxUieW3Bw/WSTFy+Hh9328SLrZsSzbcfwwSkRs/8P68q2nQzj9nvicGmEkDJkgwmehDLyBIAgM/FioJHB5q5JRzxfkPKAQOiFBTFXgNKTinx4EhUkQ6BBOotknxgLhRxC+qwJnvkePWCg4C+LS/X7p3sAPL4yKRSSkKv0XVSddMwj3iCAxS7gr62KfcDanYkaT2O5lKSY+DTwocOJP8ybpW+mEJIGIS6GVkvPgOYkiP5zWxIgZ2HDGAm85K3kX81TNyHnMgkRQu1eOjwWN4sy42e+2lH0S+tL8dshCaucRtztLacHavnfCynWwZ3B5UGRnX8H1nnF9nfVLer2Q15WsVdcM8suqhWwWlZ1GBbosYPyP1AKXJEjM5mvvyGIJp8SM/DuJhT9Z5NGJYGY04z4NtaPWRXPqJS79WFMnWSsSuIUnFd1OYvEnE3+CA9pGTLmrlTPJm+1SR+rG5e68U9ORtqDXotScipWjIJn64WfK47RZB4UXC8KnVJSrXWoVvY64vCdO5i8uknMaUOGEUJwTkjm9cQOwJ+W/fbV/Sf9+tXvW3+wcQtLUX+3bl+VGTRifE+V6wp9TJ6Zg9LicxA8hK2j8O+IKxk0n0TalUxp67/J8yjdzOlF4Yb+uunXhWcIXCmjs0yQMYQBav+v6GkNccBLGEbhQKExE0XGLSN0dg7foYlKDI4Q8IoTs3Q9ChgghWwYhXA86J2JefD8U6Vl+CIMtdOlv/4OE2YWEF+vjqdWQueOyJBRrl/DUAHZzk5V2e/soYHYCdkUg6w6QHd4PyPYQyLYUyFyYehA/hIyfDM7yIp4rqB3lNkZo6w609Qf3w7YRYtuWYNsl+BpAwJyEZApoI5gggaMD10W3niUHI4W0gCEJZKsuHTZxrhgPvK2CpdeqBdZnZRgAJ8BkxKUO4dL+/XBpH3EJcamXwZJHYx/MvM24dKybgMDUUWAa3HNh/ACBaUuAiQNWlFfHXTUAHMo54+tO1L784jiyBCcDlG8vtweMNBJYb6UhHheKSOCT+Eh6vPK4olvPCY8r+Xkknn2g4VTIEdrfLYXTuuSrcQ78FrIqBbznxPN1c3YfDhSlmVTdDtpBPVAcFkChurFcEthpGr6OP6Ubq4In1MiWXE6XGy+HbMLTXllKPifXNcnnflgTyv3pTJz5Xk2pAN9XNaGQ9t++p7oo27DN3EFyfU0ZOQ7IrCrWAdzMI0uBV2kp/bsgVlbhhMHAyVBEBhQOEFHwDsDEKa14TyTvkq6QxKWJrwqvOhuY2qOcKjiZBMwYIZxGVA1I0xy2Ht8fLwFdwGJLtYYB5dI6T4eR5F5UypaEGFHvAxilOgZ/GAM5jHOJRAyMpLf9tVqrh8JlJHwIEFrnC+tFXowjYaNA0o3CpHVzU67HDyKnqW6+mIg6p+8Ah5adQYafzfyJqEakEPwxq5V1SgxFBGMqO0FxY45MwGOvZmwyLpYQSY09JwNqP/T8S9+DGXGVaLM0SoEUFbgm1/4SLJ4n7oV2uSUAyISdbK6pPOQ2dDV1Pb7kYJGrv6KKC3JN7/D0wnVYtOQyAQG/rVRBRrDpGxJTE2YMDK0k1yBaCTaaUidzulzLzLefK31XVkYb+HuM/I383XX+VlOZp6XxLE3PKhMp0niZxvVkCskcyRzJ/KnIfLDqNGt/r57MjfYgmyObt8DmMz8WDNB57nxPCPB4QF/svhqPe9Z9aN5LUuqLqctCL3Y0iG+Y8XtWQF8i7ZdpP+saazwWMyvrW5QA60mAbRAASP+dW4of9Bvov4/0j/Tfqcm8PAYP5XFRJfc7DpI+Grdvlsahsffg7d4PmPCK8QvKn4MNdUsbzPhms/LnhHnWmXReXO/A9Q4UPC1uXjQqngEqHlQ827Hg0bIUamFZ41H00MMM3CSUNrSStHETd1MuHafmR8mE60Molza/PnTYoJaGm1NL8pU4b+eRWNRH/YdyhhILJdaDFpVACHAwSLFl1JaSKiogKaFnpQ8H6TP97W0dycqooFJ9urKZdKZ7z8IjobiqgjKhE6sqTTphD3UC6oSfSSe0dIAU1cEa6gBPmqJGQI3Q8knT4ahBIoxQIqBE+Al3a+6xxnDXXsJ6T9Y/REi0fEx129TEMR5gxQ0KVBXd2KDYGzSoin08zoECoVMHWN0pZ0mUv/frIRsOT6QFNvRUajefSDlS3WGl3VFYxeK4ToDrBMjore0lNFL6AVI6Uvp2nNBcxfUtTPzxgdS76B/n8jiXR+Yv/A7w1uV+lL0pVR7IDfwJdRduQC16CYRgwaCXA5vxllcAxg1yAd8niXKhY4+wgiM6nMovRrY1/e/+K3cf/2GCD2/fW9rqOLfHuT0yfJtz+yayxpdHIllvy9OXBotn5PizPx3YPSbPGoNsjnN1ZPKNn+Yb7Td8Bh3fG4lEvoVE/lwe8+8ekXv4hD8SORJ5W0S+P2wg8ge9AXKZClcxeYW0f5DKl+Men8vNL7Mhmbe2hA4MJBw3kAMBKPs79I140EN4jV/MdKELZNjg1auWPlI3uL5+xHc9Prmhhq0ZathoqDddNNRea4baazTUURcNNWrNUKNGQx1vVhCfnhzh86m4N4VCuD0hfNDw/eIBvhgUV7Q6vqJVQ9HrrWitc+QUDHfesxIetHLaVJau/kAFurLcdaq7BD99ggtfyPcb4vvDVXTf8BXTwRDXvXDdq23KjzhzaRwDmcs+BFaf0znjC+d8IWi83qbU89tt+kPZDml2PZpVrrcFVNuRej5zuj28L93uId0i3bZNt2oSnXGuGyX5nPlpXvP4/Cj46OQvK4mlqyELrzfZnUlLbMGEtxv1fOYsXLwFaRUNj5CGkYbbpuEpc6aMs0RIe+Ms92EU+z63HHIscmxrHLuTerkcB9LF1cGdXe2TNmdXhb/bsTujc/KZ8li/ZUB/LACgZKE92iP8Qt8H6Fj0nBr9PKRymp+XJug8Cgi4/zT3NlseLzb6+6aOmkqmFfRaGdajE5IES6ZVrmXG1vWUeo3w0iDK+sQ+BklgnWlNUPVvUzEYsUy9g6GMEhD8PaF8ka4ULiuLkvH7pdApld1tm0mzkWsUvzx0Skog+JwaYmlAGjY1DbJSBxVGMx8W9UPAfI++DuroOrdoygtmXnPoGr/mnszKNbesMrEqztHQ8iKJbpY46LZnpZm+rO+CwaouSJHX7CrweGXn+F9ZLexybKW6Mqw+cdrBuj1GRBLTTzqjWtmzoV7fber1JWhfNYxWdXzG14/Y8706iXJbbJ7/HP6wk6H7rUZaX9kzxdiJ1ip2yK5+7WfED7ojDbNLt0W+e6HkaXpzapJ8ZcRkZ3tk6Jj+rnExNC/68+L/yPjfNy+Gu2aMIU8Gxv++p3HvW9YGqRQNB1lZipnxvpmxWcpgz7woPmpqH3hmfbO6lMz3X6bEtf3Xp6OUFjPt97v2SB2aqBmdPewPDyekPxp5o3F/dLA/npx71DscHxJ3dzzxxjrxZc6+uzu3/wfcn4zpC7cAAA== + H4sIAAAAAAAC/+1dW2/bOBZ+z68QhH1oAW8R23ESLzAPbXrBYjrTTNrpAtsWBiPRNieyqFJUYm+Q+e17SOpCWVKcqy0nJ0/RIcXLOYffd3gRfbnjOO5oxMIokbH7L+fbj46RCPozYYJaMhKGXBLJeKiElyACYcBiqbPA45XORn0myWlAQTomQUy1cBJS+W8fRGESBEYiSDT9wnkgWQTyXS2cMp8e8VAKHsSl91np3YCFZ1bDIhJSnf+bbpNpGch9IknME+GptrjHgs+onNIkdjtZjjGjgQ8Vjtkk75J5lY5JEsi4JK3UnItnJIpYOKlJkVPQ4ZQH/nJR6i3u65aR05gHiaRup5weSxrFmXKzv6tS8UnIlPrdkIfUzROudpbygraZf8zLbXCn8HhQFOdewPOe9TzP7JI+L9RzpWhtml7+WNWQy6Oy02ihxwMufks1cE6CxO6+8Y4slQhK7MS/kliy8SJPTiS3k7lgNDSOWpcsqJ949FNNm1SrSOAVnlSYncTydy5/Bwd0rZSyqbUzqZfdkiFN53J33qkxpCvpXJa6U9FyFCQTFn6lIk67dVB4sSRiQmW52aVe0Xkk1DtxMntxlpzSgMpRCNWNQjKjl14A+qTil+/uP9J/v7sd5y9+CpI093f36mW5U2MuZkS7nmQzOoopKD0uZ2EhFAWdf088yYXtJEandEJD/31eTvllQccaL9zXVbcuPEsyqYHGPUnCEAag86tprzXEpSBhHIELhdJGFJO2iPTbMXiLqSZVOELIA0LI3s0gpI8QsmUQIsygG0Xcj2+GIh2HhTDYQo/+8jdkzB4UvDifTpyGwkceT0J57xoeG8AuL7Parq4eBMyOQa8IZO0BssObAdkeAtmWApkHUw/CQij40eAsr+K5gtpRpgBEtvYgW7d3M2gbILRtCbSdg68BAsxISCYANpJLEoyM8L7g1nHUYKSQFyAkgWL144iPRxdcBP5WodJr3QPnq1YMYBNAMuJSi3Bp/2a4tI+4hLjUyWDJpzEDNW8zLr01XUBgaikw9W64Ln6AwLQlwCQAK8qL454eACMqBBf3nad9G8x+vNwe+DFj33mnuv6w4EMCRuIj5ePaxwpDnhIRV8rzSTz9SMOJVGOyu1uS07rsq5ENPBWKKgk+COIz053d20NDae5Ut2V2UA8NhwU0aDOWawI9TcLX8Zd0J1WKhFrFkvPJcufVIE1EapWl7DMyr8k+Y2GNVLDJVH5mfk2tANgXNVLI+x/maxNlO7SZOyh2r6kjH/mqqIp2ACnzxJLwIq2lex2oqiYccxg4GW4oQeEAEQXvABSc0Ir3ROot5QpJXJrqannV2UDVPhVUA8g44NYIETSiekDa6nDN+P50DngCGltqNQwoj9Z5Oowk76xSt6LAiPofQSnVMXhn1BMwzhUScVCS2ec38Vk9+C1j3/XQ55wunBd5wSMFFAVarhUYncvLcjvuiJV2BPPNxtAZfQ/Is2x+Jf88ZWNZTUhB91PWKueEWFEPjKLskMSlPRYBgf2a0ciFXMIgPdpGGTSz0GfnzIdZb5VMszw6yigaMCdztgSEp4l3ZpxsachnwZvqrh1dqJ3mau56RMnhIY/wiiYuyJxe49uF6/BoyWUCAp5aaYJK4JM3JKY2sFioWcluYLMitrpSF8q0uZWZbz9Xwq6sfjYw9hAZGxm77YytpysPTdyZtOOUqROJu0zcZsKE9I30jfT9WPTdW3VEtbtXT99Wf5C/kb83wN9TFksO6Dwb/UwIMHdAX+y+Gg47zk2I3U9S6oupx0M/HhkQf3SO7zgBfYlEXyb6zBjOcCinTmZNJP37kf42UD4SfusW2HvdBsLvIuEj4bdqwq5Os0N9Qlbp/JrzoPdg8/USN3TvBkzduYPSLrg4o+Jpas30rUFxb9Yb4hxz3/msHBRXMXAVA4OaDW5CNEY1PYxqMKrZjmWMtYc7G1iseJCY53YqbQqGHm1FaO1KbWdI9DZVOIZFuM6DIdH613kOGyKi/voiInVDzbtZJBf1Sf+lgmMYhWHUrRaHgPoFKKTY7FlftFRUqUig46Sf55hT9Zvb9FGN0aJSe9qyDfTZ2MvBA5u4VoKBQSvWSpoigz2MDDAyeEqRwdqOd2I8cI94AM+BYlSAUcGGz4H2Bw1BwQCDAgwKnuCuyw3WEa7bIbjt1+y3CR02fIh02+KHt3i8FLcdMI5ox7bDXq8hjtjHgxgYErTqeKk3ETyJ8tu1brON8GDsv6bvQtv5hciRNoCTGqDQiiNwLQDXApDDN7ZD0EjiB0jiSOLbcZpyFbuvZXKPn4ReR/g4X8f5OnJ94XeAsJ5gUXbnqDpKG7Ax9RZeQB16DhTgwKBXA5uLDc/yhw0BAt7TiAFCyz4iBUccCap+enF9U/z2X1778Af/P7774Bg94/wd5+/I6ZucvzfRM17KiPS8LV9DWrydkePT+3avfdyddQb5G+fjyN1rP4c32G/4zXC8jxGpewup++l+dt8+6vbxi3ukbqTuTVH3fr+Bum91s+Iy+a3i7gpN35G8l9Menr3tXy5D+t7YwjgwkBx5gRoIQNI/wTbyVp/INf6ipAcmULLeq1dr+xG33nz+gHcoPrpq+mtUTb9RNW/aqJq9Napmr1E1R21UzWCNqhk0qubtegPbk+Mj/CYU95EwoN1cQHvQ8Du9PbxUE9eiWr4WVUPK91uLut0RUFDVacdJRLCR05+qdv0PNKAtC1Unxgj4YyC4ZIUMvyaGP1xF8A2/3dnr44oVrlhtmuQjwT0ax0DfyobA4zM642IxOl1IGt9vA+n57RP9pnWHNHs/mtWutwVU25J2PnO6Pbwp3e4h3SLdbppu9bQ541wvSvJZ8kNdn/j8SPfo+E8niZVzIe/eb3o7VZrYgiluO9r5zHm3uGtoFfEOkHiReDdNvBM+mnDBE6n0jfPa21Hsh1xzyLHIsRvj2J3Uy9U4UC6uD+DsGp90Bb8o/N2NvSmdka9UxOY7f3PtPkDJwni0T8SZeQ/QsbCcHv0ipGpin9cm6SwKCLj/JPc2Vx0Ftux9WUdNJdVKOteK9emYJMGSarVr2al1ltLX8y4NonqbZG5tBwpWKteXH5TBAcQ/EyoW6ZLgckBR0nm3JJ1QZWXXzpoNWKv65RFTCgCCr2n/l7phqdLWw8rwp9CV/c0mCwHqffo6qGPpTJFuSgd2WTOwCKt5J9NyzSurVKyrGxlEeZFEl0vUc9Vx0kJf1pugt8oEKeDapgJH13qO/8ha4ZZTK81VsvrMqYFNf6yEJKZfTEG10c6arL7bZPUlRG8YPavsnbHznQ2efcIhklDHDPk9QtcuAzhVJ6m7CWF7vWQng/orA7tMqzsF3LEJXNyQX/yzm0UBEISkMrf0WsS8Mx2rpi+nKsk/cbWp2h1YQU1313ro2w/dWfH/wPq/az/0d+0UK1bpWf93fYOGP7I+GBfI/WdlLXbB+3bBdi29Pfuh+D1Q98C325u1paS+/3Edabt/fjlKOTILBH81zmekiZ7euf1u/3BMuoOBPxh2Bwf7w/GpT/3D4SHxdodjf2gyn+dUvLtz9X/j7g7C47UAAA== namespace-by-pod.json.gz: - H4sIAAAAAAAC/+1dWW/bOhZ+768Q1E6RYpJbS16SFAgGWacF2t5Mk3Yw0wYBLdG2bmRJl6KyNMj89uGihaRI2U7SLNfqQ1uT1CF51o+HpHT9wrLs09MgSjKc2u+s7yervATBP7MAQaEMRFGMAQ7iiBZekyJSGAYppk3YLysvZTXDLAjxh4hUOqtVqQ8wSOMMeZBU2IconkI8gVlqC21gBIYhrccog0L5JPA1pYEXR7txGCNKEI2HYKWzarmOQ/7q91ct541IOgJT1vF2NRfrtbUdQoSlIeCrhLXzQToZxgD5dl53w/49IX/fMKZAP8DKaO1xBPEHn5REWRjyEgSSyXEchzhISHmHFdLp7MYRRnFIGToCYcqfD6RnwyA6E6SQgAiy9pzlBcNtLw5DkKRQpCRV+LWaMQr8w7gSJmeyIrAL8tvtCQWXxQTy31f0d8Gdkjabg1v+rEZ9UpYhmECAhakKpR8wREw8+uov8YXEYFaVTuKL4wDXNMfGeaG9myEEI2ztgMi/CHw8seU2R8FP1m4yECpyTUDxBdeBfI4V44E3gcfBFMZZbTIe1csd4J2NUZxFvk42MfoGwgzqq1LBtEjZS3dz0+sNREVlCu9214miu5urVo/ofue3jU1J6V/6vR7ogkKFKwnMNEbbh14wBUxypcztUYymTHCEa1N4mkIUQOGZMcjGUFaqKbgsZul0ROWZBlFRIRZTUaocobKYEIc0iUP/IxhC2WjUFp8AOoOMf1QVauppVP1NRfUdd4bqOzXaExiMJ1ikxa2hW/2MiHqfg1BVF9nUc84lSRCNj7kWOrpyWUmq+Qju7pzy2MKxheElFlTDyquEeQgzMRBDIBrPIOZWxGpKR3Rhj+jdYUy4kMoKQdXhiLgKie82ZRFr/Slm/p+YRhRBDxOPJrU5psNROBoncrzi+hvA0P9dU8OsOfRkhnLJABLmhKITado+HIEsxCoxPlk+xeLPqlIfRLJOyQ7rFeV5mgBioEqTLAqYBe4kqTiuG2lcuZLIKsX5cg4R4uH0Wn6mtKA6F+S5FT6Kh0l0tjZGEEbKOJmy+/CyPklBW6jEpLqb1UX6vYJhSNyFsWPH2HG/FMsdukeCHtb6do19rxv6NioZe7D0eTUDq/xPEqd4FNDebVstPCB4o4hy/c7fhHoE68+wMuMjzA98ItDC7H9GJKTQ56iEZU+BubHan99uKxVx+UCDE0lrXoKUoDPiQKFq62Eoo8MuQYaOs0H+2tikwdLZkILliPZcDyyUskiHk3FJuHU2uxIBMXbVZMOwIqGTTSOZ1xigMcQNrISXCes7zaYrAcFGcIW4QQzIsNApwZsXMTojmN2DwTk8HV5hmJ5ignDDay/MUhJttn7Yr/L//rBXS7ey9b8fgpP5Yd98f1UEp3eviCMgI6Vu8uTNG1lKzRAgN4EUg0j1yFL8OwAeZhyVTNQO4RhG/kHZhUyXKCXDfvZ2k4JIbkzkMxnuAddKKVDQ8qNJMML1CgU8fiG8t+KRtUOZbH3hLPfrgJHDoLKYma9oSxuiLbHaZluK6crB3prTjAo3M9OSWMPPeWj3+BxboNsCXblgMaTba5Fui3RbpNsi3Rbptkj3lyBdTESSTgPcQt2Hg7rHOc/xcqBdproNIywGQxGxYTSsaibaKAgxfGi9NE2trF6U4E4zwZ2FCe42E9xdmOBeM8G9hQnuNxPcX5jgQTPBg/kJJrFvoEVrGlR39hqGRgAJOo8EAyTITLDAuZcFM7d+HMPeT1/E+jBV92RoId+CEQesWxaUQLkaSh0o0wILpNZPiGIhbhPfChFk0WQUxhWEt1MPxYxZ0qBoQHsPgc+ekGtihGVmEf+gcMOHqWcMhyzofITRmM+4o8Z3YXIpvgqbljogDECq9zwMr+U8qYWiaqGtYHri5asI9B/yZ+3Tp7W9Pev9+3fT6btUCXbCStmVYxkRXh1P5BXV5qe9h4ieWn58IUNo1uwrCuthMAEk5qBIP2UZxcszK4LTJPB9BbCXywryLMKzzbdkerl3WE/9LIcI9GFqHjFE2XRITEsrBml1t4AQdKBkqeSw85hyKABiYQrWIfDOIE6XUxS79yuK5FaiEAxiuaWx9xSkoRqGtYfiJFlWX7X/RA1kuaVy8JhSOYyfINflE4a3Yrr/dh04G53Bugc3e13Q6wDo9nv93mC97zmbo1Hfe3uWER5GEMN0LU9rBdF4jawC/xGj8Qd/y3mN4IjKYqvbSV+fA7RW5bKqRBarIA9tvTo99aCafBSkXVt43lXQKnKuJ6ae+p62NbyyVghfzBk/drTTkOuraUkt1+feLtcnlZPZJeIy22xTj543fZLs3PmV7CyUM+FxZBn4ufsg6rlEDN17SAX1OdBZBr7uP6iiLhFjD2Yyto4E7rwVdYQBFnPdBUDhDDDt4DzejQhnU58WHzyxKxH3ehWCofrdAqNXxy3sIUC1Q0zsbo0uI07Loa75nPsfrlTwTwT8ADJ76PyC4/+uYftjvdrTYFYld0UYNY620+P85pC6BQ/Ox5rSYruyXkNvEu1PE3ylvzP1X7opUqvhB3ZUUvyYjlqK6DmvI36KRq3LTwEo1FPS+N/5/o68jLOZi9QQ0h77uJl/G8mdbxtpwfNWpn2bBBJNJGFhXPcvCSVJ1S5LpY2wpDgIJjc371Lp3QE/gfB7fq5JmeesfSbxHAkmsau20UW8OYljH/nxEnmgf4H1nPkEx0LHNK6vSS83N7dewTWf4fh+H4Ezzzxq77qVmYxr0YwBYhFRNWS+8+mqhnpaRIMg8oPzwM9AefDipn46hN5/rAZwCS4DxfkOM56qrnmLaW6aWN75K85oKq31zqj0LN9PakO8ApdNm62V7tCdEVkp6NlcXZYqjMc7IIU1LeIet9ZcczJSnMdMvPqUhlgo9rLCg9qhaRM+2GjxQYsPWnzw5DKUywMQin2wFiG0COGpIAQhZSN5HTFjI1XcX8LGNSRsNjUJm4Y9TQ28MUKcJpjTAHXmgztayNMIe4wMrZiqXt9RMZCOuyWHux3j9R3GaWXyGlw0BzYy46NmjDQDJzVjJTNeMmOmWbjJ7Kxm4KcmDNV0vUaViQ5PmTGVAVcZsdW8+KoJY83AWU1YqwFvNWMuM+6aib0a8Zcegxlx2CwsZsRj2mteT3Wrc05wNgdAmxukNQE1A1hrvszWeMpCD94aAZyS5ZnrAKYOzjVBOgOsmwva1R2JHuIZYF4j1GuCe2bIN8OT1qFffQp1CKi1o0pVk0Sjoias1YS3GjCX+VqtAXvNc+PzOUxBtLX5Tpq1qKyWnZofljktLGthWQvLniYse5QTUy0umxeXzbyN0UKzFpr95aFZLZv98OecVOO7ywtfHy9J2TXsJjpum6W8pyxlbxYc7rZwuIXDLRx+HlnKhzyW3aLi22Yr9fcvW2jcQuM2a9miNG3WciZM67UwrYVpLUx7JlnLFqc9k+xlC9VaqNZmMR8ri7mPEEWEt0livshHbucvFaG13Q6nRZsJnxhLvQmcgm8QpXz8zgYvpu8lLF4GzZ8jkbLSWrt6oYld9obhNAkJI6LxPN+MqxCV9t2Z+dvHDe/PLGp14Lr4dpyYUC1UzaYvabeOOADWfCtOgMdCbfW+ddH92n9mEF0x7dZ+2q5iviOVjuGlcuNa+ARd2b2a/JVujITfhJeMa6+FiMuIuT7ElzNNPNkdRCQi+3Bb/+porfHaUyKVQNO8YHAe4xfhLuvplHvWlSy5/iMeErTgAf88IGGFYIJVK6f6Rs9+dxb78+gkKgzRdsbj9F/FMGy5tjZeWqZvnAuXT0ioyFJ4zAlpb7noJW7/9neRBsjYq8Vrt4ZI8akXZ0wXuh21hntJ2+lIKjDDIqnJr6VXhNFTg1WKLbSWOY8iEtMOoiB3p7Lw73JSkWhJiSPf2M3eQlJ85fbALfVe98mDxTT/l0x+ESM5C5KvKDy6ijzdzS9uQs5zMCFFdA9oQn2T5fTvYDCzFPiePHe12jKo8AsTEiSL25AvzvV5hYI73Y4KEisG0ToTojb3pkHoBlGYhDF3V00TcybGvkjVCx0a1foDwoTV/nRVIvfcTLj8SlBrxY+Bv3TsX9SGG6yqZ9b03iKa3lsSFVe+MM3SEOXaqfzOTHyxVtp88SkZvtirHksCAgJQ9XDOs9NiMKJ47b6wrhR1X3LBtlPpuajztiP+6HbEGuFapyv838m/dnBSzIHm4epqZ+5FJDwQCYu9uD3xh3Bve90Xx1uMRWLfz5jlQu2vx7v5crdYh38u39dpvbU+FyjKWjmM/ZRjKTtj2xD2xnAdbAy77sBfB4ORM+r03F6vMxh0BxveYB2MeOPzcq3deXHzf4qvyjuTfQAA + H4sIAAAAAAAC/+1dW2/bOBZ+768Q1G6RYpOpJV+SFAgWuW4LtJ1sk3ax2wYGLdG2JrKkoahcGmR/+5LUjaRIyU7SxBmrD21NUofkuX48JKWbF4ZhDodeECU4Nt8Z38/W0xIE/0w8BLkyEAQhBtgLA1p4Q4pIoe/FmDZhv4yslNWMEs/HHwJSaa2XpS7AIA4T5EBSYR6jcAbxFCaxybWBARj5tB6jBHLlU89VlHpOGOyHfogoQTQZgbXOumFbFvmr3183rDc86QDMWMe75VyM18auDxEWhoCvI9bOBfF0FALkmlndLfv3jPx9y5gCXQ9LozUnAcQfXFISJL6fliAQTU/D0MdeRMo7rJBOZz8MMAp9ytAx8OP0eU941veCc04KEQgga5+yPGe46YS+D6IY8pSECrdSM0GeexyWwkyZLAnskvy2e1zBVT6B7Pc1/Z1zp6DN5mAXP8tRnxVlCEYQYG6qXOkHDBETj7r6S3gpMJhVxdPw8tTDFc0xcVZo7icIwQAbeyBwLz0XT02xzYn3k7WbDriKTBNQeJnqQDbHkvHAmcJTbwbDpDIZh+rlHnDOJyhMAlclmxB9A34C1VUxZ1qk7KW9ve30BryiMoW3u5tE0e3tdaNHdL/z29a2oPQv3V4PdEGuwqUEGo3RdKHjzQCTXCFzcxyiGRMc4doMDmOIPMg9MwHJBIpKNQNX+SytDq88My/IK/hiKkqZI1QWU+KQpqHvfgQjKBqN3OITQOeQ8Y+qQkU9taq/Lam+ZTeovlWhPYXeZIp5Wqk1dMufAVHvC+DL6iKaesa5KPKCyWmqhZaqXFSScj6cu7ugPDZwaGB4hTnVMLIqbh7cTDTEEAgmDcTsklhF6YguHBC9Ow4JF2JRIag6nBBXIfDdpCxirT+FzP8T0wgC6GDi0YQ2p3Q4EkfDSIxXqf560Hd/V9Qwa/YdkaGpZAAJc1zRmTBtF45B4mOZWDrZdIr5n3Wp3gtEnRId1ivK8zgCxEClJkngMQvci2J+XLfCuDIlEVUq5csFRCgNpzfiM4UFVbkgzi33UWmYROcbEwRhII2TKbsLr6qT5LSFSkyou11fpN9r6PvEXWg7trQd9wux3KN7xOlhpW9b2/empm+tkrEHC59XMbDS/0RhjMce7d005cIjgjfyKNfv/I2rR7D6DCvTPsL8wCcCLfT+Z0xCCn2OSlj0FDg1VvPz212pIiweqHEiccVLkBJ0ThwolG3d90V02CXI0LK2yF9b2zRYWltCsBzTnquBhVLm6aRkbBJure2uQICPXRXZMKxI6CSzQOQ1BmgCcQ0r4VXE+o6T2ZpHsBFcI24QAzIsNCR48zJE5wSzO9C7gMPRNYbxEBOE6984fhKTaLPzw3yV/feHuV64lZ3//eCczA/z9vurPDi9e0UcARkpdZNnb96IUqqHAJkJxBgEskcW4t8RcDDjqGCipg8nMHCPii5EukQpGfYzd+sURHBjPJ/JcI9SrRQCBS0/mXpjXK2QwOMXwnsjHBt7lMnGl5TlbhUwpjCoKGbmy9vSFm9LrLbelkK6cjB35jSj3M00WhJr+DkL7U46xxbotkBXLFgM6fZapNsi3Rbptki3Rbot0v0lSBcTkcQzD7dQ9/Gg7mnGc7waaJepbs0I88FQRKwZDatqRBs5IYYPjZe6qRXVixLcqye4tzDB/XqC+wsTPKgneLAwwcN6gocLEzyqJ3g0P8EodDW0aE2N6javYWgEEKDzmDNAgsw4C5x7WdC49WNp9n76PNaHsbwnQwvTLRh+wKplQQGUy6FUgTItMEBs/IQo5OI28a0QQRZNxn5YQngzdlDImCUMiga09xC47AmxJkRYZBbxDxI3XBg72nDIgs5HGEzSGXfk+M5NLsbXft1SB/geiNWeh+G1jCeVUFQutCVMT7x8GYH+Q/5sfPq0cXBgvH//bjZ7F0vBjlsp22IsI8Kr4omsotz8NA8Q0VPDDS9FCM2afUV+NQxGgMQcFKinLKJ4cWZ5cJp6risB9mJZQZ5FuNl8C6YXe4fV1M9qiEAdpuYRQ5DMRsS0lGIQVncLCEEFSlZKDntPKYccIOamYBwD5xzieDVFsf+woojuJArOIFZbGgfLIA3ZMIwDFEbRqvqqwyU1kNWWytFTSuU4XEKuiycM78R09+0msLY6g00Hbve6oNcB0O73+r3BZt+xtsfjvvP2PCE8DCCG8UaW1vKCyQZZBf4jRJMP7o71GsExlcVOtxO/vgBoo8xllYksVkEe2nk1HDpQTj5y0q4sPO8raBk5VxNTy76nbYyujTXCF33Gjx3t1OT6KlpSyfXZd8v1CeVkdhG/zNbb1JPnTZeSnXu/kp25ckZpHFkFfu4/inquEEMPHlNB3RTorAJfDx9VUVeIsUeNjK0igXtvRZ1ggPlcdw5QUgbodnCe7kaEta1Oiw+W7ErEg16FYKh+P8fo5XELcwRQ5RATu1ujyojTcqhqPuf+hy0U/BMB14PMHjq/4Pi/rdn+2Cz3NJhViV0RRk2C3fg0uzkkb8GDi4miNN+urNbQm0SHswhfq+9M/ZduilRq0gM7Mqn0mI5ciug5r5P0FI1cl50CkKjHpPG/s/0dcRlnMhepIKQ89nE7/zaSPd820oLnrXT7NhEkmkjCwqTqXyJKkqpdEgsbYVF+EExsrt+lUruD9ATC79m5JmmeTftM/DkSTGJXZaOLeHMSxz6mx0vEgf4F1nP6ExwLHdO4uSG93N7eeQVXf4bj+0MEzizzqLzrVmQybngzBohFRNmQ051PWzbUYR4NvMD1Ljw3AcXBi9vq6RB6/7EcwBW48iTnO0rSVHXFW8wy08Tizl9+RlNqrXZGhWf5flYZ4jW4qttsLXWH7oyISkHP5qqyVH442QMxrGhR6nErzRUnI/l5NOLVZRpirtirCg8qh6Z1+GCrxQctPmjxwdJlKFcHIOT7YC1CaBHCsiAELmUjeB0+YyNUPFzCxtYkbLYVCZuaPU0FvNFCnDqYUwN15oM7SshTC3u0DC2ZKl/fkTGQirsFh7sd7fUdxmlp8gpcNAc20uOjeozUgJPqsZIeL+kxUxNu0jurBvxUh6HqrtfIMlHhKT2m0uAqLbaaF1/VYawGnFWHtWrwVj3m0uOuRuxVi7/UGEyLw5qwmBaPKa95LetW55zgbA6ANjdIqwNqGrBWf5mt9pSFGrzVAjgpyzPXAUwVnKuDdBpYNxe0qzoSNcTTwLxaqFcH9/SQr8GTVqFfdQpVCKi0o1JVo0ihojqsVYe3ajCX/lqtBnvNc+PzOUyBt7X5Tpq1qKySnZofllktLGthWQvLlhOWPcmJqRaXzYvLGm9jtNCshWZ/eWhWyWY//jkn2fju88LXp0tSdjW7iZbdZikfKEvZa4LD3RYOt3C4hcPPI0v5mMeyW1R812yl+v5lC41baNxmLVuUpsxaNsK0XgvTWpjWwrRnkrVscdozyV62UK2Fam0W86mymIcIUUR4lyTmi2zkZvZSEVrb7aS0aDPuE2OxM4Uz8A2iOB2/tZUW0/cS5i+DTp8jkbLUWrN8oYlZ9IbhLPIJI4LJPN+MKxGV8t2Z2dvHNe/PzGtV4Dr/dhyfUFWrWvHydw4Vc7Xla9Z5r2v+mUB0zZRa+UW7kueWUDqBV9JFa+7Lc0X3cs5XuCjif+PeLa68DcKvHub6/l7GK/5AtxeQQOzCXfUbo9WMnBFheIrmOYOz0L4Id1lPw9ShriXRzR/hiIAEB7gXHokmBAqsGxnVN2r2203sz4ISrydEyRmP43/lwzDF2sp4aZm6cSbcdEJcRRLD05SQ8nKLWuLmb3/naYCEvVG8clmIFA+dMGG60O3INalzNK2OoAINhkgtfSO+JoyeaYyRb6E0yHkUkVi0F3iZFxWFf58DikRLCvj4xqx3EoLiS5cG7qj3qi8dLKb5v2TyixjJuRd9Rf7JdeCoLnylJmQ9BxOSRPeIJtTXWU7/HgbTpMAP5LnLRZZGhV/oACBZ0/rpmlydTsi50+3I2LBkEK3TAWl9bwpgrhGFThhzd1U3MWuq7YtUvVCBUKU/IExY78/WBXLPzYSLjwO1VvwU+EvF/kVtuMaqenpN7y2i6b0VUXHpw9Is+1AsmYrPy4SXG4XN51+QSdd45WORR0AAKh/OeDbMB8OL1+xzy0le9wUXbFqlnvM6b1r8j26Hr+Fuc9rc/63sIwdn+Rxo+q2qdvpeeMIDnjDfi93jf3DXtTddfrz5WAT2/QxZCtT8erqfrXLz5ffn4jWdxlvjc46ijLXj0I1TLGUmbPfB3Bptgq1R1x64m2Awtsadnt3rdQaD7mDLGWyCcdr4olhid17c/h8c4vp8in0AAA== namespace-by-workload.json.gz: - H4sIAAAAAAAC/+1dbW/bOBL+nl8h6HpFcufcWrbjpAGKQ16bAtu9XpN2cdcGBi3RtjaypKWoJG6Q++1HUm+UREp24vgl5n7oxjMyRc4Mhw9nOPTDlqbpvZ7t+iEO9EPt+3UjoiD4Z2gjyNGA63oYYNtzKfGBkAjRsQNMH2GftJjKOP3QdvBHlzCNRka1AAaBFyITEob+GXljiEcwDHTuGeiCvkP5GIWQo49sS0C1Tc898RwP0QbRsA+2mw2tZRjkn729hmbs8E27YMxefJSNRXurHTkQ4VwX8MRnz1kgGPU9gCw95j2y/1+Tfx+ZUKBl40Jv9aEL8UeLUNzQcSIKAv7oyvMcbPuE3mREOpwTz8XIc6hAB8AJou/bue86tnvDacEHLmTPRyJPBK6bnuMAP4B8SzmGVeIMkW199jJlRkIuKOyOfG51OMJ9MoD484R+TqSTts3G0Eo/Zr2+TmkI+hBgbqgc9SOGiKlHzP7i3eUEzFjByLu7snHJcnQcE/WTECHoYu0YuNadbeGRnn/m0v7Jnht1OUZsCci7i2wgHmMqeODYIGAGyCSZiaAPGCXXFWpPv0J3iJmYmzk6DEoaqp0t+sB2nJykKeEDApZNBsprSq7tdwVtG60abRtibbfTjw4cQtfKv4mIaegeBVfiqQ1uh8XBU9ON9CX2BGdjH0/ErP9C5JU5Y3AveMfYdgVUZA9H+FLob6iZCajk2d+ZSeWtkrI8RIeQjqbAOoWBWW4PE/fkCDp2C5wQJmZV0gPxFQIrotS7uG8GT70pzEgii0syU3MzXqej+ezZLv7kMXEwgs7NbBOKLNqHxGpdDIZlj+TT1qiJhvT9e3l6uftEaBZEkPn3geNlApR4kAAiGwb/uoWIWDwsDDGQ9TcoDTzAwLwp9SbA0Peh9SuRaYmHARpCHHDrIb8i0tXi3mfDoHrvkb6Z20E43raJq4PbZBnDgLSKemT5uPPQDVmCTWjfwl5/gmHQYxbxYDoh6QF6/0N/E//5Q2/QVY0NjJLTDz/0x+9viEQhIkZz+Ias5Z4TUp96vfPD/Zvmudp2+mzD9yxCHSIv9HsOHOBt2gPHA1Yj+aNHPeGOln6llzLIdw9vwj6kf/S8OzKEQwQd0IczdrehJU2+/98P/R9/5yns7fR5+n8ysh2tP9HSTu6QvnPzirpBD42ZaRDPPoa9yCbyjySiOQcmZujByLEjJ3aetvPwkHZGe3zMN4XggK1H+lGeTo2FmVlKfIz/yiwSj4hmRp5jFSyV9vucePyieVP65cge4DKjsMx9IWaleQPtmNqP9iWyJotb2lJA8sC7N4AYVig6uMiLtYq+qJcskLZr2be2FRK3VfJLyTMMB2UduAf3dmFB6ofmTTSFCl50HHufkiYTTFf0upGblvrP77yeEtec0q5LQ5iAe1gxszN7O/YLdsZmQqmDlOENjwFDbHnLixaq0uPRStXMG1i2GGUm1lj9Lm7xYt48RJUjVECqjoJUClIpSDU/SEVk5AZjGytMpTDVHDDVVWxOWMEqBatWHVaZxFWN3QphY3jPxnFFZkN+IEy9KatWJElD3+jXtL8cSRpL2bM2eFzd4PHMDZ5UN3gyc4On1Q2eztzgWXWDZzM3eF7d4PnMDX6obvDDzA1eVDd4MX2DyRojaTBlV6woU+8bMjQ4IOgjiSi/a/5Vn33fUBt3NySB970SfM3tmV4AvWog0H7SPcIUGDMwkcdklesUdWgXEFjsG3lOtEZywiKurCANK4L8bF0qyWQmhDpxqhYktjcVO0mT7lhjmZRcuJlsZzPpJnYFMzz0H/Lf7qdPu6en2sXF4Xh8GBRWFAua9hiwTEorv6IQ5ZXX5ZiRZZ70U0TMVLMIotRLj31FVKp6nuEDgm6QKx6yBGflwAnZPVqw8LbQtdl4yXcRrp/HqdBLCZxyVGszVCFeWadRhxuO+2SKCdVBAcxzlCGCwxulj+Nl6uOIbM/BEKrJIQZ1y1bGxk+O02XqI9m3J1NC+wyiHfFGquJsvqrwn6QKbkJstjbOV0EbxYmhnSKPRmU3UyUfVnSCbLZWLpapld+FQYRVEH0+pPwkyVu/7LcO+oP9fdM0jG7Xag3aRrvdH7T2Djrdg/2uaf5CsxHIhRgGu3E2xXaHu0ns5J8eGn603htvERxQrbxvN4O3twDtZomJLC3BGCz5wFIP7GOapnjT65mQS4cVbUEczHmuLRR3ouWUgjrbs4J5KJb6LmSgAgwECW5RcqpVmZx6cjqqoTKVG20hx/OzEHA7VC7k1RnIyWIMRLmQtbWQ08UsMokL8aPdhTKRNTKRswXjEGUj62cj58txI1YUpFCmskam8mFJ7kTZyvrZysUTTmQ+++TlJQaYP2+TRHUiAchOvCU1sbkDJXxJbI4xv4pY4534ZE5XUBJbEXgUVEHIKiGqqiEqKiKmrcoWVEZUVkdI5Sk67SSslBDJNpVvq8nR8rJhct4X2HW5G5UVFPIqispKippqiuqKCnlVhbyyoqa6Ql5hUVNlUVNpUV1tUVVxIa26KOtSWH0hP8MmOccmPcs2RTVGVUVGTVVGVWVGRXVG9ek5eZVGbaVG5Vk48Xk4edVGXeWGNGhedHQq8rVI5DBlIccUxRyzFnRURdElMIKHEjk4UZ/yEcONSsiRhx3JAZ6aIlppxUdV1Yek8mOq6o+yjxRXgUgqQSqrQaorQuRVIRWVIfLqEHGFCF8lUh5quVpE6E3kJRmVZRlVpRkV5RmyEg1pmUZpXGs6hC2Ryirz6ArOiojT4NkDhWcVnlV49nXiWZWoU4B2SYBWeipdYVqFaRWmlYWzF393YamgZB53Fy710siWJET+bsVujcxKeC4+BmTdmCzi1sjirUWv4NrI9oFY39x4prjlqHjzRN01R0XOQu85KvJmvuio9s4iccFzbdV3a7qq70yn5V0FgbEuNHM3n7yyi4u40SdbhnxduroKclOvLUrettqXFsXRaiFmWMdLinD+AoLqK4oKY8jwurqE6GW7+Ky7HV8B7ild7igFPoYCPgr4KOCjyqAU8pkz8knCmgr6KOizQvcvLus0alsSajNacz2OKsrEb8h51E5N/t5oPyWBL0o8TJPBF3GXmsIX8Z+cw58qDc/jQ3keXnR1QhkrTpeGz9vKNLhxE3PxBSmJ8OVLp+JVReS65uIl4HPNMvHT3UX2KhPwuHiF6DTpd8E4y5B2Dol1f/0T6/7mHBbdELApPC1aizY7Cm0qtKnQ5qqjTXW5goKbC4ObtfetKsSpEOerR5wrcJSzOPnmc4Bz0THljuS3R4w9FVOeU0zZqEH5XYXyFcpXKH/dYsrqyiMF9hcdWxZfHK8Qv0L8KsasYsytp6DPfYU+FfpU6HPtYswKfir4uYRYs0KgCoGqmPOyYs5nCFGg+5SQ81bccz3+aSfKbTejtuhjmSPRA3MEx+AbREHUf+MgItMfVWY/MwXQTfQ9ggAyq9Wz35bS07dhOPYdIgh3mM5CglgCzNn6g6juSfgL4BYcgNDBkh8AT7iiPQODkYXwd2Jq+imB9dplhOvLh/N51M9xPZ9qt+h+9T9DiCbMukW7A074Ro46hPeFm5pTFXKvL4bqc2VmzrdYEIVqAk6mvECm2MskQuPLJmyXLNgWPHJEZV3iyauPiVZsweOJgGMIMIt02Zt6kWfdDv2HP7w+QQEmsG5tsqwQINDQ4lZ3xOJv1Yk/Xp14gyHWzmQc/Dvphp7nlvpLaeKHY+VGA+IYYQCvooaEpXFTaRyE2BOVGhJyz/RCZgvtZpETeUndaOZMoGZG0im/G0yIoMeSWck/IZyZ0xgimdq2a8fuNK/85xz+JVaS4swdvdpbvIThpy9/sum/yOhnmSU3tv8VOZcT1xTVi0ZzyFBzqHpV8x1vMi7eY8gvbOkDLzGDFrp1eyzs3ZYx7ZjhPHXGrY605j1N1VJXOU33ZCvc3jMWtgVZfBbkkdj9lmzHpgfQiYKD4rhmIp12s3QhXyogypPtfOVvE93cKlaFTBlTv6pqYMZI+i7C2hLtGoVOhAihsTdu5Jpbt5U2CYmpWbyUfZJI/LPO4YpZ1ZFbemcWS+9siIlvJfGhxyjSYTMtxTGOQRSA1V3vbjed8zozc0rTc1/zbYLVUfblWGa9pDO8evU9Lv7D237OBetGZue8zesG/6Hd5Dnc3QYt7m/DinR/nYyBBs3LZid/C99wl2+Yf0urw3/gLmXZt/j+Jn3Jie+nx3Ix+terkzgslcTLfkt/4lz7RfstQVradvIr8BGe0kOWCtX7/X4L7Hf3QLfVBrB9YLSbrWbX3LfedYDRbQ6ih2/TuFhz6/H/xs1ceXmnAAA= + H4sIAAAAAAAC/+1dbW/bOBL+nl8h6HpFcufcWrbjpAGKQ16bAtu9XpN2cdcGBi3RtjaypKWoJG6Q++1HUm+UREp24vgl5n7oxjMyRc4Mhw9nOPTDlqbpvZ7t+iEO9EPt+3UjoiD4Z2gjyNGA63oYYNtzKfGBkAjRsQNMH2GftJjKOP3QdvBHlzCNRka1AAaBFyITEob+GXljiEcwDHTuGeiCvkP5GIWQo49sS0C1Tc898RwP0QbRsA+2mw2tZRjkn729hmbs8E27YMxefJSNRXurHTkQ4VwX8MRnz1kgGPU9gCw95j2y/1+Tfx+ZUKBl40Jv9aEL8UeLUNzQcSIKAv7oyvMcbPuE3mREOpwTz8XIc6hAB8AJou/bue86tnvDacEHLmTPRyJPBK6bnuMAP4B8SzmGVeIMkW199jJlRkIuKOyOfG51OMJ9MoD484R+TqSTts3G0Eo/Zr2+TmkI+hBgbqgc9SOGiKlHzP7i3eUEzFjByLu7snHJcnQcE/WTECHoYu0YuNadbeGRnn/m0v7Jnht1OUZsCci7i2wgHmMqeODYIGAGyCSZiaAPGCXXFWpPv0J3iJmYmzk6DEoaqp0t+sB2nJykKeEDApZNBsprSq7tdwVtG60abRtibbfTjw4cQtfKv4mIaegeBVfiqQ1uh8XBU9ON9CX2BGdjH0/ErP9C5JU5Y3AveMfYdgVUZA9H+FLob6iZCajk2d+ZSeWtkrI8RIeQjqbAOoWBWW4PE/fkCDp2C5wQJmZV0gPxFQIrotS7uG8GT70pzEgii0syU3MzXqej+ezZLv7kMXEwgs7NbBOKLNqHxGpdDIZlj+TT1qiJhvT9e3l6uftEaBZEkPn3geNlApR4kAAiGwb/uoWIWDwsDDGQ9TcoDTzAwLwp9SbA0Peh9SuRaYmHARpCHHDrIb8i0tXi3mfDoHrvkb6Z20E43raJq4PbZBnDgLSKemT5uPPQDVmCTWjfwl5/gmHQYxbxYDoh6QF6/0N/E//5Q2/QVY0NjJLTDz/0x+9viEQhIkZz+Ias5Z4TUp96vfPD/Zvmudp2+mzD9yxCHSIv9HsOHOBt2gPHA1Yj+aNHPeGOln6llzLIdw9vwj6kf/S8OzKEQwQd0IczdrehJU2+/98P/R9/5yns7fR5+n8ysh2tP9HSTu6QvnPzirpBD42ZaRDPPoa9yCbyjySiOQcmZujByLEjJ3aetvPwkHZGe3zMN4XggK1H+lGeTo2FmVlKfIz/yiwSj4hmRp5jFSyV9vucePyieVP65cge4DKjsMx9IWaleQPtmNqP9iWyJotb2lJA8sC7N4AYVig6uMiLtYq+qJcskLZr2be2FRK3VfJLyTMMB2UduAf3dmFB6ofmTTSFCl50HHufkiYTTFf0upGblvrP77yeEtec0q5LQ5iAe1gxszN7O/YLdsZmQqmDlOENjwFDbHnLixaq0uPRStXMG1i2GGUm1lj9Lm7xYt48RJUjVECqjoJUClIpSDU/SEVk5AZjGytMpTDVHDDVVWxOWMEqBatWHVaZxFWN3QphY3jPxnFFZkN+IEy9KatWJElD3+jXtL8cSRpL2bM2eFzd4PHMDZ5UN3gyc4On1Q2eztzgWXWDZzM3eF7d4PnMDX6obvDDzA1eVDd4MX2DyRojaTBlV6woU+8bMjQ4IOgjiSi/a/5Vn33fUBt3NySB970SfM3tmV4AvWog0H7SPcIUGDMwkcdklesUdWgXEFjsG3lOtEZywiKurCANK4L8bF0qyWQmhDpxqhYktjcVO0mT7lhjmZRcuJlsZzPpJnYFMzz0H/Lf7qdPu6en2sXF4Xh8GBRWFAua9hiwTEorv6IQ5ZXX5ZiRZZ70U0TMVLMIotRLj31FVKp6nuEDgm6QKx6yBGflwAnZPVqw8LbQtdl4yXcRrp/HqdBLCZxyVGszVCFeWadRhxuO+2SKCdVBAcxzlCGCwxulj+Nl6uOIbM/BEKrJIQZ1y1bGxk+O02XqI9m3J1NC+wyiHfFGquJsvqrwn6QKbkJstjbOV0EbxYmhnSKPRmU3UyUfVnSCbLZWLpapld+FQYRVEH0+pPwkyVu/7LcO+oP9fdM0jG7Xag3aRrvdH7T2Djrdg/2uaf5CsxHIhRgGu3E2xXaHu0ns5J8eGn603htvERxQrbxvN4O3twDtZomJLC3BGCz5wFIP7GOapnjT65mQS4cVbUEczHmuLRR3ouWUgjrbs4J5KJb6LmSgAgwECW5RcqpVmZx6cjqqoTKVG20hx/OzEHA7VC7k1RnIyWIMRLmQtbWQ08UsMokL8aPdhTKRNTKRswXjEGUj62cj58txI1YUpFCmskam8mFJ7kTZyvrZysUTTmQ+++TlJQaYP2+TRHUiAchOvCU1sbkDJXxJbI4xv4pY4534ZE5XUBJbEXgUVEHIKiGqqiEqKiKmrcoWVEZUVkdI5Sk67SSslBDJNpVvq8nR8rJhct4X2HW5G5UVFPIqispKippqiuqKCnlVhbyyoqa6Ql5hUVNlUVNpUV1tUVVxIa26KOtSWH0hP8MmOccmPcs2RTVGVUVGTVVGVWVGRXVG9ek5eZVGbaVG5Vk48Xk4edVGXeWGNGhedHQq8rVI5DBlIccUxRyzFnRURdElMIKHEjk4UZ/yEcONSsiRhx3JAZ6aIlppxUdV1Yek8mOq6o+yjxRXgUgqQSqrQaorQuRVIRWVIfLqEHGFCF8lUh5quVpE6E3kJRmVZRlVpRkV5RmyEg1pmUZpXGs6hC2Ryirz6ArOiojT4NkDhWcVnlV49nXiWZWoU4B2SYBWeipdYVqFaRWmlYWzF393YamgZB53Fy710siWJET+bsVujcxKeC4+BmTdmCzi1sjirUWv4NrI9oFY39x4prjlqHjzRN01R0XOQu85KvJmvuio9s4iccFzbdV3a7qq70yn5V0FgbEuNHM3n7yyi4u40SdbhnxduroKclOvLUrettqXFsXRaiFmWMdLinD+AoLqK4oKY8jwurqE6GW7+Ky7HV8B7ild7igFPoYCPgr4KOCjyqAU8pkz8knCmgr6KOizQvcvLus0alsSajNacz2OKsrEb8h51E5N/t5oPyWBL0o8TJPBF3GXmsIX8Z+cw58qDc/jQ3keXnR1QhkrTpeGz9vKNLhxE3PxBSmJ8OVLp+JVReS65uIl4HPNMvHT3UX2KhPwuHiF6DTpd8E4y5B2Dol1f/0T6/7mHBbdELApPC1aizY7Cm0qtKnQ5qqjTXW5goKbC4ObtfetKsSpEOerR5wrcJSzOPnmc4Bz0THljuS3R4w9FVOeU0zZqEH5XYXyFcpXKH/dYsrqyiMF9hcdWxZfHK8Qv0L8KsasYsytp6DPfYU+FfpU6HPtYswKfir4uYRYs0KgCoGqmPOyYs5nCFGg+5SQ81bccz3+aSfKbTejtuhjmSPRA3MEx+AbREHUf+MgItMfVWY/MwXQTfQ9ggAyq9Wz35bS07dhOPYdIgh3mM5CglgCzNn6g6juSfgL4BYcgNDBkh8AT7iiPQODkYXwt9jUkinOg32O6/lUqUWvq/8ZQjRhRi3aFHAyN3LUIbwvXNCcao57fTFCn6suc77F4y8MgxMlL4cptjCJrPhqCdsl67QFjxxRNZdEkGOiDFvweCLgeOWfRbrsTb3IoW6H/sMfXp8s/iawbm2ympD1v6HFre6Ixd+qE3+8KPF2QoycyTj4d9INPc8t9ZfSxA/Hyo0GxDHCAF5FDQkr4qbSOAixJ6owJOSe6YXMFtrNIidyjrrRzJlAzUSkM303mBBBjyWTkX9COCGnMUQyo23Xjr1oXvnPOfNLrCSFlzt6tZN4CcNPX/5k03+R0c8yS25s/ytyLieuKSoTjeaQoeZQ9WLmO95kXLy+kF/P0gdeYgYtdMf2WNiyLWPaMcN56oxbHWnNe5qqpa5ymu7JVri9ZyxsC7L4LLYjsfst2UZND6ATxQTF4cxEOu1m6R6+VECUJ9vwyt8murBVrAqZMqZ+VdXAjJH0XYS1JdosCp0IEUJjb9zINbduK20SCVOzeCn7JJH4Z53DFbOqI7f0ziyW3tkQE99KwkKPUYDDZlqKQxuDKO6qu97dbjrndWbmlKbnvubbBKuj7MuxzHpJZ3j16ntc2Ie3/ZwL1o3Mznmb1w3+Q7vJc7grDVrc34YV6f46GQONlZfNTv4WvuEu3zD/llaH/8DdxbJv8f1N+pIT30+PpWD0r1cncTQqCZP9lv6yufaL9luCtLTt5MffIzylhywDqvf7/RbY7+6BbqsNYPvAaDdbza65b73rAKPbHEQP36bhsObW4/8BAYppRXCnAAA= persistentvolumesusage.json.gz: - H4sIAAAAAAAC/+1bX2/bNhB/96cQiA1IBneV7LixB/QhTZFhQNtlS9o9LIVBS7TMRSZVisqfBvnuO5KSTEl0bCNZmnQG2iK6O1LHu+Pd7y7qTcfz0HhMWZrLDP3i/f25ayiCfMmpIBYNM8YllpQzRbwBEhATmkktAo+3WoxEVOJJQoA6xUlGNDFmRP4WAYnlSWIoAqezU84TSVOg+5o4oxE55EwKnmS19bS2NqHs3FJMkCkoOgMC6vsZMjR+qSW0lkZXoIY8SXCa1XSrMaIWJ8WMJIut7O00HycUZ4c84UJb5bZrMydYZM0dNSPC2ewdYbFUWgd+i0eWLZM447kI1QnQseBzImckN2eupKY0SdS2LeKvAkeUMFnau2LGgkbH3HEAbfdencQkERdYvQEF8/qbExITFlXRYRspZgfZaREXUuS1cymJi9jNCHMhjMoO5hxfLWFQ5mYIGs/kCYSZm53N+OUSDqz5i0baYWUgWmwJVyNx+Qx4YKxcO1Rta7HqpoagroRajMvizUGTY92DiqzUO+bgp/dcn1MT6o5KCcQQkzgmzjhL1WoVLLnaftDmucMTHBURQYR65zThEjW4KcHSYT6UEUFJ9vsFERCHxHGiLMUhWXZfgKmcPaoTJQ7PXdbMJElTEr0DozrPILGIiaxf+Oal14LkKtUH3Tljnpflc++SyhnP5Q5l8HIWkq7HwP673o7k6flO0PV2zvMJSYgcX/Akn5MxiMlsHGI4HJXX48m1JNlNmOSgonh9hn4ofjxDXe8fPgFKsV4RGJ4TbRUlWD0oDrg2g6QM3jWvCRNM50rKPJ6h293dXaXzi3sqji8wTdSNfmTN4Q/qNr0x5WKugwtJqjTUEdUWK5PXEQ4lF80LZeWwo2q/j1AVvBOlY3s7qDy6qqEDVGPddteLnWdo/cc0/ZEgZKXp3zRMbz3Vk4icKZTAk8iRYJTmR1BOXdlJ8U5mdCrdTKlLGvqkrWS09T5mKrPWBSuoc9MsOVi0cUfJ5KJVrcuKMpbXqX41ZRG9oFEOBWhpdSllNeyqa3aFr2jW1muSh+cmETrK3byoLMo2Dd8gFVXuVcura1UiNZB0H+EaX5E18vIiHPXFaEcO3BqSOBVUTB6/wRogtsPToA3nMoM3/BbdOvFGCeJ5HMK+ah3H0WpAOcThjJxCvECyc92kUGHoN1C1Y8Fz5rwQRuaTipXl7HaIIBFP8M7A73rBfq/r7Q26nv/zaH+3GbharNffB7neCOR8JTccLZFT2wz2zF/YbhctTTxrovbK5QU4q7NjnGuwdtNCwKU9At9vo+CS6a99Fas0+U6FWLYkMVVS77E4J2Ilur27w+g/QofRRP53AXEXqp7jNKUsPjWJNFjGWyNDFQnSJD1Pck+SK9m+4helXzfKHOXmArN4jc17axZPiLO3EMbHJfhvBFu75wg5YySUUNlacqdKI0cGSHkmp1SlJ4RcjCPOwIFf9fYD/8eGDAAC51pNv3OpNtV7aP/XKC4GI7Q7KnMpzMHQh5cHDiavFq5p86Kx6TeJ4jwx3UvjDqgWX08itP9VkuoDfAyCIfwzHKksFQxbyWyaJ8v6VvUWez+zXU/l0VG/tVGRUfRGy2GIup+HCiqxtqs2br0gKNvwuUDPZ0w3ZiWIfiqtV9V43anYtwD3urF6ecaensV+UtnmQVuP3srWY4Neb72GAw3h3ox89J92DyvhfwZVKiHKl/X9dTmwk+SwmSS1xHpJkis90esN82NZkTZKkXrRh6LklaPCTnNdtcY9gyqov0GQ6PG2m/0nv6xNsKukd1o4sT43rlz7FmezCcci8mAHVOeX1p69shiFqwRIdyxPbofY9xpi722H2Nsh9naI/fCDSKqksnEOeee7HELq+a855MMPgO/7ywOj1zP6pcG3iJZHDhhPj61XRszTmlsbdbeD6ycxuGackec+t37CZ9iOrbdj6wcZWw+2Y+vt2Ho7tt6OrR9ybH3n/PWx4fPqifAjdiD/p0nwvfH4dhT8PY6CO8UJUAaodI4/qUujtQz2DFlem1dFkG7Nl9ASxwtH6bsoGFl8U4LCjKJqW0nmaQInZ3H7E+9O08fWWLMGmUvfRmSK88QVXjbXhcD1h+B1+Fe1DkgVdO/EAFOLXyIIC7ZaXJ6WH67b+Ah9yYm41uDVBW+tL8uDGjUmzZJd+cp6fadxLstyOElKiFvHE7ZNbYOsAcZLo1kpDfIdZOOIHLiK5cKiRca295qDa6hjTWllx5JVJtavG5uGea1PYRvV47brFW/ddfuot8pHxVDC9iXcDu2I7I9STVTnts6jaG7hIgLMgS0GVOtTs5GdmR87LPyNw+JDWZ83CQzmWvTgobEW3Li18MY2ZB4lZI4rFGdwzKFCcZuEj3H6t4+d5VD1dglW/T4irFNOxcx/5tJj0goJVG0kv3wRlMPYskM0GGWxLKXhuf4tWrG4MMm4hOA2ekSDRfihwLce+vaDNVVBA+vnwH7o+zZnMTJGPevnIDLV+XN5BtU0WJG28i32xq/sje239Pbsh8VIAe1Htr6lLjXzfeW6V0YfTw8LHFeiyMUd88wlK/7LW65nUmgUjCajHh6Sob8XTKLBq308HZEIT4JeONzzQyN8USFHv3P7L0EqTdcFOAAA + H4sIAAAAAAAC/+1bX2/bOBJ/96cQiF0gOXh3JSdu7AX2oc0ihwXaXm6b9h4uC4OWaJkXmtRSVP5skO9+Q1KSKYmqbaSbpq2BtohmhqPhzHDmx6lyPwgCNJtRnhUqRz8H//1jaCmS/FlQSRwa5lworKjgmngPJCAymisjAo8PRowkVOE5I0BdYJYTQ0w5Ub8lQOIFY5Yicba8EIIpmgE9NMQlTcip4EoKljfW08ZaRvmVY5gkCzB0CQR0FObI0sSNkTBWWluBGgvGcJY3bGswkg4nw5ywtSpXneFjRnF+KpiQxisPQ5c5xzJvazSMBOfL14SnSlsdhR0e6VumcC4KGesdoHMpVkQtSWH3XEstKGNabYf4T4kTSriq/F0zU0mTc+HZgPH7qEFiJCU8qTPAdUTKX+YXZeyVLBq2a4nr1M+ICymtWR7mCt/2MCj3MyRNl+odpJLPh8DPl+LGvzKHRf+hiYlKlW0OW0H+sx6l15gVJmparcNq+hMytxbqMG7KN0dtjpPsNVmbdy4oV2+E2aghNPMgI5AoXOHU6wiU6dU6Iwqtftzl+XMQIpUQSaR+54IJhVrcjGDlcR/KiaQk/9c1kZBsxLOjPMMx6TsUwNTRnjaJCsdXPm/mimQZSV6DU717UFimRDVPdftkG0Fym5mNHlzyIMiLVXBD1VIU6oByeDmPyTDg4P/D4ECJ7OogGgYHV8WcMKJm14IVKzIDMZXPYgybo+puNr9TJL+PWQEmyl8u0Xflj5doGPxPzIFSrtcEjlfEeEUL1g+aA6HNofJCdO1rYobpSkvZx0v0cHh4qG3+4ZGG42tMmT7ST2w5/EHDdjQWQq5MciFFtYUmo7pikLpEwnk8w7ESsn2gnCJ2Vut7D6U/eKdt7KqD9mJaF3qJGqyH4Xa58wV6/yldfyYJ2ej6Vy3XO0/NIqKWGgoIlngKjLb8DHqmrzpp3rslXSg/U5mehj4YL1lrg/e5rqxNwRrP3LdbDpZdcFExhey05KqjzNRdZl5NeUKvaVJAA+rtLpWswVZNy27xLc27ds2L+MoWQk+7W5WdRfumFRuks8q/qr+71i3SoEX/Fu7wLdmiLq/T0RyMbubAqSHMa6BmivQVNiiwm54WbniXWcARdujOjncqEF/GJtyjNvBsrYGGYxwvyQXkCxQ730mKNVB+BV07laLg3gNhZT7oXOlnd1MEyXSOD8bhMIhORsPgeDwMwh+nJ4ftxDVio6MTkBtNQS7UcpNpj5xWMz62f0HdIeotPFtC8zrkJThrslNcGLB234HAlT+iMOzC4IoZbn0U6zL5WqdY3lOYaqk3WF4RuRHdfvwacdQklZ3ClyU9kHeFs4zy9MJWuaiPt0X5KKuXrUiBEoEit6p7/q4rp+90rCvlEvN0C+WjLTsbJMGvkGPnFTJvZUL3QhALzkmsoO105C60RR7HZyJXC6prB0I+xhnczd/Rv4z6cfh9Swa6tXetoX90qXHVG7iAb1H5bQPvXndsxtqNobc/vfQwRb1wS5+Xt46jNlFeMXu1aB1Ufck2swATf11BjgDbRdEE/plMdQmJJp1KsyhY36VSv8XVZ9WNdJGbHnUUlcfdKOrHCBpOnmocw7uh2vleBEnZxbYltL3k5tZUIdznci+qb0UfNexzIG9z6/npkj8/j/1DV5tPei8YbbwX7HAR2+42gCZwbqYh+luh/UZsnkOXYkTHsqnftAO3SE7aRdJIbFckhbYT/bJjfaw60k4l0ix6W7a8apA3aK+r1/gHRCX1N0gSM2D2s38XN40Zcl30LsogNie3dWh/xflyLrBMAtCAmvzK28sXDqMMlQTpgRPJ/Rj5UWPk4/0YeT9G/rbHyI8YBVItlc8KKC5f5RjQTGDtJj/9CPax43tr1xc0tv8c2fLECROYwfHGjHlek2Nr7n50/CxGx1xw8qVPjp/xHvaD4/3g+JMMjsf7wfF+cLwfHH9Dg+OPTkCfGttunsk+4fXgW5rFPhos74exX+MwdlDuAOUAGVf4gz40xsro2JLVnX1VAuXWfg2scLoOlDmLkpP1Jxcozimq1SqyyhjsnKfdz5wH7Rg7Q8cGnq1im5AFLpgvvVyuDx6bj6Gb2KwH19fAwYGSDldk1TfbLixCfxZE3hlA6YOczkfVUYOaknanrkPkvH7Q2o7jMMxYBTtb23Bc6fphC4Bc+cqpZFDmoAgn5KWvR9aORGWhdnWtICLUs6bysmfJJheb183sJXabD0SHQfmSQ39IRptCUs4F3NDBGTB+z/9dWYWa3I75muYXLgNu9+cwoCdfWEVu/X3qLAh3zoK3VRfeJQ+4b9GnzoTtQMWDgyr2KfMkKXNeYzWLVk41VtslfWzQP3/u9APShx5E+nVk2KAaTNlfWzKTyrrf15dFcfNDVM1Dq3ugRSLrZRmNr8x/ZJWLS5fMKqDtYkQ0XqcfikLn4ch9iNaphMbOz5H7cBS6nPXUFo2cn6PENuM/qj3oq4GTaRvf4ip+4Sp23zI6dh/WgwN0krj2VrY03PeXMDdi9P7itERrFVZcn7HAHrLyl7sKMxZC02g6n47whEzC42iejF+c4MWUJHgejeLJcRhb4esaH4aDh/8D0iZvTe82AAA= pod-total.json.gz: - H4sIAAAAAAAC/+2dW2/bOBaA3/srBE530WLdGcuXXAr0Ie1MFwXaTrfJdIFtg4CWaJsbWdRQVBJP4P3ty4suJEXKTnpJg6oPaUxSvByec/jx0GKuHwQBODvDaVawHDwNPp4OVApFfxaYIi0NpilhkGGSisRrnsQTE5wzUUR+CspUmTMrcMJepTwzHDSpMWQwJwWNEM8A7yhZIbZERQ60MiiFs0TkM1ogLX2JY0cqjkj6giSEigrpYgYfDQfBKAz5j+l0EISP9apTuJINHzVjCf4eHCWIMqMLbJ3JcjHMlzMCaQzKvI38/5T/3EihoBgzq7dgkSL2KuYpaZEkKoXCbHlCSMJwxtOHMlEM5wVJGSWJEOgcJrl6HhvPJjg912YhgymS5ZXIK4GDiCQJzHKk12RkxK2cBcXxO9JMphKyNWGX/PNooiVcVQMoP6/F50o6dd1yDKP6Y9Pr0zqNogxBpg1VS33FEJXT485+Ty4NAcusfEkuTzBraQ5gZSJ4UVCKUhY8h2l8iWO2BGaZY/yXLLfc0zJKTaDkUulAOcZG8DBaohO8QqRoDSYSevkcRucLSoo0ds0NoR9gUiB3Vq6ZFk/7aXR4GE32dEWVCj8a73NFHx0OggnX/eHPB4eG0v8UTyZwDCsVbmZgqzGCGEV4BeXM1XMO5oSu5MRxqa3QWY4oRtozC1gskKlUK3hVjTIc6sqzwmmVoSeLqbQlIuZiyR3SkiTxazhDptHYJd5Aeo6k/IQqtNTTq/qHluqHoy2qH7bqXiK8WDK9LmUN4+ZjytX7Aia2upimXkouy3C6OFFaGLrSTSVpxqO5uwsh44CRgKErpqlGUGZp49BG4qmMwnSxpbJRU1lL6bgu/Mr17h3hUshNhRDqcMxdhSF3IEQkS78h0v9z00hTFDHu0YwyJ6I7lkRJZq5XSn8xSuLfHTnSmpPIFKiaGciXOS3p1Bh2jOawSJhdmRqsGmL1b2Dl49TUKdNhPRQyzzMYoafBw4zEwCpYpFja4fMs13u3MXpXqoqpWEo6F4hStahem8/UdtSWhTnCylOpxZKeP1lQhFKrn1LlY3TVHqqmM2LejLzN4CbtrlGScKfhbTj0NjytJ+czmqco9rc98ra972nbq2rywdrztcys8UIZydkci9YBsBNfcuqo1rrp8G9aPkXtZ2Sa9xHpDd5wwPB7oTlfWMRzYoZNf8GUyYK3vxxZGaR+oMOV5C1fwVPoOXejyLb4JDEZccz5MAwP+I+DQ7FkhgfGkjkXLbeXF1GzXo+qZsQX3fBwbFSgr2CtuZHEyOspVqkpawbpArEOUaKrTLadF6tHmBMSesSdIYO8W/SMU+cloeec3COEL9DZbM1QfsY45ybXUVLkfM159gk8LH/9BAa1c3n2v0+aq+E5Afc1MpH//wlsPj6sVqynD7lf4B0XvvP08WNz0rq5oLSInMHUdtPGovgSRkwK2LBYkKAFSuOXdRNmvVxHJRCCoy59MbyaLnbe3ZdKSY3VQ6QfL/GctTMsonzPpyIg8+C5kHnwXs1A3KZIxUZ1srRm3bQOdNOSud2mRcR2Ajzb0aoqr7PVsGTBt+V6H6kx9vTb06+ZcDP8nfT42+Nvj789/vb42+PvV8RfxqckX2HW8++d8e9JOQXsx0Dgu4y8h57Q+/Q7C71/0ZA7TDDMX1RbhGbtBjNIW1wsz3Beo3TBpICHRjpyFd+2LRDu1DjdEAn/pDDGSJr28GuEmUP3PO819CwdhNkUF9QiPcpPyhMq25/Di4UjtVL2do44sfptlbG1+2zuP4iSdo5iQLsqRX52KhVbh2OFZHZeuaRYtee88L+lXtkuFUjP76jIyRAbfReCcluPReJl2cyoc8NyW4QXK49LSzPENZGvcYu2f8lElULtity0+GpvYRbnkxojiuQCOk8IA1vcgVq/fi8h2Rpn7uuvA0oYjM5tgfIlOMtQ/FqxitnR+xYFC2br4BEvcxscuNGaf33NW9lsvEu/kS4krC8QW4FAm93bA0EZ+nIesdbn39e6VUMqF0jbrgll5tZB2e1ZtTjwzQW+wHEB61V800YNcezedOAKXmHLF8+K6FypmuU8VqWlihE7vkBglXb7ptrRfDxtdXENr7oiF43uiL2tqRQi+uMCyIQsnsMctbRIOeBWccfeWx/H1njI99TFSrF/VFpoh+U8uLDf40KPCz0u3G3UoOeFhheqUEEPDD0wfC/AoAV0DCekx3OMjC8Xzhl5wjkHjnCOW8o+2vESTxf1dJDPrt/mdRBQJwV5BdoI1T4vspHIJd1GwqH3pEhK+tDhUdv96CYlPy11E9MWauomJz89+QlqG0X5fdUWmuoiqq6DHHtKXHTlJywPZXlJa1fa6iKuLdTVRV4d9NVNYH4K20pinTTmJjIvlW0jMy+dOQ8UbxLU4f0Xq/BdYNqOqLYDru2MbF3Y5kG37lNUD8Z1oVwnzlkhoPIsqPoWVPBOzZZ9xNmGuy7A80DeTqDX9itu4PNAXyf4dcGfHwC3ONY2CLaH0AZCp1k1qpplDhX1kVcXfXUQmP9bHR4S2+WrBvdhCLqtbSXhntHcoaudIc0afU9pPaX1lPbdUFodS+sx7R5gmvZlnZ7UelL7QUmtFer+9l+Rso3vc95JvsMIpufkMQz7EOYXCmGOR1voeNTTcU/HPR3fjxhmTIlovIfk+xTLDH5Vs9aTck/KfUyzh7YdYppbqW3cU1tPbT213ZOYZo9t9zO22ZNbT259jPOuYpy/USoA8TYhzgdlz4UbEr5D5I6Hqi5RTLsjM4+WaAU/IJqr/ocHKpmtE1RdWaCe4wtno7XgvJghyj0+ktdJqLoYWmUJF0S62OXS0wawrl2vI5c3ZXheSa5yXaxdXX6qh1srVQPiQpHgWPGw47JTjZa13OZuEN39gj8LRNdSu513szbCD43UBbqy3krX7lCtm7dDw8a7J8kH7SoM5wsm+q5ip5tkS6HpXwrHKV+gY3TkvuDAabxgxWcFO4pXAi6X/JtIV7Z0pjzroyK7/i+ZcXiIYHyB+bLCmWAQlLU+dot/tE385eqkKwzXdinj/F9VN4CZ2+qvSHMXLidXDUjLKHJ0oipyvi/jnnHw8z/0OmAhL8BovX/Ek88iUkhdGA/tHOUlQTg0VGCLRQqTf5KvuaBXHqvUSzgtcxdF5KaNU1y6U3PyP+dbjlxLaqx8DLq9haH41osHt9T7uu1ba/5XGfxNjOQcZ3/Q5HidRq53yJQJhb0JdZmQx26+O2Px78c2ckN2IwP6QiuHeanVt7OdG8nih7QnayK/oT1NfSvR9DMWoG+kz030wqPWD3w7K5CjRMW+3GG7Sjrjob3pagQk8nw7VH9rjh2vZyp8k7FzU10DC5fetnjWA9fuzukjuBAG09XAqO6+mXB9Q2RvxXexn3GJ/6Y23GFVE7+mT26i6ZMfRMWtPzkiw3p1LKK+XZBcPqltvrpAUAVPmscyzMGANg+XMjurOqNPL5hqcRpd9w0XDMJGz3WdB6H+YTzUc7Q3rEfa72H5R1ZOqzGIuHZb7fyt6BXv6RXrrYwm+gftRoX9WO9v1RdDfH8RedQA/jh5UYaPqrjWW0VgOF0EvwTvSqgDhTzYA/swPBju7UfocDKGkyFEo+lkOtnbn0bh4Xw+jVThizpcNXyw+T+Bl8Fjl2cAAA== + H4sIAAAAAAAC/+2daW/bOBqAv/dXCJrsIsG6M5aPHAX6IclMFwXaTrbJdIFtg4CWaJsbWdRQVBJP4P3ty0MHSZGykx5JEPVDGpMUj5fv8fCVpdy+8Dz/4gIlaU4z/5X3+bwnSwj8M0cEKmUgSTAFFOGEF96yIlYYo4zyJuKTV5SKmkmOYvo2YZVBry6NAAUZzkkIWYV/QvAC0jnMM19pAxMwiXk9JTlUyucospSiECfHOMaEd0hmE7Dd73mDIGA/xuOeF+yoXSdgIQY+rNfi/d07jCGh2hToMhXtIpDNJxiQyC/qVuL/c/ZzJYQCI0SN2fqzBNK3EStJ8jiWJQSk8zOMY4pSVt4XhXw5xzihBMdcoFMQZ/J6pF0bo+RS2YUUJFC0lyIvBe6HOI5BmkG1J60iatTMCIpOcL2ZUsjGhl2zz4ORUnBTLqD4vOSfS+lUfYs1DKqP9azPqzICUwioslSl9C2FRGyPvfojvtYELKqyOb4+Q7ShOT4tCv3jnBCYUO8IJNE1iujc19ucor9Eu/muUlFoAsHXUgeKNdaCB+EcnqEFxHljMSHXyyMQXs4IzpPItjeYfAJxDu1VmWJarOynwcFBONpVFVUo/GC4xxR9cNDzRkz3+z/vH2hK/1M0GoEhKFW43oG1xuhHMEQLIHau2nN/islCbByT2gJeZJAgqFwzA/kM6kq1ADflKoO+qjwLlJQVajHfSlMifC/mzCHNcRy9AxOoG43Z4j0gl1DIj6tCQz2dqn9gqH4wWKP6QaPvOUSzOVX7ktYwrD8mTL2vQGyqi27qheTSFCWzM6mFga1cV5J6PYq7u+Iy9ij2KLyhimp4RZWyDmUljs4ISGZrOhvUnTWUjunCr0zvTjCTQqYrBFeHU+YqNLn7XESi9Xss/D8zjSSBIWUeTWtzxqdjSBSneryS+otgHP1uqRHWHIe6QOXOABbmlKJzbdkRnII8pmZncrFyieW/nlGPEl2ndIe1xWWepSCEr7ytFEe+0TBPkLDDozRTZ7fSZleoiq5YUjpXkBAZVG/1ayo7aspCX2HpqWSwJJcvZwTCxJinUPkI3jSXqugM3zetbtW7y7hLGMfMaTgHDpwDj6vN+YrhCYzcYw+cY+85xnaqmriw8nwNM6u9UIozOkV8dN83C98w6ihj3bj/N6WewOY1osx5ifAG7xlguL3QlAUWfh3fYd1fUGmy/odfDo0KXF3Q4kqyhq9gJeSSuVFoWnwc64w4ZHwYBPvsx/4BD5nBvhYyp3zkZnjhPav9yG4GLOgGB0OtAzWCNfZGECPrJ18kuqwpIDNIW0QJb1IxdpYvthEjJLjNnCEFbFrkglHnNSaXjNxDiK7gxWRJYXZBGefGt2GcZyzmvP7ibxW/fvF7lXN5/b8viqthNR7zNaKQ/f/FX33eKiPWqy3mF9jEue8839nRN62dCwqLyChITDetBcU3IKRCwJrF+jGcwSR6Uw2h98t0VAChf9imL5pXU8XOpvtGKqkWPXj56RxNabPCIMqPbCs8PPWOuMy9j3IHoiZFSjaqioU1q6a1r5qWqG03LcyPE/7rDa2q9DprDUs0/FDE+1CusaPfjn71grvh76jD3w5/O/zt8LfD3w5/vyP+UrYl2QLRjn8fjH/Pii2gzwOBHzLzHjhS7+NHlnr/pil3ECOQHZdHhDp2+xNAGlws7uG8g8mMCgH3tXJoa77uWMDdqXZ3gxf8k4AIQWHa/e+RZg7s+7xb07NwEPpQTFCz5DA7K+5Qmf4cXM0spaWyN2v4HavfFild2u/N/QcS3KyRDGh2JcnPLCX86HAqkcysK0KK0XvGGv9b6JXpUn3h+S0dWRlipZ5CYGbqMS+8LoYZtB5Y7ovwPPLYtDSFTBNZjJs1/UvKu+Rql2e6xZdnC70529QIEigC6DTG1F/jDmT8+r2AZGOdmWu+FiihILw0BcpCcJrC6J1kFX2iTy0L5k2W3jZrcx8cuFPMv71lo6xWztCvlXMJqwFiLRAou3t/IChSX9ZbrNX971vVqgERAdK0a0yofnSQdntRBgd2uEBXKMpBFcVXTdTgt93rCdyAG2T44kkeXkpVM5zHorBUvmLLFwiM1nbfVDmaz+eNKS7BTVvmotYdfrbVlYJnf2wAGePZEchgQ4ukA240t5y91XWszYc8pimWiv1caaGZlnPgwl6HCx0udLjwsFmDjhdqXihTBR0wdMDwWIBBSehoTkjN52gV3y6dM3Ckc/Yt6Ry7lF204ySeNuppIZ9Nv81rIaBWCnIKtBaqeb/IRCKbdGsJB847RULSBxaP2pxHOym5aamdmNZQUzs5uenJTVDrKMrtq9bQVBtRtd3IMbfERlduwnJQlpO0NqWtNuJaQ11t5NVCX+0E5qawtSTWSmN2InNS2Toyc9KZ9YbiXZI6bP48Cj8Epm2Iahvg2sbI1oZtDnRrv4vqwLg2lGvFOSMFVNwLKr8F5Z3I3TJvcTbhrg3wHJC3Eeg1/Yod+BzQ1wp+bfDnBsA1jrUJgs0lNIHQala1qqapRUVd5NVGXy0E5v5Wh4PENvmqwVNYgmpra0m4YzR76mpjSDNW31FaR2kdpT0aSqtyaR2mPQFMU76s05FaR2rPlNQaqe4f/xUp0/i+5pnkB8xgOu48BkGXwvxGKczhYA0dDzo67ui4o+OnkcOMCOaDd5D8lHKZ3q9y1zpS7ki5y2l20LZBTnMttQ07auuoraO2J5LT7LDtaeY2O3LryK3LcT5UjvM3Qjgg3ifF+aKYOXdD3Hfw2mFf9sWbKe/IzMI5XIBPkGRy/sG+LKbLGJavLJDXscBZa61/mU8gYR4fitdJyL4oXKQxE0Qy2+SlpzVg3doeRy7elOF4JLmstbF2+fJTNd1qV7XqRSUKJCu19StBVK/r/5lDshRKbX0lay3zQCudwRvjYXTl1anV8GZGWHvkJP6kvAHD+lyJepjY6AWyhazU74KjhMXlCB7a32tgF+SCbQayNC8FXET6u0hXjHQhHep2nt7+F08YM4QgukIsmjAU6HlFrzt28Q/Wib8ISqqeMCUXMs7+VU7D12sb8+Vl9sbF5soFKRV5Bs9kR9bHZOw77v/8D7UPkIv3XjQeO2LFFyHOhS4M+2aNdI5+0NdUYI0hckt/mS2ZoBcOY1RbWA1yE0VkFo0SVHhRffO/5suNTEsqmtzx252EpvjG8wb31Ptq7Htr/ndZ/F2M5BKlf5D4dJmEtkfHpAkFnQm1mZDDbh6dsbiPYStxDruTAX2jyKG/y+rH2c6dZPEs7cnYyB9oT2NXJBp/RQD6QfpcJy0cav3CdaDyMxjLlJc9W1dKZ9g3z1q1gHid62DqHs1y0HVshWszNh6qbWHB3DkWq3phO9RZfQQTQm+86GndPTUTrl4M2VnxQ5xnbOK/qw23WNXIremju2j66JmouPGXRkQ2r0pBVC8VxNcvK5sv3xsocyb1ZSliYEDqiwuZXZSTUbfXHyvpGVX3NRfsB7WeqzrvB+qHYV+tUR6sHii/B8XfVjkv18DT2U21c4+idryrdqyOMhipH5QXKexF6nzLuWji+wuLOwz+H2fHRdaoTGd9kASGkpn3i3dSQJ2fi/t5/h4I9vu7eyE8GA3BqA/gYDwaj3b3xmFwMJ2OQ9n4qspS9V+s/g+n66MejmcAAA== proxy.json.gz: - H4sIAAAAAAAC/+1d3W/bthZ/z18haLtACni9luy48YA+pOlaFGu7rPF2H9bCoCVa1iKLKkkl9gLvb7+H1IcpiYrtfsVu+BLY55DU4fn88cgBb48syx6PwzhJObN/tv760MkoFH9MQ4oVGopjwhEPSSyIt0ACYhQyLofA15Uchv2Qo0mEgTpFEcOSGMSYv/KBFKdRlFEoSmYjQiIeJkDvSuIs9PE5iTklEavMDytzozC+UgSjeAqCzoBg97rMzmjkRo6QUmayAtUjUYQSVpGtwvAbnATFOFovpS6XzUTeDI/COSYpV2Rc80lE6DPkXQWUpHFj/fWYP1GU4nZ2VQRJ/8EdDr3+wO5UyTSYoGO396RjOe6wY/W7Hav7+HT4qD7uB7/fRz1kK9QPlef6iCNGUuoJsewLSuaYz3DKKgvZU0LnSGzdjkmMq7wApQEunaUkz9Gi2K3T7dbEmodxwayz2Izc6DQELD4TLkAi/zWa4Krz6Ea9QfQKS51ymmJl0KoqPw39CyLdvcqQ/uhWSTHH9BpFQhPOvKqHCAdY2r6mCBSFQXzGRnm8CFnqxgyDGb+EsNgkajUmFFUnSRgHo2Uitd3Ga3pXVVIYHaO59INrYRyLE4vjBa/5lJWzxaMqjFVnu8UpioMtFneri7d6MDjac3DiCwLGYU1vs0W0SuYbIjUMoRbH2OOQBxrjRkIiTYAnhPFpuBCzbR3jBeSzy/AfufxJ9z+1MZC7tHMl/c6pUlVvIGltNt0UQlcGKMjeVCrPNma//e+ZhknKiVvqnEHGbAQHEOkVuKgmGUzDKDoXGU7aX+SungOpyzmFP6dDkbuc00bumgqB9DEunqKuly3nQhZ0hr3GQmpKaY0tWc9gzXQeN03FEQ0w38IKeJFImVg6P06TWy9KGWSMp+/tH/OP7+2O9TeZAOUqneCfEkoWy/f26lHTKuucy6HyjBmmIWbNYUVKeoE8LvXhNoZkmelFuV5zEfBEWbnts209oMyyTKOtkMtUZ/+R1BglFLitWwjRdV1ut1GW4WwGGS3CDIBKdX2ZOtSAOq0HlByxXUARIaf9dMdYKrLXTuEkJ73N0yPUxvU8Zf8VRAJFBbHzAjPUCtcEUW1xhGLPZq9xHPCZzJMNHm6bthVGCGW4Og3iS4r8EMe8Xuw3VN7eN6i86DpoyTBeSmkms44LNaeNE8YtnHqV1ycpHQfm/C/0pdHqlUkGFUdteVI6FtsYVyKdMs3DJeMmf7SzDRRpFNtGObITDI4UcxTogXAiZguPScXyJ02e3kfBWD6mWCbfaURqiYHiBCNtZc/S6m/XmIIzYs2OoKp5uC1o8jpYlRLyknelFRIKQJJg/3VWI5v8TyozFHF8LIqJrCVjtoy9cfaRppAkx35K5YEOKgggH5+NPTij8K1LU8cKY9hR7OGn/8LY4gvUrL9+hBMkPHxcxOaHR9+0jIlnf/FSVrO+kPlFhqwariN4l7NwyvXMvAa+E0awLsEq1ru6wNsWxHpCIJRrTk4y2MdFjQxjP7wO/RRVqpC+nspzelWyBVqErCnXJPWuMg/VZKJ5HvNCMXUMloN/zaz2xFdmL9l50G9hiRbbHGzWXkgSjfdF4kCpFU8wSfAMscbRSikE2mlZJeg26Mp+dzo+HcIW1BgzGOZnq78nGEYz7eAgzKbOiEEwh4hgZiHjBOrPfPwxRTEPI3zcfTwcdnZGNVll+g5gze1tIcxqdRjwJkM3r0Fnsbe0hkM+s37PjWngzj3DnYMHO4cFdY5qEajP2zn1FeQGmcT07HfkpvI6rZRnlAdf9SVWGZLPAQVNCKK+BSvYVX7RGJsNFEYeFhRGHynG+Zpv1B4A9jsx/SvTvzoM9DfYo/5VjPkNoVcC7AlYCP4YmBbW/WK8t5lJrIu1SUwvy/SyTC/rQeGZgellmV6WQTOf0MuyPhXmfNWeljVZWsfFmI4VYdPm2g0CmYaXaXiZhpdpeJmGVwYQn5iGl2l4HQZE7N97wwvACh97kQgz+W848JWNpWd8PtzreGBLQXMfP96DJpe7WOwA8Dr7qMjeXiiyd6cinx2CIvt7ocj+nYo8PwRFnuyFIk/uVOTz/Ti7/QqKtc4uXlnvMmOY3vUD7F1rWQfXvt5tF6aDXT2gnJoDijmgHMYB5XR/e9gaePP5nes2uANaFGMvfrsc3dXGFsM6Vkqj++hji4evVtbtLTz/EHrZkCB5iYVME9s0sU0T2zSxTRM7w4hD8ysH8yuHQ0CIjvvAIGJrS8zKQeLLXwxG/FIY8SU2ENFARAMRDUQ0ELEOEWvymT6i6SOaHzrUQSKgNg8zBvBPOAngwDmeE7ocT5Ycs8/uC67ML1XbkNsbqWcDz+4Znkk//w5ecO75Pswrzho2cQw2MdjEYJM7sYnsURUAxUvSsiW144+0dvhvG4NX2vDK+cUfVsqEXxvIcs8dpZnQ3qF3lfZ4Ewas1MCKa8CKASsGrNwJVgIyDgglKRcGNI2Tr/jKq9SyQSIPDYl8pebJnu/je37TdZTvwGbeDM/Rn5iyTEqnn5H5MnuUj+hVdp0U1Ku1l8psSmMsul/lWhzPkwi2GwfNy7GO6t6g4IsKsCuuj/DxFKWR7gYLlavDifIKrWpyKX3KFnfhWJcZllP4xeU7CtJTuCQprvxS06r9McV0mXfzm4hQuZPLqVADXL/tpjSQ8vij2r4UzaEoKq6Hqnq0qlNVIVvg10JpStmC/AtF1Mdnuntm1hrNC6261hxME2rmFFrWTNmkYvm4cZZOxdUxmktiOla+7iO9FdxNVsiLkmotcHqpavZ7IYhd5TYkFjT94NzG2ZYURsrwKFtIxajf2vDdNsNXq1pLdt5o8gLqfFmblybffIXQ+gdS34d7HBV1KbtyUAKWMuuWt12Rm5+cAhYVF1llRWA9LQkBENH15FwlZXtOxSb2ydp3bKerfOmpX5TzqX2ifHbUL72uylmDN9tVPjt+lgk/FHsQIFxxnI1PURceqAurT3H76pf1zWf2E1+Vt5Clor5/SJzd5jQ6zwtl+Z8EwvUyWipP9Pag52J3cOLjrjscnPa9fnfiOt7kyWDiof502M8GX5fVuHu0+j/y+qz8nnIAAA== + H4sIAAAAAAAC/+1dW3PbthJ+96/gsD0zzoyao3uszuTBcU4ymSapm6jtQ5PRQCRE8ZgCGAC0pXp0fvtZgBeBJGjLcR1JMV4y4i4ui93F7oclHVwfOY47mYQkTgR3f3b++txKKQx/SUKGNRoihAokQkok8RpIQIxCLlQTeFyrZtgPBZpGGKgzFHGsiAHB4o0PJJJEUUphKJ6PKY1EGAO9rYjz0MdnlAhGI17qH5b6RiG50ARjeAaCzoHg9trcTWn0SrVQUqayAtWjUYRiXpKtxPBrnBgRHG2G0odLeyJvjsfhAtNEaDJu+DSi7AXyLgJGE1Ibf9PmDxQluJldFkHRf+iORl5/6LbKZBZM0XG396zldLqjltNvt5z205PRk2q7H/x+H/WQq1E/l+b1kUCcJsyTYrnnjC6wmOOElwZyZ5QtkFy6SyjBZV6AkgAXzlKQF2iZr7bTblfEWoQkZ1ZZfE6vTBoClphLF6CR/xZNcdl5TK3eIXaBlU4FS7DWaF2Wn4X+OVXuXmYof+yWSURgdokikxOUHVbTQxyHJBivYqWKJl7d9GV9QmuCFspIl1JzjqCOwEtRMbiTseVUJca6td3gDJFgi8G75cEb3Qu84CV42DkFzfG6K7hSi4r5jvpqfo8Sgj0Bm7TWbiwlMig+plzMwqXs7ZoYryDYfAz/VsMP2v+qtIHAYuyr6Dd2Vap6BxHldtPNYF+p3QOy15Uq0oW57/99amDSouOWOucQzmqeC0R2AS5q2KmzMIrOZPhR9peBpdeBuNI5gX9ORjKwdE5qgWUmBTJvQDmLPl46XBdCVGfUqw2k7/fGPaqSDYyZLEjdVAKxAIstrICXsZKJJ4vjJL72ooTDdn7+yf0x+/nJbTn/pVOgXCRT/FPM6HL1yV0/qVtlExAFpIUJxyzEvN4sjxevkCeUPrq1JhEOMPFfFePVBwFPVGnVPd3WA4oQyA3aCoXK2+7vcYVR5OnrqoUQ2yTNZhulEc7lENEizAFFlMdXoUPfUCfVDaVabLehqJTTfX7HvZRHrzttJ9XpfRYeIXFt+mnrL8EFFIWIn+UJvZJVpogZMxdkYj5/i0kg5ipO1ni4qdtWCTxU27VTI75myA8xEdVMfEta7JVTn/LhuueAIgJyysdVqKg3uQwaOF7CWCqYiQuJpYkTkgYOC4M5eJ/fJEsWiiRaqHKg05+hr0xTzT9q6wjUFA2V+/Bbd48MmtwwuWJcZVN3tgEctZRaSzpujMFdiECBGYvGsrf0i0QOP6jzzJ4I1vIxwyrEziJa2f4MxxgZ83caPH+9xAxcDhtWBLnLw01bI8t2ZSkh+ngXRiEhzMcx9t+mmbDO/6pkwpDAxzJlqIwx4SviTdKfLIFQOPETps5UkCcA3/h84sExQWydgFpOSGBFxMPP/wdt8wfITH8NFp+ffNP0JJf6j6eoir2lzK9SxFRzFsn7OA9nwszMctsHqXbnI9jB+VAVeNtEVw0BlAnDcUVt70me+0Lih5ehn6BSdjHnSXU4Lku2RMuQ1+WaJt5F6pOG2LPIdrlUTBVbZaDe0Ks51BXxSh33zUtYoeU2B5aNF9LY4H2RPMUZxZNMGrxAvHZk0mK/sVsa/Ns1urbeOx2LDmEJ+h6z2ORnp/812MSwE7aBJoZu90UmJvjxoMDkxuqIxSUHikvmIRcUcsxi8iVBRIQRPm4/HY1ad8YqafY5SLByfZ1Pv14fBmhJMctbsBHxVs5oJObOb5n5LIjZMYg5eAhzWADmqLIDzZE6o76B2KDClpn9gV6V3kwV8oyzzVd+H1RsyZeAbaYUMd+BEdwyPy9jzYcaI9sWDFofacZ5yJdTjwDRDWy1yVab9gnVDfeo2kSwuKLsQoI4CffAIQNbcPrW2O19agTnfGMEW3mylSdbeXpUOGVoK0+28vToMUpT5cn5WvDyD1egnOnKOc6pLSfCtih1N2Bjy1O2PGXLU7Y8ZctTKex7ZstTtjy1T9Cvv/PyFEASMfEiuZnUX5nAI58oz7g/jGt5YEtJ6z59upOSVHe5vANwa+2j6no7Ul3vRtW9OATV9Xekuv6Nqjs7BNUNdqS6wY2qe7kfJ61fQJXO6fkb50Oqfls/foT1YyPr4ErId1uFrSKXjxMn9jhhjxP7dJw42d9KsgG23L9+3ARjQIuy7fmvH8flYrJktJyERbuoJsvJ12vn+hrmP4SKMgQ+UWAcW0q2pWRbSralZFtKTrHfyH5BYL8g2Bfc1+k+MuDXWMByMuj3+j8W+X098nuNLfCzwM8CPwv8LPCrAr+KfLbqZ6t+j/UjAkBjHuYcYJ10EsB3C7ygbDWZrgTm967ire23nk347J3SswVhOwZhys+/g5eOe74O+9qxgkA6FoFYBGIRSPY3p8c5DPHipCgo3fGDqBv/DsXikCYccnb+u5Nw6ckWiuy4HjSX2jv0mtAeL8KCkAoI6VoQYkGIBSEyz9BJQBlNhDSgLXs84GupQssWbzw2vPFApY89X8f3/DbqKFuBy705XqA/MOOplJ1+ShardCofsYv0HiHIVxsvVdGUESxrV8VYAi/iCJZLgvqtSEdVb9AARgm+5VcT+HiGksh0O4LONaFBdXdSObg0+FRxn4sG4zQujfMrnvRo6n5JMFtlJfg63NPuYOqUqAGuXqBS2EWb/qiyHE1hKIry64Aqy9BUqethC3Ca60rLVhB2IXf6+NR0dUmhSDfLr/pYC7BIaOiTa9nQ5TYVq+kmaRRV/yHGJKb+JCQz2nKy4Z6Yld+9TflZCtKNBC6uNMx/y+d3y9yaoJJmbpyZNl2Jxkg4HqcD6Yj0W9u73WTvcg5r2je3WToHNvcy9cP93yfrzTdM34f3HOVJKr14TqGXIgQX1yrRq586OUbKb0xKM8KmWxyCJtmmc6aSSY5vdaDiDjau5Xba2kNPf+gsNr8H2u+O/tBr65wNknO72u+On8bHz/kaJCLX/OrWWfSBh/rA+izdvv6wuWLLfebr8uaylNT3NyXptUHjsyxrFh/0S9dLaYk6xLvDXhd3hwMft7uj4Unf67en3Y43fTaceqg/G/XTxpdFam4frf8PZcHkVKRwAAA= scheduler.json.gz: - H4sIAAAAAAAC/+1d3W/bOBJ/z18haPeABPD2LH+k8QJ9SNNrUVy7m228ew+7hUFLtKyLLKoklTgb+P72G1JflEQ5duomccOXIJrhx3BmOPMbSpZuDyzLnkyCKE44s3+2/vzcSSkUf0kCihUaiiLCEQ9IJIi3QAJiGDAum8DlSjbDXsDRNMRAnaGQYUn0I8zfe0CKkjBMKRTF8zEhIQ9ioHclcR54+IxEnJKQVfoHlb5hEF0qglE8A0HnQLD7XWanNHItW0gpU1mB6pIwRDGryFZheA1OjCIclkOpw6U9kTvH42CBScIVGUs+CQl9jdxLn5IkaoxftvkDhQluZ1dFkPQfeqOROzi2O1Uy9afosNd/2bGc3qhjDbodq/viZHRUb/eDNxigPrIV6ufKvB7iiJGEukIs+5ySBeZznLDKQPaM0AUSS7cjEuEqz0eJjwtnKcgLtMxX63S7NbEWQZQz6yw2J9c6DQGLz4ULkND7gKa46jy6Vh8RvcRSp5wmWGm0qspPA++cSHevMqQ/9qqkiGN6hUKhCWdR1UOIfSxtX1MECgM/OmXjbL8IWerGDPw5v4BtcZeo1T2hqDqOg8gf38RS2228pndVJYXWEVpIP7gSxrE4sThe8ppPWRlbTFVhrDqbDU5R5G8weK86eKsHg6O9ASc+J2Ac1vQ2W+xWyfxIpIZhq0URdjnEgUa7sZBIs8FjwvgsWIreto7xFuLZRfC3HH7Y/UetDcQubV9JX9tVquojBK27TTeDrSs3KMjeVCpPF2b/8s9TDZMUHTfUOYOI2dgcQKSX4KKaYDALwvBMRDhpfxG7+g6ELucE/pyMROxyThqxayYE0u9xMYs6XjpcD6KgM+o3BlJDSuvekvkMxkwWUdNUHFEf8w2sgJexlIkli8MkvnXDhEHEePWX/WP27192x/ovmQLlMpninxgkFi8JBX111LRMGXc5ZJ8JwzTArNksD0tvkculTnqNJml0eluM1xwEvFFmb/t0Uy8oIi3TaCzgMtzZv8c1RgEHbutWQrTMze12SqOczSCqhZgBWKmOL8OHuqlO6ptKtthsUxEhp/1qy/2UR7CttpTs9EsWIiE/lv2U9VdQCSQWxM5y3FBLXlNEtQkSEj6bf8CRz+cyVjZ4uK3bRjghkFvWaRDfUeQFOOL1hH9H9u0/QPZFV35LlHETSlOZNd0g7bR0E8bTc+qJXh+ndBzo85/AkzarJye5pzhqC5XSr9gGyKJo1GBcZzM7m4CRRrptJCQ7xuBGEUe+HgrHorfwl0QMP2zy9B4KpvIwxTL8zkJSCwsUxxhpc3saVH+9whRcEWtWBHnNxW1bJsuEVSkhKrmXWiEhBcQx9j6kWbLJv1eioYjjwyKTTHAPMkV6BTFy4iVUFnSQPQD5eGziQo3Ct0pNHSuIYE2Ri1/9D9rnF5Cz/vwRqkiYfpLvzc9HR9b0xjrMxix7Pmh2u82Xt1pZt7e5CHAButki83XubYNpEHlG+TXlZ0pZY4DXOzOAsgFQ6BMa8PnCWEO1Rqkhq9DQGtOc7cw0VwJkmxDVZpdUPWss8eZe+LyW1MRy3qYlYyMjCt7FPJhxPTMD9hel/3wC9d4L5ddhDqFccyQkIcwkB/4iiFwFXoIq0FpfJMgDyKpkS7QMWFOuaeJepolXg68WGZQRiqkXl9mphqZXO5wrMJk8UtUv4QYtNzmxKR2UxBrHDMVJmVY8wST+a8QaZ0YKvNV2S/Ftt0FX1rtVqNiHJah7zBRmP1sDU5iZwswUZmuAzzxgnED+WUy+JCjiQYgPuy9Go451j4otTU7b4aGvh0MdK8RH33HZto2FWuu5Z2+ab1bUbWOezaq9Z2+rhyn5tjHcBrXgs7faHhaEIeg4cm+s0YjPrd8yLzAV4iNXiHtfH+5XdXhQ24J6qJtR38PGl3FPz/5EriuPVhXyjLPdV32gqdiTb6BwnBJEPQtGsKv8/Abp/FhhZNuCQusDxTjf8umqZ1AuD59IuaytivetXl7/fIApmL+uYB48+p1MwCh84oZim8knVOGSTaRn7OaOQMcFewpa78WLFkz4kLivt1x+w1uSD6bM/pNQZn+tMl/vizIHT0KZg7XKPNsXZQ6fhDKHa5X5REq4f4NyrdPz99an1CDmzp65s2fu7D2rUuXYlCqmVNmPUuXk6d7b04Cb3Zxst6IdC3Qpepz/ejFed9ItmnWshIaPccItJpfH2zD/arXz3yDsHA9BmOQFFvpgDrXNobY51DaH2uZQWyLFl+YZMPMM2D7gRKdngGITKL77l8GJu8KJ77CBiQYmGphoYKKBiXWYeGIOFM2Bonn2YS1OjClxMWOAAIWTABRc4AWhN5PpDcdsR7+DfFgEVz6/+vTR20epbAPRHhmiSWd/SJSjZX09Unvi6zA3PKv4ZGTwicEnBp+sxSfyqCoHKW6cFCdTO3xeS3cUZTBLG2Y5O//dSphwbgNbnhls6X4PmMU8oXVvwFKTzyAWg1gMYqkjFp9MfEJJwoUBd/PjYQNGWm9/FZo2aOSx73PNhfb2/xDlia/je77rdZCtwBZBcYH+wJSlUjqDlMxv0qk8RC/TLzFAziq9VEZUGmGBKIuxOF7EISw38pvflTioe4OCMSrgLn/rsodnKAl1L35WuTqsKL8+UQ0uhU/Z4jXy1kWK5xR+/t56Be0pXBLnX8tQw6r9JcH0Jjvab6JC5XMWToXq4/qL4gsDKdMf1NalaA6FYf5lhapHqzpVFbIBhs2VpqQtiL+QSD18qntFe6nRLNmqYy3ANIGmT65lTZe7VCynm6ThVLx1veX96h0rG/tIb4neXZbIEpNqMXB8qW72Wy6MXeU2pBY0fePMzumyFEbC8DgdSMWqD238bpvxq5mtJULfafYc7uze7oXZK+hvpX/t5/66w0Gei9Iv9EiQUkTa4uMQ5PonJ4dC+Xcf0sBfdosDAEG07JyppDiWU/GIPSx9xXa6ykVfvVDqUnuo/O+oF/2uyikBm91T/ne8NPp9ztcggLfiKHfOog58rA6sztIbqBflh0Lsl54qby5LRX1/kyj98MH4LEuOtRfkZMHNTmQ1b/fw8fQY9aeDqefNnEHv5bSPhkOEnL7juMfDNFXbV0UW7h6s/g/BmvPv0WkAAA== + H4sIAAAAAAAC/+1dW3PbthJ+96/gsD0z9oyaI+riWJnJg+OcZDJNWjdR24c2o4FIiOIxRTAAaMv16Pz2swBvIAnakqL4EuHFI+4C4GJ3sfstSIM3B5ZlTyZBFCec2S+svz53UgrFX5KAYoWGoohwxAMSCeINkIAYBozLJnC5ks2wF3A0DTFQZyhkWBL9CPN3HpCiJAxTCkXxfExIyIMY6F1JnAcePiMRpyRklf5BpW8YRBeKYBTPQNA5EOx+l9kpjVzJFlLKVFaguiQMUcwqslUYXoMTowiH5VDqcGlP5M7xOFhgknBFxpJPQkJfIffCpySJGuOXbf5AYYLb2VURJP2H3mjkDo7tTpVM/Sk67PWfdyynN+pYg27H6j47GR3V2/3gDQaoj2yF+rlyXw9xxEhCXSGWfU7JAvM5TlhlIHtG6AKJqdsRiXCV56PEx4WzFOQFWuazdbrdmliLIMqZdRabkyudhoDF58IFSOi9R1NcdR5dqw+IXmCpU04TrDRaVeWngXdOpLtXGdIfe1VSxDG9RKHOCaoOq+ghjoPIH1/HUhVtvKbpq/qE1hFaSCNdCs1ZnFgcL3nN4FbGFreqMFad9QanKPLXGLxXHbzVvcALXoOHnRPQHGu6gi20KJkfiCfv75Iowi6HRdpoNxYSaRQfE8ZnwVL0tnWMNxBsPgX/yOGH3X/V2kBg0faV9Fu7SlV9gIhyt+lmsK7k6gHZm0rl6cTsX/59qmGSouOaOmcQzhqeC0R6AS6qWamzIAzPRPiR9heBpe9AXHFO4M/JSAQW56QRWGZCIP0CFHdRx0uH60GIckb9xkDqem9dozLZwJjJImqaiiPqY76GFfAyljKxZHGYxDdumDBYzi//tn/Mfv5td6z/kilQLpIp/olB1PeSUNBXR03LlEGRQ2qYMEwDzJrN8pjxBrlc6qTXaBJiH0fem2K85iDgjTK12qfrekERBplGYwGXudv+Pa4xilx9U7cSomXibLdTGuVsBlEtxAyQRHV8GT7URXVSX1SyxXqLigg57Zcbrqc8gm20pGSnX7IQCcmr7KfMvwIZUBggdpYn9VpmmSKqzV6Qjdn8PY58PpexssHDbd3WSuKBXLJOg/iWIi/AEa9n4ztSY7+a/qQPNz0HFOFHp2ycwUWRimsRAF36LZHETShN5dJ0g9TS0k0YSM+hgT8H3/NaJMlikY4Dff4MPGmXegKS64ajtnAofedOFCKCJtPcWzKusjs76wCORkptJB07xuAqEUe+HovGorfwiUQMP2zy9F4IpvIwxTLEzkJSW/oUxxhp83caOH+9xBTcDWtmBLnLxW3LIst2VSkh8rgXWiEhzMcx9t6nmbDJ3yqZUMTxYZEtJrgH2SC9gjg48RIqKyrIEIBuPDZxoUjgG6WfjhVEMKfIxS//B+3zC8hLfw0Xn4+OrOm1dZiNUra915x1k09otbJubnIR4AK0sUE+62yt9WkQeUbdVqaGW1T+amcqV5wchT6hAZ8v9l3/pU6sQie3GONsZ8a4FPDYBJ7SEqlCbtH9662wdC05iem8Scu7RmYTvE/zYMb1zAyEfyo95iOYdCtEXocrhHLN3oqEIpMcpItAcRl4CarAYD2glzt5VcmWaBmwplzTxL1IE6gGJy0ySCIUUy8Esx0ITa92WFZgK7k3qZ/CNVqus7tSOiiJNY4Zii0nrXiCSfxXiDX2dxSYqu2W4tRug67Md6Pg8BSmoK4xU0S9sAamiDJF1N4XUfOAcQI5ZjH5kqCIByE+7D4bjTrWFtVVmoA2QznbgJyOFeKj77jE2sQmrbXXHhrjmxVgmxhkvcpsD61zP+XZJqZao27bQzs9weItBLtG7rU1GvG59Vtmd1PNPXA19+RruadVyR3UlqAesmbUd7DwZaTTsz+Sq8r7RIU842z1Vd/iKdbkayjypgRRz4IR7Co/f/A4P1YY2bKg0PpAMc63fKVoD0rb4TalrS7+rVPbakvYryxutbH4m1a3tz9dN+Xt15W3gwd/RghIhE/cUCwm+fIlXLKJ9Izd7Mp3XLCnoPWePSuw3n3iud5y+Q0f9t2b+voPpL7+rep79VTUN3gg9Q1uVd/ZU1Hf8IHUN7xVfY+k4PoZ1Gmdnr+zPqYmMM/MzDMz88xsrwqLY1NYmMLiMRUWJ4/3uZkGtOxmf7kVxVigS9Hj/NdP4+p+s2B0rISGD7HPLG4uN5nh/qvVzt+w3znOgfDHC4zz3mwtm61ls7VstpbN1rJEgM/NW1PmranHgv6cnoF/Tfj39j8G/W2P/t5iA/4M+DPgz4A/A/7q4O/EbP+Z7T/zXsELcUgJcTFjgOyEkwDEW+AFodeT6TXHbEf/53e/OK18A/TxY7QPUtkGiD0wEJPOfp9YRsv6ejz2yOdhHkJWUcjIoBCDQgwKkUfGcXyYQxE3Top9pR2+G5VuJBks0oZFzs5/txIm3NnAkT2DI93vAYuYt6G2BiI1+QwSMUhkX5GITyY+oSThwoC7+XdZAzlaH1EVmjaY46GfRc2F9p7+Fsgjn8f3/GTqIJuBLYLiAv2BKUuldAYpmV+nt/IQvUiP34ecVXqpjKg0wgI3FmNxvIhDmG7kNz8mcFD3BgVkVCBcfpqvh2coCXUHCqtcHSKUnxyoBpcWnyqOQVegnMIlcf5lBDWa2l8STK+z/fgm5FM+XeBUqD6unzte2EW5/UFtOorCUBjmp+jXpqGoUtXDGgA115WSrSDsQv708KnuxO9CkXaWY9WxFmCRQNMn17Kmy10qlrebpFFUHOLdclx3x8rGPtJboneXJbJ8pFoM/F2qm/2WC2NXuQ2pBU3fOLNzOi2FkTA8TgdSIep9G7/bZvxqQmtbRHeZPUc5W9t9N5tNK/2BlU/XSw7yzJR+pEVCliLuFp8gIFc/OTkwyr8ukKaBslscACSiZedMJZMc1KroxB6WLmQ7XeWir1445TE79lD57agX/a7KKeGb3VN+O14aFD/ncxAwXPGfO++iDnysDqzepTdQL8rPUdjPPVXeXJaK+v4hUXq8/vgsS5W142KymGcnsoK3e/h4eoz608HU82bOoPd82kfDIUJO33Hc42GauO3LIid3D1b/B+dQ2oPUZwAA workload-total.json.gz: - H4sIAAAAAAAC/+2dX3PbuBHA3/0pOJxMx26VVpQs25eZe3CSS5uZ610a++5mmng0EAlJqCmCB4KWdR71sxcA/wEkQEmuLEsxXmxxQYHAYnfx4wIUH44cxx0OURSnNHHfOF9uOpmEwN9TRKAkA1GEKaAIR1z4wERMGKKE8lPEkZNLRckoRSH9GLFCr1NJA0BBglPiQ1bgfiJ4BukUpokrnQMjMAp5OSUplORTFGikyMfROxxiwiskkxE47nacnuexP4NBx/FO5KojMBMXvqz64vzJuQwhoUoT6CIW5wUgmY4wIIGbly3F/xv2dymUAgNEa611JxGkHwMmidIwzCQExNNrjEOKYibvCiHvzjscUYJDrtAxCJPs+0j5boiiW2kUYhBBcX6m8kLhro/DEMQJlGtSCoJGyYSg4BOuBjNTcm3A5uy4dyoJ7osO5McLflxop6xb9KFXHlatvillBMYQUKmrkvQjhUQMj774M54rChZFyRTPrxFtWI5Lc6H7LiUERtR5C6JgjgI6ddVzrtAf4rzpmVSQWwLB88wG8j6WigchAokwQKHJSgUjICRKU7g9/QijCRVq7ipymDRGaKW3uGMUhoqmueDvBASIdVQeKfNof1cbba+3YrQ9/Wj3y8MQTmAUqFdiappEl8m13rXB3aTeeW662XjpI8EPs5gu9EX/hgQ3S2bgXnONGYo0UoImU3qljTfczDRSdu5vwqRUq+RFmPAulL2pFb2Hid+sj7LwFGoadgfCFBZm1RgHFis0VsSl87xtniy9rXkk08UV81TF413em08YRfSfWKhDCFzJs32os+gYMquNKJg0I1LMa+MmmvLrD1R5s/lMaQEkUMT3cYgrBRoiSAIJgsnPd5Awi4e1Liam9iaNjicU+LeN1iQUxjEMfmQ6bZRRQCaQJtJ8KM+IfLa4j0U3+LgPWdv84ySdHSMW6uAxm8YoYLWSIZs+5pjcsinYh+gODkcLCpOhsIiH/+DR919dHwR3iFXy1e04fpiyNhEmfZV/ZFI+z4mufv9fJi+PvrrLL6+YkiFhdvTmFZvecZjyMHtz8jX6s4Mj57g8txPjgEknBKfxMIRjeswbFWIQdIoPQx4cT5zyK8OygH33zW06gvzDEM9Zr94QGIIRDB82a2/HKeoUJcWBXCBawavj/1kPT5zRwjnmrWfNl7yNB0dMZsJgWLyfwWFmKeophXY+AJ8KpvCU4iy0fSjreXhw2JWc5VKthcCxmKDcS1XOrUfYXSlc5p8qE6VTNi5THAY10+VN/sCmgLq9c/nVFI1ps6A2731mdubgsfOWG5TzOTOvQJrrSkJ5kOMdIAIe6hEvC2u9enAaFjMmigJ0h4KUxbFGoCrOEWBUNeAe3KPaDDVK/dvMp2phdZaHo8YgFpBXD8NZ3DYG1C/yOBWxupTdNLqwAPewxdUrU3sb10xM+EGjgbwAT94CgXCq0WUzV+P0bOrqqgZWzU6ViXX2v4lHsppfHmIpghbGOrWMZRnLMtb2GIvpKEpmiFrIspD1FJB1ndsXtZxlOWvfOUvKISpIJacQlYLtZRA9QwpxoEkh6rVsgkQTKLbBYgswrpvF1oBjKzwa9amDSC1I6nRb6deTZKpuhJ7PNAG42YxWwDRDZitoroDNduA0Q6cZPFfApxlAV0DoChBth9E2IDVCaXMstXBqBlQDpBpBdQ1YbQPWFdDaBq4t8NoOsGaIXQmyrTCrB1oz1K4CWyPc1gOdAXKZ49lE4q4Yd03OXYN1N+DdNuY1cK885Sv828bAbRzcysIqD18yn2JOvirpaATiNig2gPFacNwMmnpINoByKyy3A7MZmlvA2QzPeoCWIbrZ1SZMa8OLmVhbqbWNXFvo1USwRopt9OtAu3CkGzLTXYTlW12ydG3APbeAawHXAu63Cbg2i2sJd18IV5fxtZBrIddCrgy5jQWZ3e8GLdx3q7tBn3UbrmfYNXCxZ/twS407//iYsCljsYt9uPVtH9/CRtyefry/22SXSH2hbtU2kXrJTveJ1Ms23iiycs+HKDBu+lDsWb7j6K2356Ma0uZdBsPaCPrKQvE3tvFD6n1xC6Eo1O6ttds+lJsEdqH93vSRp7O1DHGImzx4j9fe4lHrQ8XvdhPHHm+W/QY4qLlb1gBCUocsCVkSsiRkc6cWhZ4IhYq8p2Uhy0J2Q6vTM+TiPG+rO1p1a/cvZEtrr9e+4q9WtO6Sv25lYp01f13psy7668ofveq/1sK9DIzmlfvGHhUtPK63cF8b4jVA8iWu3te0pAPOp168XycVx3rCJ2RLoAewet+k0QNbuy/W7Iv9qM6nzPpewJJ9jWvXW7DX9LPJuFtYio8Pfyk+fjn7TV8IfWo3nK7Ez77FT4ufFj/3HT/L/KflT8ufO+dPabeoRVCLoC8UQfdgN2jd+bazB3TnWWfTQvipzTpvKevcX4H9A4v9Fvst9h9a1jkgmDfD0r+l/+fLPjvvMyu0twD2FsBmoS2NrpGFXoWjZxZHLY5aHD24LLTlUcuj+5CNtkhqkdRmpZ8rK/0DIZx8H5OUPspbzsMQjx28tN/N6uKnSe8US/wpnIFfIUmy9nsXmZguwvwtaOQ2+x5DgspqXR7qCZvBmCWXV6NwFodMEdFknZfEVeioUD+F98JiAjgGaVgDBeFecqnuJqJ4WZycIC9MzX3PON+5ykBf83I46TZAKsVx8eo7Ofy6v6eQLIR1a99lVynfU6QTyK3X1b9zrrx8PZmvPLsW/porovZEgqTT5YZv3suVJj96gSI2XQfwMtQ9K6Z3XnfGRgVpTi8UnAPAJtoVVxpmkfW4RAwUjXHORlz2mlEdha9ZxwjyE8YEJSad6Mekt2pM8ilLtiLmAkLxyb+KtrlqaaMTXKY/OR/xrJdSQZrA66wi7UN4ejNw//oXuQ6QUqx7qJGJhz5OhYH0u/WSLHS6XlexixVumul+wRQ9M7iqfIbWXdexTubvKEJ5jFUtYru7ipndlCx64rYHFcU/ak8BPdI9yms/2kF2pI5NHOkWxb+Q8GoR+bqHVzM38w7BzWqDuUM3M/jWkzjUU97cLaubuI28a0uzT3HtR3vXc+jGutpOXS2AcYgXs/oPnMrsWZ5waO5nzK0sa8mV53BOYUD76JiP09q23bZr3bbNbQcm/hz8H9i5I8uvsrIG+z8yJVncBIZZgl+/NlFop99t/FBnqSBeZkpWma+m+4ln/VCYBmPtS7V1zJsar8WKjnSJHm0wYUroDGYdpbpDm3mLBLb14mdJbejUv6kPt3jVqdnSTzex9NMXYuJHRUp3mSUnkRilPC05ztZM3AjPX5c+7woz5zJX+VqM2I0zqb6c62xYNEYeXncgpWxl21dCsOtVdi7bvOvJB/2uXCL9pElP+uwF2djfFH3gS1xNszNfRa74TK5YvkrvVD6QfpzpPJDbW7RFUd8fWKynur9cv8szyUWK+6csQ4GiifM35zf53sxNxRYG97x3MRqfn/u+552dBb1x3+v3R+Pe4OL07OL8zPezk+/K9HX3aPk/kZkRRteEAAA= + H4sIAAAAAAAC/+2dX3PbNhLA3/0pOJxMx75TW1GybDczfXDS5i4zvbtc7LYzTTwaiIQkjCmCBUHLqkf32Q8A/wEkQFGOLNsxXhJzlwKBxe7ipwUk3R04jjseoyhOaeK+dj5d9TIJgX+miEBJBqIIU0ARjrjwjomYMEQJ5beIKyeXCs0kRSF9HzGl16ukAaAgwSnxIVO4HwheQDqHaeJK98AITEKupySFknyOAo0U+Th6i0NMeINkNgGH/Z4z8Dz2z2jUc7wjuekILMSDz6uxON845yEkVOkCXcXivgAk8wkGJHBz3Vr8f8X+XQujwADRWm/dWQTp+4BJojQMMwkB8fwS45CimMn7QsiH8xZHlOCQG3QKwiR7PVJeG6LoWpqFGERQ3J+ZvDC46+MwBHEC5ZYURdDQzAgKPuBqMjMj1yZsya4Hx5LgthhAfr3i14V1yrbFGAblZdXrq1JGYAwBlYYqSd9TSMT06NUf8VIxsFAlc7y8RLThOS7Nhe7blBAYUecNiIIlCujcVe+5QH+J++YnkiL3BIKXmQ/kYywND0IEEuGAwpKVCSZASJSucH/6BUYzKszcV+QwaczQxmhxpygMFUtzwT8ICBAbqDxT5tn+oTbb3mDDbHv62R6WlyGcwShQn8TMNIvOk0t9aIObWX3w3HWz+dJngp8XMV3pVX9AgpuaBbjVPGOBIo2UoNmcXmjzDXczjZTd+7twKdUruQoTPoRyNDXVTzDxm+1Rlp5CTcduQJjCwq0a88ByhcaLuHSZ982Tpde1iGS2uGCRqkS8y0fzAaOI/gsLcwiBK0W2D3UeHUPmtREFs2ZGinlr3EVT/vyRKm92nxktgASK/D4NcWVAQwZJIEEw+c8NJMzjYW2Iiam/SWPgCQX+daM3CYVxDINfmE0bOgrIDNJEWg/lFZGvFrexGAaf9zHrm3+YpItDxFIdPGTLGAWsVTJmy8cSk2u2BPsQ3cDxZEVhMhYeceeHKesB+fGz+yr/87Pb46uaGNiP/2Py8uqzu/70ipkUEuY1r1+xxRyHKU+qV0efo785OHIOy3t7MQ6YdEZwGo9DOKWHvAshBkGv+GPMU+GRU75kXCrYa19fpxPI/xjjJRvDawJDMIHb9rfnFG0KTXEhK0QveHP8fzbCI2eycg5571n3pdjiqRCThXAPlt0XcJz5hXpLYZ13wKeCIDxFnSWyd2U7d3cOe5KzXqutEDgVy5F7rsq5rwgvK4Xr/K/KIemczcsch0HNUXmX37GEX/duLr+YoyltKmqr3EfmVQ6eOm+4+zgfM2cKpJWt5JE7ObsBIlChnt+yJDaop6JxsT6iKEA3KEhZ1mqkpeIegUFVB27BLaqtR5PUv84iqJZEF3nyaUxigXT1pJtlaWP6/CTPU5GZS9lVYwgrcAtbArtytTdxzcVEHDQ6yBV49gYIYFOdLlunGrdnC1VfdbBqLapcrPf0u3ggm/nlAZUiaCGqY0tUlqgsUe2OqJiNomSBqEUqi1RfjlSXuTdRS1WWqp46VUn1QQWg5PKgothdddAzlAdHmvKg3somJDRhYRsatuBh1wq1BhNbUdFoTx0yarFRZ9vKvp4kU20j7HyiScDNbrTipBkpW7FyA1q246UZMc2YuQE1zbi5ATk3YGc7erbhpxFBm3OpRVEzjhqQ1IilHdC0DU83IGobpragajuumpF1I7a2oqseX80IuwljjShbT3QGpGWBZ4uED0O0Ham2A9luQbdthGugXHmBV2i3jXjbqLeVfFX6PWcRxEJ6U0HRiL9tCGzA4E4o3EyReiQ2YHErGrfjsRmRWzDZjMp6XJaRuTnUJjprk4mZT1sZtY1TW1jVxKtGZm2M65kO4UA3Zab3DJZmdYXQzjh7anHW4qzF2a8TZ22F1vLs4/CsrpprkdYirUVaGWkbmy37P8VZhO9OT3E+6vFZz7D/f/bEzs+WFnf++T5hS8ZqH+dn6wc4voYDtAP9fP+wzXmP+ibcpgMfdc1eT3zUdVsf+dh4ekMojMc3FH+W318Mup3eqKa0+Z6CQWwEfWUT+Cs7wiGNvnjDoBjUnol90Qc42IOe9vGNvFStJYbneFyDj7jzYY3aGCpat8cxnvAh16+AepqnXA3YIw3Ico/lHss9ti5qwWcn4FPUNC35WPKxB1GdgaHO5nk7PYmq24V/IUdRB4P2vXu1oa6b97pdhy679zrto27f6/T33r/vtAUv46F5D75x2kSLit224GtT3AEbX+I+fM1KOrx86G34LmU2NhK+IFvefHL78E32fGa78MXue3GO1PmQ+doL2HyvUWy3rXfNOJtEu4NN9fj5b6rHL+ec6AthTe1B0Y2wObSwaWHTwuZTh82ytmlp09LmA9OmdMrTAqcFzhcKnE/gFGc9+HZzdnPvFWXTlvaxrSjvqKI83AD5Iwv5FvIt5D+3inJAMO+GZX3L+vuqLDs/ZT5ngd8Cv60wW/bsUGHeBJ8nFj4tfFr4fHYVZkuflj73X2m2AGoB1FacH6vi/DMhnHPvU3A+yHvO0xDPHVw77Gdt8duk3+dK/DlcgN8gSbL+e2eZmK7C/BfFyHX2OgYAlde6PNUTtl4xTy6fRuEiDpkholmXH1yrQFFhfApvhccEcArSsIYFIrxkre4tQ/HDa3LxW+9q5e+rSawvaXFc/HqcnHXdP1NIVsKptT8HV9ncU6QzyJ3W1f9sW/n4en1e+WBZ+Fs+/towJFOut/zxutxW8iclUMRW6QCeh7oPchkMuWCTgTS3FwbO1/1trCueNM4S6mFJFiia4p6TN3ekt/tgk93z1Uh2EObdwrjJf4vnu6q20VEu09+cz2o2EkmRJvAya0j7KTj9VLvf/V1uA6QU6z5VyMRjH6fCCYb9uibLiq7XV+Z+QwRym3+brJihF4YolO/QRmIXD2ShjCKUp0911r/koO+6V1HqkdueHRSPr30M554OXz773i7/IIPfJkiuUfwrCS9Wka/7ZGgWQt5zCKHa1O0xhAxx8yDB8pDvydbVe6+tYmlHq0fx7HvH0mPYxobaXkMtgHGIV4v6N4PKyFje8NzCz1gSWddqIo8RnMKBnmJg3s9quw7bvg3btrAdmdhy9AVIuSfPr4qpBv8/MNVG3ASGWRVev4FQWGfYb3znZWkgrjPVmMxP0303sn4qTJPR+VFtA/Pmxmcx1YGuPqNNJswIvdGipzT33Fbeou5so/hRShM6828bwy1RdWz29ONtPP34hbj4QVGJXWc1RSRmKa8mTrOtDjfCy2/LmHeFm3OZq7wsRuxtMqlenNtsXHRGnl53JFVaZd9XUrDrVX4u+7zryRfDvqyRvkFkIP3tBdncXxVj4DtTTbczP0Vu+ERuWH7K4Fi+kL756DSQ+1v0RTHfX1hserq/Xr7NC8BFZfrfWT0CRTPne+d3+b2Zm4pzBu7p4GwyPT31fc87OQkG06E3HE6mg9HZ8cnZ6YnvZzfflFXn/sH6/+UubNfagwAA diff --git a/charts/kubezero-metrics/templates/istio-authorization-policy.yaml b/charts/kubezero-metrics/templates/istio-authorization-policy.yaml index ecc0ea3..78d84be 100644 --- a/charts/kubezero-metrics/templates/istio-authorization-policy.yaml +++ b/charts/kubezero-metrics/templates/istio-authorization-policy.yaml @@ -18,9 +18,13 @@ spec: - source: notIpBlocks: {{- toYaml $service.ipBlocks | nindent 8 }} + to: + - operation: + hosts: ["{{ $service.url }}"] when: - key: connection.sni - values: ["{{ $service.url }}"] + values: + - '*' --- {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/templates/rules/etcd-mixin.yaml b/charts/kubezero-metrics/templates/rules/etcd-mixin.yaml index be7404e..4fd4bab 100644 --- a/charts/kubezero-metrics/templates/rules/etcd-mixin.yaml +++ b/charts/kubezero-metrics/templates/rules/etcd-mixin.yaml @@ -49,9 +49,9 @@ spec: severity: warning - alert: etcdGRPCRequestsSlow annotations: - description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile of gRPC requests is {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}} for {{`{{`}} $labels.grpc_method {{`}}`}} method.' + description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": gRPC requests to {{`{{`}} $labels.grpc_method {{`}}`}} are taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' summary: etcd grpc requests are slow - expr: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type)) + expr: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) without(grpc_type)) > 0.15 @@ -95,7 +95,7 @@ spec: severity: warning - alert: etcdHighFsyncDurations annotations: - description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fsync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' expr: 'histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 1 @@ -118,7 +118,7 @@ spec: severity: warning - alert: etcdBackendQuotaLowSpace annotations: - description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.' + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": database size exceeds the defined quota on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.' expr: '(etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100 > 95 ' @@ -127,7 +127,7 @@ spec: severity: critical - alert: etcdExcessiveDatabaseGrowth annotations: - description: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Observed surge in etcd writes leading to 50% increase in database size over the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.' + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": Observed surge in etcd writes leading to 50% increase in database size over the past four hours on etcd instance {{`{{`}} $labels.instance {{`}}`}}, please check as it might be disruptive.' expr: 'increase(((etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes)*100)[240m:1m]) > 50 ' diff --git a/charts/kubezero-metrics/templates/rules/kubernetes.yaml b/charts/kubezero-metrics/templates/rules/kubernetes.yaml index 9422520..69d21e2 100644 --- a/charts/kubezero-metrics/templates/rules/kubernetes.yaml +++ b/charts/kubezero-metrics/templates/rules/kubernetes.yaml @@ -85,7 +85,7 @@ spec: severity: warning - alert: KubeContainerWaiting annotations: - description: pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting summary: Pod container waiting longer than 1 hour expr: 'sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0 @@ -173,7 +173,7 @@ spec: severity: warning - alert: KubeMemoryOvercommit annotations: - description: Cluster has overcommitted memory resource requests for Pods by {{`{{`}} $value | humanize {{`}}`}} bytes and cannot tolerate node failure. + description: Cluster has overcommitted memory resource requests for Pods by {{`{{`}} $value {{`}}`}} bytes and cannot tolerate node failure. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit summary: Cluster has overcommitted memory resource requests. expr: 'sum(namespace_memory:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0 @@ -191,7 +191,7 @@ spec: description: Cluster has overcommitted CPU resource requests for Namespaces. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit summary: Cluster has overcommitted CPU resource requests. - expr: "sum(min without(resource) (kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=~\"(cpu|requests.cpu)\"}))\n /\nsum(kube_node_status_allocatable{resource=\"cpu\", job=\"kube-state-metrics\"})\n > 1.5\n" + expr: "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"cpu\"})\n /\nsum(kube_node_status_allocatable{resource=\"cpu\"})\n > 1.5\n" for: 5m labels: severity: warning @@ -200,7 +200,7 @@ spec: description: Cluster has overcommitted memory resource requests for Namespaces. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit summary: Cluster has overcommitted memory resource requests. - expr: "sum(min without(resource) (kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=~\"(memory|requests.memory)\"}))\n /\nsum(kube_node_status_allocatable{resource=\"memory\", job=\"kube-state-metrics\"})\n > 1.5\n" + expr: "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"memory\"})\n /\nsum(kube_node_status_allocatable{resource=\"memory\",job=\"kube-state-metrics\"})\n > 1.5\n" for: 5m labels: severity: warning @@ -247,7 +247,7 @@ spec: description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup summary: PersistentVolume is filling up. - expr: "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n" + expr: "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\n" for: 1m labels: severity: critical @@ -256,7 +256,7 @@ spec: description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup summary: PersistentVolume is filling up. - expr: "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n" + expr: "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\n" for: 1h labels: severity: warning @@ -367,7 +367,7 @@ spec: rules: - alert: KubeClientCertificateExpiration annotations: - description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days. + description: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration summary: Client certificate is about to expire. expr: 'apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 @@ -377,7 +377,7 @@ spec: severity: warning - alert: KubeClientCertificateExpiration annotations: - description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours. + description: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration summary: Client certificate is about to expire. expr: 'apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 @@ -385,21 +385,21 @@ spec: ' labels: severity: critical - - alert: KubeAggregatedAPIErrors + - alert: AggregatedAPIErrors annotations: - description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. - runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors - summary: Kubernetes aggregated API has reported errors. + description: An aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/aggregatedapierrors + summary: An aggregated API has reported errors. expr: 'sum by(name, namespace)(increase(aggregator_unavailable_apiservice_total[10m])) > 4 ' labels: severity: warning - - alert: KubeAggregatedAPIDown + - alert: AggregatedAPIDown annotations: - description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. - runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown - summary: Kubernetes aggregated API is down. + description: An aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/aggregatedapidown + summary: An aggregated API is down. expr: '(1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85 ' @@ -419,9 +419,9 @@ spec: severity: critical - alert: KubeAPITerminatedRequests annotations: - description: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + description: The apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests - summary: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + summary: The apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. expr: 'sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20 ' @@ -460,7 +460,7 @@ spec: expr: "count by(node) (\n (kube_pod_status_phase{job=\"kube-state-metrics\",phase=\"Running\"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job=\"kube-state-metrics\"})\n)\n/\nmax by(node) (\n kube_node_status_capacity{job=\"kube-state-metrics\",resource=\"pods\"} != 1\n) > 0.95\n" for: 15m labels: - severity: info + severity: warning - alert: KubeNodeReadinessFlapping annotations: description: The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes. @@ -593,46 +593,33 @@ spec: for: 15m labels: severity: critical - - name: kubernetes-system-kube-proxy - rules: - - alert: KubeProxyDown - annotations: - description: KubeProxy has disappeared from Prometheus target discovery. - runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeproxydown - summary: Target disappeared from Prometheus target discovery. - expr: 'absent(up{job="kube-proxy"} == 1) - - ' - for: 15m - labels: - severity: critical - name: kube-apiserver-burnrate.rules rules: - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[1d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[1d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n" labels: verb: read record: apiserver_request:burnrate1d - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[1h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[1h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n" labels: verb: read record: apiserver_request:burnrate1h - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[2h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[2h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[2h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[2h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[2h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[2h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n" labels: verb: read record: apiserver_request:burnrate2h - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[30m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n" labels: verb: read record: apiserver_request:burnrate30m - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[3d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[3d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[3d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[3d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[3d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[3d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n" labels: verb: read record: apiserver_request:burnrate3d - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[5m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[5m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[5m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[5m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[5m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[5m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n" labels: verb: read record: apiserver_request:burnrate5m - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[6h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[6h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"}[6h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[6h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[6h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[6h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n" labels: verb: read record: apiserver_request:burnrate6h @@ -717,32 +704,16 @@ spec: labels: verb: write record: code:apiserver_request_total:increase30d - - expr: 'sum by (cluster, verb, scope) (increase(apiserver_request_duration_seconds_count[1h])) - - ' - record: cluster_verb_scope:apiserver_request_duration_seconds_count:increase1h - - expr: 'sum by (cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_duration_seconds_count:increase1h[30d]) * 24 * 30) - - ' - record: cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d - - expr: 'sum by (cluster, verb, scope, le) (increase(apiserver_request_duration_seconds_bucket[1h])) - - ' - record: cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase1h - - expr: 'sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase1h[30d]) * 24 * 30) - - ' - record: cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d - - expr: "1 - (\n (\n # write too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n -\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n ) +\n (\n # read too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"\ - })\n )\n ) +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d)\n" + - expr: "1 - (\n (\n # write too slow\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~\"POST|PUT|PATCH|DELETE\"}[30d]))\n -\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[30d]))\n ) +\n (\n # read too slow\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~\"LIST|GET\"}[30d]))\n -\n (\n (\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30d]))\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30d]))\n )\n ) +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{code=~\"5..\"}\ + \ or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d)\n" labels: verb: all record: apiserver_request:availability30d - - expr: "1 - (\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n # too slow\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"})\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n" + - expr: "1 - (\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30d]))\n -\n (\n # too slow\n (\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30d]))\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30d]))\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n" labels: verb: read record: apiserver_request:availability30d - - expr: "1 - (\n (\n # too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n -\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\"})\n" + - expr: "1 - (\n (\n # too slow\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~\"POST|PUT|PATCH|DELETE\"}[30d]))\n -\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[30d]))\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\"})\n" labels: verb: write record: apiserver_request:availability30d @@ -786,19 +757,19 @@ spec: record: node_namespace_pod_container:container_memory_cache - expr: "container_memory_swap{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n max by(namespace, pod, node) (kube_pod_info{node!=\"\"})\n)\n" record: node_namespace_pod_container:container_memory_swap - - expr: "kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" + - expr: "kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests - expr: "sum by (namespace, cluster) (\n sum by (namespace, pod, cluster) (\n max by (namespace, pod, container, cluster) (\n kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"}\n ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n" record: namespace_memory:kube_pod_container_resource_requests:sum - - expr: "kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" + - expr: "kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests - expr: "sum by (namespace, cluster) (\n sum by (namespace, pod, cluster) (\n max by (namespace, pod, container, cluster) (\n kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"}\n ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n" record: namespace_cpu:kube_pod_container_resource_requests:sum - - expr: "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" + - expr: "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits - expr: "sum by (namespace, cluster) (\n sum by (namespace, pod, cluster) (\n max by (namespace, pod, container, cluster) (\n kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"}\n ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n" record: namespace_memory:kube_pod_container_resource_limits:sum - - expr: "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n )\n" + - expr: "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n )\n" record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits - expr: "sum by (namespace, cluster) (\n sum by (namespace, pod, cluster) (\n max by (namespace, pod, container, cluster) (\n kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"}\n ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n" record: namespace_cpu:kube_pod_container_resource_limits:sum diff --git a/charts/kubezero-metrics/templates/rules/node-exporter.yaml b/charts/kubezero-metrics/templates/rules/node-exporter.yaml index a844027..f358862 100644 --- a/charts/kubezero-metrics/templates/rules/node-exporter.yaml +++ b/charts/kubezero-metrics/templates/rules/node-exporter.yaml @@ -14,7 +14,7 @@ spec: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup summary: Filesystem is predicted to run out of space within the next 24 hours. - expr: "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 20\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n" + expr: "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 40\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n" for: 1h labels: severity: warning @@ -188,13 +188,13 @@ spec: severity: critical - name: node-exporter.rules rules: - - expr: "count without (cpu, mode) (\n node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}\n)\n" + - expr: "count without (cpu) (\n count without (mode) (\n node_cpu_seconds_total{job=\"node-exporter\"}\n )\n)\n" record: instance:node_num_cpu:sum - - expr: "1 - avg without (cpu) (\n sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\"}[5m]))\n)\n" + - expr: "1 - avg without (cpu, mode) (\n rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\"}[5m])\n)\n" record: instance:node_cpu_utilisation:rate5m - expr: "(\n node_load1{job=\"node-exporter\"}\n/\n instance:node_num_cpu:sum{job=\"node-exporter\"}\n)\n" record: instance:node_load1_per_cpu:ratio - - expr: "1 - (\n (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"}\n +\n node_memory_Cached_bytes{job=\"node-exporter\"}\n +\n node_memory_MemFree_bytes{job=\"node-exporter\"}\n +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n )\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n" + - expr: "1 - (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n" record: instance:node_memory_utilisation:ratio - expr: 'rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) diff --git a/charts/kubezero-metrics/templates/rules/prometheus-operator.yaml b/charts/kubezero-metrics/templates/rules/prometheus-operator.yaml index b5e57f8..f8d3f77 100644 --- a/charts/kubezero-metrics/templates/rules/prometheus-operator.yaml +++ b/charts/kubezero-metrics/templates/rules/prometheus-operator.yaml @@ -86,19 +86,4 @@ spec: for: 5m labels: severity: warning - - name: config-reloaders - rules: - - alert: ConfigReloaderSidecarErrors - annotations: - description: 'Errors encountered while the {{`{{`}}$labels.pod{{`}}`}} config-reloader sidecar attempts to sync config in {{`{{`}}$labels.namespace{{`}}`}} namespace. - - As a result, configuration for service running in {{`{{`}}$labels.pod{{`}}`}} may be stale and cannot be updated anymore.' - runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/configreloadersidecarerrors - summary: config-reloader sidecar has not had a successful reload for 10m - expr: 'max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0 - - ' - for: 10m - labels: - severity: warning diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index a3dfe36..4a8fa62 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -1,8 +1,4 @@ kube-prometheus-stack: - global: - rbac: - pspEnabled: false - defaultRules: create: false @@ -75,18 +71,17 @@ kube-prometheus-stack: nodeExporter: enabled: true - serviceMonitor: - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: node - replacement: $1 - action: replace prometheus-node-exporter: - rbac: - pspEnabled: false + prometheus: + monitor: + relabelings: + - sourceLabels: [__meta_kubernetes_pod_node_name] + separator: ; + regex: ^(.*)$ + targetLabel: node + replacement: $1 + action: replace resources: requests: cpu: 20m @@ -127,8 +122,6 @@ kube-prometheus-stack: # Custom Grafana tweaks grafana: enabled: true - rbac: - pspEnabled: false # Disabled as we use the upstream kube-mixin dashboards directly defaultDashboardsEnabled: false @@ -190,8 +183,6 @@ kube-prometheus-stack: # Assign state metrics to control plane kube-state-metrics: - podSecurityPolicy: - enabled: false tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule @@ -209,6 +200,34 @@ kube-prometheus-stack: group_wait: 30s group_interval: 5m repeat_interval: 6h + routes: + - matchers: + - alertname = Watchdog + receiver: 'null' + - matchers: + - alertname = InfoInhibitor + receiver: 'null' + inhibit_rules: + - equal: + - namespace + - alertname + source_matchers: + - severity = critical + target_matchers: + - severity =~ warning|info + - equal: + - namespace + - alertname + source_matchers: + - severity = warning + target_matchers: + - severity = info + - equal: + - namespace + source_matchers: + - alertname = InfoInhibitor + target_matchers: + - severity = info alertmanagerSpec: # externalUrl: logFormat: json