From c2b929f952bf645e5a507e35346f4288071e220f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 16 Sep 2022 11:21:38 +0200 Subject: [PATCH] feat: another round of upgrade to 1.23.11 --- admin/upgrade_cluster.sh | 3 + charts/clamav/README.md | 2 +- charts/kubeadm/Chart.yaml | 2 +- charts/kubezero-addons/Chart.yaml | 2 +- charts/kubezero-metrics/Chart.yaml | 4 +- .../charts/kube-prometheus-stack/Chart.yaml | 10 +- .../charts/kube-prometheus-stack/README.md | 27 + .../charts/grafana/Chart.yaml | 4 +- .../charts/grafana/README.md | 15 +- .../charts/grafana/templates/_helpers.tpl | 13 +- .../charts/grafana/templates/_pod.tpl | 269 +++++++--- .../charts/grafana/templates/configmap.yaml | 14 + .../charts/grafana/templates/deployment.yaml | 2 +- .../grafana/templates/headless-service.yaml | 2 +- .../templates/image-renderer-deployment.yaml | 4 +- .../image-renderer-network-policy.yaml | 2 +- .../templates/image-renderer-service.yaml | 3 + .../grafana/templates/networkpolicy.yaml | 20 +- .../grafana/templates/podsecuritypolicy.yaml | 4 +- .../charts/grafana/templates/service.yaml | 16 +- .../grafana/templates/servicemonitor.yaml | 2 +- .../templates/tests/test-configmap.yaml | 2 +- .../tests/test-podsecuritypolicy.yaml | 2 + .../charts/grafana/values.yaml | 145 +++++- .../charts/kube-state-metrics/Chart.yaml | 4 +- .../templates/deployment.yaml | 6 + .../charts/kube-state-metrics/values.yaml | 3 +- .../prometheus-node-exporter/Chart.yaml | 2 +- .../charts/prometheus-node-exporter/README.md | 21 +- .../templates/NOTES.txt | 2 +- .../templates/_helpers.tpl | 27 +- .../templates/daemonset.yaml | 20 +- .../templates/endpoints.yaml | 3 +- .../templates/service.yaml | 5 +- .../templates/serviceaccount.yaml | 6 +- .../templates/servicemonitor.yaml | 3 +- .../prometheus-node-exporter/values.yaml | 15 +- .../crds/crd-alertmanagerconfigs.yaml | 2 +- .../crds/crd-alertmanagers.yaml | 489 ++++++++++++++++-- .../crds/crd-podmonitors.yaml | 2 +- .../crds/crd-probes.yaml | 2 +- .../crds/crd-prometheuses.yaml | 175 ++++++- .../crds/crd-prometheusrules.yaml | 2 +- .../crds/crd-servicemonitors.yaml | 2 +- .../crds/crd-thanosrulers.yaml | 83 ++- .../job-patch/job-createSecret.yaml | 4 + .../job-patch/job-patchWebhook.yaml | 4 + .../additionalAlertmanagerConfigs.yaml | 2 +- .../templates/prometheus/prometheus.yaml | 8 +- .../charts/kube-prometheus-stack/values.yaml | 24 +- .../jsonnet/dashboards/apiserver.json | 97 ++-- .../jsonnet/dashboards/cluster-total.json | 2 +- .../dashboards/controller-manager.json | 57 +- .../dashboards/k8s-resources-cluster.json | 132 +++-- .../dashboards/k8s-resources-namespace.json | 107 +++- .../dashboards/k8s-resources-node.json | 60 ++- .../jsonnet/dashboards/k8s-resources-pod.json | 108 ++-- .../dashboards/k8s-resources-workload.json | 134 +++-- .../k8s-resources-workloads-namespace.json | 134 +++-- .../jsonnet/dashboards/kubelet.json | 52 +- .../jsonnet/dashboards/namespace-by-pod.json | 2 +- .../dashboards/namespace-by-workload.json | 2 +- .../dashboards/persistentvolumesusage.json | 22 +- .../jsonnet/dashboards/pod-total.json | 2 +- .../jsonnet/dashboards/proxy.json | 72 +-- .../jsonnet/dashboards/scheduler.json | 68 ++- .../jsonnet/dashboards/workload-total.json | 28 +- .../kubezero-metrics/jsonnet/jsonnetfile.json | 2 +- .../jsonnet/jsonnetfile.lock.json | 68 ++- .../jsonnet/k8s-dashboards.yaml | 4 +- .../jsonnet/rules/alertmanager-prometheusRule | 3 +- .../rules/kube-prometheus-prometheusRule | 14 +- .../rules/kube-state-metrics-prometheusRule | 2 +- .../jsonnet/rules/kubernetes-prometheusRule | 199 ++++--- .../rules/node-exporter-prometheusRule | 12 +- .../rules/prometheus-operator-prometheusRule | 24 +- .../jsonnet/rules/prometheus-prometheusRule | 29 +- .../templates/grafana-dashboards-k8s.yaml | 38 +- .../templates/rules/kube-prometheus.yaml | 22 +- .../templates/rules/kubernetes.yaml | 198 ++++--- .../templates/rules/node-exporter.yaml | 10 +- .../templates/rules/prometheus-operator.yaml | 19 +- .../templates/rules/prometheus.yaml | 22 + charts/kubezero-storage/Chart.yaml | 4 +- .../charts/aws-ebs-csi-driver/CHANGELOG.md | 8 +- .../charts/aws-ebs-csi-driver/Chart.yaml | 4 +- .../templates/controller.yaml | 26 +- .../templates/csidriver.yaml | 1 + .../templates/node-windows.yaml | 2 - .../aws-ebs-csi-driver/templates/node.yaml | 2 - .../charts/aws-ebs-csi-driver/values.yaml | 22 +- .../jsonnet/jsonnetfile.lock.json | 6 +- charts/kubezero-storage/values.yaml | 5 - charts/kubezero/Chart.yaml | 2 +- charts/kubezero/README.md | 2 +- charts/kubezero/values.yaml | 4 +- charts/manticore/README.md | 2 +- 97 files changed, 2458 insertions(+), 862 deletions(-) diff --git a/admin/upgrade_cluster.sh b/admin/upgrade_cluster.sh index 1651f6e3..f44cb526 100755 --- a/admin/upgrade_cluster.sh +++ b/admin/upgrade_cluster.sh @@ -133,6 +133,9 @@ control_plane_upgrade kubeadm_upgrade kubectl delete ds kube-multus-ds -n kube-system +# Required due to chart upgrade to 4.X part of prometheus-stack 40.X +kubectl delete daemonset metrics-prometheus-node-exporter -n monitoring + control_plane_upgrade "apply_network, apply_addons" kubectl rollout restart daemonset/calico-node -n kube-system diff --git a/charts/clamav/README.md b/charts/clamav/README.md index c1a7d949..f7de6a30 100644 --- a/charts/clamav/README.md +++ b/charts/clamav/README.md @@ -39,4 +39,4 @@ Kubernetes: `>= 1.18.0` | service.port | int | `3310` | The port to be used by the clamav service | ---------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.9.1](https://github.com/norwoodj/helm-docs/releases/v1.9.1) +Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0) diff --git a/charts/kubeadm/Chart.yaml b/charts/kubeadm/Chart.yaml index 598fa2c1..031c568a 100644 --- a/charts/kubeadm/Chart.yaml +++ b/charts/kubeadm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubeadm description: KubeZero Kubeadm cluster config type: application -version: 1.23.10 +version: 1.23.11 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-addons/Chart.yaml b/charts/kubezero-addons/Chart.yaml index a35246c3..255dd1f1 100644 --- a/charts/kubezero-addons/Chart.yaml +++ b/charts/kubezero-addons/Chart.yaml @@ -3,7 +3,7 @@ name: kubezero-addons description: KubeZero umbrella chart for various optional cluster addons type: application version: 0.6.2 -appVersion: v1.23.10 +appVersion: v1.23.11 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index 0bb82dc5..f731e7d1 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-metrics description: KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all Kubernetes integrations. type: application -version: 0.8.1 +version: 0.8.4 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -18,7 +18,7 @@ dependencies: version: ">= 0.1.5" repository: https://cdn.zero-downtime.net/charts/ - name: kube-prometheus-stack - version: 39.9.0 + version: 40.0.0 # Switch back to upstream once all alerts are fixed eg. etcd gpcr # repository: https://prometheus-community.github.io/helm-charts - name: prometheus-adapter diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml index 79485c14..60067124 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml @@ -6,20 +6,20 @@ annotations: url: https://github.com/prometheus-operator/kube-prometheus artifacthub.io/operator: "true" apiVersion: v2 -appVersion: 0.58.0 +appVersion: 0.59.1 dependencies: - condition: kubeStateMetrics.enabled name: kube-state-metrics repository: https://prometheus-community.github.io/helm-charts - version: 4.15.* + version: 4.18.* - condition: nodeExporter.enabled name: prometheus-node-exporter repository: https://prometheus-community.github.io/helm-charts - version: 3.3.* + version: 4.2.* - condition: grafana.enabled name: grafana repository: https://grafana.github.io/helm-charts - version: 6.32.* + version: 6.38.* description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus @@ -51,4 +51,4 @@ sources: - https://github.com/prometheus-community/helm-charts - https://github.com/prometheus-operator/kube-prometheus type: application -version: 39.9.0 +version: 40.0.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md index fa6e2aa0..76982e2e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md @@ -80,6 +80,33 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. +### From 39.x to 40.x + +This version upgrades Prometheus-Operator to v0.59.1, Prometheus to v2.38.0, kube-state-metrics to v2.6.0 and Thanos to v0.28.0. +This version also upgrades the Helm charts of kube-state-metrics to 4.18.0 and prometheus-node-exporter to 4.2.0. + +Run these commands to update the CRDs before applying the upgrade. + +```console +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +``` + +Starting from prometheus-node-exporter version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i kube-prometheus-stack prometheus-community/kube-prometheus-stack +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + ### From 38.x to 39.x This upgraded prometheus-operator to v0.58.0 and prometheus to v2.37.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml index e34c67e3..bf9e324e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 9.0.5 +appVersion: 9.1.4 description: The leading tool for querying and visualizing time series and metrics. home: https://grafana.net icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png @@ -19,4 +19,4 @@ name: grafana sources: - https://github.com/grafana/grafana type: application -version: 6.32.10 +version: 6.38.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md index bbe3f0c9..45046f0d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md @@ -67,6 +67,7 @@ This version requires Helm >= 3.1.0. | `service.type` | Kubernetes service type | `ClusterIP` | | `service.port` | Kubernetes port where service is exposed | `80` | | `service.portName` | Name of the port on the service | `service` | +| `service.appProtocol` | Adds the appProtocol field to the service | `` | | `service.targetPort` | Internal service is port | `3000` | | `service.nodePort` | Kubernetes service nodePort | `nil` | | `service.annotations` | Service annotations (can be templated) | `{}` | @@ -127,6 +128,7 @@ This version requires Helm >= 3.1.0. | `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` | | `plugins` | Plugins to be loaded along with Grafana | `[]` | | `datasources` | Configure grafana datasources (passed through tpl) | `{}` | +| `alerting` | Configure grafana alerting (passed through tpl) | `{}` | | `notifiers` | Configure grafana notifiers | `{}` | | `dashboardProviders` | Configure grafana dashboard providers | `{}` | | `dashboards` | Dashboards to import | `{}` | @@ -233,13 +235,14 @@ This version requires Helm >= 3.1.0. | `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | | `imageRenderer.service.enabled` | Enable the image-renderer service | `true` | | `imageRenderer.service.portName` | image-renderer service port name | `http` | -| `imageRenderer.service.port` | image-renderer service port used by both service and deployment | `8081` | -| `imageRenderer.grafanaProtocol` | Protocol to use for image renderer callback url | `http` | +| `imageRenderer.service.port` | image-renderer port used by deployment | `8081` | +| `imageRenderer.service.targetPort` | image-renderer service port used by service | `8081` | +| `imageRenderer.appProtocol` | Adds the appProtocol field to the service | `` | | `imageRenderer.grafanaSubPath` | Grafana sub path to use for image renderer callback url | `''` | | `imageRenderer.podPortName` | name of the image-renderer port on the pod | `http` | | `imageRenderer.revisionHistoryLimit` | number of image-renderer replica sets to keep | `10` | -| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | -| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | +| `imageRenderer.networkPolicy.limitIngress` | Enable a NetworkPolicy to limit inbound traffic from only the created grafana pods | `true` | +| `imageRenderer.networkPolicy.limitEgress` | Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods | `false` | | `imageRenderer.resources` | Set resource limits for image-renderer pdos | `{}` | | `imageRenderer.nodeSelector` | Node labels for pod assignment | `{}` | | `imageRenderer.tolerations` | Toleration labels for pod assignment | `[]` | @@ -273,7 +276,7 @@ ingress: ### Example of extraVolumeMounts Volume can be type persistentVolumeClaim or hostPath but not both at same time. -If none existingClaim or hostPath argument is givent then type is emptyDir. +If neither existingClaim or hostPath argument is given then type is emptyDir. ```yaml - extraVolumeMounts: @@ -482,7 +485,7 @@ grafana.ini: ## How to securely reference secrets in grafana.ini -This example uses Grafana uses [file providers](https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider) for secret values and the `extraSecretMounts` configuration flag (Additional grafana server secret mounts) to mount the secrets. +This example uses Grafana [file providers](https://grafana.com/docs/grafana/latest/administration/configuration/#file-provider) for secret values and the `extraSecretMounts` configuration flag (Additional grafana server secret mounts) to mount the secrets. In grafana.ini: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl index 7b146e56..369e69fa 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl @@ -141,22 +141,11 @@ Return the appropriate apiVersion for ingress. {{- end -}} {{- end -}} -{{/* -Return the appropriate apiVersion for podSecurityPolicy. -*/}} -{{- define "grafana.podSecurityPolicy.apiVersion" -}} - {{- if and (.Capabilities.APIVersions.Has "policy/v1beta1") (semverCompare ">= 1.16-0" .Capabilities.KubeVersion.Version) -}} - {{- print "policy/v1beta1" -}} - {{- else -}} - {{- print "extensions/v1beta1" -}} - {{- end -}} -{{- end -}} - {{/* Return the appropriate apiVersion for podDisruptionBudget. */}} {{- define "grafana.podDisruptionBudget.apiVersion" -}} - {{- if and (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">= 1.21-0" .Capabilities.KubeVersion.Version) -}} + {{- if $.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" -}} {{- print "policy/v1" -}} {{- else -}} {{- print "policy/v1beta1" -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl index 1a71c831..adc025c6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl @@ -1,17 +1,16 @@ - {{- define "grafana.pod" -}} {{- if .Values.schedulerName }} schedulerName: "{{ .Values.schedulerName }}" {{- end }} serviceAccountName: {{ template "grafana.serviceAccountName" . }} automountServiceAccountToken: {{ .Values.serviceAccount.autoMount }} -{{- if .Values.securityContext }} +{{- with .Values.securityContext }} securityContext: -{{ toYaml .Values.securityContext | indent 2 }} + {{- toYaml . | nindent 2 }} {{- end }} -{{- if .Values.hostAliases }} +{{- with .Values.hostAliases }} hostAliases: -{{ toYaml .Values.hostAliases | indent 2 }} + {{- toYaml . | nindent 2 }} {{- end }} {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} @@ -31,8 +30,10 @@ initContainers: runAsNonRoot: false runAsUser: 0 command: ["chown", "-R", "{{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }}", "/var/lib/grafana"] + {{- with .Values.initChownData.resources }} resources: -{{ toYaml .Values.initChownData.resources | indent 6 }} + {{- toYaml . | nindent 6 }} + {{- end }} volumeMounts: - name: storage mountPath: "/var/lib/grafana" @@ -50,13 +51,19 @@ initContainers: imagePullPolicy: {{ .Values.downloadDashboardsImage.pullPolicy }} command: ["/bin/sh"] args: [ "-c", "mkdir -p /var/lib/grafana/dashboards/default && /bin/sh -x /etc/grafana/download_dashboards.sh" ] + {{- with .Values.downloadDashboards.resources }} resources: -{{ toYaml .Values.downloadDashboards.resources | indent 6 }} + {{- toYaml . | nindent 6 }} + {{- end }} env: {{- range $key, $value := .Values.downloadDashboards.env }} - name: "{{ $key }}" value: "{{ $value }}" {{- end }} + {{- with .Values.downloadDashboards.securityContext }} + securityContext: + {{- toYaml . | nindent 6 }} + {{- end }} {{- if .Values.downloadDashboards.envFromSecret }} envFrom: - secretRef: @@ -86,6 +93,14 @@ initContainers: {{- end }} imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} - name: METHOD value: "LIST" - name: LABEL @@ -94,6 +109,10 @@ initContainers: - name: LABEL_VALUE value: {{ quote .Values.sidecar.datasources.labelValue }} {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} - name: FOLDER value: "/etc/grafana/provisioning/datasources" - name: RESOURCE @@ -110,12 +129,14 @@ initContainers: - name: SKIP_TLS_VERIFY value: "{{ .Values.sidecar.skipTlsVerify }}" {{- end }} + {{- with .Values.sidecar.resources }} resources: -{{ toYaml .Values.sidecar.resources | indent 6 }} -{{- if .Values.sidecar.securityContext }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} securityContext: -{{- toYaml .Values.sidecar.securityContext | nindent 6 }} -{{- end }} + {{- toYaml . | nindent 6 }} + {{- end }} volumeMounts: - name: sc-datasources-volume mountPath: "/etc/grafana/provisioning/datasources" @@ -129,10 +150,26 @@ initContainers: {{- end }} imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} env: + {{- range $key, $value := .Values.sidecar.notifiers.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.notifiers.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} - name: METHOD value: LIST - name: LABEL value: "{{ .Values.sidecar.notifiers.label }}" + {{- if .Values.sidecar.notifiers.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.notifiers.labelValue }} + {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.notifiers.logLevel }} + {{- end }} - name: FOLDER value: "/etc/grafana/provisioning/notifiers" - name: RESOURCE @@ -149,20 +186,22 @@ initContainers: - name: SKIP_TLS_VERIFY value: "{{ .Values.sidecar.skipTlsVerify }}" {{- end }} -{{- if .Values.sidecar.livenessProbe }} + {{- with .Values.sidecar.livenessProbe }} livenessProbe: -{{ toYaml .Values.livenessProbe | indent 6 }} -{{- end }} -{{- if .Values.sidecar.readinessProbe }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} readinessProbe: -{{ toYaml .Values.readinessProbe | indent 6 }} -{{- end }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} resources: -{{ toYaml .Values.sidecar.resources | indent 6 }} -{{- if .Values.sidecar.securityContext }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} securityContext: -{{- toYaml .Values.sidecar.securityContext | nindent 6 }} -{{- end }} + {{- toYaml . | nindent 6 }} + {{- end }} volumeMounts: - name: sc-notifiers-volume mountPath: "/etc/grafana/provisioning/notifiers" @@ -190,6 +229,14 @@ containers: {{- end }} imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} env: + {{- range $key, $value := .Values.sidecar.dashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.dashboards.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} - name: METHOD value: {{ .Values.sidecar.dashboards.watchMethod }} - name: LABEL @@ -198,9 +245,9 @@ containers: - name: LABEL_VALUE value: {{ quote .Values.sidecar.dashboards.labelValue }} {{- end }} - {{- if .Values.sidecar.logLevel }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} - name: LOG_LEVEL - value: {{ quote .Values.sidecar.logLevel }} + value: {{ default .Values.sidecar.logLevel .Values.sidecar.dashboards.logLevel }} {{- end }} - name: FOLDER value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" @@ -227,27 +274,35 @@ containers: value: "{{ .Values.sidecar.dashboards.script }}" {{- end }} {{- if .Values.sidecar.dashboards.watchServerTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchServerTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} - name: WATCH_SERVER_TIMEOUT value: "{{ .Values.sidecar.dashboards.watchServerTimeout }}" {{- end }} {{- if .Values.sidecar.dashboards.watchClientTimeout }} + {{- if ne .Values.sidecar.dashboards.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.dashboards.watchClientTimeout with .Values.sidecar.dashboards.watchMethod %s" .Values.sidecar.dashboards.watchMethod) }} + {{- end }} - name: WATCH_CLIENT_TIMEOUT value: "{{ .Values.sidecar.dashboards.watchClientTimeout }}" {{- end }} -{{- if .Values.sidecar.livenessProbe }} + {{- with .Values.sidecar.livenessProbe }} livenessProbe: -{{ toYaml .Values.livenessProbe | indent 6 }} -{{- end }} -{{- if .Values.sidecar.readinessProbe }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} readinessProbe: -{{ toYaml .Values.readinessProbe | indent 6 }} -{{- end }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} resources: -{{ toYaml .Values.sidecar.resources | indent 6 }} -{{- if .Values.sidecar.securityContext }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} securityContext: -{{- toYaml .Values.sidecar.securityContext | nindent 6 }} -{{- end }} + {{- toYaml . | nindent 6 }} + {{- end }} volumeMounts: - name: sc-dashboard-volume mountPath: {{ .Values.sidecar.dashboards.folder | quote }} @@ -264,6 +319,14 @@ containers: {{- end }} imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} env: + {{- range $key, $value := .Values.sidecar.datasources.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.datasources.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} - name: METHOD value: {{ .Values.sidecar.datasources.watchMethod }} - name: LABEL @@ -272,6 +335,10 @@ containers: - name: LABEL_VALUE value: {{ quote .Values.sidecar.datasources.labelValue }} {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.datasources.logLevel }} + {{- end }} - name: FOLDER value: "/etc/grafana/provisioning/datasources" - name: RESOURCE @@ -288,6 +355,10 @@ containers: - name: SKIP_TLS_VERIFY value: "{{ .Values.sidecar.skipTlsVerify }}" {{- end }} + {{- if .Values.sidecar.datasources.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.datasources.script }}" + {{- end }} {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} - name: REQ_USERNAME valueFrom: @@ -308,20 +379,36 @@ containers: - name: REQ_METHOD value: POST {{- end }} -{{- if .Values.sidecar.livenessProbe }} + {{- if .Values.sidecar.datasources.watchServerTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchServerTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.datasources.watchClientTimeout }} + {{- if ne .Values.sidecar.datasources.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.datasources.watchClientTimeout with .Values.sidecar.datasources.watchMethod %s" .Values.sidecar.datasources.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.datasources.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} livenessProbe: -{{ toYaml .Values.livenessProbe | indent 6 }} -{{- end }} -{{- if .Values.sidecar.readinessProbe }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} readinessProbe: -{{ toYaml .Values.readinessProbe | indent 6 }} -{{- end }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} resources: -{{ toYaml .Values.sidecar.resources | indent 6 }} -{{- if .Values.sidecar.securityContext }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} securityContext: -{{- toYaml .Values.sidecar.securityContext | nindent 6 }} -{{- end }} + {{- toYaml . | nindent 6 }} + {{- end }} volumeMounts: - name: sc-datasources-volume mountPath: "/etc/grafana/provisioning/datasources" @@ -335,6 +422,14 @@ containers: {{- end }} imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} env: + {{- range $key, $value := .Values.sidecar.plugins.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + {{- if .Values.sidecar.plugins.ignoreAlreadyProcessed }} + - name: IGNORE_ALREADY_PROCESSED + value: "true" + {{- end }} - name: METHOD value: {{ .Values.sidecar.plugins.watchMethod }} - name: LABEL @@ -343,6 +438,10 @@ containers: - name: LABEL_VALUE value: {{ quote .Values.sidecar.plugins.labelValue }} {{- end }} + {{- if or .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + - name: LOG_LEVEL + value: {{ default .Values.sidecar.logLevel .Values.sidecar.plugins.logLevel }} + {{- end }} - name: FOLDER value: "/etc/grafana/provisioning/plugins" - name: RESOURCE @@ -355,6 +454,10 @@ containers: - name: NAMESPACE value: "{{ .Values.sidecar.plugins.searchNamespace | join "," }}" {{- end }} + {{- if .Values.sidecar.plugins.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.plugins.script }}" + {{- end }} {{- if .Values.sidecar.skipTlsVerify }} - name: SKIP_TLS_VERIFY value: "{{ .Values.sidecar.skipTlsVerify }}" @@ -379,20 +482,36 @@ containers: - name: REQ_METHOD value: POST {{- end }} -{{- if .Values.sidecar.livenessProbe }} + {{- if .Values.sidecar.plugins.watchServerTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchServerTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.plugins.watchClientTimeout }} + {{- if ne .Values.sidecar.plugins.watchMethod "WATCH" }} + {{- fail (printf "Cannot use .Values.sidecar.plugins.watchClientTimeout with .Values.sidecar.plugins.watchMethod %s" .Values.sidecar.plugins.watchMethod) }} + {{- end }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.plugins.watchClientTimeout }}" + {{- end }} + {{- with .Values.sidecar.livenessProbe }} livenessProbe: -{{ toYaml .Values.livenessProbe | indent 6 }} -{{- end }} -{{- if .Values.sidecar.readinessProbe }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.readinessProbe }} readinessProbe: -{{ toYaml .Values.readinessProbe | indent 6 }} -{{- end }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.resources }} resources: -{{ toYaml .Values.sidecar.resources | indent 6 }} -{{- if .Values.sidecar.securityContext }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.sidecar.securityContext }} securityContext: -{{- toYaml .Values.sidecar.securityContext | nindent 6 }} -{{- end }} + {{- toYaml . | nindent 6 }} + {{- end }} volumeMounts: - name: sc-plugins-volume mountPath: "/etc/grafana/provisioning/plugins" @@ -410,10 +529,10 @@ containers: - {{ . }} {{- end }} {{- end}} -{{- if .Values.containerSecurityContext }} + {{- with .Values.containerSecurityContext }} securityContext: -{{- toYaml .Values.containerSecurityContext | nindent 6 }} -{{- end }} + {{- toYaml . | nindent 6 }} + {{- end }} volumeMounts: - name: config mountPath: "/etc/grafana/grafana.ini" @@ -466,6 +585,13 @@ containers: subPath: {{ . | quote }} {{- end }} {{- end }} +{{- if .Values.alerting }} +{{- range (keys .Values.alerting | sortAlpha) }} + - name: config + mountPath: "/etc/grafana/provisioning/alerting/{{ . }}" + subPath: {{ . | quote }} +{{- end }} +{{- end }} {{- if .Values.dashboardProviders }} {{- range (keys .Values.dashboardProviders | sortAlpha) }} - name: config @@ -511,11 +637,8 @@ containers: mountPath: {{ .mountPath }} {{- end }} ports: - - name: {{ .Values.service.portName }} - containerPort: {{ .Values.service.port }} - protocol: TCP - name: {{ .Values.podPortName }} - containerPort: 3000 + containerPort: {{ .Values.service.targetPort }} protocol: TCP env: {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} @@ -595,30 +718,40 @@ containers: optional: {{ .optional | default false }} {{- end }} {{- end }} + {{- with .Values.livenessProbe }} livenessProbe: -{{ toYaml .Values.livenessProbe | indent 6 }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.readinessProbe }} readinessProbe: -{{ toYaml .Values.readinessProbe | indent 6 }} + {{- toYaml . | nindent 6 }} + {{- end }} {{- if .Values.lifecycleHooks }} lifecycle: {{ tpl (.Values.lifecycleHooks | toYaml) . | nindent 6 }} {{- end }} + {{- with .Values.resources }} resources: -{{ toYaml .Values.resources | indent 6 }} + {{- toYaml . | nindent 6 }} + {{- end }} {{- with .Values.extraContainers }} {{ tpl . $ | indent 2 }} {{- end }} {{- with .Values.nodeSelector }} nodeSelector: -{{ toYaml . | indent 2 }} + {{- toYaml . | nindent 2 }} {{- end }} {{- $root := . }} {{- with .Values.affinity }} affinity: {{ tpl (toYaml .) $root | indent 2 }} {{- end }} +{{- with .Values.topologySpreadConstraints }} +topologySpreadConstraints: + {{- toYaml . | nindent 2 }} +{{- end }} {{- with .Values.tolerations }} tolerations: -{{ toYaml . | indent 2 }} + {{- toYaml . | nindent 2 }} {{- end }} volumes: - name: config @@ -744,6 +877,10 @@ volumes: {{- else if .hostPath }} hostPath: path: {{ .hostPath }} + {{- else if .csi }} + csi: + data: + {{ toYaml .data | nindent 6 }} {{- else }} emptyDir: {} {{- end }} @@ -753,6 +890,6 @@ volumes: emptyDir: {} {{- end -}} {{- if .Values.extraContainerVolumes }} -{{ toYaml .Values.extraContainerVolumes | indent 2 }} +{{ tpl (toYaml .Values.extraContainerVolumes) . | indent 2 }} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml index 0746a39c..3f519957 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml @@ -56,6 +56,14 @@ data: {{- end -}} {{- end -}} +{{- if .Values.alerting }} +{{ $root := . }} + {{- range $key, $value := .Values.alerting }} + {{ $key }}: | +{{ tpl $value $root | indent 4 }} + {{- end -}} +{{- end -}} + {{- if .Values.dashboardProviders }} {{- range $key, $value := .Values.dashboardProviders }} {{ $key }}: | @@ -86,6 +94,12 @@ data: {{- if $value.token }} -H "Authorization: token {{ $value.token }}" \ {{- end }} + {{- if $value.bearerToken }} + -H "Authorization: Bearer {{ $value.bearerToken }}" \ + {{- end }} + {{- if $value.gitlabToken }} + -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \ + {{- end }} -H "Content-Type: application/json;charset=UTF-8" \ {{ end }} {{- $dpPath := "" -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/deployment.yaml index d0465b1e..fee9c335 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/deployment.yaml @@ -46,5 +46,5 @@ spec: {{ toYaml . | indent 8 }} {{- end }} spec: - {{- include "grafana.pod" . | nindent 6 }} + {{- include "grafana.pod" . | indent 6 }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/headless-service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/headless-service.yaml index 1df42e96..b5faddcf 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/headless-service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/headless-service.yaml @@ -18,5 +18,5 @@ spec: ports: - protocol: TCP port: 3000 - targetPort: 3000 + targetPort: {{ .Values.service.targetPort }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml index 1a9d4c58..c558a86a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml @@ -77,7 +77,7 @@ spec: {{- end}} ports: - name: {{ .Values.imageRenderer.service.portName }} - containerPort: {{ .Values.imageRenderer.service.port }} + containerPort: {{ .Values.imageRenderer.service.targetPort }} protocol: TCP livenessProbe: httpGet: @@ -85,7 +85,7 @@ spec: port: {{ .Values.imageRenderer.service.portName }} env: - name: HTTP_PORT - value: {{ .Values.imageRenderer.service.port | quote }} + value: {{ .Values.imageRenderer.service.targetPort | quote }} {{- range $key, $value := .Values.imageRenderer.env }} - name: {{ $key | quote }} value: {{ $value | quote }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-network-policy.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-network-policy.yaml index 8136ff88..0d9bdfe4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-network-policy.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-network-policy.yaml @@ -19,7 +19,7 @@ spec: - Ingress ingress: - ports: - - port: {{ .Values.imageRenderer.service.port }} + - port: {{ .Values.imageRenderer.service.targetPort }} protocol: TCP from: - namespaceSelector: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-service.yaml index f29586c3..fcf707a3 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-service.yaml @@ -24,6 +24,9 @@ spec: port: {{ .Values.imageRenderer.service.port }} protocol: TCP targetPort: {{ .Values.imageRenderer.service.targetPort }} + {{- if .Values.imageRenderer.appProtocol }} + appProtocol: {{ .Values.imageRenderer.appProtocol }} + {{- end }} selector: {{- include "grafana.imageRenderer.selectorLabels" . | nindent 4 }} {{ end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/networkpolicy.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/networkpolicy.yaml index d277a9d3..b751d943 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/networkpolicy.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/networkpolicy.yaml @@ -6,13 +6,13 @@ metadata: namespace: {{ template "grafana.namespace" . }} labels: {{- include "grafana.labels" . | nindent 4 }} -{{- if .Values.labels }} -{{ toYaml .Values.labels | indent 4 }} -{{- end }} -{{- with .Values.annotations }} + {{- with .Values.labels }} + {{ toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.annotations }} annotations: -{{ toYaml . | indent 4 }} -{{- end }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: policyTypes: {{- if .Values.networkPolicy.ingress }} @@ -39,10 +39,10 @@ spec: - podSelector: matchLabels: {{ template "grafana.fullname" . }}-client: "true" - {{- if .Values.networkPolicy.explicitNamespacesSelector }} - namespaceSelector: - {{ toYaml .Values.networkPolicy.explicitNamespacesSelector | indent 12 }} - {{- end }} + {{- with .Values.networkPolicy.explicitNamespacesSelector }} + - namespaceSelector: + {{- toYaml . | nindent 12 }} + {{- end }} - podSelector: matchLabels: {{- include "grafana.labels" . | nindent 14 }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/podsecuritypolicy.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/podsecuritypolicy.yaml index 881df6f6..d9905c62 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/podsecuritypolicy.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/podsecuritypolicy.yaml @@ -1,5 +1,6 @@ {{- if .Values.rbac.pspEnabled }} -apiVersion: {{ include "grafana.podSecurityPolicy.apiVersion" . }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} +apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: name: {{ template "grafana.fullname" . }} @@ -47,3 +48,4 @@ spec: max: 65535 readOnlyRootFilesystem: false {{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/service.yaml index ba84ef97..d0a1756c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/service.yaml @@ -9,9 +9,10 @@ metadata: {{- if .Values.service.labels }} {{ toYaml .Values.service.labels | indent 4 }} {{- end }} +{{- $root := . }} {{- with .Values.service.annotations }} annotations: -{{ toYaml . | indent 4 }} +{{ tpl (toYaml . | indent 4) $root }} {{- end }} spec: {{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} @@ -40,12 +41,15 @@ spec: port: {{ .Values.service.port }} protocol: TCP targetPort: {{ .Values.service.targetPort }} -{{ if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + {{- if .Values.service.appProtocol }} + appProtocol: {{ .Values.service.appProtocol }} + {{- end }} + {{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} nodePort: {{.Values.service.nodePort}} -{{ end }} - {{- if .Values.extraExposePorts }} - {{- tpl (toYaml .Values.extraExposePorts) . | indent 4 }} - {{- end }} + {{ end }} + {{- if .Values.extraExposePorts }} + {{- tpl (toYaml .Values.extraExposePorts) . | nindent 4 }} + {{- end }} selector: {{- include "grafana.selectorLabels" . | nindent 4 }} {{ end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/servicemonitor.yaml index 56bc68ed..0876a637 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/servicemonitor.yaml @@ -5,7 +5,7 @@ kind: ServiceMonitor metadata: name: {{ template "grafana.fullname" . }} {{- if .Values.serviceMonitor.namespace }} - namespace: {{ .Values.serviceMonitor.namespace }} + namespace: {{ tpl .Values.serviceMonitor.namespace . }} {{- else }} namespace: {{ template "grafana.namespace" . }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/tests/test-configmap.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/tests/test-configmap.yaml index ff53aaf1..772ecbbe 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/tests/test-configmap.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/tests/test-configmap.yaml @@ -11,7 +11,7 @@ data: @test "Test Health" { url="http://{{ template "grafana.fullname" . }}/api/health" - code=$(wget --server-response --spider --timeout 10 --tries 1 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + code=$(wget --server-response --spider --timeout 90 --tries 10 ${url} 2>&1 | awk '/^ HTTP/{print $2}') [ "$code" == "200" ] } {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/tests/test-podsecuritypolicy.yaml index 58b46498..6ccc7a70 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/tests/test-podsecuritypolicy.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -1,4 +1,5 @@ {{- if and .Values.testFramework.enabled .Values.rbac.pspEnabled }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: @@ -27,3 +28,4 @@ spec: - csi - secret {{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml index 985bcf51..f1aaea1b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml @@ -136,6 +136,7 @@ downloadDashboards: env: {} envFromSecret: "" resources: {} + securityContext: {} ## Pod Annotations # podAnnotations: {} @@ -158,9 +159,12 @@ service: port: 80 targetPort: 3000 # targetPort: 4181 To be used with a proxy extraContainer + ## Service annotations. Can be templated. annotations: {} labels: {} portName: service + # Adds the appProtocol field to the service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" serviceMonitor: ## If true, a ServiceMonitor CRD is created for a prometheus operator @@ -249,6 +253,11 @@ tolerations: [] ## affinity: {} +## Topology Spread Constraints +## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +## +topologySpreadConstraints: [] + ## Additional init containers (evaluated as template) ## ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ ## @@ -468,6 +477,13 @@ extraVolumeMounts: [] # mountPath: /mnt/volume1 # readOnly: true # hostPath: /usr/shared/ + # - name: grafana-secrets + # csi: true + # data: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "grafana-env-spc" ## Container Lifecycle Hooks. Execute a specific bash command or make an HTTP request lifecycleHooks: {} @@ -502,6 +518,71 @@ datasources: {} # authType: default # defaultRegion: us-east-1 +## Configure grafana alerting (can be templated) +## ref: http://docs.grafana.org/administration/provisioning/#alerting +## +alerting: {} +# rules.yaml: | +# apiVersion: 1 +# groups: +# - orgId: 1 +# name: {{ .Chart.Name }}_my_rule_group +# folder: my_first_folder +# interval: 60s +# rules: +# - uid: my_id_1 +# title: my_first_rule +# condition: A +# data: +# - refId: A +# datasourceUid: '-100' +# model: +# conditions: +# - evaluator: +# params: +# - 3 +# type: gt +# operator: +# type: and +# query: +# params: +# - A +# reducer: +# type: last +# type: query +# datasource: +# type: __expr__ +# uid: '-100' +# expression: 1==0 +# intervalMs: 1000 +# maxDataPoints: 43200 +# refId: A +# type: math +# dashboardUid: my_dashboard +# panelId: 123 +# noDataState: Alerting +# for: 60s +# annotations: +# some_key: some_value +# labels: +# team: sre_team_1 +# contactpoints.yaml: | +# apiVersion: 1 +# contactPoints: +# - orgId: 1 +# name: cp_1 +# receivers: +# - uid: first_uid +# type: pagerduty +# settings: +# integrationKey: XXX +# severity: critical +# class: ping failure +# component: Grafana +# group: app-stack +# summary: | +# {{ `{{ template "default.message" . }}` }} + ## Configure notifiers ## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels ## @@ -562,6 +643,12 @@ dashboards: {} # url: https://example.com/repository/test-b64.json # token: '' # b64content: true + # local-dashboard-gitlab: + # url: https://example.com/repository/test-gitlab.json + # gitlabToken: '' + # local-dashboard-bitbucket: + # url: https://example.com/repository/test-bitbucket.json + # bearerToken: '' ## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. ## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both. @@ -590,6 +677,8 @@ grafana.ini: mode: console grafana_net: url: https://grafana.net + server: + domain: "{{ if (and .Values.ingress.enabled .Values.ingress.hosts) }}{{ .Values.ingress.hosts | first }}{{ end }}" ## grafana Authentication can be enabled with the following values on grafana.ini # server: # The full public facing url you use in browser, used for redirects and emails @@ -667,15 +756,21 @@ sidecar: enableUniqueFilenames: false readinessProbe: {} livenessProbe: {} - # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. - logLevel: INFO + # Log level default for all sidecars. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. Defaults to INFO + # logLevel: INFO dashboards: enabled: false + # Additional environment variables for the dashboards sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true SCProvider: true # label that the configmaps with dashboards are marked with label: grafana_dashboard # value of label that the configmaps with dashboards are set to labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) folder: /tmp/dashboards # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead @@ -725,10 +820,16 @@ sidecar: sizeLimit: {} datasources: enabled: false + # Additional environment variables for the datasourcessidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true # label that the configmaps with datasources are marked with label: grafana_datasource # value of label that the configmaps with datasources are set to labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO # If specified, the sidecar will search for datasource config-maps inside this namespace. # Otherwise the namespace in which the sidecar is running will be used. # It's also possible to specify ALL to search in all namespaces @@ -737,8 +838,20 @@ sidecar: watchMethod: WATCH # search in configmap, secret or both resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # # Endpoint to send request to reload datasources reloadURL: "http://localhost:3000/api/admin/provisioning/datasources/reload" + # Absolute path to shell script to execute after a datasource got reloaded + script: null skipReload: false # Deploy the datasource sidecar as an initContainer in addition to a container. # This is needed if skipReload is true, to load any datasources defined at startup time. @@ -747,10 +860,16 @@ sidecar: sizeLimit: {} plugins: enabled: false + # Additional environment variables for the plugins sidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true # label that the configmaps with plugins are marked with label: grafana_plugin # value of label that the configmaps with plugins are set to labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO # If specified, the sidecar will search for plugin config-maps inside this namespace. # Otherwise the namespace in which the sidecar is running will be used. # It's also possible to specify ALL to search in all namespaces @@ -759,8 +878,20 @@ sidecar: watchMethod: WATCH # search in configmap, secret or both resource: both + # watchServerTimeout: request to the server, asking it to cleanly close the connection after that. + # defaults to 60sec; much higher values like 3600 seconds (1h) are feasible for non-Azure K8S + # watchServerTimeout: 3600 + # + # watchClientTimeout: is a client-side timeout, configuring your local socket. + # If you have a network outage dropping all packets with no RST/FIN, + # this is how long your client waits before realizing & dropping the connection. + # defaults to 66sec (sic!) + # watchClientTimeout: 60 + # # Endpoint to send request to reload plugins reloadURL: "http://localhost:3000/api/admin/provisioning/plugins/reload" + # Absolute path to shell script to execute after a plugin got reloaded + script: null skipReload: false # Deploy the datasource sidecar as an initContainer in addition to a container. # This is needed if skipReload is true, to load any plugins defined at startup time. @@ -769,8 +900,16 @@ sidecar: sizeLimit: {} notifiers: enabled: false + # Additional environment variables for the notifierssidecar + env: {} + # Do not reprocess already processed unchanged resources on k8s API reconnect. + # ignoreAlreadyProcessed: true # label that the configmaps with notifiers are marked with label: grafana_notifier + # value of label that the configmaps with notifiers are set to + labelValue: "" + # Log level. Can be one of: DEBUG, INFO, WARN, ERROR, CRITICAL. + # logLevel: INFO # If specified, the sidecar will search for notifier config-maps inside this namespace. # Otherwise the namespace in which the sidecar is running will be used. # It's also possible to specify ALL to search in all namespaces @@ -824,6 +963,8 @@ imageRenderer: # image-renderer service port used by both service and deployment port: 8081 targetPort: 8081 + # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp" + appProtocol: "" # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana grafanaProtocol: http # In case a sub_path is used this needs to be added to the image renderer callback diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml index 188e7edd..6636f4f4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 2.5.0 +appVersion: 2.6.0 description: Install kube-state-metrics to generate and expose cluster-level metrics home: https://github.com/kubernetes/kube-state-metrics/ keywords: @@ -18,4 +18,4 @@ name: kube-state-metrics sources: - https://github.com/kubernetes/kube-state-metrics/ type: application -version: 4.15.0 +version: 4.18.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml index 0dfc963f..cdc62f19 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml @@ -92,7 +92,9 @@ spec: {{- if .Values.selfMonitor.telemetryHost }} - --telemetry-host={{ .Values.selfMonitor.telemetryHost }} {{- end }} + {{- if .Values.selfMonitor.telemetryPort }} - --telemetry-port={{ .Values.selfMonitor.telemetryPort | default 8081 }} + {{- end }} {{- if or (.Values.kubeconfig.enabled) (.Values.volumeMounts) }} volumeMounts: {{- if .Values.kubeconfig.enabled }} @@ -105,7 +107,11 @@ spec: {{- end }} {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.image.sha }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}@sha256:{{ .Values.image.sha }}" + {{- else }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + {{- end }} ports: - containerPort: {{ .Values.service.port | default 8080}} name: "http" diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml index cf74d67e..1fe0660a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml @@ -2,7 +2,8 @@ prometheusScrape: true image: repository: registry.k8s.io/kube-state-metrics/kube-state-metrics - tag: v2.5.0 + tag: v2.6.0 + sha: "" pullPolicy: IfNotPresent imagePullSecrets: [] diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml index ef9937fa..efa2bb66 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml @@ -15,4 +15,4 @@ name: prometheus-node-exporter sources: - https://github.com/prometheus/node_exporter/ type: application -version: 3.3.1 +version: 4.2.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/README.md index 59d78e71..02de7b14 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/README.md @@ -1,17 +1,17 @@ -# Prometheus Node Exporter +# Prometheus `Node Exporter` Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors. -This chart bootstraps a prometheus [Node Exporter](http://github.com/prometheus/node_exporter) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. +This chart bootstraps a prometheus [`Node Exporter`](http://github.com/prometheus/node_exporter) daemonset on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. -## Get Repo Info +## Get Repository Info ```console helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update ``` -_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ +_See [`helm repo`](https://helm.sh/docs/helm/helm_repo/) for command documentation._ ## Install Chart @@ -19,7 +19,7 @@ _See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation helm install [RELEASE_NAME] prometheus-community/prometheus-node-exporter ``` -_See [configuration](#configuration) below._ +_See [configuration](#configuring) below._ _See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ @@ -41,6 +41,17 @@ helm upgrade [RELEASE_NAME] [CHART] --install _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ +### 3.x to 4.x + +Starting from version 4.0.0, the `node exporter` chart is using the [Kubernetes recommended labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). Therefore you have to delete the daemonset before you upgrade. + +```console +kubectl delete daemonset -l app=prometheus-node-exporter +helm upgrade -i prometheus-node-exporter prometheus-community/prometheus-node-exporter +``` + +If you use your own custom [ServiceMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#servicemonitor) or [PodMonitor](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#podmonitor), please ensure to upgrade their `selector` fields accordingly to the new labels. + ### From 2.x to 3.x Change the following: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/NOTES.txt b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/NOTES.txt index dc272fa9..df05e3fb 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/NOTES.txt +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/NOTES.txt @@ -9,7 +9,7 @@ export SERVICE_IP=$(kubectl get svc --namespace {{ template "prometheus-node-exporter.namespace" . }} {{ template "prometheus-node-exporter.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') echo http://$SERVICE_IP:{{ .Values.service.port }} {{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ template "prometheus-node-exporter.namespace" . }} -l "app={{ template "prometheus-node-exporter.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export POD_NAME=$(kubectl get pods --namespace {{ template "prometheus-node-exporter.namespace" . }} -l "app.kubernetes.io/name={{ template "prometheus-node-exporter.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") echo "Visit http://127.0.0.1:9100 to use your application" kubectl port-forward --namespace {{ template "prometheus-node-exporter.namespace" . }} $POD_NAME 9100 {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/_helpers.tpl index b6cbc91c..21030894 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/_helpers.tpl @@ -26,13 +26,28 @@ If release name contains chart name it will be used as a full name. {{/* Generate basic labels */}} {{- define "prometheus-node-exporter.labels" }} -app: {{ template "prometheus-node-exporter.name" . }} -heritage: {{.Release.Service }} -release: {{.Release.Name }} -chart: {{ template "prometheus-node-exporter.chart" . }} +helm.sh/chart: {{ template "prometheus-node-exporter.chart" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/component: metrics +app.kubernetes.io/part-of: {{ template "prometheus-node-exporter.name" . }} +{{- include "prometheus-node-exporter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} {{- if .Values.podLabels}} {{ toYaml .Values.podLabels }} {{- end }} +{{- if .Values.releaseLabel }} +release: {{ .Release.Name }} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-node-exporter.selectorLabels" }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/name: {{ template "prometheus-node-exporter.name" . }} {{- end }} {{/* @@ -58,8 +73,12 @@ Create the name of the service account to use The image to use */}} {{- define "prometheus-node-exporter.image" -}} +{{- if .Values.image.sha -}} +{{- printf "%s:%s@%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }} +{{- else -}} {{- printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }} {{- end }} +{{- end }} {{/* Allow the release namespace to be overridden for multi-namespace deployments in combined charts diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml index c49d930d..a742f24c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -7,8 +7,7 @@ metadata: spec: selector: matchLabels: - app: {{ template "prometheus-node-exporter.name" . }} - release: {{ .Release.Name }} + {{- include "prometheus-node-exporter.selectorLabels" . | indent 6 }} {{- if .Values.updateStrategy }} updateStrategy: {{ toYaml .Values.updateStrategy | indent 4 }} @@ -147,13 +146,21 @@ spec: {{- end }} {{- if .Values.sidecars }} {{ toYaml .Values.sidecars | indent 8 }} - {{- if .Values.sidecarVolumeMount }} + {{- if or .Values.sidecarVolumeMount .Values.sidecarHostVolumeMounts }} volumeMounts: {{- range $_, $mount := .Values.sidecarVolumeMount }} - name: {{ $mount.name }} mountPath: {{ $mount.mountPath }} readOnly: {{ $mount.readOnly }} {{- end }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- if $mount.mountPropagation }} + mountPropagation: {{ $mount.mountPropagation }} + {{- end }} + {{- end }} {{- end }} {{- end }} {{- if .Values.imagePullSecrets }} @@ -204,6 +211,13 @@ spec: medium: Memory {{- end }} {{- end }} + {{- if .Values.sidecarHostVolumeMounts }} + {{- range $_, $mount := .Values.sidecarHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- end }} {{- if .Values.configmaps }} {{- range $_, $mount := .Values.configmaps }} - name: {{ $mount.name }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/endpoints.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/endpoints.yaml index d070c492..ef3e270b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/endpoints.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/endpoints.yaml @@ -4,8 +4,7 @@ kind: Endpoints metadata: name: {{ template "prometheus-node-exporter.fullname" . }} namespace: {{ template "prometheus-node-exporter.namespace" . }} - labels: -{{ include "prometheus-node-exporter.labels" . | indent 4 }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} subsets: - addresses: {{- range .Values.endpoints }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/service.yaml index cfccaa8e..fbed05ca 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/service.yaml @@ -3,11 +3,11 @@ kind: Service metadata: name: {{ template "prometheus-node-exporter.fullname" . }} namespace: {{ template "prometheus-node-exporter.namespace" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} {{- if .Values.service.annotations }} annotations: {{ toYaml .Values.service.annotations | indent 4 }} {{- end }} - labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} spec: type: {{ .Values.service.type }} ports: @@ -19,5 +19,4 @@ spec: protocol: TCP name: {{ .Values.service.portName }} selector: - app: {{ template "prometheus-node-exporter.name" . }} - release: {{ .Release.Name }} + {{- include "prometheus-node-exporter.selectorLabels" . | indent 4 }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/serviceaccount.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/serviceaccount.yaml index 07e9f0d9..dc3fee6a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/serviceaccount.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/serviceaccount.yaml @@ -5,11 +5,7 @@ kind: ServiceAccount metadata: name: {{ template "prometheus-node-exporter.serviceAccountName" . }} namespace: {{ template "prometheus-node-exporter.namespace" . }} - labels: - app: {{ template "prometheus-node-exporter.name" . }} - chart: {{ template "prometheus-node-exporter.chart" . }} - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} annotations: {{ toYaml .Values.serviceAccount.annotations | indent 4 }} imagePullSecrets: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/servicemonitor.yaml index 302b7b5d..6cf5aa43 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/servicemonitor.yaml @@ -15,8 +15,7 @@ spec: {{- if .Values.prometheus.monitor.selectorOverride }} {{ toYaml .Values.prometheus.monitor.selectorOverride | indent 6 }} {{ else }} - app: {{ template "prometheus-node-exporter.name" . }} - release: {{ .Release.Name }} + {{ include "prometheus-node-exporter.selectorLabels" . | indent 6 }} {{- end }} endpoints: - port: {{ .Values.service.portName }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml index 80b399a0..ac22d28e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml @@ -6,6 +6,7 @@ image: # Overrides the image tag whose default is {{ printf "v%s" .Chart.AppVersion }} tag: "" pullPolicy: IfNotPresent + sha: "" imagePullSecrets: [] # - name: "image-pull-secret" @@ -139,6 +140,9 @@ podAnnotations: # Extra labels to be added to node exporter pods podLabels: {} +## set to true to add the release label so scraping of the servicemonitor with kube-prometheus-stack works out of the box +releaseLabel: false + # Custom DNS configuration to be added to prometheus-node-exporter pods dnsConfig: {} # nameservers: @@ -170,7 +174,7 @@ extraArgs: [] # - --collector.diskstats.ignored-devices=^(ram|loop|fd|(h|s|v)d[a-z]|nvme\\d+n\\d+p)\\d+$ # - --collector.textfile.directory=/run/prometheus -## Additional mounts from the host +## Additional mounts from the host to node-exporter container ## extraHostVolumeMounts: [] # - name: @@ -204,6 +208,15 @@ sidecarVolumeMount: [] ## mountPath: /run/prometheus ## readOnly: false +## Additional mounts from the host to sidecar containers +## +sidecarHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + ## Additional InitContainers to initialize the pod ## extraInitContainers: [] diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml index 5c391fd9..7863a2bc 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml index 1851cba3..236f26b9 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -981,15 +981,365 @@ spec: x-kubernetes-map-type: atomic alertmanagerConfiguration: description: 'EXPERIMENTAL: alertmanagerConfiguration specifies the - global Alertmanager configuration. If defined, it takes precedence - over the `configSecret` field. This field may change in future releases.' + configuration of Alertmanager. If defined, it takes precedence over + the `configSecret` field. This field may change in future releases.' properties: + global: + description: Defines the global parameters of the Alertmanager + configuration. + properties: + httpConfig: + description: HTTP client configuration. + properties: + authorization: + description: Authorization header configuration for the + client. This is mutually exclusive with BasicAuth and + is only available starting from Alertmanager v0.22+. + properties: + credentials: + description: The secret's key that contains the credentials + of the request + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: + description: Set the authentication type. Defaults + to Bearer, Basic will cause an error + type: string + type: object + basicAuth: + description: BasicAuth for the client. This is mutually + exclusive with Authorization. If both are defined, BasicAuth + takes precedence. + properties: + password: + description: The secret in the service monitor namespace + that contains the password for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + description: The secret in the service monitor namespace + that contains the username for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + bearerTokenSecret: + description: The secret's key that contains the bearer + token to be used by the client for authentication. The + secret needs to be in the same namespace as the Alertmanager + object and accessible by the Prometheus Operator. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + followRedirects: + description: FollowRedirects specifies whether the client + should follow HTTP 3xx redirects. + type: boolean + oauth2: + description: OAuth2 client credentials used to fetch a + token for the targets. + properties: + clientId: + description: The secret or configmap containing the + OAuth2 client id + properties: + configMap: + description: ConfigMap containing data to use + for the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + description: Secret containing data to use for + the targets. + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + clientSecret: + description: The secret containing the OAuth2 client + secret + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpointParams: + additionalProperties: + type: string + description: Parameters to append to the token URL + type: object + scopes: + description: OAuth2 scopes used for the token request + items: + type: string + type: array + tokenUrl: + description: The URL to fetch the token from + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object + proxyURL: + description: Optional proxy URL. + type: string + tlsConfig: + description: TLS configuration for the client. + properties: + ca: + description: Struct containing the CA cert to use + for the targets. + properties: + configMap: + description: ConfigMap containing data to use + for the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + description: Secret containing data to use for + the targets. + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + cert: + description: Struct containing the client cert file + for the targets. + properties: + configMap: + description: ConfigMap containing data to use + for the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + description: Secret containing data to use for + the targets. + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keySecret: + description: Secret containing the client key file + for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object + type: object + resolveTimeout: + description: ResolveTimeout is the default value used by alertmanager + if the alert does not include EndsAt, after this time passes + it can declare the alert as resolved if it has not been + updated. This has no impact on alerts from Prometheus, as + they always include EndsAt. + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + type: object name: description: The name of the AlertmanagerConfig resource which - is used to generate the global configuration. It must be defined - in the same namespace as the Alertmanager object. The operator - will not enforce a `namespace` label for routes and inhibition - rules. + is used to generate the Alertmanager configuration. It must + be defined in the same namespace as the Alertmanager object. + The operator will not enforce a `namespace` label for routes + and inhibition rules. minLength: 1 type: string type: object @@ -1595,13 +1945,13 @@ spec: Cannot be updated. type: string ports: - description: List of ports to expose from the container. Exposing - a port here gives the system additional information about - the network connections a container uses, but is primarily - informational. Not specifying a port here DOES NOT prevent - that port from being exposed. Any port which is listening - on the default "0.0.0.0" address inside a container will be - accessible from the network. Cannot be updated. + description: List of ports to expose from the container. Not + specifying a port here DOES NOT prevent that port from being + exposed. Any port which is listening on the default "0.0.0.0" + address inside a container will be accessible from the network. + Modifying this array with strategic merge patch may corrupt + the data. For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. items: description: ContainerPort represents a network port in a single container. @@ -2880,13 +3230,13 @@ spec: Cannot be updated. type: string ports: - description: List of ports to expose from the container. Exposing - a port here gives the system additional information about - the network connections a container uses, but is primarily - informational. Not specifying a port here DOES NOT prevent - that port from being exposed. Any port which is listening - on the default "0.0.0.0" address inside a container will be - accessible from the network. Cannot be updated. + description: List of ports to expose from the container. Not + specifying a port here DOES NOT prevent that port from being + exposed. Any port which is listening on the default "0.0.0.0" + address inside a container will be accessible from the network. + Modifying this array with strategic merge patch may corrupt + the data. For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. items: description: ContainerPort represents a network port in a single container. @@ -4509,6 +4859,19 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod label keys to select + the pods over which spreading will be calculated. The keys + are used to lookup values from the incoming pod labels, those + key-value labels are ANDed with labelSelector to select the + group of existing pods over which spreading will be calculated + for the incoming pod. Keys that don't exist in the incoming + pod labels will be ignored. A null or empty list means only + match against labelSelector. + items: + type: string + type: array + x-kubernetes-list-type: atomic maxSkew: description: 'MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, @@ -4549,11 +4912,31 @@ spec: minimum\" is treated as 0. In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the - three zones, it will violate MaxSkew. \n This is an alpha - field and requires enabling MinDomainsInPodTopologySpread - feature gate." + three zones, it will violate MaxSkew. \n This is a beta field + and requires the MinDomainsInPodTopologySpread feature gate + to be enabled (enabled by default)." format: int32 type: integer + nodeAffinityPolicy: + description: "NodeAffinityPolicy indicates how we will treat + Pod's nodeAffinity/nodeSelector when calculating pod topology + spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector + are included in the calculations. - Ignore: nodeAffinity/nodeSelector + are ignored. All nodes are included in the calculations. \n + If this value is nil, the behavior is equivalent to the Honor + policy. This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + nodeTaintsPolicy: + description: "NodeTaintsPolicy indicates how we will treat node + taints when calculating pod topology spread skew. Options + are: - Honor: nodes without taints, along with tainted nodes + for which the incoming pod has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + \n If this value is nil, the behavior is equivalent to the + Ignore policy. This is a alpha-level feature enabled by the + NodeInclusionPolicyInPodTopologySpread feature flag." + type: string topologyKey: description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered @@ -4561,10 +4944,11 @@ spec: as a "bucket", and try to put balanced number of pods into each bucket. We define a domain as a particular instance of a topology. Also, we define an eligible domain as a domain - whose nodes match the node selector. e.g. If TopologyKey is - "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each - zone is a domain of that topology. It's a required field. + whose nodes meet the requirements of nodeAffinityPolicy and + nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, if TopologyKey + is "topology.kubernetes.io/zone", each zone is a domain of + that topology. It's a required field. type: string whenUnsatisfiable: description: 'WhenUnsatisfiable indicates how to deal with a @@ -6153,8 +6537,55 @@ spec: web: description: Defines the web command line flags when starting Alertmanager. properties: + httpConfig: + description: Defines HTTP parameters for web server. + properties: + headers: + description: List of headers that can be added to HTTP responses. + properties: + contentSecurityPolicy: + description: Set the Content-Security-Policy header to + HTTP responses. Unset if blank. + type: string + strictTransportSecurity: + description: Set the Strict-Transport-Security header + to HTTP responses. Unset if blank. Please make sure + that you use this with care as this header might force + browsers to load Prometheus and the other applications + hosted on the same domain and subdomains over HTTPS. + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security + type: string + xContentTypeOptions: + description: Set the X-Content-Type-Options header to + HTTP responses. Unset if blank. Accepted value is nosniff. + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options + enum: + - "" + - NoSniff + type: string + xFrameOptions: + description: Set the X-Frame-Options header to HTTP responses. + Unset if blank. Accepted values are deny and sameorigin. + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options + enum: + - "" + - Deny + - SameOrigin + type: string + xXSSProtection: + description: Set the X-XSS-Protection header to all responses. + Unset if blank. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection + type: string + type: object + http2: + description: Enable HTTP/2 support. Note that HTTP/2 is only + supported with TLS. When TLSConfig is not configured, HTTP/2 + will be disabled. Whenever the value of the field changes, + a rolling update will be triggered. + type: boolean + type: object tlsConfig: - description: WebTLSConfig defines the TLS parameters for HTTPS. + description: Defines the TLS parameters for HTTPS. properties: cert: description: Contains the TLS certificate for the server. diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml index c967172a..f6e208c1 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml index 34ee1a16..014ef274 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml index 8f7bdace..e3798352 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -110,6 +110,31 @@ spec: - key type: object x-kubernetes-map-type: atomic + additionalArgs: + description: AdditionalArgs allows setting additional arguments for + the Prometheus container. It is intended for e.g. activating hidden + flags which are not supported by the dedicated configuration options + yet. The arguments are passed as-is to the Prometheus container + which may cause issues if they are invalid or not supporeted by + the given Prometheus version. In case of an argument conflict (e.g. + an argument which is already set by the operator itself) or when + providing an invalid argument the reconciliation will fail and an + error will be logged. + items: + description: Argument as part of the AdditionalArgs list. + properties: + name: + description: Name of the argument, e.g. "scrape.discovery-reload-interval". + minLength: 1 + type: string + value: + description: Argument value, e.g. 30s. Can be empty for name-only + arguments (e.g. --storage.tsdb.no-lockfile) + type: string + required: + - name + type: object + type: array additionalScrapeConfigs: description: 'AdditionalScrapeConfigs allows specifying a key of a Secret containing additional Prometheus scrape configurations. Scrape @@ -2004,13 +2029,13 @@ spec: Cannot be updated. type: string ports: - description: List of ports to expose from the container. Exposing - a port here gives the system additional information about - the network connections a container uses, but is primarily - informational. Not specifying a port here DOES NOT prevent - that port from being exposed. Any port which is listening - on the default "0.0.0.0" address inside a container will be - accessible from the network. Cannot be updated. + description: List of ports to expose from the container. Not + specifying a port here DOES NOT prevent that port from being + exposed. Any port which is listening on the default "0.0.0.0" + address inside a container will be accessible from the network. + Modifying this array with strategic merge patch may corrupt + the data. For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. items: description: ContainerPort represents a network port in a single container. @@ -3440,13 +3465,13 @@ spec: Cannot be updated. type: string ports: - description: List of ports to expose from the container. Exposing - a port here gives the system additional information about - the network connections a container uses, but is primarily - informational. Not specifying a port here DOES NOT prevent - that port from being exposed. Any port which is listening - on the default "0.0.0.0" address inside a container will be - accessible from the network. Cannot be updated. + description: List of ports to expose from the container. Not + specifying a port here DOES NOT prevent that port from being + exposed. Any port which is listening on the default "0.0.0.0" + address inside a container will be accessible from the network. + Modifying this array with strategic merge patch may corrupt + the data. For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. items: description: ContainerPort represents a network port in a single container. @@ -6347,6 +6372,29 @@ spec: notice in any release. \n This is experimental and may change significantly without backward compatibility in any release." properties: + additionalArgs: + description: AdditionalArgs allows setting additional arguments + for the Thanos container. The arguments are passed as-is to + the Thanos container which may cause issues if they are invalid + or not supporeted the given Thanos version. In case of an argument + conflict (e.g. an argument which is already set by the operator + itself) or when providing an invalid argument the reconciliation + will fail and an error will be logged. + items: + description: Argument as part of the AdditionalArgs list. + properties: + name: + description: Name of the argument, e.g. "scrape.discovery-reload-interval". + minLength: 1 + type: string + value: + description: Argument value, e.g. 30s. Can be empty for + name-only arguments (e.g. --storage.tsdb.no-lockfile) + type: string + required: + - name + type: object + type: array baseImage: description: 'Thanos base image if other than default. Deprecated: use ''image'' instead' @@ -6756,6 +6804,19 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod label keys to select + the pods over which spreading will be calculated. The keys + are used to lookup values from the incoming pod labels, those + key-value labels are ANDed with labelSelector to select the + group of existing pods over which spreading will be calculated + for the incoming pod. Keys that don't exist in the incoming + pod labels will be ignored. A null or empty list means only + match against labelSelector. + items: + type: string + type: array + x-kubernetes-list-type: atomic maxSkew: description: 'MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, @@ -6796,11 +6857,31 @@ spec: minimum\" is treated as 0. In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the - three zones, it will violate MaxSkew. \n This is an alpha - field and requires enabling MinDomainsInPodTopologySpread - feature gate." + three zones, it will violate MaxSkew. \n This is a beta field + and requires the MinDomainsInPodTopologySpread feature gate + to be enabled (enabled by default)." format: int32 type: integer + nodeAffinityPolicy: + description: "NodeAffinityPolicy indicates how we will treat + Pod's nodeAffinity/nodeSelector when calculating pod topology + spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector + are included in the calculations. - Ignore: nodeAffinity/nodeSelector + are ignored. All nodes are included in the calculations. \n + If this value is nil, the behavior is equivalent to the Honor + policy. This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + nodeTaintsPolicy: + description: "NodeTaintsPolicy indicates how we will treat node + taints when calculating pod topology spread skew. Options + are: - Honor: nodes without taints, along with tainted nodes + for which the incoming pod has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + \n If this value is nil, the behavior is equivalent to the + Ignore policy. This is a alpha-level feature enabled by the + NodeInclusionPolicyInPodTopologySpread feature flag." + type: string topologyKey: description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered @@ -6808,10 +6889,11 @@ spec: as a "bucket", and try to put balanced number of pods into each bucket. We define a domain as a particular instance of a topology. Also, we define an eligible domain as a domain - whose nodes match the node selector. e.g. If TopologyKey is - "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each - zone is a domain of that topology. It's a required field. + whose nodes meet the requirements of nodeAffinityPolicy and + nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, if TopologyKey + is "topology.kubernetes.io/zone", each zone is a domain of + that topology. It's a required field. type: string whenUnsatisfiable: description: 'WhenUnsatisfiable indicates how to deal with a @@ -8404,11 +8486,58 @@ spec: web: description: Defines the web command line flags when starting Prometheus. properties: + httpConfig: + description: Defines HTTP parameters for web server. + properties: + headers: + description: List of headers that can be added to HTTP responses. + properties: + contentSecurityPolicy: + description: Set the Content-Security-Policy header to + HTTP responses. Unset if blank. + type: string + strictTransportSecurity: + description: Set the Strict-Transport-Security header + to HTTP responses. Unset if blank. Please make sure + that you use this with care as this header might force + browsers to load Prometheus and the other applications + hosted on the same domain and subdomains over HTTPS. + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security + type: string + xContentTypeOptions: + description: Set the X-Content-Type-Options header to + HTTP responses. Unset if blank. Accepted value is nosniff. + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Content-Type-Options + enum: + - "" + - NoSniff + type: string + xFrameOptions: + description: Set the X-Frame-Options header to HTTP responses. + Unset if blank. Accepted values are deny and sameorigin. + https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options + enum: + - "" + - Deny + - SameOrigin + type: string + xXSSProtection: + description: Set the X-XSS-Protection header to all responses. + Unset if blank. https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-XSS-Protection + type: string + type: object + http2: + description: Enable HTTP/2 support. Note that HTTP/2 is only + supported with TLS. When TLSConfig is not configured, HTTP/2 + will be disabled. Whenever the value of the field changes, + a rolling update will be triggered. + type: boolean + type: object pageTitle: description: The prometheus web page title type: string tlsConfig: - description: WebTLSConfig defines the TLS parameters for HTTPS. + description: Defines the TLS parameters for HTTPS. properties: cert: description: Contains the TLS certificate for the server. diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml index 35d16f5d..b28f61cd 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml index f9dd94cc..f64292b1 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml index eefc3592..12713a04 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.58.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.59.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1508,13 +1508,13 @@ spec: Cannot be updated. type: string ports: - description: List of ports to expose from the container. Exposing - a port here gives the system additional information about - the network connections a container uses, but is primarily - informational. Not specifying a port here DOES NOT prevent - that port from being exposed. Any port which is listening - on the default "0.0.0.0" address inside a container will be - accessible from the network. Cannot be updated. + description: List of ports to expose from the container. Not + specifying a port here DOES NOT prevent that port from being + exposed. Any port which is listening on the default "0.0.0.0" + address inside a container will be accessible from the network. + Modifying this array with strategic merge patch may corrupt + the data. For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. items: description: ContainerPort represents a network port in a single container. @@ -2955,13 +2955,13 @@ spec: Cannot be updated. type: string ports: - description: List of ports to expose from the container. Exposing - a port here gives the system additional information about - the network connections a container uses, but is primarily - informational. Not specifying a port here DOES NOT prevent - that port from being exposed. Any port which is listening - on the default "0.0.0.0" address inside a container will be - accessible from the network. Cannot be updated. + description: List of ports to expose from the container. Not + specifying a port here DOES NOT prevent that port from being + exposed. Any port which is listening on the default "0.0.0.0" + address inside a container will be accessible from the network. + Modifying this array with strategic merge patch may corrupt + the data. For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. items: description: ContainerPort represents a network port in a single container. @@ -4731,6 +4731,19 @@ spec: type: object type: object x-kubernetes-map-type: atomic + matchLabelKeys: + description: MatchLabelKeys is a set of pod label keys to select + the pods over which spreading will be calculated. The keys + are used to lookup values from the incoming pod labels, those + key-value labels are ANDed with labelSelector to select the + group of existing pods over which spreading will be calculated + for the incoming pod. Keys that don't exist in the incoming + pod labels will be ignored. A null or empty list means only + match against labelSelector. + items: + type: string + type: array + x-kubernetes-list-type: atomic maxSkew: description: 'MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, @@ -4771,11 +4784,31 @@ spec: minimum\" is treated as 0. In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the - three zones, it will violate MaxSkew. \n This is an alpha - field and requires enabling MinDomainsInPodTopologySpread - feature gate." + three zones, it will violate MaxSkew. \n This is a beta field + and requires the MinDomainsInPodTopologySpread feature gate + to be enabled (enabled by default)." format: int32 type: integer + nodeAffinityPolicy: + description: "NodeAffinityPolicy indicates how we will treat + Pod's nodeAffinity/nodeSelector when calculating pod topology + spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector + are included in the calculations. - Ignore: nodeAffinity/nodeSelector + are ignored. All nodes are included in the calculations. \n + If this value is nil, the behavior is equivalent to the Honor + policy. This is a alpha-level feature enabled by the NodeInclusionPolicyInPodTopologySpread + feature flag." + type: string + nodeTaintsPolicy: + description: "NodeTaintsPolicy indicates how we will treat node + taints when calculating pod topology spread skew. Options + are: - Honor: nodes without taints, along with tainted nodes + for which the incoming pod has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + \n If this value is nil, the behavior is equivalent to the + Ignore policy. This is a alpha-level feature enabled by the + NodeInclusionPolicyInPodTopologySpread feature flag." + type: string topologyKey: description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered @@ -4783,10 +4816,11 @@ spec: as a "bucket", and try to put balanced number of pods into each bucket. We define a domain as a particular instance of a topology. Also, we define an eligible domain as a domain - whose nodes match the node selector. e.g. If TopologyKey is - "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each - zone is a domain of that topology. It's a required field. + whose nodes meet the requirements of nodeAffinityPolicy and + nodeTaintsPolicy. e.g. If TopologyKey is "kubernetes.io/hostname", + each Node is a domain of that topology. And, if TopologyKey + is "topology.kubernetes.io/zone", each zone is a domain of + that topology. It's a required field. type: string whenUnsatisfiable: description: 'WhenUnsatisfiable indicates how to deal with a @@ -4832,6 +4866,11 @@ spec: - key type: object x-kubernetes-map-type: atomic + tracingConfigFile: + description: TracingConfig specifies the path of the tracing configuration + file. When used alongside with TracingConfig, TracingConfigFile + takes precedence. + type: string volumes: description: Volumes allows configuration of additional volumes on the output StatefulSet definition. Volumes specified will be appended diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml index 566fc6ed..abe36380 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -42,6 +42,10 @@ spec: - --host={{ template "kube-prometheus-stack.operator.fullname" . }},{{ template "kube-prometheus-stack.operator.fullname" . }}.{{ template "kube-prometheus-stack.namespace" . }}.svc - --namespace={{ template "kube-prometheus-stack.namespace" . }} - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + {{- with .Values.prometheusOperator.admissionWebhooks.createSecretJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} resources: {{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} restartPolicy: OnFailure diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml index 968d4740..560dc11f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -43,6 +43,10 @@ spec: - --namespace={{ template "kube-prometheus-stack.namespace" . }} - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission - --patch-failure-policy={{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- with .Values.prometheusOperator.admissionWebhooks.patchWebhookJob }} + securityContext: + {{ toYaml .securityContext | nindent 12 }} + {{- end }} resources: {{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} restartPolicy: OnFailure diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/additionalAlertmanagerConfigs.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/additionalAlertmanagerConfigs.yaml index 8aebc96c..2fe8fdb8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/additionalAlertmanagerConfigs.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/additionalAlertmanagerConfigs.yaml @@ -12,5 +12,5 @@ metadata: app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-confg {{ include "kube-prometheus-stack.labels" . | indent 4 }} data: - additional-alertmanager-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs | b64enc | quote }} + additional-alertmanager-configs.yaml: {{ tpl (toYaml .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs) . | b64enc | quote }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml index 4cd6e296..8e29e0c8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml @@ -112,8 +112,10 @@ spec: {{- if .Values.prometheus.prometheusSpec.retentionSize }} retentionSize: {{ .Values.prometheus.prometheusSpec.retentionSize | quote }} {{- end }} -{{- if .Values.prometheus.prometheusSpec.walCompression }} - walCompression: {{ .Values.prometheus.prometheusSpec.walCompression }} +{{- if eq .Values.prometheus.prometheusSpec.walCompression false }} + walCompression: false +{{ else }} + walCompression: true {{- end }} {{- if .Values.prometheus.prometheusSpec.routePrefix }} routePrefix: {{ .Values.prometheus.prometheusSpec.routePrefix | quote }} @@ -354,7 +356,7 @@ spec: name: "{{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) . | trunc 63 | trimSuffix "-" }}" {{- end }} {{- if .Values.prometheus.prometheusSpec.excludedFromEnforcement }} -{{ toYaml .Values.prometheus.prometheusSpec.excludedFromEnforcement | indent 4 }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.excludedFromEnforcement | indent 4) . }} {{- end }} {{- end }} {{- if .Values.prometheus.prometheusSpec.queryLogFile }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml index 0196a54c..74e7c6df 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml @@ -1530,6 +1530,7 @@ prometheus-node-exporter: ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards ## jobLabel: node-exporter + releaseLabel: true extraArgs: - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ @@ -1610,7 +1611,7 @@ prometheusOperator: enabled: true image: repository: k8s.gcr.io/ingress-nginx/kube-webhook-certgen - tag: v1.2.0 + tag: v1.3.0 sha: "" pullPolicy: IfNotPresent resources: {} @@ -1631,6 +1632,14 @@ prometheusOperator: runAsNonRoot: true runAsUser: 2000 + # Security context for create job container + createSecretJob: + securityContext: {} + + # Security context for patch job container + patchWebhookJob: + securityContext: {} + # Use certmanager to generate webhook certs certManager: enabled: false @@ -1841,7 +1850,7 @@ prometheusOperator: ## image: repository: quay.io/prometheus-operator/prometheus-operator - tag: v0.58.0 + tag: v0.59.1 sha: "" pullPolicy: IfNotPresent @@ -1856,10 +1865,9 @@ prometheusOperator: ## Prometheus-config-reloader ## prometheusConfigReloader: - # image to use for config and rule reloading image: repository: quay.io/prometheus-operator/prometheus-config-reloader - tag: v0.58.0 + tag: v0.59.1 sha: "" # resource config for prometheusConfigReloader @@ -1875,7 +1883,7 @@ prometheusOperator: ## thanosImage: repository: quay.io/thanos/thanos - tag: v0.27.0 + tag: v0.28.0 sha: "" ## Set a Field Selector to filter watched secrets @@ -2308,7 +2316,7 @@ prometheus: ## image: repository: quay.io/prometheus/prometheus - tag: v2.37.0 + tag: v2.38.0 sha: "" ## Tolerations for use with node taints @@ -2500,7 +2508,7 @@ prometheus: ## Enable compression of the write-ahead log using Snappy. ## - walCompression: false + walCompression: true ## If true, the Operator won't process any Prometheus configuration changes ## @@ -3139,7 +3147,7 @@ thanosRuler: ## image: repository: quay.io/thanos/thanos - tag: v0.27.0 + tag: v0.28.0 sha: "" ## Namespaces to be selected for PrometheusRules discovery. diff --git a/charts/kubezero-metrics/jsonnet/dashboards/apiserver.json b/charts/kubezero-metrics/jsonnet/dashboards/apiserver.json index 1c940dcb..946d237d 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/apiserver.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/apiserver.json @@ -56,7 +56,11 @@ }, "gridPos": { }, "id": 3, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -128,13 +132,14 @@ "fillGradient": 0, "gridPos": { }, "id": 4, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -236,7 +241,11 @@ }, "gridPos": { }, "id": 5, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -307,13 +316,14 @@ "fillGradient": 0, "gridPos": { }, "id": 6, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -406,13 +416,14 @@ "fillGradient": 0, "gridPos": { }, "id": 7, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -488,13 +499,14 @@ "fillGradient": 0, "gridPos": { }, "id": 8, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -516,7 +528,7 @@ "steppedLine": false, "targets": [ { - "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}", + "expr": "cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ resource }}", @@ -594,7 +606,11 @@ }, "gridPos": { }, "id": 9, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -665,13 +681,14 @@ "fillGradient": 0, "gridPos": { }, "id": 10, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -764,13 +781,14 @@ "fillGradient": 0, "gridPos": { }, "id": 11, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -846,13 +864,14 @@ "fillGradient": 0, "gridPos": { }, "id": 12, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -874,7 +893,7 @@ "steppedLine": false, "targets": [ { - "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}", + "expr": "cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ resource }}", @@ -940,13 +959,14 @@ "fillGradient": 0, "gridPos": { }, "id": 13, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": false, "sideWidth": null, "total": false, @@ -968,7 +988,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1021,13 +1041,14 @@ "fillGradient": 0, "gridPos": { }, "id": 14, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": false, "sideWidth": null, "total": false, @@ -1049,7 +1070,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1102,6 +1123,7 @@ "fillGradient": 0, "gridPos": { }, "id": 15, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -1130,7 +1152,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name, le))", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1196,13 +1218,14 @@ "fillGradient": 0, "gridPos": { }, "id": 16, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -1277,13 +1300,14 @@ "fillGradient": 0, "gridPos": { }, "id": 17, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -1305,7 +1329,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])", + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1358,13 +1382,14 @@ "fillGradient": 0, "gridPos": { }, "id": 18, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -1452,7 +1477,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1470,7 +1495,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(apiserver_request_total, cluster)", + "query": "label_values(up{job=\"apiserver\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -1490,7 +1515,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(apiserver_request_total{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "query": "label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/jsonnet/dashboards/cluster-total.json b/charts/kubezero-metrics/jsonnet/dashboards/cluster-total.json index 86e147d2..9cab13f7 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/cluster-total.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/cluster-total.json @@ -1619,7 +1619,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/jsonnet/dashboards/controller-manager.json b/charts/kubezero-metrics/jsonnet/dashboards/controller-manager.json index 373d6ab9..56bffc9e 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/controller-manager.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/controller-manager.json @@ -36,7 +36,11 @@ }, "gridPos": { }, "id": 2, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -106,6 +110,7 @@ "fillGradient": 0, "gridPos": { }, "id": 3, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -134,7 +139,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name)", + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} {{name}}", @@ -200,6 +205,7 @@ "fillGradient": 0, "gridPos": { }, "id": 4, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -228,7 +234,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name)", + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} {{name}}", @@ -294,6 +300,7 @@ "fillGradient": 0, "gridPos": { }, "id": 5, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -322,7 +329,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name, le))", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} {{name}}", @@ -388,13 +395,14 @@ "fillGradient": 0, "gridPos": { }, "id": 6, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -416,28 +424,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -490,13 +498,14 @@ "fillGradient": 0, "gridPos": { }, "id": 7, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -518,7 +527,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -584,6 +593,7 @@ "fillGradient": 0, "gridPos": { }, "id": 8, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -612,7 +622,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -678,13 +688,14 @@ "fillGradient": 0, "gridPos": { }, "id": 9, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -759,13 +770,14 @@ "fillGradient": 0, "gridPos": { }, "id": 10, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -787,7 +799,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}[5m])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -840,13 +852,14 @@ "fillGradient": 0, "gridPos": { }, "id": 11, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -934,7 +947,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-cluster.json b/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-cluster.json index d08fa6c6..e69207cc 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-cluster.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-cluster.json @@ -24,10 +24,12 @@ "id": 1, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -47,7 +49,7 @@ "steppedLine": false, "targets": [ { - "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[$__rate_interval]))", + "expr": "cluster:node_cpu:ratio_rate5m{cluster=\"$cluster\"}", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -99,11 +101,14 @@ "fill": 1, "format": "percentunit", "id": 2, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -123,7 +128,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"cpu\",cluster=\"$cluster\"})", + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -175,11 +180,14 @@ "fill": 1, "format": "percentunit", "id": 3, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -199,7 +207,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"cpu\",cluster=\"$cluster\"})", + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -251,11 +259,14 @@ "fill": 1, "format": "percentunit", "id": 4, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -275,7 +286,7 @@ "steppedLine": false, "targets": [ { - "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{cluster=\"$cluster\"})", + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -327,11 +338,14 @@ "fill": 1, "format": "percentunit", "id": 5, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -351,7 +365,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"memory\",cluster=\"$cluster\"})", + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -403,11 +417,14 @@ "fill": 1, "format": "percentunit", "id": 6, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -427,7 +444,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"memory\",cluster=\"$cluster\"})", + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -490,11 +507,14 @@ "datasource": "$datasource", "fill": 10, "id": 7, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -578,11 +598,14 @@ "datasource": "$datasource", "fill": 1, "id": 8, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -741,7 +764,7 @@ ], "targets": [ { - "expr": "sum(kube_pod_owner{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -861,11 +884,14 @@ "datasource": "$datasource", "fill": 10, "id": 9, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -885,7 +911,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -949,11 +975,14 @@ "datasource": "$datasource", "fill": 1, "id": 10, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1112,7 +1141,7 @@ ], "targets": [ { - "expr": "sum(kube_pod_owner{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1130,7 +1159,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1148,7 +1177,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1166,7 +1195,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1234,10 +1263,12 @@ "id": 11, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1381,7 +1412,7 @@ ], "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1390,7 +1421,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1399,7 +1430,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1408,7 +1439,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1417,7 +1448,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1426,7 +1457,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1492,11 +1523,14 @@ "datasource": "$datasource", "fill": 10, "id": 12, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1516,7 +1550,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1568,11 +1602,14 @@ "datasource": "$datasource", "fill": 10, "id": 13, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1592,7 +1629,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1656,11 +1693,14 @@ "datasource": "$datasource", "fill": 10, "id": 14, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1680,7 +1720,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "avg(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1732,11 +1772,14 @@ "datasource": "$datasource", "fill": 10, "id": 15, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1756,7 +1799,7 @@ "steppedLine": false, "targets": [ { - "expr": "avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "avg(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1820,11 +1863,14 @@ "datasource": "$datasource", "fill": 10, "id": 16, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1844,7 +1890,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1896,11 +1942,14 @@ "datasource": "$datasource", "fill": 10, "id": 17, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1920,7 +1969,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -1984,11 +2033,14 @@ "datasource": "$datasource", "fill": 10, "id": 18, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2008,7 +2060,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2060,11 +2112,14 @@ "datasource": "$datasource", "fill": 10, "id": 19, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2084,7 +2139,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2149,11 +2204,14 @@ "decimals": -1, "fill": 10, "id": 20, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2173,7 +2231,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\"}[5m])))", + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2225,11 +2283,14 @@ "datasource": "$datasource", "fill": 10, "id": 21, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2249,7 +2310,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -2313,11 +2374,14 @@ "datasource": "$datasource", "fill": 1, "id": 22, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2465,7 +2529,7 @@ ], "targets": [ { - "expr": "sum by(namespace) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2474,7 +2538,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_writes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2483,7 +2547,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_reads_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2492,7 +2556,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2501,7 +2565,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2510,7 +2574,7 @@ "step": 10 }, { - "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, diff --git a/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-namespace.json b/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-namespace.json index a5f8e4e6..8c304a3c 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-namespace.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-namespace.json @@ -22,11 +22,14 @@ "fill": 1, "format": "percentunit", "id": 1, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -46,7 +49,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -98,11 +101,14 @@ "fill": 1, "format": "percentunit", "id": 2, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -122,7 +128,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -174,11 +180,14 @@ "fill": 1, "format": "percentunit", "id": 3, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -198,7 +207,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -250,11 +259,14 @@ "fill": 1, "format": "percentunit", "id": 4, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -274,7 +286,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -337,11 +349,14 @@ "datasource": "$datasource", "fill": 10, "id": 5, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -464,11 +479,14 @@ "datasource": "$datasource", "fill": 1, "id": 6, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -699,11 +717,14 @@ "datasource": "$datasource", "fill": 10, "id": 7, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -746,7 +767,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -826,11 +847,14 @@ "datasource": "$datasource", "fill": 1, "id": 8, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1004,7 +1028,7 @@ ], "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1022,7 +1046,7 @@ "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1040,7 +1064,7 @@ "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1049,7 +1073,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1058,7 +1082,7 @@ "step": 10 }, { - "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "expr": "sum(container_memory_cache{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1067,7 +1091,7 @@ "step": 10 }, { - "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "expr": "sum(container_memory_swap{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1135,10 +1159,12 @@ "id": 9, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1282,7 +1308,7 @@ ], "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1291,7 +1317,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1300,7 +1326,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1309,7 +1335,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1318,7 +1344,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1327,7 +1353,7 @@ "step": 10 }, { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1393,11 +1419,14 @@ "datasource": "$datasource", "fill": 10, "id": 10, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1469,11 +1498,14 @@ "datasource": "$datasource", "fill": 10, "id": 11, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1557,11 +1589,14 @@ "datasource": "$datasource", "fill": 10, "id": 12, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1633,11 +1668,14 @@ "datasource": "$datasource", "fill": 10, "id": 13, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1721,11 +1759,14 @@ "datasource": "$datasource", "fill": 10, "id": 14, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1797,11 +1838,14 @@ "datasource": "$datasource", "fill": 10, "id": 15, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1886,11 +1930,14 @@ "decimals": -1, "fill": 10, "id": 16, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1910,7 +1957,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m])))", + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1962,11 +2009,14 @@ "datasource": "$datasource", "fill": 10, "id": 17, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1986,7 +2036,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -2050,11 +2100,14 @@ "datasource": "$datasource", "fill": 1, "id": 18, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -2202,7 +2255,7 @@ ], "targets": [ { - "expr": "sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2211,7 +2264,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_writes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2220,7 +2273,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_reads_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2229,7 +2282,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2238,7 +2291,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2247,7 +2300,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2336,7 +2389,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -2359,7 +2412,7 @@ "multi": false, "name": "namespace", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-node.json b/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-node.json index c2263308..7cc77908 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-node.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-node.json @@ -21,11 +21,14 @@ "datasource": "$datasource", "fill": 10, "id": 1, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -38,12 +41,32 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ ], + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], "spaceLength": 10, "span": 12, "stack": true, "steppedLine": false, "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, { "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "time_series", @@ -109,11 +132,14 @@ "datasource": "$datasource", "fill": 1, "id": 2, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -344,11 +370,14 @@ "datasource": "$datasource", "fill": 10, "id": 3, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -361,12 +390,32 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ ], + "seriesOverrides": [ + { + "alias": "max capacity", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hiddenSeries": true, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], "spaceLength": 10, "span": 12, "stack": true, "steppedLine": false, "targets": [ + { + "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max capacity", + "legendLink": null, + "step": 10 + }, { "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", "format": "time_series", @@ -432,11 +481,14 @@ "datasource": "$datasource", "fill": 1, "id": 4, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -762,7 +814,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -785,7 +837,7 @@ "multi": true, "name": "node", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, node)", + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-pod.json b/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-pod.json index 798184a1..5d8f7a1b 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-pod.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-pod.json @@ -21,11 +21,14 @@ "datasource": "$datasource", "fill": 10, "id": 1, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -72,7 +75,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "requests", @@ -80,7 +83,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "limits", @@ -144,11 +147,14 @@ "datasource": "$datasource", "fill": 10, "id": 2, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -168,7 +174,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[5m])) by (container) /sum(increase(container_cpu_cfs_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[5m])) by (container)", + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{job=\"cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -241,11 +247,14 @@ "datasource": "$datasource", "fill": 1, "id": 3, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -476,11 +485,14 @@ "datasource": "$datasource", "fill": 10, "id": 4, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -521,7 +533,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -529,7 +541,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "requests", @@ -537,7 +549,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "limits", @@ -601,11 +613,14 @@ "datasource": "$datasource", "fill": 1, "id": 5, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -779,7 +794,7 @@ ], "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -797,7 +812,7 @@ "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -815,7 +830,7 @@ "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -824,7 +839,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "expr": "sum(container_memory_rss{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -833,7 +848,7 @@ "step": 10 }, { - "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "expr": "sum(container_memory_cache{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -842,7 +857,7 @@ "step": 10 }, { - "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "expr": "sum(container_memory_swap{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -910,10 +925,12 @@ "id": 6, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -933,7 +950,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -987,10 +1004,12 @@ "id": 7, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1010,7 +1029,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1076,10 +1095,12 @@ "id": 8, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1099,7 +1120,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1153,10 +1174,12 @@ "id": 9, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1176,7 +1199,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1242,10 +1265,12 @@ "id": 10, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1265,7 +1290,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1319,10 +1344,12 @@ "id": 11, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1342,7 +1369,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1407,11 +1434,14 @@ "decimals": -1, "fill": 10, "id": 12, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1431,7 +1461,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[5m])))", + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Reads", @@ -1439,7 +1469,7 @@ "step": 10 }, { - "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[5m])))", + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Writes", @@ -1491,11 +1521,14 @@ "datasource": "$datasource", "fill": 10, "id": 13, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1515,7 +1548,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Reads", @@ -1523,7 +1556,7 @@ "step": 10 }, { - "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[5m]))", + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "Writes", @@ -1588,11 +1621,14 @@ "decimals": -1, "fill": 10, "id": 14, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1612,7 +1648,7 @@ "steppedLine": false, "targets": [ { - "expr": "ceil(sum by(container) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m])))", + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -1664,11 +1700,14 @@ "datasource": "$datasource", "fill": 10, "id": 15, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1688,7 +1727,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -1752,11 +1791,14 @@ "datasource": "$datasource", "fill": 1, "id": 16, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1904,7 +1946,7 @@ ], "targets": [ { - "expr": "sum by(container) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1913,7 +1955,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_writes_total{job=\"cadvisor\",device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1922,7 +1964,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_reads_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1931,7 +1973,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1940,7 +1982,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -1949,7 +1991,7 @@ "step": 10 }, { - "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=\"$pod\"}[5m]))", + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"cadvisor\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))", "format": "table", "instant": true, "intervalFactor": 2, @@ -2038,7 +2080,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -2061,7 +2103,7 @@ "multi": false, "name": "namespace", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "sort": 1, @@ -2084,7 +2126,7 @@ "multi": false, "name": "pod", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-workload.json b/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-workload.json index 73349891..bfc912fd 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-workload.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-workload.json @@ -21,11 +21,14 @@ "datasource": "$datasource", "fill": 10, "id": 1, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -109,11 +112,14 @@ "datasource": "$datasource", "fill": 1, "id": 2, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -251,7 +257,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -260,7 +266,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -269,7 +275,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -278,7 +284,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -344,11 +350,14 @@ "datasource": "$datasource", "fill": 10, "id": 3, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -432,11 +441,14 @@ "datasource": "$datasource", "fill": 1, "id": 4, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -574,7 +586,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -583,7 +595,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -592,7 +604,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -601,7 +613,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -669,10 +681,12 @@ "id": 5, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -816,7 +830,7 @@ ], "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -825,7 +839,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -834,7 +848,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -843,7 +857,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -852,7 +866,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -861,7 +875,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -927,11 +941,14 @@ "datasource": "$datasource", "fill": 10, "id": 6, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -951,7 +968,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1003,11 +1020,14 @@ "datasource": "$datasource", "fill": 10, "id": 7, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1027,7 +1047,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1091,11 +1111,14 @@ "datasource": "$datasource", "fill": 10, "id": 8, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1115,7 +1138,7 @@ "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1167,11 +1190,14 @@ "datasource": "$datasource", "fill": 10, "id": 9, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1191,7 +1217,7 @@ "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1255,11 +1281,14 @@ "datasource": "$datasource", "fill": 10, "id": 10, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1279,7 +1308,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1331,11 +1360,14 @@ "datasource": "$datasource", "fill": 10, "id": 11, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1355,7 +1387,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1419,11 +1451,14 @@ "datasource": "$datasource", "fill": 10, "id": 12, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1443,7 +1478,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1495,11 +1530,14 @@ "datasource": "$datasource", "fill": 10, "id": 13, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1519,7 +1557,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -1606,7 +1644,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -1629,30 +1667,7 @@ "multi": false, "name": "namespace", "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "", - "value": "" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "workload", - "options": [ ], - "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload)", + "query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "sort": 1, @@ -1675,7 +1690,30 @@ "multi": false, "name": "type", "options": [ ], - "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}, workload)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-workloads-namespace.json b/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-workloads-namespace.json index 9654b4fa..3e995a54 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-workloads-namespace.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-workloads-namespace.json @@ -21,11 +21,14 @@ "datasource": "$datasource", "fill": 10, "id": 1, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -148,11 +151,14 @@ "datasource": "$datasource", "fill": 1, "id": 2, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -329,7 +335,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -338,7 +344,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -347,7 +353,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -356,7 +362,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -422,11 +428,14 @@ "datasource": "$datasource", "fill": 10, "id": 3, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -469,7 +478,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}} - {{workload_type}}", @@ -549,11 +558,14 @@ "datasource": "$datasource", "fill": 1, "id": 4, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -721,7 +733,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -730,7 +742,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -739,7 +751,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -748,7 +760,7 @@ "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -757,7 +769,7 @@ "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -825,10 +837,12 @@ "id": 5, "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -987,7 +1001,7 @@ ], "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -996,7 +1010,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1005,7 +1019,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1014,7 +1028,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1023,7 +1037,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1032,7 +1046,7 @@ "step": 10 }, { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload))\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1098,11 +1112,14 @@ "datasource": "$datasource", "fill": 10, "id": 6, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1122,7 +1139,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1174,11 +1191,14 @@ "datasource": "$datasource", "fill": 10, "id": 7, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1198,7 +1218,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1262,11 +1282,14 @@ "datasource": "$datasource", "fill": 10, "id": 8, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1286,7 +1309,7 @@ "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(avg(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1338,11 +1361,14 @@ "datasource": "$datasource", "fill": 10, "id": 9, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1362,7 +1388,7 @@ "steppedLine": false, "targets": [ { - "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(avg(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1426,11 +1452,14 @@ "datasource": "$datasource", "fill": 10, "id": 10, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1450,7 +1479,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1502,11 +1531,14 @@ "datasource": "$datasource", "fill": 10, "id": 11, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1526,7 +1558,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1590,11 +1622,14 @@ "datasource": "$datasource", "fill": 10, "id": 12, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1614,7 +1649,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1666,11 +1701,14 @@ "datasource": "$datasource", "fill": 10, "id": 13, + "interval": "1m", "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, "values": false @@ -1690,7 +1728,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}}", @@ -1777,7 +1815,30 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-state-metrics\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "sort": 1, @@ -1814,29 +1875,6 @@ "tagsQuery": "", "type": "query", "useTags": false - }, - { - "allValue": null, - "current": { - "text": "", - "value": "" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "namespace", - "options": [ ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false } ] }, diff --git a/charts/kubezero-metrics/jsonnet/dashboards/kubelet.json b/charts/kubezero-metrics/jsonnet/dashboards/kubelet.json index a5f43f84..324f71db 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/kubelet.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/kubelet.json @@ -154,7 +154,7 @@ "refId": "A" } ], - "title": "Running Container", + "title": "Running Containers", "transparent": false, "type": "stat" }, @@ -294,7 +294,7 @@ "pluginVersion": "7", "targets": [ { - "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m]))", + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -347,7 +347,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (operation_type, instance)", + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -432,7 +432,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -517,7 +517,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -602,14 +602,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} pod", "refId": "A" }, { - "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} worker", @@ -694,14 +694,14 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} pod", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} worker", @@ -788,7 +788,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", @@ -875,7 +875,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", @@ -962,7 +962,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin, le))", + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", @@ -1047,7 +1047,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{operation_type}}", @@ -1132,7 +1132,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{operation_type}}", @@ -1218,7 +1218,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1303,7 +1303,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1388,7 +1388,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1473,28 +1473,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -1579,7 +1579,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{verb}} {{url}}", @@ -1749,7 +1749,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1893,7 +1893,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1927,11 +1927,11 @@ "datasource": "$datasource", "hide": 0, "includeAll": true, - "label": null, + "label": "instance", "multi": false, "name": "instance", "options": [ ], - "query": "label_values(kubelet_runtime_operations_total{cluster=\"$cluster\", job=\"kubelet\"}, instance)", + "query": "label_values(up{job=\"kubelet\",cluster=\"$cluster\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/jsonnet/dashboards/namespace-by-pod.json b/charts/kubezero-metrics/jsonnet/dashboards/namespace-by-pod.json index cbe880cd..1d2e1f18 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/namespace-by-pod.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/namespace-by-pod.json @@ -1137,7 +1137,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/jsonnet/dashboards/namespace-by-workload.json b/charts/kubezero-metrics/jsonnet/dashboards/namespace-by-workload.json index c05bda2f..67f48514 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/namespace-by-workload.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/namespace-by-workload.json @@ -1349,7 +1349,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/jsonnet/dashboards/persistentvolumesusage.json b/charts/kubezero-metrics/jsonnet/dashboards/persistentvolumesusage.json index e9ab9c20..cf722b5e 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/persistentvolumesusage.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/persistentvolumesusage.json @@ -26,13 +26,14 @@ "fillGradient": 0, "gridPos": { }, "id": 2, + "interval": "1m", "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -124,7 +125,11 @@ }, "gridPos": { }, "id": 3, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -207,13 +212,14 @@ "fillGradient": 0, "gridPos": { }, "id": 4, + "interval": "1m", "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -305,7 +311,11 @@ }, "gridPos": { }, "id": 5, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -388,7 +398,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -406,7 +416,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kubelet_volume_stats_capacity_bytes, cluster)", + "query": "label_values(kubelet_volume_stats_capacity_bytes{job=\"kubelet\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/jsonnet/dashboards/pod-total.json b/charts/kubezero-metrics/jsonnet/dashboards/pod-total.json index e854c415..e5c0b6e4 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/pod-total.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/pod-total.json @@ -903,7 +903,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/charts/kubezero-metrics/jsonnet/dashboards/proxy.json b/charts/kubezero-metrics/jsonnet/dashboards/proxy.json index 8cf589a2..d7ad8acc 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/proxy.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/proxy.json @@ -36,7 +36,11 @@ }, "gridPos": { }, "id": 2, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -106,13 +110,14 @@ "fillGradient": 0, "gridPos": { }, "id": 3, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -134,7 +139,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "rate", @@ -187,6 +192,7 @@ "fillGradient": 0, "gridPos": { }, "id": 4, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -215,7 +221,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -281,13 +287,14 @@ "fillGradient": 0, "gridPos": { }, "id": 5, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -309,7 +316,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "rate", @@ -362,6 +369,7 @@ "fillGradient": 0, "gridPos": { }, "id": 6, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -390,7 +398,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -456,13 +464,14 @@ "fillGradient": 0, "gridPos": { }, "id": 7, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -484,28 +493,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -558,13 +567,14 @@ "fillGradient": 0, "gridPos": { }, "id": 8, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -586,7 +596,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\",verb=\"POST\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -652,6 +662,7 @@ "fillGradient": 0, "gridPos": { }, "id": 9, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -680,7 +691,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -746,13 +757,14 @@ "fillGradient": 0, "gridPos": { }, "id": 10, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -827,13 +839,14 @@ "fillGradient": 0, "gridPos": { }, "id": 11, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -855,7 +868,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\"}[5m])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -908,13 +921,14 @@ "fillGradient": 0, "gridPos": { }, "id": 12, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -1002,7 +1016,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1020,7 +1034,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(up{job=\"kube-proxy\"}, cluster)", "refresh": 2, "regex": "", "sort": 1, @@ -1040,7 +1054,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\"}, instance)", + "query": "label_values(up{job=\"kube-proxy\", cluster=\"$cluster\", job=\"kube-proxy\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/jsonnet/dashboards/scheduler.json b/charts/kubezero-metrics/jsonnet/dashboards/scheduler.json index de71562f..b480c682 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/scheduler.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/scheduler.json @@ -36,7 +36,11 @@ }, "gridPos": { }, "id": 2, - "interval": null, + "interval": "1m", + "legend": { + "alignAsTable": true, + "rightSide": true + }, "links": [ ], "mappingType": 1, "mappingTypes": [ @@ -106,6 +110,7 @@ "fillGradient": 0, "gridPos": { }, "id": 3, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -134,28 +139,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} e2e", "refId": "A" }, { - "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} binding", "refId": "B" }, { - "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} scheduling algorithm", "refId": "C" }, { - "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} volume", @@ -208,6 +213,7 @@ "fillGradient": 0, "gridPos": { }, "id": 4, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -236,28 +242,28 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} e2e", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} binding", "refId": "B" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} scheduling algorithm", "refId": "C" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{cluster}} {{instance}} volume", @@ -323,13 +329,14 @@ "fillGradient": 0, "gridPos": { }, "id": 5, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -351,28 +358,28 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -425,13 +432,14 @@ "fillGradient": 0, "gridPos": { }, "id": 6, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -453,7 +461,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -519,6 +527,7 @@ "fillGradient": 0, "gridPos": { }, "id": 7, + "interval": "1m", "legend": { "alignAsTable": true, "avg": false, @@ -547,7 +556,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{verb}} {{url}}", @@ -613,13 +622,14 @@ "fillGradient": 0, "gridPos": { }, "id": 8, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -694,13 +704,14 @@ "fillGradient": 0, "gridPos": { }, "id": 9, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -722,7 +733,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -775,13 +786,14 @@ "fillGradient": 0, "gridPos": { }, "id": 10, + "interval": "1m", "legend": { - "alignAsTable": false, + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": false, + "rightSide": true, "show": true, "sideWidth": null, "total": false, @@ -869,7 +881,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -907,7 +919,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-scheduler\"}, instance)", + "query": "label_values(up{job=\"kube-scheduler\", cluster=\"$cluster\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/jsonnet/dashboards/workload-total.json b/charts/kubezero-metrics/jsonnet/dashboards/workload-total.json index f1ce7029..9cf5daf7 100644 --- a/charts/kubezero-metrics/jsonnet/dashboards/workload-total.json +++ b/charts/kubezero-metrics/jsonnet/dashboards/workload-total.json @@ -89,7 +89,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ pod }}", @@ -184,7 +184,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ pod }}", @@ -290,7 +290,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ pod }}", @@ -385,7 +385,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ pod }}", @@ -506,7 +506,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -597,7 +597,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -699,7 +699,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -790,7 +790,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -901,7 +901,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -992,7 +992,7 @@ "steppedLine": false, "targets": [ { - "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{pod}}", @@ -1061,7 +1061,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1079,7 +1079,7 @@ "multi": false, "name": "cluster", "options": [ ], - "query": "label_values(kube_pod_info, cluster)", + "query": "label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)", "refresh": 2, "regex": "", "sort": 0, @@ -1099,14 +1099,14 @@ "value": "kube-system" }, "datasource": "$datasource", - "definition": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "definition": "label_values(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\"}, namespace)", "hide": 0, "includeAll": true, "label": null, "multi": false, "name": "namespace", "options": [ ], - "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", + "query": "label_values(container_network_receive_packets_total{job=\"cadvisor\", cluster=\"$cluster\"}, namespace)", "refresh": 2, "regex": "", "skipUrlSync": false, diff --git a/charts/kubezero-metrics/jsonnet/jsonnetfile.json b/charts/kubezero-metrics/jsonnet/jsonnetfile.json index 16193091..2a66a20a 100644 --- a/charts/kubezero-metrics/jsonnet/jsonnetfile.json +++ b/charts/kubezero-metrics/jsonnet/jsonnetfile.json @@ -8,7 +8,7 @@ "subdir": "jsonnet/kube-prometheus" } }, - "version": "release-0.9" + "version": "release-0.11" } ], "legacyImports": true diff --git a/charts/kubezero-metrics/jsonnet/jsonnetfile.lock.json b/charts/kubezero-metrics/jsonnet/jsonnetfile.lock.json index 35d1520b..3fe6e0bf 100644 --- a/charts/kubezero-metrics/jsonnet/jsonnetfile.lock.json +++ b/charts/kubezero-metrics/jsonnet/jsonnetfile.lock.json @@ -8,8 +8,8 @@ "subdir": "grafana" } }, - "version": "90f38916f1f8a310a715d18e36f787f84df4ddf5", - "sum": "0kZ1pnuIirDtbg6F9at5+NQOwKNONIGEPq0eECzvRkI=" + "version": "d039275e4916aceae1c137120882e01d857787ac", + "sum": "515vMn4x4tP8vegL4HLW0nDO5+njGTgnDZB5OOhtsCI=" }, { "source": { @@ -18,9 +18,19 @@ "subdir": "contrib/mixin" } }, - "version": "74aa38ec10bc22d34ffd204f46df6e460b78d855", + "version": "19002cfc689fba2b8f56605e5797bf79f8b61fdd", "sum": "W/Azptf1PoqjyMwJON96UY69MFugDA4IAYiKURscryc=" }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafana.git", + "subdir": "grafana-mixin" + } + }, + "version": "3eed09056849ab873b867b561b7ce580ef2c75ba", + "sum": "MkjR7zCgq6MUZgjDzop574tFKoTX2OBr7DTwm1K+Ofs=" + }, { "source": { "git": { @@ -38,7 +48,7 @@ "subdir": "grafana-builder" } }, - "version": "c132c4afcf17491718539db4c2d94c0ea4346120", + "version": "dbf6fc14105c28b6fd0253005f7ca2da37d3d4e1", "sum": "tDR6yT2GVfw0wTU12iZH+m01HrbIr6g/xN+/8nzNkU0=" }, { @@ -48,8 +58,8 @@ "subdir": "" } }, - "version": "fb9d8ed4bc4a3d6efac525f72e8a0d2c583a0fe2", - "sum": "xjKkdp+5fkekCNBUIgZCHTRmVdUEmQNFKslrL2Ho8gs=" + "version": "b8f44bb7be728423836bef0e904ec7166895a34b", + "sum": "LCgSosxceeYuoau5fYSPtE5eXOFe46DxexfkrctUv7c=" }, { "source": { @@ -58,7 +68,7 @@ "subdir": "lib/promgrafonnet" } }, - "version": "eb98d4f74e8ac9c30b1f0e815b07bed31da76c8f", + "version": "5e44626d70c2bf2d35c37f3fee5a6261a5335cc6", "sum": "zv7hXGui6BfHzE9wPatHI/AGZa4A2WKo6pq7ZdqBsps=" }, { @@ -68,8 +78,8 @@ "subdir": "jsonnet/kube-state-metrics" } }, - "version": "f170cc73f11c1580d7f38af746be0f2fa79c6a1e", - "sum": "S5qI+PJUdNeYOv76jH5nxwYS9N6U7CRxvyuB1wI4cTE=" + "version": "0567e1e1b981755e563d2244fa1659563f2cddbc", + "sum": "P0dCnbzyPScQGNXwXRcwiPkMLeTq0IPNbSTysDbySnM=" }, { "source": { @@ -78,7 +88,7 @@ "subdir": "jsonnet/kube-state-metrics-mixin" } }, - "version": "f170cc73f11c1580d7f38af746be0f2fa79c6a1e", + "version": "0567e1e1b981755e563d2244fa1659563f2cddbc", "sum": "u8gaydJoxEjzizQ8jY8xSjYgWooPmxw+wIWdDxifMAk=" }, { @@ -88,8 +98,8 @@ "subdir": "jsonnet/kube-prometheus" } }, - "version": "452aaed72e36acb31cae93cfa85a5d9c3d3d2ec7", - "sum": "pupXEvlRbhLdEO9b8LfFZB66+Z7fEqvRZ9m3MyEvsv4=" + "version": "e3066575dc8be21f578f12887563bda3ee7a2eff", + "sum": "nNEMDrb5sQDOxJ20ITDvldyfIbbiGcVr8Bq46PH2ww8=" }, { "source": { @@ -98,8 +108,8 @@ "subdir": "jsonnet/mixin" } }, - "version": "83fe36566f4e0894eb5ffcd2638a0f039a17bdeb", - "sum": "6reUygVmQrLEWQzTKcH8ceDbvM+2ztK3z2VBR2K2l+U=", + "version": "5db6996d3ca995e66301c53c33959fd64c3f6ae6", + "sum": "GQmaVFJwKMiD/P4n3N2LrAZVcwutriWrP8joclDtBYQ=", "name": "prometheus-operator-mixin" }, { @@ -109,8 +119,8 @@ "subdir": "jsonnet/prometheus-operator" } }, - "version": "83fe36566f4e0894eb5ffcd2638a0f039a17bdeb", - "sum": "J1G++A8hrtr3+OZQMmcNeb1w/C30bXqqwpwHL/Xhsd4=" + "version": "5db6996d3ca995e66301c53c33959fd64c3f6ae6", + "sum": "pUggCYwO/3Y/p6Vgryx8Y4KO3QkJ+GqimrZtn/luzzI=" }, { "source": { @@ -119,8 +129,8 @@ "subdir": "doc/alertmanager-mixin" } }, - "version": "b408b522bc653d014e53035e59fa394cc1edd762", - "sum": "pep+dHzfIjh2SU5pEkwilMCAT/NoL6YYflV4x8cr7vU=", + "version": "14b01e6a34dd3155768c7e9bd5c4376055de9419", + "sum": "f3iZDUXQ/YWB5yDCY7VLD5bs442+3CdJgXJhJyWhNf8=", "name": "alertmanager" }, { @@ -130,8 +140,8 @@ "subdir": "docs/node-mixin" } }, - "version": "832909dd257eb368cf83363ffcae3ab84cb4bcb1", - "sum": "MmxGhE2PJ1a52mk2x7vDpMT2at4Jglbud/rK74CB5i0=" + "version": "a2321e7b940ddcff26873612bccdf7cd4c42b6b6", + "sum": "MlWDAKGZ+JArozRKdKEvewHeWn8j2DNBzesJfLVd0dk=" }, { "source": { @@ -140,10 +150,20 @@ "subdir": "documentation/prometheus-mixin" } }, - "version": "751ca03faddc9c64089c41d0da370a3a0b477742", - "sum": "AS8WYFi/z10BZSF6DFkKBscjB32XDMM7iIso7CO/FyI=", + "version": "d7e7b8e04b5ecdc1dd153534ba376a622b72741b", + "sum": "APXOIP3B3dZ3Tyh7L2UhyWR8Vbf5+9adTLz/ya7n6uU=", "name": "prometheus" }, + { + "source": { + "git": { + "remote": "https://github.com/pyrra-dev/pyrra.git", + "subdir": "config/crd/bases" + } + }, + "version": "3738a607a42a0c9566587a49cec7587cc92d61bd", + "sum": "GQ0GFKGdIWKx1b78VRs6jtC4SMqkBjT5jl65QUjPKK4=" + }, { "source": { "git": { @@ -151,8 +171,8 @@ "subdir": "mixin" } }, - "version": "ff363498fc95cfe17de894d7237bcf38bdd0bc36", - "sum": "cajthvLKDjYgYHCKQU2g/pTMRkxcbuJEvTnCyJOihl8=", + "version": "17c576472d80972bfd3705e1e0a08e6f8da8e04b", + "sum": "dBm9ML50quhu6dwTIgfNmVruMqfaUeQVCO/6EKtQLxE=", "name": "thanos-mixin" } ], diff --git a/charts/kubezero-metrics/jsonnet/k8s-dashboards.yaml b/charts/kubezero-metrics/jsonnet/k8s-dashboards.yaml index 99758679..13f2ff17 100644 --- a/charts/kubezero-metrics/jsonnet/k8s-dashboards.yaml +++ b/charts/kubezero-metrics/jsonnet/k8s-dashboards.yaml @@ -6,10 +6,10 @@ dashboards: url: https://grafana.com/api/dashboards/12539/revisions/5/download tags: ['kubernetes', 'DNS'] - name: etcd - url: https://grafana.com/api/dashboards/3070/revisions/3/download + url: https://grafana.com/api/dashboards/15308/revisions/1/download tags: ['kubernetes', 'etcd'] - name: node - url: https://grafana.com/api/dashboards/1860/revisions/23/download + url: https://grafana.com/api/dashboards/1860/revisions/27/download tags: ['kubernetes'] # cd dashboards; for f in *.json; do echo "- name: ${f%%.json}" >> ../dashboards.yaml; echo " url: file://dashboards/$f" >> ../dashboards.yaml; done; cd - - name: apiserver diff --git a/charts/kubezero-metrics/jsonnet/rules/alertmanager-prometheusRule b/charts/kubezero-metrics/jsonnet/rules/alertmanager-prometheusRule index 75e44f62..1c675801 100644 --- a/charts/kubezero-metrics/jsonnet/rules/alertmanager-prometheusRule +++ b/charts/kubezero-metrics/jsonnet/rules/alertmanager-prometheusRule @@ -4,9 +4,10 @@ "metadata": { "labels": { "app.kubernetes.io/component": "alert-router", + "app.kubernetes.io/instance": "main", "app.kubernetes.io/name": "alertmanager", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "0.22.2", + "app.kubernetes.io/version": "0.24.0", "prometheus": "k8s", "role": "alert-rules" }, diff --git a/charts/kubezero-metrics/jsonnet/rules/kube-prometheus-prometheusRule b/charts/kubezero-metrics/jsonnet/rules/kube-prometheus-prometheusRule index cfd34465..68de755a 100644 --- a/charts/kubezero-metrics/jsonnet/rules/kube-prometheus-prometheusRule +++ b/charts/kubezero-metrics/jsonnet/rules/kube-prometheus-prometheusRule @@ -41,6 +41,18 @@ "labels": { "severity": "none" } + }, + { + "alert": "InfoInhibitor", + "annotations": { + "description": "This is an alert that is used to inhibit info alerts.\nBy themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with\nother alerts.\nThis alert fires whenever there's a severity=\"info\" alert, and stops firing when another alert with a\nseverity of 'warning' or 'critical' starts firing on the same namespace.\nThis alert should be routed to a null receiver and configured to inhibit alerts with severity=\"info\".\n", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/general/infoinhibitor", + "summary": "Info-level alert inhibition." + }, + "expr": "ALERTS{severity = \"info\"} == 1 unless on(namespace) ALERTS{alertname != \"InfoInhibitor\", severity =~ \"warning|critical\", alertstate=\"firing\"} == 1", + "labels": { + "severity": "none" + } } ] }, @@ -86,7 +98,7 @@ "record": "cluster:node_cpu:sum_rate5m" }, { - "expr": "cluster:node_cpu_seconds_total:rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu))", + "expr": "cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu))", "record": "cluster:node_cpu:ratio" } ] diff --git a/charts/kubezero-metrics/jsonnet/rules/kube-state-metrics-prometheusRule b/charts/kubezero-metrics/jsonnet/rules/kube-state-metrics-prometheusRule index 210c4a36..2e33a787 100644 --- a/charts/kubezero-metrics/jsonnet/rules/kube-state-metrics-prometheusRule +++ b/charts/kubezero-metrics/jsonnet/rules/kube-state-metrics-prometheusRule @@ -6,7 +6,7 @@ "app.kubernetes.io/component": "exporter", "app.kubernetes.io/name": "kube-state-metrics", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "2.1.1", + "app.kubernetes.io/version": "2.5.0", "prometheus": "k8s", "role": "alert-rules" }, diff --git a/charts/kubezero-metrics/jsonnet/rules/kubernetes-prometheusRule b/charts/kubezero-metrics/jsonnet/rules/kubernetes-prometheusRule index abbe47b0..2e4562bf 100644 --- a/charts/kubezero-metrics/jsonnet/rules/kubernetes-prometheusRule +++ b/charts/kubezero-metrics/jsonnet/rules/kubernetes-prometheusRule @@ -36,7 +36,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready", "summary": "Pod has been in a non-ready state for more than 15 minutes." }, - "expr": "sum by (namespace, pod) (\n max by(namespace, pod) (\n kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Pending|Unknown\"}\n ) * on(namespace, pod) group_left(owner_kind) topk by(namespace, pod) (\n 1, max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!=\"Job\"})\n )\n) > 0\n", + "expr": "sum by (namespace, pod, cluster) (\n max by(namespace, pod, cluster) (\n kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Pending|Unknown\"}\n ) * on(namespace, pod, cluster) group_left(owner_kind) topk by(namespace, pod, cluster) (\n 1, max by(namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!=\"Job\"})\n )\n) > 0\n", "for": "15m", "labels": { "severity": "warning" @@ -114,7 +114,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetrolloutstuck", "summary": "DaemonSet rollout is stuck." }, - "expr": "(\n (\n kube_daemonset_status_current_number_scheduled{job=\"kube-state-metrics\"}\n !=\n kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n ) or (\n kube_daemonset_status_number_misscheduled{job=\"kube-state-metrics\"}\n !=\n 0\n ) or (\n kube_daemonset_updated_number_scheduled{job=\"kube-state-metrics\"}\n !=\n kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n ) or (\n kube_daemonset_status_number_available{job=\"kube-state-metrics\"}\n !=\n kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n )\n) and (\n changes(kube_daemonset_updated_number_scheduled{job=\"kube-state-metrics\"}[5m])\n ==\n 0\n)\n", + "expr": "(\n (\n kube_daemonset_status_current_number_scheduled{job=\"kube-state-metrics\"}\n !=\n kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n ) or (\n kube_daemonset_status_number_misscheduled{job=\"kube-state-metrics\"}\n !=\n 0\n ) or (\n kube_daemonset_status_updated_number_scheduled{job=\"kube-state-metrics\"}\n !=\n kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n ) or (\n kube_daemonset_status_number_available{job=\"kube-state-metrics\"}\n !=\n kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n )\n) and (\n changes(kube_daemonset_status_updated_number_scheduled{job=\"kube-state-metrics\"}[5m])\n ==\n 0\n)\n", "for": "15m", "labels": { "severity": "warning" @@ -123,11 +123,11 @@ { "alert": "KubeContainerWaiting", "annotations": { - "description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} has been in waiting state for longer than 1 hour.", + "description": "pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting", "summary": "Pod container waiting longer than 1 hour" }, - "expr": "sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job=\"kube-state-metrics\"}) > 0\n", + "expr": "sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job=\"kube-state-metrics\"}) > 0\n", "for": "1h", "labels": { "severity": "warning" @@ -160,14 +160,13 @@ } }, { - "alert": "KubeJobCompletion", + "alert": "KubeJobNotCompleted", "annotations": { - "description": "Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than 12 hours to complete.", - "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobcompletion", + "description": "Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ \"43200\" | humanizeDuration }} to complete.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobnotcompleted", "summary": "Job did not complete in time" }, - "expr": "kube_job_spec_completions{job=\"kube-state-metrics\"} - kube_job_status_succeeded{job=\"kube-state-metrics\"} > 0\n", - "for": "12h", + "expr": "time() - max by(namespace, job_name, cluster) (kube_job_status_start_time{job=\"kube-state-metrics\"}\n and\nkube_job_status_active{job=\"kube-state-metrics\"} > 0) > 43200\n", "labels": { "severity": "warning" } @@ -232,7 +231,7 @@ { "alert": "KubeMemoryOvercommit", "annotations": { - "description": "Cluster has overcommitted memory resource requests for Pods by {{ $value }} bytes and cannot tolerate node failure.", + "description": "Cluster has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit", "summary": "Cluster has overcommitted memory resource requests." }, @@ -249,7 +248,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit", "summary": "Cluster has overcommitted CPU resource requests." }, - "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"cpu\"})\n /\nsum(kube_node_status_allocatable{resource=\"cpu\"})\n > 1.5\n", + "expr": "sum(min without(resource) (kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=~\"(cpu|requests.cpu)\"}))\n /\nsum(kube_node_status_allocatable{resource=\"cpu\", job=\"kube-state-metrics\"})\n > 1.5\n", "for": "5m", "labels": { "severity": "warning" @@ -262,7 +261,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit", "summary": "Cluster has overcommitted memory resource requests." }, - "expr": "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"memory\"})\n /\nsum(kube_node_status_allocatable{resource=\"memory\",job=\"kube-state-metrics\"})\n > 1.5\n", + "expr": "sum(min without(resource) (kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=~\"(memory|requests.memory)\"}))\n /\nsum(kube_node_status_allocatable{resource=\"memory\", job=\"kube-state-metrics\"})\n > 1.5\n", "for": "5m", "labels": { "severity": "warning" @@ -332,7 +331,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup", "summary": "PersistentVolume is filling up." }, - "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\n", + "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n", "for": "1m", "labels": { "severity": "critical" @@ -345,7 +344,33 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup", "summary": "PersistentVolume is filling up." }, - "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\n", + "expr": "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n", + "for": "1h", + "labels": { + "severity": "warning" + } + }, + { + "alert": "KubePersistentVolumeInodesFillingUp", + "annotations": { + "description": "The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage }} free inodes.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup", + "summary": "PersistentVolumeInodes are filling up." + }, + "expr": "(\n kubelet_volume_stats_inodes_free{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_inodes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_inodes_used{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n", + "for": "1m", + "labels": { + "severity": "critical" + } + }, + { + "alert": "KubePersistentVolumeInodesFillingUp", + "annotations": { + "description": "Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup", + "summary": "PersistentVolumeInodes are filling up." + }, + "expr": "(\n kubelet_volume_stats_inodes_free{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_inodes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_inodes_used{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_inodes_free{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n", "for": "1h", "labels": { "severity": "warning" @@ -376,7 +401,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeversionmismatch", "summary": "Different semantic versions of Kubernetes components running." }, - "expr": "count(count by (git_version) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"},\"git_version\",\"$1\",\"git_version\",\"(v[0-9]*.[0-9]*).*\"))) > 1\n", + "expr": "count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~\"kube-dns|coredns\"},\"git_version\",\"$1\",\"git_version\",\"(v[0-9]*.[0-9]*).*\"))) > 1\n", "for": "15m", "labels": { "severity": "warning" @@ -389,7 +414,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclienterrors", "summary": "Kubernetes API server client is experiencing errors." }, - "expr": "(sum(rate(rest_client_requests_total{code=~\"5..\"}[5m])) by (instance, job, namespace)\n /\nsum(rate(rest_client_requests_total[5m])) by (instance, job, namespace))\n> 0.01\n", + "expr": "(sum(rate(rest_client_requests_total{code=~\"5..\"}[5m])) by (cluster, instance, job, namespace)\n /\nsum(rate(rest_client_requests_total[5m])) by (cluster, instance, job, namespace))\n> 0.01\n", "for": "15m", "labels": { "severity": "warning" @@ -468,7 +493,7 @@ { "alert": "KubeClientCertificateExpiration", "annotations": { - "description": "A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days.", + "description": "A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration", "summary": "Client certificate is about to expire." }, @@ -480,7 +505,7 @@ { "alert": "KubeClientCertificateExpiration", "annotations": { - "description": "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.", + "description": "A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration", "summary": "Client certificate is about to expire." }, @@ -490,25 +515,25 @@ } }, { - "alert": "AggregatedAPIErrors", + "alert": "KubeAggregatedAPIErrors", "annotations": { - "description": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.", - "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/aggregatedapierrors", - "summary": "An aggregated API has reported errors." + "description": "Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors", + "summary": "Kubernetes aggregated API has reported errors." }, - "expr": "sum by(name, namespace)(increase(aggregator_unavailable_apiservice_total[10m])) > 4\n", + "expr": "sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total[10m])) > 4\n", "labels": { "severity": "warning" } }, { - "alert": "AggregatedAPIDown", + "alert": "KubeAggregatedAPIDown", "annotations": { - "description": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.", - "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/aggregatedapidown", - "summary": "An aggregated API is down." + "description": "Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown", + "summary": "Kubernetes aggregated API is down." }, - "expr": "(1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85\n", + "expr": "(1 - max by(name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85\n", "for": "5m", "labels": { "severity": "warning" @@ -530,9 +555,9 @@ { "alert": "KubeAPITerminatedRequests", "annotations": { - "description": "The apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.", + "description": "The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.", "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests", - "summary": "The apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests." + "summary": "The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests." }, "expr": "sum(rate(apiserver_request_terminations_total{job=\"apiserver\"}[10m])) / ( sum(rate(apiserver_request_total{job=\"apiserver\"}[10m])) + sum(rate(apiserver_request_terminations_total{job=\"apiserver\"}[10m])) ) > 0.20\n", "for": "5m", @@ -578,10 +603,10 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubelettoomanypods", "summary": "Kubelet is running at capacity." }, - "expr": "count by(node) (\n (kube_pod_status_phase{job=\"kube-state-metrics\",phase=\"Running\"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job=\"kube-state-metrics\"})\n)\n/\nmax by(node) (\n kube_node_status_capacity{job=\"kube-state-metrics\",resource=\"pods\"} != 1\n) > 0.95\n", + "expr": "count by(cluster, node) (\n (kube_pod_status_phase{job=\"kube-state-metrics\",phase=\"Running\"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job=\"kube-state-metrics\"})\n)\n/\nmax by(cluster, node) (\n kube_node_status_capacity{job=\"kube-state-metrics\",resource=\"pods\"} != 1\n) > 0.95\n", "for": "15m", "labels": { - "severity": "warning" + "severity": "info" } }, { @@ -591,7 +616,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodereadinessflapping", "summary": "Node readiness status is flapping." }, - "expr": "sum(changes(kube_node_status_condition{status=\"true\",condition=\"Ready\"}[15m])) by (node) > 2\n", + "expr": "sum(changes(kube_node_status_condition{status=\"true\",condition=\"Ready\"}[15m])) by (cluster, node) > 2\n", "for": "15m", "labels": { "severity": "warning" @@ -617,7 +642,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletpodstartuplatencyhigh", "summary": "Kubelet Pod startup latency is too high." }, - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job=\"kubelet\", metrics_path=\"/metrics\"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"} > 60\n", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job=\"kubelet\", metrics_path=\"/metrics\"}[5m])) by (cluster, instance, le)) * on(cluster, instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"} > 60\n", "for": "15m", "labels": { "severity": "warning" @@ -752,98 +777,98 @@ "name": "kube-apiserver-burnrate.rules", "rules": [ { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[1d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[1d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[1d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[1d]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[1d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate1d" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[1h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[1h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[1h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[1h]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[1h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate1h" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[2h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[2h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[2h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[2h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[2h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[2h]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[2h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate2h" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[30m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[30m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[30m]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[30m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate30m" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[3d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[3d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[3d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[3d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[3d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[3d]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[3d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate3d" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[5m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[5m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[5m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[5m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[5m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[5m]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[5m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate5m" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[6h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[6h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[6h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[6h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[6h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[6h]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[6h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n", "labels": { "verb": "read" }, "record": "apiserver_request:burnrate6h" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1d]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[1d]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1d]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[1d]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[1d]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1d]))\n", "labels": { "verb": "write" }, "record": "apiserver_request:burnrate1d" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1h]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[1h]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1h]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[1h]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[1h]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1h]))\n", "labels": { "verb": "write" }, "record": "apiserver_request:burnrate1h" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[2h]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[2h]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[2h]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[2h]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[2h]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[2h]))\n", "labels": { "verb": "write" }, "record": "apiserver_request:burnrate2h" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[30m]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[30m]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[30m]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[30m]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[30m]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[30m]))\n", "labels": { "verb": "write" }, "record": "apiserver_request:burnrate30m" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[3d]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[3d]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[3d]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[3d]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[3d]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[3d]))\n", "labels": { "verb": "write" }, "record": "apiserver_request:burnrate3d" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[5m]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[5m]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[5m]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[5m]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[5m]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[5m]))\n", "labels": { "verb": "write" }, "record": "apiserver_request:burnrate5m" }, { - "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[6h]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[6h]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[6h]))\n", + "expr": "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[6h]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[6h]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[6h]))\n", "labels": { "verb": "write" }, @@ -855,41 +880,20 @@ "name": "kube-apiserver-histogram.rules", "rules": [ { - "expr": "histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))) > 0\n", + "expr": "histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[5m]))) > 0\n", "labels": { "quantile": "0.99", "verb": "read" }, - "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" + "record": "cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[5m]))) > 0\n", + "expr": "histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[5m]))) > 0\n", "labels": { "quantile": "0.99", "verb": "write" }, - "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT\"}[5m])) without(instance, pod))\n", - "labels": { - "quantile": "0.99" - }, - "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" - }, - { - "expr": "histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT\"}[5m])) without(instance, pod))\n", - "labels": { - "quantile": "0.9" - }, - "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" - }, - { - "expr": "histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",subresource!=\"log\",verb!~\"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT\"}[5m])) without(instance, pod))\n", - "labels": { - "quantile": "0.5" - }, - "record": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile" + "record": "cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile" } ] }, @@ -916,21 +920,37 @@ "record": "code:apiserver_request_total:increase30d" }, { - "expr": "1 - (\n (\n # write too slow\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~\"POST|PUT|PATCH|DELETE\"}[30d]))\n -\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[30d]))\n ) +\n (\n # read too slow\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~\"LIST|GET\"}[30d]))\n -\n (\n (\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30d]))\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30d]))\n )\n ) +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d)\n", + "expr": "sum by (cluster, verb, scope) (increase(apiserver_request_slo_duration_seconds_count[1h]))\n", + "record": "cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase1h" + }, + { + "expr": "sum by (cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase1h[30d]) * 24 * 30)\n", + "record": "cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d" + }, + { + "expr": "sum by (cluster, verb, scope, le) (increase(apiserver_request_slo_duration_seconds_bucket[1h]))\n", + "record": "cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase1h" + }, + { + "expr": "sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase1h[30d]) * 24 * 30)\n", + "record": "cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d" + }, + { + "expr": "1 - (\n (\n # write too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n -\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n ) +\n (\n # read too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"})\n )\n ) +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d)\n", "labels": { "verb": "all" }, "record": "apiserver_request:availability30d" }, { - "expr": "1 - (\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30d]))\n -\n (\n # too slow\n (\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30d]))\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30d]))\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n", + "expr": "1 - (\n sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n # too slow\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"})\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n", "labels": { "verb": "read" }, "record": "apiserver_request:availability30d" }, { - "expr": "1 - (\n (\n # too slow\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~\"POST|PUT|PATCH|DELETE\"}[30d]))\n -\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[30d]))\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\"})\n", + "expr": "1 - (\n (\n # too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n -\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\"})\n", "labels": { "verb": "write" }, @@ -992,7 +1012,7 @@ "record": "node_namespace_pod_container:container_memory_swap" }, { - "expr": "kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", + "expr": "kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", "record": "cluster:namespace:pod_memory:active:kube_pod_container_resource_requests" }, { @@ -1000,7 +1020,7 @@ "record": "namespace_memory:kube_pod_container_resource_requests:sum" }, { - "expr": "kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", + "expr": "kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", "record": "cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests" }, { @@ -1008,7 +1028,7 @@ "record": "namespace_cpu:kube_pod_container_resource_requests:sum" }, { - "expr": "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", + "expr": "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n", "record": "cluster:namespace:pod_memory:active:kube_pod_container_resource_limits" }, { @@ -1016,7 +1036,7 @@ "record": "namespace_memory:kube_pod_container_resource_limits:sum" }, { - "expr": "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n )\n", + "expr": "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n )\n", "record": "cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits" }, { @@ -1043,6 +1063,13 @@ "workload_type": "statefulset" }, "record": "namespace_workload_pod:kube_pod_owner:relabel" + }, + { + "expr": "max by (cluster, namespace, workload, pod) (\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"Job\"},\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n)\n", + "labels": { + "workload_type": "job" + }, + "record": "namespace_workload_pod:kube_pod_owner:relabel" } ] }, @@ -1118,7 +1145,7 @@ "name": "node.rules", "rules": [ { - "expr": "topk by(namespace, pod) (1,\n max by (node, namespace, pod) (\n label_replace(kube_pod_info{job=\"kube-state-metrics\",node!=\"\"}, \"pod\", \"$1\", \"pod\", \"(.*)\")\n))\n", + "expr": "topk by(cluster, namespace, pod) (1,\n max by (cluster, node, namespace, pod) (\n label_replace(kube_pod_info{job=\"kube-state-metrics\",node!=\"\"}, \"pod\", \"$1\", \"pod\", \"(.*)\")\n))\n", "record": "node_namespace_pod:kube_pod_info:" }, { @@ -1128,6 +1155,10 @@ { "expr": "sum(\n node_memory_MemAvailable_bytes{job=\"node-exporter\"} or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"} +\n node_memory_Cached_bytes{job=\"node-exporter\"} +\n node_memory_MemFree_bytes{job=\"node-exporter\"} +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n) by (cluster)\n", "record": ":node_memory_MemAvailable_bytes:sum" + }, + { + "expr": "sum(rate(node_cpu_seconds_total{job=\"node-exporter\",mode!=\"idle\",mode!=\"iowait\",mode!=\"steal\"}[5m])) /\ncount(sum(node_cpu_seconds_total{job=\"node-exporter\"}) by (cluster, instance, cpu))\n", + "record": "cluster:node_cpu:ratio_rate5m" } ] }, @@ -1135,21 +1166,21 @@ "name": "kubelet.rules", "rules": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"})\n", + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"})\n", "labels": { "quantile": "0.99" }, "record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"})\n", + "expr": "histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"})\n", "labels": { "quantile": "0.9" }, "record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile" }, { - "expr": "histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"})\n", + "expr": "histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"})\n", "labels": { "quantile": "0.5" }, diff --git a/charts/kubezero-metrics/jsonnet/rules/node-exporter-prometheusRule b/charts/kubezero-metrics/jsonnet/rules/node-exporter-prometheusRule index 799ebaa4..32de17be 100644 --- a/charts/kubezero-metrics/jsonnet/rules/node-exporter-prometheusRule +++ b/charts/kubezero-metrics/jsonnet/rules/node-exporter-prometheusRule @@ -6,7 +6,7 @@ "app.kubernetes.io/component": "exporter", "app.kubernetes.io/name": "node-exporter", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "1.2.2", + "app.kubernetes.io/version": "1.3.1", "prometheus": "k8s", "role": "alert-rules" }, @@ -25,7 +25,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup", "summary": "Filesystem is predicted to run out of space within the next 24 hours." }, - "expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 40\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n", + "expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 15\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n", "for": "1h", "labels": { "severity": "warning" @@ -38,7 +38,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup", "summary": "Filesystem is predicted to run out of space within the next 4 hours." }, - "expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 15\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 4*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n", + "expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 10\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 4*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n", "for": "1h", "labels": { "severity": "critical" @@ -255,11 +255,11 @@ "name": "node-exporter.rules", "rules": [ { - "expr": "count without (cpu) (\n count without (mode) (\n node_cpu_seconds_total{job=\"node-exporter\"}\n )\n)\n", + "expr": "count without (cpu, mode) (\n node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}\n)\n", "record": "instance:node_num_cpu:sum" }, { - "expr": "1 - avg without (cpu, mode) (\n rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\"}[5m])\n)\n", + "expr": "1 - avg without (cpu) (\n sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\"}[5m]))\n)\n", "record": "instance:node_cpu_utilisation:rate5m" }, { @@ -267,7 +267,7 @@ "record": "instance:node_load1_per_cpu:ratio" }, { - "expr": "1 - (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n", + "expr": "1 - (\n (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"}\n +\n node_memory_Cached_bytes{job=\"node-exporter\"}\n +\n node_memory_MemFree_bytes{job=\"node-exporter\"}\n +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n )\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n", "record": "instance:node_memory_utilisation:ratio" }, { diff --git a/charts/kubezero-metrics/jsonnet/rules/prometheus-operator-prometheusRule b/charts/kubezero-metrics/jsonnet/rules/prometheus-operator-prometheusRule index 39c0894c..1283aea7 100644 --- a/charts/kubezero-metrics/jsonnet/rules/prometheus-operator-prometheusRule +++ b/charts/kubezero-metrics/jsonnet/rules/prometheus-operator-prometheusRule @@ -6,7 +6,7 @@ "app.kubernetes.io/component": "controller", "app.kubernetes.io/name": "prometheus-operator", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "0.49.0", + "app.kubernetes.io/version": "0.57.0", "prometheus": "k8s", "role": "alert-rules" }, @@ -38,7 +38,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorwatcherrors", "summary": "Errors while performing watch operations in controller." }, - "expr": "(sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job=\"prometheus-operator\",namespace=\"monitoring\"}[10m])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job=\"prometheus-operator\",namespace=\"monitoring\"}[10m]))) > 0.4\n", + "expr": "(sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job=\"prometheus-operator\",namespace=\"monitoring\"}[5m])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job=\"prometheus-operator\",namespace=\"monitoring\"}[5m]))) > 0.4\n", "for": "15m", "labels": { "severity": "warning" @@ -90,7 +90,7 @@ "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatornotready", "summary": "Prometheus operator not ready" }, - "expr": "min by(namespace, controller) (max_over_time(prometheus_operator_ready{job=\"prometheus-operator\",namespace=\"monitoring\"}[5m]) == 0)\n", + "expr": "min by (controller,namespace) (max_over_time(prometheus_operator_ready{job=\"prometheus-operator\",namespace=\"monitoring\"}[5m]) == 0)\n", "for": "5m", "labels": { "severity": "warning" @@ -110,6 +110,24 @@ } } ] + }, + { + "name": "config-reloaders", + "rules": [ + { + "alert": "ConfigReloaderSidecarErrors", + "annotations": { + "description": "Errors encountered while the {{$labels.pod}} config-reloader sidecar attempts to sync config in {{$labels.namespace}} namespace.\nAs a result, configuration for service running in {{$labels.pod}} may be stale and cannot be updated anymore.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/configreloadersidecarerrors", + "summary": "config-reloader sidecar has not had a successful reload for 10m" + }, + "expr": "max_over_time(reloader_last_reload_successful{namespace=~\".+\"}[5m]) == 0\n", + "for": "10m", + "labels": { + "severity": "warning" + } + } + ] } ] } diff --git a/charts/kubezero-metrics/jsonnet/rules/prometheus-prometheusRule b/charts/kubezero-metrics/jsonnet/rules/prometheus-prometheusRule index 404d4b40..13109a14 100644 --- a/charts/kubezero-metrics/jsonnet/rules/prometheus-prometheusRule +++ b/charts/kubezero-metrics/jsonnet/rules/prometheus-prometheusRule @@ -4,9 +4,10 @@ "metadata": { "labels": { "app.kubernetes.io/component": "prometheus", + "app.kubernetes.io/instance": "k8s", "app.kubernetes.io/name": "prometheus", "app.kubernetes.io/part-of": "kube-prometheus", - "app.kubernetes.io/version": "2.29.1", + "app.kubernetes.io/version": "2.36.1", "prometheus": "k8s", "role": "alert-rules" }, @@ -226,6 +227,32 @@ "severity": "warning" } }, + { + "alert": "PrometheusScrapeBodySizeLimitHit", + "annotations": { + "description": "Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf \"%.0f\" $value }} scrapes in the last 5m because some targets exceeded the configured body_size_limit.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapebodysizelimithit", + "summary": "Prometheus has dropped some targets that exceeded body size limit." + }, + "expr": "increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job=\"prometheus-k8s\",namespace=\"monitoring\"}[5m]) > 0\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, + { + "alert": "PrometheusScrapeSampleLimitHit", + "annotations": { + "description": "Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf \"%.0f\" $value }} scrapes in the last 5m because some targets exceeded the configured sample_limit.", + "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapesamplelimithit", + "summary": "Prometheus has failed scrapes that have exceeded the configured sample limit." + }, + "expr": "increase(prometheus_target_scrapes_exceeded_sample_limit_total{job=\"prometheus-k8s\",namespace=\"monitoring\"}[5m]) > 0\n", + "for": "15m", + "labels": { + "severity": "warning" + } + }, { "alert": "PrometheusTargetSyncFailure", "annotations": { diff --git a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml index 91600f8f..bdec216b 100644 --- a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml +++ b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml @@ -12,40 +12,40 @@ binaryData: coreDNS.json.gz: H4sIAAAAAAAC/+1dW2/bOBZ+z68QhHaR7Hgzkm+1C/ShTaczxbZpt+kMsGgDg5Zom40sqiKVxA08v314kSzq4ji25UZO+JCLSOmI/Hgu3zmmrJsDwzAHA+QHESXmc+MLOzaMG/Gb9fhgClmr+fps8PHTh/e/ff7jtz/PzEbS7YEh9Hj/xxBPIZ3AiKSdLiROiAKKsM9PSTvoLBBCXUABwVHowLQv8KIx8t+6vD8oESr7T+NhKbcVJ8zZ7/OGnFIIv0cohCWTSu4/DsEI+CAVjtzS5gSE3/MdlzAk8eyeHVvHrXgQjfLbBcBnYBVvFkxKb6U2Kze6/R5lkKKlYPpFGMtuabO5WRvMjVBAizc7y7QWZ7ZYQuD7mJ3LevkayluaHiJ0saLpQFjPMEIefcsl2Y20VQGkfKLsHOiDocf7aRhBpX2C3JJW5GD/BHs45ALD8RAcWg2jadvsV6fTMOwjVXQy6ZfpXIx/GS89GNLMENL1I5MhBqFrxn1z8ff8IIY+b1QvjcUVxgiHBpubcYJD+Pr0zOA/BIYMYeMK0YkRBQwM6BoMgRA5RJwfw2/Yx0yBfzk2jFeAsFNYiyopvcdwZgyjCZgdi8Gb0EU0h5059iEV9ms3O62+bOLa/Bljj6KAdViiUeiIH3mePKIwBPGs7E6/1e50re6zXqsrej3kX5QY8pMnEzay/8IZhwIPv0GHPu+3FTV0JErwmkn3gWdwOYofAmMh9FxpCceQvvIAOy278CZFVMzTjEE5Rrjo0bLyo1D4xgmlAXn+668Ou9D1Cb8wp+nCekrm52DPAwFbENY1Ah5JR7NSrRnmyP2IU8uRGp2zjit23GwrDdfJ8sTHHFkrUcWseTd7qVNejP+8iFcRpRBflXuT1bMaIei5J9gfoXF2Zi4cgcij2flyCCNC8TTXyp2Lh8Z+rIBKz7yhXuwiEnhglkSbm5vYXOZzM3PeFAQB8sdEcUx595SMPxSDMa2MgBRUq9CMA24Y0tsUL6JMtZf0YOG6SzrkKjQz7XPl6DwzNzphMXSCPZcUQZxi4SBNMCTYiyjM3YwFABgUQSnCIlVdznEcQugXRs0jBfAiWFyxwqqtugEzwtvE96y88Mzx+VJliXwklsLHPjQPSs4xMVMfZpWSkpwXrGqpxbZyFrvKYO1yg20tzmNKlQ2rKULv4yWVcCiChQ9PesUclc5vzMrQaJZ0O9BnDlc9AYeItSUO3gQRxWo3W5PIgR9KRsXHBTynqEWMexJ6iukpUwdzqfYKf0EKFiKnRxK3epBf63mOb/6VJ3q5iJEdXmb08DoQahd7/wGnKS7j2yN8g3zGkHwHvvj7q/kkOfhq5pwLC9NTIDSLoikcsIiOIMmeIq+lBa4iuthKsNkWIfDgGPrum4X0pd4thCPJx1+aBaBUfz+Fb6R3S4L6ov1sgka02JENqEYCciFeCB5ZGjDqESDbSwJkt/IAyaIWIIJ8inGnNxyCkJRgQCbvoD+mYlZWph2SDSAroXuyPQyFb83K2yJWz5eGVx2c7hqcNglBI+R5qgWIBpaLutx7q5ovrCergkvt6VnOnnorzKlTGBZLxFzon0nPl1czaWtpSUK4tewowOU4f5lQtzCU08r3TMF1WSvyS1qZ+l0Vk0TKsj2v5OzSuJNOlOUPojdjXbzxCrnCiptq60XOo3Dl+4iZx1/EYez7LCdSdKo88nPLf7eQV4yBMOQRHYxhAfxV4THgw+H6E3HZnWx7cS3ZergwhMIURh5W6gQy7H1Q1bc83goXKaKlWIKMu5iBa0Qy3LcYx0gAHFjmNVkUcgpZIXciAXQZeEVs7kwNSDQ9ZBwfHiYcgf/w8hUkdODgyKcDMZmlhOFLZ3p+dMST88MgxBQfGTj86t8mlqwrcQNOsox4JD1vgCPzmuYKXiJGcAsrybTzFWHNXes2rqKGjS934zCf4Dg2nNwFt5ObTzHexqEA/EgJ9ot6yI3qTUAo2EzOnxAc0ixS0pUMEsqAfBddIjcCCzI8b5TUO9Wi4nVsDZkamnMhVVadjRJM+YRL6lu5s8t94sL3lbgYZpm3GXVZocfu2suUMghyyujhMS9sZcndwtFnR7/w9FZWr9I5lUXWOwy3ee/DLRjCrKgCSWEkH7lE+zt4Ker9Cs2ZP26iujcsq1cBy2prlnU/LKvuLKqxQgpm9hnuPxfjgXRdQsav4XzMqIyQCYk7I1tc+t5zLQbT9yxMmm5tS7es9l7RLauj6ZamW/dGt+xuBXyrq/mW5luPoWr1A/uw0qKVELgzjsSlPwSOlEVJU6RtKVJrvypSLV2R0hTp51Eku7lq30pzA47U0xxJc6RqOFJ2xLsnSS5ehyctZUdMzFoEaUe06PWHtQhR21oZtKpmmtXwyx3BV6LUCwRfrUSwHpTy9QdjiOjOGKUTTSMPUHQJHw2j7PT2ilF2+rsdbqFZk8pHTiq7+bpbswpSqSyOZpWPhVU6wfO+tfVHnUJM36hETsfaD5Y7QcxRsMg7HXyPgE+RBw+t436/YdzK3Aj6AQfDGYV8UzyPycuIV0NsAHvx1YzcIFvm86DsOzra9eaz4pquUfBrbAKfdf/wrQ+StRaD3Rajzj5i1FmG0cmeFI45qA2DAbUl0bc00V8w56a9lOgL9a0d1W/aS6k+Ex1SXT7WTH/HTL9XCdPXT45opq84Drsipm9Xw/TtR8r0GSZ3o2H38iTKQ0gG6o7w3mcSNQf4YaQhDGSdhWyUhdQpwdC5g84d7nnrSX+T3EE/D/Xwcoc67qklAQMPDkKHIbzu1lpx0a17H6R0sq7InTEUIX7N7bX12QsRoyn21+aA0htst6ySdtt7tR2i2955jVRviNBUZwXVKe6I2ITrtHSd9FFzndrUEt1Ifk/sgEAGvruqHJMrwBjf8HCL+ssdOUy//3RXe2E3Kw9uBdpPwMt6uvHO151U+2qOV2cpXif7wo4ThHX1rsIn9HvWUr5Zwwf0e7Zmx5od16wQ2OxuUgjUD6E9uk0Ekcs/tX9a0af/T6vZd1yVnP7Th7nxOC6kVrot1PgJuw1qk07sKYD1yS9+AoDxa2tyvn+7vQI5/OQrZIortH95iN7MvINExG7292s3s223LL2dWWcitarTb5SKNPWTizoV2UUqYleUitgVpSL2Y01FbtlXauy68PywcpKaIPkAkpMyJI2fD+WDz1P0dme93VnnFjq32PZTjlZrk9RCbwHS250r3+6cMBAHOBNJPZbussi+UEHuZ85ezgDkM7yLhDq9QKEufOOEgyi5hv4St+q+26G99FUELnRq+vUO7a6uh2rOUqt66GakRT+j9ejqoVNESN6n7stLFCSPmaAtXwhV1WfAfCDPt+Y0jY1AkMtYCxjKNGpPvp035nRsITnE+mG06h5G6z/bq4fR+r2H/LaHg1isSZi6T0H6RupmRzbTmbQHF4QXYrbMSY/TJTcvoiEMfZiScfP16Zm5EEvhNOBpi59SLhZoCVV0JjMJ76/4le3m8b/NUmqhvuR+NUWU7A75iMYv2vbAEHoDqdOHUXDzDQ9ffDVjD8ocY8NIvKTqExOPqBaEkO94kQtfCpKXsxlxF367t7EwVRZL5CgqWarYKE1Uck3KMlIvxpq/RzCcbTEt5oy5R81qLGsdw+tcSm+SCxT8GXpnM98pI23S0bVUxgbGYjHJ/5JBmtnewmx4W/nJsSeU01U6IgI/S0EZ+if+cm81l0qIBLCx+o1koDB9fPUfO/GrzK3HbWbmsgAxzxqmF8eADZJAqDo/07aU3KSlHtjT9P+O8r+tHrQstUfx+E3lf9s1S2bH30i0AE0JYmz9mTka9vGzY+sX2RkJfm9e/b899D69//2HbL1cGL4g/opiiIkczP8BlFHZZPGQAAA= etcd.json.gz: - H4sIAAAAAAAC/+1dbW/bOBL+vr+CJ+CAFPDmbCdpmzssDm3T3hXXNyS5XRzawqAl2iIik1qSSuIt8t9vSEq2LFOO7Dqx3OWHFrGGIueNMw+HlPTtJ4SCwYCyNFMy+Dv6DL8R+mb+BwrDEwJXg7OLwafzj+9fX/779X8vgk5BTvCQJJqeCj4hKiaZnBMjIkNBU0U5003mBDVNTacRVljyTIRkTkuTbEzZ26iuU0v/kLP1ad7CNLiD/792rEiC/J5RQRxCFeOPBR5hhued08h5uVDCv6qEayJkLt3x4fFhL2ei4x4uxQyUtTxYGjuHKl8uDbR6DJdKaa0y2bIaXUP2DruH3Q1kk5SNEyIVVstDXjhoy1LOzIkZ49AWqNqedvggoVJp6379KWcsIBFVeJjoEZTIiLk2ZkQZfzrqPuvaK1q5l5wniqZw3V6MaURecaYET/QQI5xIe7+RhWVJYn4llF0Zl7J8CX7jcLCQJwlOJSn3YwgxoeNY89w/6c5dWutu3ku5J9sbDmNySSeEZ6rEypzOEy5e4vBqLHjGouqg8za/4iQrq2aRusiBuS7GQ3zQPz7poJNj+697ePok6LhaHT3roF7/tIOOu7rZ85p2IDfqPetDsxPT27MnQanV1wW2Ss7sdlLTasTFBGvFBIwzskgb42xMZg4zuzzBt4Uuet1uhc0JZQWxSpIxv3GpV0+EGEJNzJPonQ6J8r5W77G4Ikbl2hilRncL/BvfOz5evMYUEdc4cbnConeW5E1TmG+XdrL26mjLHrCot9L0vdYaQoojRW5VxdAoJ+uhFgh3nWadC8zGDTrvL3Ze60Zg7TPwpE8cNCeXTR5oLRriex6Z8UPOGAkViYKldpeaI4fiUy7ViN4u5Lky4Q1Elgv6h+n+pPvXShtB3Pea6ytvNap6D7HmftONYP6YWQK8LytVWcGCD3974SDy2Y0NdS4hqGnXrV4UV+Cijhk5oknySkchY38dKI56ECh6z+G/56c6UvSeLwWUkWbIPdH0KOX+bHd9HXxOj5Y6Ks/r2rloEgv0mU3YsqkUFmOiGliB3KaGJ3DKA6LCaCBhMhMxiLEcJARHRDxZ1v88xClIA/oWSuRysyIyvMGhMpL3l5qAR1ls5bCyVESnw6fdbkMjz6KZljvodnoVnVBlMnHwGuREICDCyIr4z0rDHEQ48MJ8wpenwfPqNDAtmk0DrmUMfqmfAf97fVEfc4JesFZIu3e0Dx9XDNZtOt3MHQUoxtfj+X0lBh8WVNSQPapoK6roe1ThUYVHFT80qrDpdhDG2qASrhI2ULCQTvYVZNQgjMuYwOyZDIlAfJSDDJRLjbTUbQYczskxwwDrzA8PAzwMWBcGHHkY4GGAhwEeBvwgMMDIVQIDI0wTEqFUcJgtYAMPB0r3zNqDsVKCq9Env/oWTI3zbTwH+ZzfvI2qJO31l7ldFvdAZtY6wzIeciwiBD0Ei/RC0/FT975Tkx2WfuMdFpxQLF8VOOXb4tQcYuHMuIAgZPyOsLGKTdxfopG625oAD8dO1pwmhJlyy33rSFeFFjbP9xfzfELGxMC5SqjUXuIOemEmBGGqhgoRp45CWQ1lFeDRk7iGZrxa3htRdZCWLu1pwg2NjNX6TQBO0xSeErApU3jsxsGp7kLgiGZ6jJNlmttdQOkQu4mJ66OEV0KTDc8fIcgLGhEH85AWQ1LnpXkifbp4UQHcd7Kig3ZKonc2yS7T185TMpscQGQhB2ORhkWygvEF6NYmqG+GogP0L1+CjGEx/RLcfT6ZfH3yoKnLTo83s/7OP71C58DocmcweQUN7U5+jQzBZplxLXS7UqOQ9qNklUY75loI7v0XuPjxPzvT8RubrNdR9YJwK1T9srGqm2GQyjzT2nhjQfASfte0i5iOlJuYp0Onj0EkLI4pVJeZ8pxIAIx5XnYHWCxI5IiBmsaFcixCTXAdFICIsohe0yjDC5DjzgmeFg+yGNItvqVymfNhFl7ZMFFVhhYrj7FaZ1UknS/hHHflqcQh5yxbmFMibhGm+LbJ8nTu+jx1uHxxFmqZPU3k45dYLi2QS7nTeZtNnk5SSeS1IsVcCuhBqPbKUZ6G99ax/gQI7rjnEZxHcOsguCXFPQiAuwe7aQoNNdjQFQnbLh0e/oZVGBfII0cjQxpR6EIQPAEE8gT9jKr9O5DM5v0/Jrgx7KALM7bcNrR5EBS5mVnfEcgOD2jWBv0/plkNOw9l1tYj1hehotdu8T1ubSNuvQfxBYHHrQ+FW9tQcH2IMmvQP+mmt8E+l1ojEtIJTlwTe5sgfiwMjP/m2IT1yN4j+xpkf7yr2qzZP4zIMBuPKRsPJtdhOIiGFrQMJMSNAWWD4VS5kFRDwKUfuXE70uJJhGArWO3bN8pAfywkd3fo7CUysW8FZNsIcPf67YBmLvl2i8nCbJIlWANGj8nKM6RmCu0T8HooVqtJxBcFG+IJv9fbKjzR23s8saTYteFETKXiENcng98zzBRNyEH38PS0g2Y7lhZwUHk1uAGEMZJTBoAjs+sggBBgiEgObLy3m5NoOEUHRU7voIR8x35lIyiyKdj47cU7ZORZBTfWEH87yKSzfcMNwZtADYOQTyZUPZ71toAG17BPMyk3Kuy1Bj6CnOgCNILOcuk8kmx/dc+jSI8iNzsceOoBoy9AtawAlQoeEikHkOeAe6YGEzLhYvqdNafvhQrnOTfoveFmFWJYU4A9qi45ZPcbfvtUXPJH1f4kW34fyA0Sfttv6yDqxAGi+h5EeRDlBlFHOzqfN6/PMKJuuLgahAnVUMScRxIkJLCKjiwisVt7pjizM3T1ynCHLgUejWiI3rJ7KzJN5dprvLVaLR567eFZKw+9/FMCD4JC/Fkij0L2CoVI/VdbEcjHTG0EQapC/Ujwo6oTjz/aiD9e+mcUPfp49I2krkcfHn2sgz4e8y0TC9k6JUTUV0AWj6fsDo58Ai43KYfUSrfXQGSVNjwK8SjEo5C9RCEteADrqcctHrfsD25x1ky2hVl2+FjWQoJfp/bi1MqPg3XaVXLxB3DXATv+CK4/MeNPzDjfaOWrRR51tfadpOUXaM/eHj2wr5Muo67HrA19yvkwL8rMBLn3TZn3C/HY75Vyc5SCmJSNd6PMT3ZwdOnWx33azHnfypucOltxUvtkmmqBn74ynGzophUxVij41SMrGKdpQlug3hfAx7RGuzPlnLX+zbd4pFAhk3+PmH+PmK/q/lhV3e6DLiN6DZ9eTOiYvZCX+dCugPGYKw2h14UXtaU2vxL5069E8s/1NPuSz+detMNTcwY5o3f2u3uvEzA/ZGUJwFqgMzxtivxqhdvKGmVH4GYN1Xiws4dgx29i+4rwZhXh/kl3v1+bKkNB01zfQe2yHQkSchFJpBxfZwOwIAmTmSx9oG12/yG6jKlE5jubCAyVJRGiLBTmpdscMgbSgQPRkek6TDKI/ALBHTHBiYqnh+iCQCsYML+AJkQPLPXImCGde2a3TfAUxfiaoIiORkSDvZzZGTslFrFCHDJfzmBEQRMETBhOTTdD6MR+wFKLDhzo4pL+QiLS25Rw90gPaF6qDpQOSrB5xybcF0MQNsLYVNhBXMBVTZtfBM5NG0iBXBDNc0GfcKlyduUhequ0KugkhWmPjTTQgFHI79AYKDYHIxwKLoGnJLF95BqirKzUfyBsTEXhB1PJFFgGMKMZB/bUDSRqaGA/n5f3YLgEPgqmdXYJsSLaDUB7eSNgQyb8RouZscJqX9gXVlNqWduX8ru1oYi1t8UaOUEi7SBTh40R1i+EigVnPJPJ1HZQuEZIZmLXu71WQJ0YuTNnErJtYhxGTrQFDm4o2EYrcwRBR0ErCd0YLhloSCsyhuiBEo6jJ2Bi6/kOtuDyRCsEHCzTOgdXpDDLwSJVS6iKYsAk2mX1CLMZONERCzwN8C1DN7G2ti48ab8DsAo616+B/z0z4BIl9IroCXc9RfZbqwUBjGzdRt0yxNM8VsvDwLXO6zkOF2+yzHOgBLvKcxDmizwH0aY7F+F7l3iOPhut8FZ+3LfRAq+30QJv6Ruvfm237V2mnRfw18rTW3+L8g9ZtF8jXW2o0NZ/MORyXQTYcHns175+7bu/Gxb7X+hfWAjO8SUIDOshNi0HOGHwYAZIGzCpneeH6JzqtQPKN9LL3+rOxmMilbTrHY1NsQXA9llbg4ORWdDM1h0hZowXy6B5V41R5klbzyRtChO3sBHgceIucWL7TsxIdP+Zl9YX5lcL48GGBxu+0P64hfYzQCJDjkWEzn25vYqy6hHLM49YPGKpIpZevx2lrfu+aAGtP/ceuxTzRelKRAJM6i0bYFEJOszsxjwf2W9BwEIiSWCZMpyimw1PIBxto4zV6JMTO9HiSh3m66+5EnMRNipfHbXnqxiytHPhAaMHjB4wesC4b4DxuQeMHjC2FTA2flH1TvBOdffKMIQK/jTO0Qzn5Wm5O9TY6EWbLdKgtOetmmqv9VDxg9W/R4geIXqE6BHiXmzcvhgy7XD6QKLZY5UMpI+5QkW5AR0UlwYSX5P8W/DVYsST0glDXbxAVEp9BlGfh7QHCUOcSbJwZhci/5CgTD9QMkxI473aUw9kPZBtbeWTDDNzrnugZ82qGbO78p0FY7VFPM1zqYRXzP6twNodAbOLalDzEM1DNA/R9gWi/ZQzG8gwJhP8KxHSMtQ7tpfV1PYa4Xz1BWF8PHfI4CqDdSesjmdB1DyBGsz6VWSS6nfbsfFsskESlarw+jvbiprZk9NHNiIFjN/83CvmLYSS/FqwcFtKYcaK+c0QOnW8GxQRvDx5gpN5qA963dKPo/KP3mT+90np7175x1G3TJlHl6Bf+rsXWT1/LWTQOYendrejzFfdKOWOn5Y7Lo/SPy7/mG8JBc+iMr8FLwvq+4ObbB0MBb+BZJibuPCl1/pBFshUFcQbXM/8xOTFKvI1t839ECZ1qYf8QSmJ4BacFkPmZtN3a1v8dPd/fx3uAEm/AAA= + H4sIAAAAAAAC/+2d6XPbOJbAv/uvwLKntuItxdHhs7e2tnwkHc90EnecTs9sx6WCSEhCm1cToG0l5f3b5wE8xFuULEp0B18SiyAJ4OHh4YeHB/DbDkLacEht1+dM+xH9Dr8R+ib/hRQbWwSuahfXw6uPH969/vT29a/XWidKNvGImCL9ynMswqfEZ/NEgzDdoy6nji1umXLush9fvSJcN/ao88pwdPbqbrB38Aqe9Cj8mD/KZ67M1sAcM8f3dDJPc01/Qu1LQ6S7BdkG6e/DgicKJm94hH9vOkGlPfKnTz1SUO0o/4mHx9jG85dTo/ByJKafsgl3xGNh/Y/3+nsHYSE6xdkVVZeWVtTOV7Eo395ed69bna+LbWjGTJaMY57P7Dp1NZHNCjlwPDJJPotP6ctPzINahBGPkgLBfYI0lE3MZxdrDLZtB6oPqUJlgvw1kzIeK9C8VJAy8qnJL8Wbep351UQbF7cd3ENsKYIfEfd8krg+pUbBVao79rljOp54oTcZ4RfdDur3evDPwUEH9XaTr47qfjqvC/pPdGoSj6eKwLE3ITyuZtTfqUXFxV6320letzDXp6f2DJLG2GQklcjxRHaxm/TVWOfZdORgz9Di1MdkOcpue5T/3+yEt2etzWuwMujc9BknHvoAjXpHyT1iU+ee2hNkEI6pyRA8gl1ioDE0A4JmQMI4oXmjoNA0IWIbrkNtvidlpBGD8kwTaWPKdGz+i2AP+ojH3zk2n0K6FJQ2sQmXJqt3MOgeB5cg6+knxzE5deP7pNLavmkGv6DwOKxR73BwdNQ7Ou4PTnrBC0xq3xbYLswuPMc1nHs7oypSUYRsyAO818YmEm9IdBhbN32DfMYey7ajdkuIK7pLLiFq3ITyggijRrpJ3Cf06czEkGPuHZRLUQaNdhG1dKJT8lhMWtEgkX/A98yiHmzSO/LeuU+WIDAbBXLUHdPELiNGrrypLhw1lkyZeNS4cliq22jTjAUQ2ff3ExceouYPf4tu1N3J9IVANY6P5+NcXO6bvCSFmU6OB5GsPKh8oRFdaJZAv4lpnDv2mE7S9TPIGPsmZ1ljoYc2KXlR2ApHGjGNT2HonTqmwbTEDY8Zu+K60F+LzMf86dIc8Ig5ps+J1kmng01w00qbtd25OmjcwzZzsUdsnnldMGpg04+0IZP6mPp9s1OUkrR4DhgrUCOSLWG2ksLkkrx845FvNJMMlC2s5rjRCKa9xQz9TLABr0nd9ZgRGNCHCwMEJTWlFhQgbrwKef2eSyp6YabcxTfALd2KxGRjzohpQkfolN9KbYM8pPtt7h4OdlS87b2jldz0WPa01qtZUgN7ty8nHiF2ndJ2a5T2X4SVFndniUrEJiVoyp06r7tZoWvc5AxhqZEdZI3sAhvbK7axJ/F9hToXNM670MaMsH478RzfNpL0JMf26BYwGjiZ+AdACR3PomQdTAp0wKQFACK1eTTwa9jnTjLZI4avkw+F3UEDBNHzvRQmaoy/d/h7ME/JpkrbVGnhWWqQnXfWmAmKMS3Qrm/Za7EMRB1y4g6map+zc6QMNKSrk6oteSCWa2IvB8UyzfUCM/TwQjDJEEgfTOtwitnQlEbvG7VhkmPr5H/+/4v2t+jHF63zhzOSl+D/L9rjbloaIYTnOTd4AS8qCxVNDGLMy9YkE2DLN44H9jxjklP3eWQcTHkvEpycp6xo/C94TTCEjWVOoeYkySHszXLWt2VCGNMHYpynTKDre66ZG8ijgV4+oChCUYSiCEUR66aI7EytX5MiBooi/kIUwXwrRRHUaIweitLq4sM7Yo1AECXscF6LHbLveCo46BiGGOE8cnyedZgshoqMX2+7/ggdOp1j5d+BTTqxC7thUCLKQL9mRWqcz8MgOrWw9Oz00wnBW6KVDa2ljFM2zlTSTWeJDMByVb3+uFt/fEgJyrelb1sDYXlc2zZVXZyhXxkx0LWLddIEWMnqVshRg3cKddZWaKsgh1iTK3Lpr/z2oC/uJftWVW2k+rxMDLQrZ5zoUBUZFnHZov62oNdVge2SuFfVGSuBbXHuC7A4znn/YL35yloX24ZFFqIcXG92Ft21tMLeU4NPqzSn1z9ZzYI1YIWABaQZasIAJcezqp5bXIZlJL/Q1I1mPD+heaaGLksXDSlpt9saJf0B/YPM2DZVtKgEa1XQDJOsrKCN68XhQWvU4rfTn9H1zNbRyYm7FYBiCp0UOjWBTv1ud1vstN9tOTz12jMuwRRO2R9lf5T9+Z7sT7899ucyWpf73+9rYe6HwdHxUd847qq1ua2vzbV/THiOM95zx3KxzonR2My3tlQGg/bYu3ANrkF5BAs9S7tftiaSX70JsTm6IGMPT6x4PfgvMRyIoP0qk6gB6/km35odXmXUAN0p908vMNMetifFZrrThNyP1yn4KvZbcrCuGv6q5d7rrlvw39X4eLzfppn3OYYRkvLZVmbeT1xPqO/sOGiTyH/xHY7/2vLuNht7dpiJPesdZ4LPDusFnx3tJ3Z1xbFDqcibePfYPJTHwg8XmOMrsdGNpXf5Fcewie10b4PIz9yuROZ4/GwmM2g0YisfLRWFbAXxWtadrg+N0ZCDbppDRr+SIbWHUmFqRnChVygZ+fWn0PKhsKzENpZ60e5/gUTTYTvjOHIrvRk2bLoVAsqjlDcwP5ADbK8yYKwkUuysMP5l+TaA/q4TxoZABNQA/BxaxHK82VJiex4SO12TxOJAQ4OM/Al0k0mgwrcw0QuUuK62odEMvYiSd5+HFM+XkmJptOaUgkGfeNiC3go1oSZ50d07OekgIV0PcxKKmLLb4T2YhTGb2WAl/GC3LfR03bENNhz50Ml5TXn/fmDd7KaF3kEm2X0mkr/YvOQjG6o7lkX59y3+1+sRfy5Medm9Ts/RaLxp2PQCPeiBw01o5/dhiH9ajz4+ewZrdrdfiXuuuEnerqdJ1iNH9BKtqWlb0xhJp0FxC1y2qQVK5UYtMsycarMB6ZWI7O/VW04s8ibwJqbmpuL69ZSOeT6h7HyL/BaV4vYJndUEOl66xGWOz3CjlMeSU9n0BL9MF4K8HG+CbegXNbMjD/IUlrNon0fOIxMew5JtKpk2HIrzfYbDsnTQnDCp0ksj3a0LS5Dz/2qfhZ8E/XAqvMNliWdpL0Y68VxsnCtLvJAuibLU19I/U5b6JrubJpn4k1y3L0t9K1e5ylIv0zsE8+1wlEus3f2/aK5jmsTr9vb4zCTu1LFne0AiP/YHRydgD5AwCF/k+4Q9+BEd5PKK3lXkOw+1YVCpCh4RNSnXhZIRLGsQsuoxD+qu0pPMJpgqrYlCcKuUJx0RWqVIqdCtKp1KxlhUqVd2xbZK2epQQVYFSwevrDoWNEtz+jh3Clcp5nzdtkRDtYxPudgI50eXcHkmAIkVzzRofPth+rADk06m/OVIuJvVWQdVTnV11sFGQ6rUWQctiadq9qyDwYITk/ZLzjroqbMO/sInJgUexKE+FXEODK4SezkHmNbASQeBFUbnQaFqLMqUH3hQ8qrmD0xScKHgQsGFgovvEy4GNeGir+BCwUVLzld6g6lJ5AnhrsOy++OKw2fKuaP0ZU8+cWmrR1cflfTjg3pHV3/y8HhM9VafXe1ik3BOXurQkRmUdfkDox4o+zn6eEYWiETilYl1Inx/BSZH3jTC3qnYjBDe080eOeXh+2vhpQvAz85R15ia5ofQYQi2OPs8mExDmL3IYjiFvkXZ3cIVnBz3BH2nqMOFIdfRme7F6Xf0a87GFdGQqNyl6MCuY8ZGmlmOk4vllLf+JoM8cyihyeP9r8U6Tc6PromD/ckFZdyjIz/MIlfdSE1FJngBwYnAyTjMUrPJXf7MMKBbW4wPrFhAovfrt8BzRUURI6AbzH9yYo1U2HbsBRtl5kwfKdK3spc543GFj1YdVFb4+icfVHbmshrHlC1BbEcZU3+4wNIfF1v6fr8o+liM4eGPqiDkQsaLTUkxXN0UnZMXWS75gZbUDW7Sto4cLix0MUbFJiqVb6SMFowcND/Sbxio5kF1NuH3jnc71E1huYcTz9WHHtEJvSPhWs5SZCUj6paOMlgiMKlfiVrfvqGoaOgxE+0QfJYljqWoW/FSL1HqurAlmZ3c6wxdOJeFRCHmoEs7TzqJjxYp4FHAo4BHAY8Cnm0Az2FN4Okp4GkL8DDx1/cGO9lKtxF0PvhckY4iHUU6inQU6bSOdHr9mqjTVaizOdRJb5qMRn6XEO/Jjp26u9LaiD6lAmgF91xB6ZR7R0GPgh4FPQp6Wg09x/Wgp3eooKcV0PMU585TgacismjLJNRK70+Kglby/Ww3iik+Wi0bxnRUL4wptc1PxTAp5vvOmS+/H30NyAeW01TQ9118bfEJ2HeyyNQfFpv6g+MNY98KQeLfGRuu5WS8Jn1dvZUJr+6mtVWBTAAJenEtbC4xdpVXShGKIpTmCUU5pZ45nzhNO6WyRzyf1KUTFWW9NvDIBhZlCMP1+QaDiVZiCQRlZK/YUw4nXiAEg8BoTFovh6CY5aI4a4yu0AeXeOGGPUVXiq4UXSm6UnS1de/PUXbR76gmYO0fK8DarmfniQdvt8PPs3FHz3ksMlS8CNXuhbf+oOSbNof1Ft5+IjZgmIku7bGjFuAUgCkAU0FXisBKCSzz6bQmCCwVW15o8kuOfuqfKALbHIE1+q2uloSRV9axHXFTQQnRx7CE6J0sofIoKaBRQKOARgFNG4Emt3uujGh6KqKojX6nBr9i1ZZwcvRJ1A1d575ENIej1fmn04j8fdZ8MzQU2oUubfHlms2uQV6cFTWyAsbNAWP/RAGjWoLcKi9a1M7qsWLIpZcl18uPya/W1TpYc9ArWQNR/rDNkWH974Qz36obEdVr0TFTSHwPT1ZpKfIbrAf8lvwY+HOVcVgvFNSrFAgXCXqdmAjiZipYTbkWFSkq16LCwhZj4WFNLBw8O69iwcfdn99+gSkYKAfstjX808c2pyZ50d07Oemg9AkWi+hx5AMh8NWPsuggk+w2cJ7FWpkyc35FPZk04JBcvuUWgemWmu9prXRxVreR6lV/Ka5taulcgG3w2e6LsJQKbRXaKrRV+zAU2W5+H8ZCus0tmpfh7WFXrZpvAW+TX2mcghniI4K5OGzMGI4xNX2PbGg7xmorsHGREYqK+/TdsRmZmHway6K1UoBCLpJAQwvSb7K5KgxTGKYwTGGYwrAWYdhxXQzrKzdjK6MXQx5hpnM/hL5ozp7laWjoGsqPTqH89GmktoLUPIKNIbUNeJg9Y+F9hGqgy6AamyU9mb1aUlbAp4BPAZ8Cvm0C3+GSsYb7Jbh3oD56sL1gw2jVDcY2D09aHQ3X3hNvL6TwGl0MhO6pe9SNhqzTUXAmuTlDUzqZImZjF7o4R1EDohfRpSHDd+Fpernm3QWZGVQHlWBIqAOijPnwN7YNZMF7OdKxzwiCwiDdBDwhHuIOGhHkC1nDnGdPcZPiJsVNipsUN62Lmw5rcpM6s2SD2JSJ14qPjxOjbNUIuyxAbXE/wScY5INtnT9DRW19hpLWmCFnjK6hougcsIMYIrTsOkSMlXd1rNMxFBGQCslSiKMQR+02UIzTYsbJhWSVQc6++gz4mqOt0iQTLk65nuM6DMyDjDIixgpfvWw6sPwqLCIKg33QR8xJZVh5Zd2WYpbD+hH/QrzFubtQHbBYNYW6OXleBeUK2G85gYZVWioof0lZVqhqsFOAt1tbz2UhV1DWTO1KZHzesIyxXCZvtYTFSv6sQMCxjC4WymiN85CPeMxRVDa1Pq0mIWoSsolJiPpAq5qGrDANOa47Dempacj6pyETz9Xj8Dh4co5ycQoF2PgiCSm4zx3t/Ya5PgXokPcIQwU3jKhB4RUewRak1CRs9BJlSzHFtmFuthRNc5IsKLqWpWJlCFoqgyanSas1/88EM7L15l9bKZpuflnQdTf/2SaJ+lTn9C5fBUXUiqgVUavIBYXTG/qe8NLxnkcl8Z77iqW3Eu+51m/ePe2Tu+velRLWpNZ3iZv5Al7K/6jYRLGJYhMVcqDgpMVhlSV40lNxlRvEE32K7QlhqdVHk2AD/guT4Cqxl0OTntGec5nrLPqWVnhl79saGScMCZVFRK9NogfRoFfw6wLPNrDXZb5FBYYeZGF7huL1aYQ9gv70iU8MsTElWDPfQx8pExENYXhC4nYGBEwYZ2JHCzxJ4QXBBhrdFMiATAcbyPHkhheLWCOopI5t2+Hhq+evUhtfFKIp95EiNEVoDQaFlnmQFKFtdufLdgMan0xkcThWFOvYBq4qL1RzKFUaXYg8ojueIbkIBVdsX+KPMwb0sRmxmc8SIBU/v4c+TQGjJtifEARGwzcNwF/dk8uOwlIgUQtEx6lNxPBEcGDgbA9diwEKMgwvhNwl9xthGwnVix+z8AxNxR4kg47HAHBAbEFh4+Ik0ZAjBxQxLCD0WigUDHv6TL5mBC/xRV1l1aEEAhTHIEKgRpE7HosM5bI0pHSQiaXvFJ6bApDKygTY3hG8OCIibX5R7qCWjAl6TkSZo3TLYRFMsj10yYUoqOWCscWyNnADWGCJoJASzBcQ1j2HQZlMM0Gm8JydFOp/A8uKpgJ7BIIxZ1Bk6P6i4FA8fg/jEdwguNiM2VaUEsoRFXpO2nwK0gtvgmKIs4tENX07arUv9he7JEp0aV0KnxYNRYL2DkxNmMCQUJBZQRsjLI4Wn3qO7fjMnAUviFQDZl9RtcvVXgigrBqhMvvMl/vsQWGY2HGPXtxTaBshzDG5h0pC3vAaWUobJCQEKacUYi6xC00caH5BseCyJQQCCiY23QtVpICq0CLZluAZwUCTCJUVOcQ9MNivP4U6wKvvp6K1RWCs0Dsw5SBzsYgOUyVhTpBJb4nocHcz5IlJZ5wAjRyoDX+wkROfs6RmO2q2o2Y7arajZjsNxp6WzHa2exqmRbCd6/uJEzIzSRZ+UIdmtnn/0pMnU+dRqdfyde/D9XzjsX27mJ5+HIOE0OCY0mVlvdH4zPm5Ee9jzD+PMX8+zc0rjgqTUFSqqFSFSSgs3VYMZ8rlXhTEeTAogdKB8sGvf0OURJgF22LCzSa+jb1Z7W0mm9mS/fHqvHKje2nNmtxllJNpwSafjEzlNR004D/g4od/tFDKb+RJFrWF3bo9PTlNUSioUFChoHJQKhLcmoMyx4K5eIwyGDxWLLgtp6JN+L3j3Q6D+MWhHPQ9ohN6B8P9aMaX/MzMts8g/RgWfeUTRTtNCpaJv56fUK+F6i8DfQ0d0fqToL73gVxrgN9OmKsorRgtxE2DbiAkGKWnxMLzrjboB5cjCjJwmAd0usm8x2m3/oh40LaxsGUgthbnxaWe8ORgG1iUeZ9N2Cnd97zArqS0jRERJk0KcQhyeJCt8rdvF9fDq48f3r3+9Pb1r9eZQPFo4MjfVmi50nE+SfGLGnqe1JLMZYFzmRk/tXXTN8ipaeaLrpkRwl4AgqPrgMET6YF5zD9nYytskpjck6NXbP+TZl0T8QizMGQpS/hR8udIRsmUubL0Ulcn5CF7K7ul7q+eKT7kXlDsSC8Txc4a/IQmYNP8nECJTqGOpFps0TwmmLBQGP/DwUTKfygVI9wvEQb+DO+CbiDMzq62La34uzNaRhv+SN++QA1S3SvWjaUlkjR912CswUoYn7FHxULkL/Kthb1rbTrFBMSlphzCPEnFYb9EtdLSqSUJoXYGskgk+Ix8CixecjbVBpX9JkbFeFDsxJtkGlfZ9Ig/19jLsABFapt5JtJaWvDI01W3jmSU6iZVV/hIco0krg51x5eam3RyBCkWlTrZ66YU9ilDeM8qGbR7VqPDdEKDMzOslKrm0xKqulM2r62seqLyRX6qObUMh1LmUSHiP0q9Z6sVonfASssg0pbPrcD9UtLcZQ2+loodlOd1sO68et2KinXXndugIrfB2nPrTcurNl13zcrzGqw7r8PyvA7XnVevXyHE/tpzM8ozM1J57RREmc1HWOj+nZ7VObA6oMMd0KxOb9oZTDuH0w4UupPKptZ8or/01CFr9IISCxfdYzDZpNbc1aqNg0m1Zjv3L2O11eRAJ65pqcdcqt+S+aJFVMzYziZtvJYwkqnhL55OBynzfpk0PyBIK/lEMmWuF1pCR+J2uonqIFwc+bGnPJfkiw+TL07m0t9P/khEFR0ZyfJGZUmJ76vwiAtvoOfcs9AfP3dUvBah9ufhnpMPd+IEQHIf3ONLh6c2/Tr9p/H1q/1/gVNZu4sdEce9ncd/A4+bRw3sVgEA node.json.gz: - H4sIAAAAAAAC/+192XLjyBHte39FmbbvtGa0cF8c4QeJlDy67kVuqWfCywQDAosSLBDgYJFa3bf97TerAJDYQRILATAfZlokQKAqazvnZFbWtzeENKZTSVmaht74C/k3fCbkG/8/XFGEBYVvG5Pb6c2nj+8v736+/HzbOHYuy8I9ldn1G01dUOORmvr64ozqoiYtDUlV2C3rC8brkj90JhiCrpqaSNfXlrL5ICnXM3Z9GfJQ6/oHu1iu1/IbvsP/fzu2qqTR301JoyGVct6/FBQo/erREn/pg2A+uMrjGOBv3q+fqaY79bLefBz+jgdNmAuKEHiL/+vVe/wXXG8anHZOB/GvC6+SJiwfQ1/l/nrjKoU1mxTZYEqwqcJe2TptnjZ3qJsuKQ8y1Q3BCL7yNuRasJarLiMoigr3wlXWZ6zXN2RJN1Y9aF0ouPKnPz0K+uPf6St7lnr/Xyoaf2k1e8PVu+Cee1OSjWv2tpbrW5cBww0D91BFuJfZdUMzqev7R2kW8q0kqspYlVWNPVB7uBfeNo9Ju9WC//V6x6R15H60Y57zdX3J/yHnMtUMTxHW7a0/3quCNmvY177zf397YzdVg84kw1faxoNCDT6KW8N+0/qGdbc7VZUNaQnfW1/yRlRMWbY+GVQT7Pmi1W91+81es9XvD/v8qiwpTyGjmdWdFZN+gV8rgkzYfa7JRnjgP/pt/Y1kyNYAkIxH8z44L3kfYGp8hns0jKX+l7OzB/6jU1FdnGlzTZDkzpk9mE9WhtLDO3KKkvpnhc2Kav2Kl9X++2xdxjPWMv4xwMdYiJFFVZaFpU5Zc80FWafrWT6pM0PDS7MbdT2mrH7sGxMv8LnddX3xxekj9mc2zJpOB/ROAu1+a70+rCqwNqNGl1QwXB3Na9x/mJL4RMY3n8kZeU8X8P+JpD8Fba2pL+HtKgriI72TFlQ1A29JNo9vmbww9VfC5ixK1DkRZJmXTFRhJSOG+sB+ra1/PJeoPBurylx68Np3RueCKRteq1sNyWcJ95fw9ULl00rDeIT3PKqy04V99rYeYeqGumDP8H6/EJZLmHF113TpnzTdrdY89n+tsnmh8ddG4IIBI4Zd+nB2HnLRap5W4MKzIJu8UqxBGp6r312ffvNVgnW8VrPp/VZS/CXmT/2FveK9bTv+Gs8tLmtGWly411XZNKivYg3doMugKYPGdLWpNfP3YOpvDdrHpAszf/N0NDgK2MxlHFZo39Xvx9u+st0ZwDvbI3hnk71zOIp957CX/o2scr2u9R9UMvZ9o8D7PJ9/i+zppiLxfrekMH4VY92HXLc1VIAUMMNZQPO3wAwVOft1fbNfJ2Hya0VMfs01GncWyPXUpy69eMb6UpOgNs5C23iEz19V+MLde2HSnJki/Rjyez7jyWKwcwId0I0PqvHBN968I4xPWbqHEaxbS3fWl1BTw1h6uXPG1DvGPXT/euS/672gPQHks6FJwIQWnfjFj7Fda7IGU663qh5L0C9L3ivfvn0rqqZi2P9XYHRPxaU51Sks+jN9agDUkr9JCszsikj/+p/Gn9gt/2kc/1e9Z5/gn/80vh+R+1fyFn53dHRETojw/PBWNxf8SzZfHL0FeESjHs7u+OsP0kymPxwnvejffwKCBM8C4gdYBEz/2xF75Y9s5juCFTDbungb2oawvmZj/dkuy5UgGnyse+b0hkwfqDK7UrUFX8x93Uejc4s1epcIPo9ybNF84x/+IUCLrbVsBQ6u/hYvLMf6T972CCxJ0D8ALz/QI8QDiAcQD9QcD3Q2wwMtIPwICPYPCNjazVdOWRVmvQ1WyzOS8aprreZem85Xq6cB6xU8G1pez2Z5zmYBvn3VyTuwGCxxC4Z/jsq+ErdwKcalGJfig1qK+xsuxSNcicu2ErdKsxSXZZ1t7X2h/aCy9VOQZOZEIp/O35MFXajaaznX0hkVpYXAu3czcpHFlavwlatZ8MrVrOLKNUpcudi0xAb2R7sEAYeftbT1cWkrherM1yZrupy+p4s7tjJN718NqieuT+SE+H59pVG64Y/ZIpnq5Wn5qTcCIttl8zje6lBqsF3A9ufOErapCVZqewo7HhXP7y/S4A62un9mcQR7Qxvs7eT2RVgiVUeqXmHA0yoY8LQrSdVb7Q3d6C0ENKUDNGyWToFo2M/TQJqtXx+OaYrh87+e35RhYf2kqga5usW1FddWFBNqLia0epu6pLu4uJZgcfXw1rkkU/0VxvBiyrXXzda44wVTxJcqrGnw3Rl8MdfZ9PYH+KTB1D/Xg8TW9SZd+kozfNH25LeQtdheA7NdjvnsciGITw8aWCYYh8+v/2LPJyGXvP0jeY7aZN6MnM4DttkaTHDcBRZY3FONud9Xrvc0yGINC3aZJ9fdC0at5tpTZLWopwiwSP+y4mnuSF5Yp50LTd98EDJNGIkTibHhJBI5ybf9mzD8k/xww0l+Pcc7g8zfhUPmfhuR3fkQkvv7mInN2cnEp2ViqITDsOCM7Sp1tLrnPEwTlIeEh7VjZgFo+Ql09hs2c+ne5mfXbqj2iTf1WkBnNuK3h8I0/unOgpceay5V3ZhLX7xbK+0vr2CpvIW5ll3sNf/sug4zXOA3/LvIn3B7vBeWMe0w1/iwCsOXEbi4YahB3Bs0pr6EXg3dxje6YEWRvdvdOi2Yg6CrQncdsUmoNfTMU3P2nuDgYU92P8d6TJvNZ6OO5wHu8Rno/3zzGzzHXPg3um66MucZbu0ajY1GWeKt3bTDbTB3HPbYO90762fYjk8+MCN7ML8a34PDiF10543gbcF68hudzcqiqWkrjI5Lv3vpX3nUW1F44JYDSGIuGbQrBQao6frv34PZbu7iMnZ75xwSWDw+CN+iPep6R3OmKCLilb3wV+aINdKjCh1hRcVhBYcSNhfmWGJjBf0eKOx2v0yAFu0CwcNn3yKRK3KIGPADb8UODl/EarybA4wgTEmHL5rxegNTbtI5L7KCF1wkO1SJoRPho+2gxIASA2KB3bFAflJ8RbaZeaDDoHk8ajbC9furW8KXBNQfDkp/SMIH5+8RHBQLDgL6QwQ4GPQQHJQSHAwQHFQDHOwWgN7Y0+LdCI8tx0UbF23fos1CI3HVLnjVbm+2aq+pP67auGrjqr3zqr1trHqJlm0eun5Y63Ypsun2orLpdtJk070QdEn0ZNP9QI3tc+oKsiToYweIuKrKs/PBL/94eX4x7Izcw40n7LlWXwQrJvqPw1HzqtkO3KHyFLrshtbVYDhuuW+4nlm1+GOz177oBS6RE/IrPB06JYF1nei8edgnmPgfYfGhSsSPH0yqW4UanY+7Q3cXa0jR75TWlQlWV9J+51curloXnj0nDUWy+s8fx63euDXwTEnq3HB+eNnuTroe+8CAtgKF/ng1vmxPLj3XuG7IL/aaw773h6Zum7TXGvXGl41A17oXND2kt+uP76jyYPD+2/R8T/UdBocLFLYj0zqtOqikzNVsgKE3bmoFWHbCixJfdLqeL/6mCTMWD+8Bu5EzwcA3E7TaCTNBPyykY0aVW0u3Dc0AMBisERoPFPOWA8bvg3Ku39l54f1rqPD8EPKtM2kGr1ibXgLf8k0v/m816eER1oYwedlZkX3p83W4+Vdpxrthu+dB2CzQ1zvV2d9OoFcFr/BYvZD3hm4c+O7GuFT3h82wL1/sUrXi4fAOADF014XADgFY7UuI3JBgbTYRHgKRPklbFbgTgfVlk7245/0+2NGgM8yoZs0sc1l1YQDLo/DRPZTC13q+jqyyudmLw3HYfrPVepGI+VfP5KtFxMMGlxft/mSLh/ECWmFuUc+014Atn/lZd2WM9z2xed7tDMbbPvGjJwm975H9Seuq307A5yINm/cB54lP/j7FQOaSzt5ZeN7bQTaGyk4WZQcWb5BJ+T/2igfIeZdsyrlnX2zHButG9qYM82DsYNQfGFD4obIGDQ6lbbJi5GBOCyAWatBd7BY2764sN96H5f73n8bpjwCEix3dO9nu0z/0CMtNMrFcorH+4GSVtz9YY9j+YE2Sq49Oh3Q+ar+v77SoR+k7a8jqtrL4ZWZ9Nf98/sWYLIiBVsa6SqsWeY5CWtArSzD06Q4L+ok+2FDW94PbR2kerVTwXP+MDLpkidXxVN/cnEHQ6CyENVjcoOmH+VNH3JCUmfQszUxBDvJh1/lw7kPYvghfJB8mvzfFJwvj+LjGKiOCO9h1LUn7OUso81nxkhAm+ip82Tq4vt2J6m/ePZP2Vmz79EDf1+oDNEsgY4LNARuBPm2xwIbvW1d9dwvab3d3qIrX6ImVCdxu1SXwdYiQHDZ4XoO9h/PwMALOvn9Hn1el9pzttrEad77kQiug/vbo0qtdXZjzueVagqut7mTk7hqNMXNduvmC/9rMlpNag65H9Rqri4VkGM71gBLFEr64qY3r0s+CNnuBscx2lmnmEp5BTsg5j25jW4yZ+9h4FAz4HyVPVFOoTAA8KoY0l+BWQWcbkO3fsa3tBnlRtSdJebCktqur86srj2CowAwqPVul6Q273YFHibsB/szlEd0ubq953vLfMI29A0o8XdX3snk1mnjesM4JEy4T3kKvtYhvs395MfBfmq7b6LI56LUnnhtYArWIx8Kl2NfCddezgxolv2FVr/ZVb9DzKJGfFdN5upcRl1dutLw0xNSFB3qQkmMgD9dOmuOwnppjx6M5Hp6G6G3qEouIwQgjv0Lnn4TXw6vpA8OW1BWZSCa6AHzqJD8Ra4GNVPYsZ9J2j+aTblaC5vkz+/M+WiKdXE7Or1IYqwzCYmaRcwVKWRG9OENlcF9ZiVl6I/evLSS56Y9/8rzZxq8lyMOb3JqeTENZ65L7MWjeukXSTJqhPLl7fy7CCME5P0N9cZ/pPPM2XjDl5tZa4T7lr/cWJUEFbGcFLCq0d3dxK0IPSqNtoWwVIlt9ouKzNY1MgdO33fj2OPQuWY2SYNg9M01drh/Uv5w0L5vhN9nPCWo1/BYgIq4CBZWT9U32c8bj8473ljtNUPTkqrlvi6ybdVNS5Vx3RdbOuiepeq67IuunbdAmmrtNmlbU3ajbbZ6H32Q/Z9QfdLzRgpq7TZoRhdb8bRKMOjS8bdKMaBNjkzYxPG0SVTkj0CbB2hmeNomqnhFoE2+UTPGSW4i6plCDSbE8oI8sqUb4Sj4HMojhfeyGzi5aW7dIrY2RocvF0niNuPYvqqnFS3QWvFkVOzwC0Pejvah3qNN5paez0x/5xHX645kvyp99yyARNxJ9EJiD5uSfqaLVvNXIRFVahwXYU9sUlhkKRbWXh81SQoYEBfw4zJsVsfWQfPs2o8+SSL9/z01iCpqIt+1CMspvI15Sl5FII8OTlopjkB/sZRdw23wOyzBSySyo5FLfB0fbkksulznS3bLzyD26mTtRQJhtryJ8qSLMOe6cWL5O+KMTHujgzkh/yO7nMExMGUL0blJMxsmtXrtGTumNEG9gNzk6rPcGhPeLU7M4UcQCQX/43w9WLrEfsjk+JPDU3N0B376t86Slwb37RHR8Fbnlqwg/zQsRXQaIzjmbKQtUl0f0awk8BN5yFgfs3BvxPctQkfvw282ojfi9kI34McpHiKcj9c71pN3rCTvYE3exx+9kT9jNnryjPWlXe+LO9rjd7b5BFcEH4jhBDC/YrBtGhqGGcYRG8IyzEPQfxwDiWEAUEwhYKYoRBM3pZwax7CByVK5HZttXjJewL78ET370ZDAKFjOOJKyHcycEPQTL6aML/lPOV4wh5MKaNIRctKblsAt8Yg65EMMeolfM+O300Vvq47fVx7EM7zrtWUj87RRGOKJJRwTxiCYfGxCQSBKSSESCtYnchb8JJ4njJTHcJJ6f2FdD0sMkspdYBhMZwplEZCLJjH/9TLNDL/t94hsSFA9JsbBhzA3hLCY8sMk6a+eEwIIjUl2nOqFfqGhy9CApzt4YDrADD4ugOlF0J2R1yKphst1rvkujbGFztrMcLP6BfSGTZbjhWYWSzH6xd7MzvFYVs3+AsjKzwz+zNFYf793qme4TztnqGxKSaHNP9m/ubBMv5G1wXlqfya/PPjJji+piKVMjrndf7t/cGeYNyNvW2u9gaGADz5JozSDwCLZtVI+x8NXeLZx1doacrXxrFddjabsKcXb+2/7tzMSIyliZFRZszHJMEx3mZIMtiDyBIlGBgAh8MnEcjC+PVCGaqSj2wimQZ0kzTKC0X2F1pQp8UpWFXwr1NtDPe28gLnZVpYH+xgrrbSCnAVbW57kO2YQvEF63QMvFNMf15s3h+uSldRH+izgfRqwfI9aX4fVnQM39CYSDHow4L0aEJ2Mjb0aQsod7NSI8G7HejTgPR7SXI0G3CXo7glUIej1CB1+cT8EbL+KSMYI3hfsY4vwh0T6RSH/DJpPJxvUJGnyjeoT+LNxnEh0REz8MX8N7WIQjJdaZ4n1R7EbgKJ9EdOIQfjk2eQi/IyaByPp6RBIR64b4RCL8nuhkIvxysQlFLF9ObFIRfktSYpHVTdPEu+ITjPBbopOIrC7HJRKxbopMJrK6zIAe+8eTSCP+J3E5RtY3RecZ4fcEc42E3gDFYw9aFU8RdB3Gjv1L/158dBvV020U+DILv1G7e5COow46jopwHAU73d48R3HbPEJBmXu7R+hS/OPpWRh/CUsmEo+dSuKZ2mNajd0TOfh+vF0GCV8GAAYmdvrlGhPtnntgq5LvLAI8Rk6cuykEDGhDZd47uAkGxj3/VzuxwsWF5VKWRMGaWop056Vqn9LYd11yn5UNlSwAsN5T44VSZSXGCMqMLB9fdTC57MBFYTbTuOOpUL/efvr3LiqkU7i1hTmteqIU+jasQ+ITo1tLaAnduvIoPFOwPJh9Tg1WK8IOmyI6YxD3psG51is1rFsW6owTskLdeztOa6Xp9qzMwWnFRXSZZ4m3GSMjRIfFUTRM6OVciZQMnagv3LVK3vLbdCJLT5RIrNLHhJFk7fWYQOMdFeoG3HmlKk3DOAPlhp3sN+MbgKwmeMv/FFV4pGIcWY1TqM8vo1xQ+zSuXWww74Wswqxj7TIgb+npwylMOtpsJulPyba9ytW2KVJM7dO2SUJKkU690qSt2usU75Ld1jsLC/Xdue3oMLwVwataB89NLa6t+85GF7ecYKMfb69+vGCKM68fL+p6GtcXevBK7MFzMpLAysC6dkxmq9DbnVxhoTt2Vqk8Nnx44H7v00vmeEEXynY7b1q9lC6U4eG4UMrk7ECXxtYujbCTaHrDQSu4XLmdH3dhua7WE2NcvqsM/B5Bo2Tu+CgqH1bOQqYr49MJ+WTVoFDJvbCkWUUa8s6uQjbieRl5kC/RFlKhfVCh8Kl52I3r9/5kXj7KxGVw0yBvT1i2E0khb386ahTHPbbmUBEm6MWZANnWjmyrZCFiHQwRyy1ELI7gBJOSbUF8Wr3+4QSPReYu88eIhT0Xg8Qqy6jKSVmyzF3meJSyyrBWLFSPTIu2M9spI0b3pU5DjI7uCnRXIICuB05ul8IP0B6k3UrRHu0VDrtOskA3AcLdvPdEfKLCLIVX4HjzN+kzYRr6pujjY7d/xX3SK7yHge/yCjHpFfbxVyleMUusxdWw2xmneQVNfEXALb51c7cSXmFvTU3zinZSNS7Ou53zVO0tdJLecdWd9EbDVN02yVT21ts0r2gnWcraJFvyV3SSuq21eTjN+E5qi1G/PRmmGhlikqHs7expXpFkKDtLb5pZKslQ/d64N7lI9YokQ12NRp3uZapXJPao8363n6rT0lZip+W75nM11PjyfJyquWknsS0u25N2dcMGWFz+VAM8pE+dfGezzZzdtnr21//BF+wp1kc93Au+gWLW3TZKAMq8ytE221O0ADffiyaxEIE922/L4IBfeaE3MuBFDQTI64/+g5mSpcdQ5oja4+7ao6TGuv2vPxI2E9lefz6qyu74RzUy4+BpSSW8CwcP1A32P3T919j1n1rT7LYPzcOPCWBQ7AySGLakhuZ7yUPszF/rzF/qzF/pzF/o3EXnrCJr3SY6OyXh2tPmWA9puzVFloh/DhPgq4WVI9z1xZJfgyqH3BC2CRLbouI8mqXF/8zSS3LtBRgalxAwniccpsViqnyC7kf92IDzi2Vi3E/l+He4GfpxZljEWWGvdUWKjhS9Suf+jVIy9FZ7gBQdKTqG3++RREnq1EbR7gM/dgbwpHAEH75qp8L39Y3F5xDekGTpK09riti9NNi91ey24gaKPe+ZihR3ZsifTWjcEgfvR9S8HbtPthxwvRwutcAJ8KGnynrO+QEqZ6cLOyMfqAH/Zw7x9XntzsDToJolPgu+FXUWfD+bs+Dx3BU8d2V9U/K5KzkdolKVo1Cy3v/STMNEB820TLTTx7NESs0325nxTdbx34WpMtsRzfKSyGKP23Dm/xKmx3eKtk7H/vIoiY8EMKiV4B7WGiZ5s3aWrVztp4RcG0TSCdxOCYXhLsFQZznb7ym7UxagjDOeqt067HNpaktVp/s7IOK8rNY/99nePn/ANj1Pac2NvLI/OwDC1IEkwt8Me6zNbSq8pYR7XZVNg7LrlLnfBO21vu4222y2Gc/ICmgVRNpFc2HKPCYEg1hxAz2GrCJzROaIzLHCzDE0zDgD6thD6nggrspqUcfNA0ZX68f0/NY+HXLn4FPrWe8k+F/UQZOBvdFBVmLNEc2KBah6Dsly2bSE7MzX5OuV3j5faAm8x6ZlsqyKAlvYVYUv/1aitb0RXlcHK61dne6/NqtjV3suZ3ZlieqEtbqwvZ3rQXAtixWXGA55LfJa5LXIa5HXIq+trUd0mJrWjlpIa5HWokc0hUeUZaWu2lmcnsKzM2fhn5N7dnjczKEwQE8kx7P67tNnIku6sTc2uCqvoKhKZY3NCs9wCfzzCthE575Qfii7dVR4mMmP4StRNmcARoixWM518lZ/BKscZcMYd3dFV7Hbn2/Q6Tfv8uMizFzFDn++cXe3jS1TQTdOnBCBEx418BZa4Ch8CMS0yaSe0QBkQg3BF/ON4gmKJzuLJwk1KVY8qewe2fO1AjAadbvN8zj1pDdsdZvjvagnE0kzXt1pTaLUlYvB5OL8IlbqsHOG+m55LzCkbQkYvfZFr5WjFuJXMiwBIQcl487mhFGm59eZuBRT79U95CeyKlPcraubwtW2Xxbcg/B5LZ/4U7iifLJxWEB6/aTTPEj9pEJbmDGkvKy+c55YhvGvEjp4V2XzR5VLuk1a5FdyTxklWeUFYncbKpm5990VLZSsyn23WJbZrFC8tWU537t/JVefby950L1BF0tVEzQr6ZLVDvc2lNqX7MExVAlNyssV0ktfBMngjFklMEQDvZR5whJ66rgWXHo9kpn8YCFh3BCPNBpjEDAGAWMQMAYBYxDSxSCM0nPoIcYgYAxC5ULrw7LcdFudkPw+q8D5W+a5/dl8oGzC1YMEyEKcjjPSdvRy3Gq5ftfBw+RFMh7JIzyKLNmzNg2uP96pTsNS16lSGwYsobyEPNIqGLTfZ33dXJJCFtb3vEUIP4F9tb/+mVq7vBdwD50dE92ErwHIyNI9kHdpj9vleacsoZF5uRwbB0YG/3hPjRdmVB2QtibIZKmpbBM81Y8BOZq6yxfOECWj8Pr+YkG8g78og2eWDLDIuaugsBFepZvFrNh5JtsWWRWfE6gVg7Jbw2qGM94CR07b2GE80GYbtUE9wkRuLYOw/mj7XzE+ZA/CVjhoasamJ6+6BBZR50FsikcUyzDoBINOMOjksINOMhDMWhh0gkEnuGtnB/nl9rNiJ5wrozqwKhwQnxtB46yHLTzHludIFBTmF/LkKFQVdy4H3dTo/kSXT1ap2HxTRuu6ihdu3wWbpjzmXWtafMuCXvs0DdyBh+ERGB6B4RHI+JDx1ZHxIZnLLvih1UxL5ga4gQCjH3ADwU5kzp7Jxo+m8lS1PdLusgMbecfsoAPFUxVDAiBv6uQeLj8xhvJs3UoAbgvr+O45W0z3RfXs0vNVp6KWt1bME2L9q0tfqdvWNqdmJt+bc921UFfUxjzMwR0EG96VeRxm3fch2CZBco3kGsk17j3AvQe49wD3HqT1pabn363eCHcfIP9G/r0D/76A9UUso6vPKpgvnpdtLLc4tXUCOLm37kreWH5eCwJiGQVDRJF/IP9A5x4699C5h+wi3rnXSk0u2iP07iG7qPaxYasctKGnfEFZFZ2BAm4t+sAh38k/GxXeossSFhe+p3Gb9Mru8q1JjqQQYZVqeaNdcDkfWQ2FKbMJg+YDmqjZG515nO1qTyHf9lz/M6id3oM+KuSIyBHRR4U+KvRRoY8qtY8qPY3sN9FHVWoWifv9ykrm/s5XklujnGmmXaWDxdD65MQecoOdkrtHSWdBcmzF09Y72PZG66DPi0uzcolloNjjm89s3x9wPPaXE+G5SuEzewUCIInwxSvjfLIqzPhWQaAM5iHQP7v3nTHroJcQGSAyQPQSopewjF7CcPIY6iUMvxW9hBl6Cdup6V03W3oXNjWhmxDdhPUkeGtPHVtbSsXuvEUD7vHzyivH3E1M6FwCtLZkT6aMMo73So01Kdkby1sX/ZP+PCupVVnRvFZlMZ7WtjKBiByuLGAQsfNuHJOSOaw2a9M/Crqdw1iY0WNybxrQCM7NMID5DaxJVjftbTvgut63prYsaZOwokV3dOFefab8I5/L2bUzlt/4TH/Vz54XZ4o2ZY7rDDPolphvr134Y+YDgUkZ3a57Jd0JVLUR0S2rR7obDaTcSLmRciPlrqpLNTXnHrSQcqNPFSl3SsrNJ8ByUpFAapcVC4G/SxAe+7NDdKB4JfRMu4vnEDpmV27O2vtC173o1l9dJGboDUVihsQMiRnumERfqI+XddLvmCzwMNiweYtlwrtcLPnAjbr+L6qpEZeR16ErtbK8DhYsKhqwqLT+VkJG4iqdZ+OI5WayjykVdOB5kp7EUvKldquStt9XLk7WVfbUVr4oxsrdp+paufv31Fauhzd0ZREk20i2MfQYN59uSbZx8yluPkVPaR6MvDPAzadIqJFQ70CoPyss06hR0gMRXaXzLIam63t7l6Rz/OQP/HhE/cUiKKpp8IhbgTwLUCrjlf1aozB7Kvvzq75nKV7zPzFjB3u/f8dLBra+tV3SFt2zCsyClBdOulpAFhyKLNi1t0dEfwULLQjbnFp736u7WwrKjNhmwy2pyAuRF6IbFt2wtXTDWreubgpXEtBXm6GvtpueGbYxuy1SQ6SGu1DDD1e3QA/1snJDKB5xiufJkcouWLRFt3dWMp4CbwbzO/snrU2s1v5Li9jwe+DdrF/U/ZwNsBD6sJCrIFfJiau88b0O5o8lFfxDMzAs4R9JmauN9Q32wNCgbm9creKUuCGqsiws9cAITYaBobit4VvBOVxrd11f+KAah2nt9htfn7HAV7u/kuUbS0GhsneAxNG4MmDkUkHhdictEh70DwYJI9QtF9Td/CAH1TTqe4CDJhj0LQfYzwt4rTFdPiwfJCURVP/7T9Mp++3UAcm/HRWJs/kmI4DVJ9bmKvhLhX4lWD2zSBdKqP2gx1TCgMwRZVuQ/bmRCevhLLE2qV0r5Ix8NA3kHRXIIMK76NuTI2gyGO5vfzoqcNMXMpHivSYHDbdDg5JS4+12G+E2wm2E22WC2/rLsgJwm5fSxor6KpAJhu1+sTYUqwpY2ypmwHyqaRwI0Gb+aUTbiLYRbVckRgn3ruDelSyjonDvSmFemU56r0wPd64giawcifzTnx5hhP+d8qBV9f6/VDT+0m+1hkE4sKKbNw9zwZRXIvhC+C/blqHMCHQ1+Mu6GCeL2yO/GQLVXZwyS5Ia6ElFuIS4IUpPs9I2ZrFuIihfRczqlJQFtHGjMoawjT0virDnzsYkUK9qt4+krNqHd/Vt22dcG6mBXLFa67jpaR8SQ/gC3OnEjYYkMWIebM9yxRxGVLqbotKoUmyoUuBOKvsW3EmFO6lKLVSE+rNTKxWd5gB3UqFUgTupdqb2qrqYPsH43wvJySwtINSCsFpQjUjKs30EnU7qu2Xq48f35O+8wuhMLQvT6XUGqZiOGHKaXfmpDtQaqc52VGfHjVi/8Bm7cvuwOhH7sEadKu/DglVe1CQOVQKrWMk2aXXTBo32mxUD2dWICkW3XTD28xdBkzh4+/E0JAZUZFOAFc0wal4121UL/uTAm6GZL1Oqw7/spOop2EnVACqLqjLLMZfBYyTnzAOQXzrVI7x6LITMrmHx+e0sg6vzuU6Nutn5DopCrKoxG99T44Wdt81O4Zad5HfMAwo2hhEL9SUiy4xXfO55qxFgKq9ld38PWHFhLrbp7BV3ufGOd/uqiJqqSF9hnE809hMko/uN7I2hl5FdsoJhvDlxZdw5V3eCFL6tLjVDGrWQISFDqp23wIXcZFVd8j+nsIowzGZU6NSgm0dYEk7sdNGsJoS9nwiz/5q6UV8nAcdoN+/ekfNgRRGclW7bVfGYBvdZITYrt3jdSQvNWv0hQjOEZiheZwkG9VdFnDJnpFmlsyOvdUv8JKz4j45uZahEIBqFZmQZe618vORti/yVvFL9mDThD0U92pdqPdfo7yZVxNepBVYXMCCmPJ6/QoZ/x9Voy/arCpF1heqbAyGgkt7yQYNIHJE4InFE4pVSSTvpVdIuQnGE4nVWSQ1JfMrft505QLu1SrwKH7CgGqvM3sI1DEEqLGQje6bBfqJwKgjA99xQF5JIOBZ8e3d+fWQHbNQc9r6XdBGBLsYDIMgtKIp8NfxurbgvNgqZ1FG5iPFueMR4Zw1L8eSOlLr6ICWW77cRyucA5XHjZKlB/lJTRX16bwUSlOrYOZh0RKrrFDC8HebwIkgGOw2bnTN+ffaR6dyiuljK1KDFY3rLcJqpKFCkkhpOUggroOUGMIRYM1Ucn68rjXp0ybLl9JrNw9tDCrVu4R7Smkrc5VKx0wLfTquHyBeRb1mQb/7JQFf4jQMGHvVBKyTFfvtmgTny/Xt9A329eI4inEM4h3AO4Vz9ZcxWWjjX6mJ4MMK5QxIy1xngYEJ+0qe81xSa/W2jrfk7SXi8Rvbe+0NAe+SKVRjhXmngXr+bCu5Z/feM9eCqQb5+FyEfQr4iFLwMMN8IMR/GoaZAdptvCXsvfDn98Sz62KAqnl5ri4HTZ0kzAAxMFzw95fT+1aB6NkCy2ORNazxl14hYNSK69JXyrFqsZoU6zR0TA8bkJxI6NoapY0M7lyUjlrA6btFnXOFZkGTubN/AwjmdH4Q92WfncaF23rw3l9DW2XbwSV0oqZWrGCnpfuO+Z1SM6HErrhl1vSwsE7kjcsfMU2Gnj/7oIHVEd8EeSGWY6DZojuKOtr2e6ESWFpIRl4vkqt0d9ca1Iai+aJWlVKl9gx/MxT3VGJZkjbefMG3LcAyarz5VKdNfTKevTfQ2q6PdVVhu73eB6iLixpgfjPlBEF9PB1B6FN/EGG5E8WVB8d1RvxMD489Of7S38oU6kqBOis6mWW5V+sDPIj/5Z6OKjiWOQXV23jI/mtLeLujk/9hDpFLm8HR885l8+0bEpUm+fycnzjEhRF9C1yV2hYlAbOhdKAMIawK75x1OE9y/OrZnDeHeQ2s8SjqBH9d+VyjhzW/au2B18snulWfkV8scSDXKQjW6w3481ah2+peIKiPPQJ5RBM/opPcWYMJD5BnV8BbcAe4QZgfvMDAsM1RI8T6X2WGP7KRNu+z7dhtUz4QJfb82WRntaqLrAF0H6Dqod+7HVUBe5ZI/9sKTP7b7I0z+mFEY1DAlr8Et08hqMmA1Gl1SIRSyVHAztajCyvvFmOovkiE+0n1K5duI4VapiVPqPTkcWOeqwUZ0nvZdM5eGXl8S4fSYW7vHkDMSUW1kEgdyrhFG/aOQX2ohPy3eHSDeRbyLeNeRe2VVmLUKTOLd3eCASWFGWovilW9mil75TNFbFLqXet0rSmiLVqwxxtVG47baxyqK2Ls0Kn47VYhKVVX8dr+NKj7i+fzl61ZaPN/uYQKoPBA9Rt/4gmrOTn8ca5IhiYL84+lZXFTNZbs76bY3jao53ioBVdKrr4bdzrheKaocVdnWJesQt/7tG2Gwygpahw+SMldrndR+LSuTCTUESUaEi+oyqsuIRsuWj3SYejtqH+EoCsy5ANXq7PZkS58uSyKt3y7D+mK0W2er4Lr1CP1CRZPFo9+/EiqIj8wICN0qs2VwLyAPtw0iHqxLjsG0cLDVayEcRDh4QHCQI0GwjqYuX6ertMnTe8nQS3VW+KVVRFdmZ0MlmqDM1AXMo3xTFfyAAihlk2NtQZ9tBsR05dk21h8e4rax/gghHUK6AiS+dnqJD/dMIaY7OInPOfdDXJp1SiF2B2+z8oXVF+QxAdNY1ZMdYWLqdtIA3Qo6tHe84aYfTAeGuh6CwLrrep20GLCPycAw6DC/oMNQitxpJ6QWTg4PHI6aV812bcIDHUjKMlXNd8pSldtJciogLQJmpQQaXNSkZXYy4pbHTrKSlMU6H7e0SsX311+xmk4iaoroep8qa6dziCprp4Mq63YAO5CcK3Tjb2AT33tJFyuXrasfnq2r0+xitq6sAgr6qQMKUHxG5lEw8xi0+/HMYx8boyJKOkjLkWq2hYrHYzy+LFRlCgvTcirC/A1drEIZfVno7aO0ZJul4E+dKoCB2QdWneKzRLhNCb1+KsiCtsjfqnZ6s+AsWqTFnWFOzlmli09L4bd9ffqyY9lssluksekjANhD684/Szo8K8H+k9ztz2Ssw7A8rMIxlr6sturzs6DNXgSN8gWKBRCa8Dc0ssSshgpQWRSgQS/Wv2oPxBgNyNW8VdOBBj10tKKjtZBou37qaLsmCh4oeBQqeLT6vR7KCEGsKgJeYSeIzuizJMJHU5uy7b40d7Qan9U7u42+1kTAsCpDLAypSgpZJ2spXm3wWZwxhJwtnodHfBeDVv98JtWAv4ndgsRqQcT/ZcH/rf5geIB7p6Ha6AJG6F9EjGV65N9G5I/IPwXyL/MppeoLBfRsLpfy61RVWCPWBUUDvHPXjoE+q4L13WBzw+pLrPoixEOIhxCv4lF+QWdOJSP6BhHnb476GNGXEcxtNdMmMB/hkUQIc+sHc115IXlM9GyqqwxbTAVRpEuDztgxmwosWKwJapLU2xE5x+ua1ThfpNWs5Fa1ICPiXszonQrMIkRFFTL7jN7p8RnuuEB8ll8Awiqs4EqQZDqLiym4andHvXFjt/NjrhWAMtIzjX3B1WjYae74gvPExw86F1f90Y6Pn1BefsGQlIf4OowvBpM0dUh8xXh41R63G1mGcATGUz6yswPFTWWDfJzHPNQAPq4NX6rIgrjGyjswI4UhafmMSIvfy7GbAWe0pH0xfnbKexPHbsac8wWnVGaMWgPz3oWxmwElpYQDOnqdr8sGC0d4+Mwai9yyxkDx4RDFh0xdaag+7OQgswdj5VxjwwjX2KBZZdeYk1YJplcrCpU6yezfCnMYgGRBgTboR0Sdk+uzj0Sjv8NkALOoqC6WMmUn/SxZJANPIElgSBB4SWjs6lyi8mysKnPpIYSf07lgykawL3KxQzf40vDte3AUhXH6wNBtqB7vR+QsVi4/YSulDrXfY45ZCNG/qKZmpVFp0sOjcRsZl3TQKasFmWrGnYOvkmqTJHLJ5oOk/EI13Z4TBqedU2+SjHyEsNBESbtJZGFRLu1mJyEd4icqzE5/DNmkA1VVdLbc83ahDwKDyyf/bKSOvIEydePLpM+EaWiZXCLZ5UW7P8mkML2kwtwnFeby/GLYGWVSmH5SYcSkwvQvJ83LZiaFGSQVZpZoGWt/VxaFGSYVhiYWxkqsk0VhRokduJVQmN6w2x0MsihMt5lYmHaSaS7Ou53zTHpwt5VYmk5Saa66k95omElp2omDO6mhmue95nkrk8J0EgvTTmqnq9ZFM5PB3U2cg+87SeOpOei1M5mDu4lzsJjUTKN+ezLMZHB3k+fgpGbqt7qTUSeTwiTOwWJSM43Ox93heSaFSZyDZ0nN1O+Ne5OLTAqTOAfPkprpajTqdC+zKEwvcQ6eJY6m8363n8nQ7iVOwbSVOLSvRpNMVqdeO30zjS/Px5l04F7iDEw7iX3msj3J1qtZaJjhTNKfphqAfn26UjOyjCsMaPHdDUICLenk+3dyQhgfcQkthfosfVZ60SSDFmWm9nZm+pWXbSM7VXyP+ASaglx/XOpkHFrZDfwYodIJOjK2dWSEzqmtYT/Oqyepy7jsUNdMXBVm5O3JETkjfMCRtz8dVW0zEVhhgJuJahqpGe0tUOfk/pVNw7wPz+H3RNV4Lzaowk7KXbsFXA4D9BCkC1VN6yLodNBHgD6CCvkIdg+WzV3fL0C1L0CLL0BhL0A3L0ANL0DjLkK5LkKPLkBlLkA7LkARLkDnLUC9LUCTLUBpLUA/LUAVLUDrLEDBPFRdcsop1R63OqdRLi0+uG/REthneay4g7DJ2HOSIeugan46+5VMBENAPbM0emanH5sV6CJWzXSJQRUXNDuDJuqZh6NnCs9UEx4oYYOUxzOvIp4lXTfpzKdlwqd7SmAFeKazU3L3KMF9iiib8GR+G3+MvmRJou9f+Tfr5ykE/jLhlcrMfy97oCSyNMBwYXFaVZ20U44zFFPLpAOUSd0r66q4IZfZ6cWosaLGihoraqyosaLGihoraqyosaLGWlqN1ZbwWDhKCpGQnJHiAkuzTM++pbKrTYUXQTL2HouaW7PlGeladFp9V7u9JLZbHVTkc1u8+RWqSu78MivKyfuUk7vd+CNE4w5OhRqe1kJM7obtwkMxue5isqXzyvzNLFTWowMbj4JBXqhGQxVmDJFNd5xsau23h9ovar+o/aIii4osKrKoyKIii4osKrKoyOajyEqqJey9UIZT6Sy9wtcoUiCt79k9jrL2D85kb6WvKK2VR1rrtTpxqpKihh2lulLXhN/NE/1riY/5Ca9yF4W0wxHS1rvMuQrMIiYtDXilovE8tZ5ktGtdjctvqKtlGFLZSSurtXDrOepqqKthTCUqeKjgoYKHCh4qeKjgoYJXl3yaFhsrU5ZIviXdKlYJEmmWzz48i2aygeqy1/x9sKKoYe5Vwxz20iTPPPuoVy0WsBefLhQlzPpImDcrWYJJmNQ6zcza7z0zNbbN++VREh+9J2xFhgSSt/eCMuPaDDENSZa+CuxFvtO3jk7JxLpdFwxTs25RRdHU4NGPLCkn26zORzW8hYiyqvPt7K1m88/8UdZjdGsvOhRxVTAmZgiy/HpKyIVpeO4N3iwpZClocDuVj4luQiUFnXw6v54QQdOEV51ruWyu0hRyezvRj61y2YLvTIWHKqoBz5vLMGhY/SSNqbx8oMCiSWRpIRk6bpFPFyeZWtDt9FHPRT0X9VxUWVFlRZUVVVZUWVFlRZUVVdZ84ySrFR5JTsj1x32qr/A/UdBmFbWdXfr6qrNsxza55QnwJipTEQKyHmq0+9Ro+73Yo31ssmYqUoxu2fgzk6wqFm3a7w1Rqj3EaFPVNNiyMPNImgKXIYm1Ylh/6wJLF0JeBJ0YwhNVTsm1Imp0AcOBzpjiuf6xRsmD9GwdiiQsl5q61CRYV4gOPVU0nBun69SgM+p5ErzulcwlRdIfTzGINU0Qaze15tlFzRM1T9Q8UfNEzRM1T9Q8UfNEzRM1T9Q8c9M8FfWlHEezX38kUJb6bve+tritoFDV1HHTdxnFuEF7VK6AybxFuEEHD+I5EBEOda1UwXzpha1mC5WtqotTTCz+yoyIItWWK0272QnJLlKAnBVRmF5SYXIRviIK008qTC4SWURhBkmFyUVMiyjMMKkwuchuEYUZJXbgPAS68MJ0m4mFyUXKiyhNK7E0uYh+EaVpJw7uPOTBiMJ0EguTh5AYUZjEOTgXyTGiMIlzcC7iZERhkufgPGTMiMIkzsG5CJ4RhUmcg3ORRiMKkzgH5yKihhemlzgH5yK3RhQmcQrORZiNKEw7fTPtIOFGFCZxBj4MsdeOdczpIJ584zQndtnJquwliHnNM+dAPkcWrcx4EEkJrj8u9ZCeQ87Cqo/K+z6V93Yrfud+kvRevVQFUOMBau9bae9vfK+D2WpJBf9EsJ4EbmGOZPkB2GTQWF+2B5oGNXvjaj2nvGzZt7IJ+EZCsu4eqkA3fGIkF57brvXPH0xpHbIzeuPrW5aM3B6sHtVYCgqVvQOu/E4JUVoIvNAd9FbkFYXbaqd0VnT3m0kWvREYKruBF6KcrIvTBBj5VH+FZy+mwrMgydP7V4PqSRTh2ILqf/jfD5qqGnP9h+9xCMHeccdH8m7nom5FIxaqqRi8+TiVOGfV4gM/iCeooUliOIPJkaT5DT/XKN2X3YMTTUZmv4JKZUPbdrSqLn2tn1UDcVheq46rTYavVq1H+CRJhNCxuwEDjt6nggQ4HQHuDNuxBJiPuBgGHHW9zNFnUGekwBh+hoQuOfwsPaPD6LPy8T1kdZVldfxPTjHqROoYuyCsaoRVRK/vZgwGickHVscgo0IcvFccPEpzCl0jrvOWHAyPegiGEQwjGE7ybrTTYuH2sGLeDfReIM4tGufOZ1NoUGlhLhIDm3bGsztEhb0XvhB1SRWOUvX9+BfANIIsq6Jg0FmBxknG8x83McxFDZD7xF5PwS4Yw7UP6L4dIC8zFkeEjQgb5ebMIXZrhHoz6s2IwzPVm2slNa+EOmINkUPQmnfJ94Nac/GA1WouRK11Rq3eR5/x02w/KjLX5P84HDWvmu3ogbP36PnAdIMAOEOJuZM63GL/aawvF0vjFdEx6tflwc2BNsodNrOj01WY1AtEzlmA49VatM/4ecskU2hyVdvafsdznYFL+MJYLDOxZ4Y7on3Wts/05RWtt3BuR4FLyqqHkTNyGag3spJ9RsD0B2mScjZE1rmptgt5aYTkECko+qU/aGH0y1YsZ9fd0Ot5oGp7ojvNqD3R7V33RPtwLBWp9EyngGXYdDKFqjTdiQeP42+XVXfGOT/S1QRFX0jGpg8P3O99esmYabnYWzMle+s3D2b7sw4/+tVmSZ1ms6bOC9aJ34XJFduxr9IdAXTHJonQ3FV8+mDrJ7cUfRAMmKlO/lndFFYKNV5U7Wnqn3QzzL20P0LiStH0yarfntJbOVYOrD51M/OdXcH68r0PVlOyms7nkkjuX8mN1ZjI9fbrgVrG5rKyRxw7J5O8PTkCgg50/e1PR2VOcIVErRZBVOUKQkoN41vddmm8MGH9x+WGCbtcHhpQm7NIkUMgh3BxCLDXPpFtSopgOSz0sjCFqhrTLn+yNWvGBy6D9UUugFwAuQBygdIp+t30VKCDVACpAFIBpALhVGCmqcsKUwFW/LIQgYqa0vEIJNmyZjRg4q8tkgAkAUgCkASUzyGQAQtoIQtAFoAsAFlAOAtgZ4QBMtOzPdOtYC6wrkRZGEGlzbriBRvZtWbsYBxeZ+QIyBGQIyBHKJ2joJeeIvSRIiBFQIqAFCGcIixMAD6ioBsVZgirOtQ36ZIfyL4PrTLiWMSxiGMRx5ZP684AyHYRyCKQRSCLQDYcyM6luVphDMuKXxZ9u6KmXCnbCbasmaZ95a8tsgBkAcgCkAWUTs3upycBPSQBSAKQBGyUhK43CDmCHdmCxRY0gA57wLjZn0MQxiVY5Q5HC78KVBcR8B4TfvaGo7jhUT+oHGqEURPzfyKmLkBZTw2q2x2MIkdQfZiguiqwdR16LEDhqVZBdfbWABKhG4DX/JU5HKQ6DqkwYtXSHZmVIuU8yrMIJasqzw7SI0lMUIhIEuVZjNGI3jkny3q1N81BBQ4IsAaqi3AV4SrCVYSrJVA+U+PVfutgzsVBUFgNn32/34nx2X+4AgCisKKLT0SWQs/7ENmM4T70OYiZrFHcrBistBDlfLqywBRamJk8EUkWCRs9TWQXsPhjT8PMNOUdprzGiujPtQnbvQI0bVUVAXVZYhX6g2Gas0kjx3cqxJr34aT9wQhjExCh5y8oD9MLyk1E6IjQDy4AgGM4QVuWEuF++0YsZZRwafT80022OLeM4I1V8jKkkgjeSqeGXpYekyHUQqiVuRiaHmsNh4i1EGv5Cn0gWMtxVC8Mc3r/apQbcV2wAtYXa72/+4wYax8Ya0ZFaQEDKmg2T+++D+9+KwAWdR3hF8Kvuipdo/ToC5UuRF8Hjr70JaWz8uOvW1bM+uKvYPUQgSECQwSGCKzUAlgGEGyEEAwh2GFDsNVOjd9NCsu0zItXZjBGyDV7xhxsudpYTHjhiVX4+gK1f/BavgvWEvFaufCaoio0OecNAjYEbAckmbWaqQHbqInbjTeb83C7cc3i1raP+m132vHZINnJ6QAsD2Nnsq7ODcC806Wmivs+yZPdmAk0Ht98Jt++iUuTI+Mbp2p7SivvmHhm9av6GdgeMPXdsnJrtSC5CQGoyDH2uG2l3Ryk2bZiEQ6qEzY07TSb9kRYxVyb7eYQ97MgZSlAY07PWTot5CzIWXCvTZk5AQMB+lT/3aT0ax1x661dsxqHMNjA9aNpEHVO/mFCEyJ6rQl6rUC6I0SpiFL3J6y3UoPU5ghBKoJUBKnljNEwl99UsBa83mBw1FwCFi1XhIbkhGRwvPkRCiuwziHIhBeafL7ZQ0ok23p20vk8Y1rYg43IU55iE43ePL7qkgiGYgPRslasvlydlKLuXsDOAjBRT8a8ohiYkiN+fuN7HcwbSyr4R2tk8t/G+g57cGhQuTeuZnGKzHIOysJSD4zSZPgcCoQbPlzC8W+76/rCh3057u203vg6jR0mMuisvlgKCpW9gwS5xTbcopM25WoHmUUWzALJAZIDD7zVVfGJwcXp3fhmKsiyKuYftZ2ZPr0qM8snxf4F2DsjrEqhcbF5UwWPLSXF1GnFbMnLzI4tEJeOFYk9L8qv7ERNuJxN2MrOZl3QRX5GtU/ZDc6qeRkcagPm/gwIiMCfqvZKoKTEEJcxVh4XYGVVWz4KSsV6r1VoJhxYfyTPA5MCTGm8VMyMxgvz9dmj/wWot/JARFmNHfmXVff7WQ1GoP6oLhyiuoC7XtA5V+4QsrQMuuUWQpBCI4VGCp01hf48uXl3fXdZOernKTfjI7OlLBl0FwqYN7OGolbRvG7TlpFZsyLmyqxzMWo4ezZnGbHnUjMVqD8yFWQqyFSQqZTL1ddOT1T6SFSQqCBRyY+o2Ah0CqBzViHMd+sqtkujzoeelBv96Zhqs+zwL9L9hPAP4V99herU+K/dxq3OiP8Q/+Uc67Wgi6JSpmcGAFdlZjFK45uVkgq4z3gEWJS07+G8GBm10naFOmxt14uawOr3lop8K32liK73i64xVz1ia5RWvdC6k15a7SG0RmiN0Do/aH316fxvlYtQWBcaAOCVJjyUMfSDF9Ly81fNtHZ0QjBWYQ623mPYx6fzXyvXV1dlBnN+El526al1iQFhfYucEbAIspWyJKlq9Qbt8mWpypLZRNS6g1mqtuJAu+6yd0Z/5bbZt6O22Xdxm31WBLEdTRApo0KBVXhjn0wLiWMWxLE2mb3ApDdU+6QG+yeephG3UA4H/fjjND6axmEcpaHA7xgJuV5efoH/Kx9FA76pdsJcpxaMm1CRQgvNiMq/2dNhGl4jQ9/ao5W3sOOqoCzMCwZ6shEr7oL6YLUTub4h1wpwOpaB+GOwzkjt9knthsNRGmrH+rAIXVqFtrUOz5CUKp6b0RqOmkj20OFVQDBZboRmiHwG+Qx60crNC6YACF8EbSYpD9WmBet6AJ4FiDdffa7vXgUXor0Kry6C2X2C2VEv1WkabM4Dyy30qnkqRj08T6Ou4NV1rnpgYiqVUN/NC9e2WvuP8CoLcEWp/WCk9sMW0MXFcnqtvNcfKq+eszoARCbvqa5Dj9PJy6MkPhKY6gn0Qcl4JZotq5+SD6pBre0JxqOkEztwgkiKKJvQaYggy3BF1al9aUbuX4kEtrpWLjWNrVp7luRZs0FnrX672ZWAhgu0GzSM3XCCYdDFkjWDoRIdfp+iBeGFiU1YcYfA9fj92hWAtKncW70XTrevkrSPnAc5T6Fifm6kp9dEzoOcBznP4XEeCwVWnfVYtQiDzwHaQ+4BYcwovARmD/go6ORReGaOBQYYT/QlrAZzSSTUeuTbe2HGrxDxkYpPurnQjwn7TuaNekyoIZ4e1dcJwaseQhUQQyOGRgyNGLpqfoNebhAa3QYIoRFC7wyhjzd/0a0yOyys/nnGoPrECRioOlpfVQQA+/pvB6Dv2anAbA09uSbGdtfEY209ODPXSPtnKbhQ+q8GbVnFQSFvQd5y2MH6uZGTVnOE7ATZSf3YSbUQfM3Edoay1oiSh6GIqinPiKIa5J6SGZUB0GtWwIqgEGG5lCVRYB2vBCD/g3oDY73irWFXItAYK2+HCmYnS7iHvEjGI7QMkSUwpxKWICfHrFmhTfBOAqtmOyriDRzZl1lJLBvyv7Lv1ONGhj33k/h8b87rMJm4a2L3Yfg8p5rjfttAFJgUNF/cKrOaWN1dk1Crs+DC+p5VzuqLHs1ybabqtjvtMm6myj0VAFQc876hV3TPXtF+bmnPRig8oPCAblF0i+YgqtyJTFS5pYXv/mHvMsJHYz6SC6sjT+T1sIC+v6ZFx/ZWHxa+aO3zWRF/SbHQ/Cm5W20NWm8M0v1PYkLBOgE0hRreyxIsfzP4paIAUuGDdr+qDWtvGE17aPDsvbL+FmX+2GBrrhvF3Qw8hJV+8d4K1w1BUtgXqiKzeFc+7heSYWySQq/iDl92lhU6fNHhiw5fpGQJlEwXNYlj0CCnKZc3uJ+fN7iLuduQtKG3uMTEhiUbfsc9dawGULuXYjHvY+RoyQMQ+2rKDieFwul8/5blryS/m9QEQDwngn1gDVHt2/ceMupprommLuvfVLyWjL/884POHKHvrm/vLj+sjz19UFQttl0uimsX4Aa3r8oniw9Vmzh6qmKZ/wT+Ozsf/91F+HiL3GtUeCIz9UVZX9F1Th8BbMOPjqHH6MYZqwDD4a6fZ3PwUyqab1cxO6q/XWCA6/Ve2cXNqU+s6ABJP+bzlGIu7qnGZih+sLLzG/cvPBwdJjONLGCYkKVGnyXV1IG1b8XZJ40MLc5jIfYXhREqb61ELPKWnj6cWhtvmXVXe3GPYt3U1dAt0BddBdWi/MeKoUKBIelZO4YHuaWPx+2yqCHUz/EbGuLT68YfWPVe+DJWFYX8eBrivRXZ5MRu/ONw1LxqhkVJWRNGs2L+3AACHcOAvWSOx/zO0i2WKq8qxPQMhlpdLjx2dvE6R43j5GO2oISlfJTga41c3t6dXwC3/vlywtjC+N3H28sT8uv59V3xx0i7m8rusXVpKGcAnpB3EpAv5nZlrcLnXh+vc7ehK7uQKChEN5c80P7tRAJUK4msHf/TOGn9p3FUb8frOCJCAFnMPiNq+6N0pyjHRH2U+nyC7gDP1kLv7d69t7kxpxEevIXOWyRecYGwUNqDi4Rlnq7bV0VU1Sew3JUgyXRWYzekv6rsWDAFHiHNmG+L2NfKklQo0D6fqPh8EK3DKmo5Krdpk4t9tAmbNg6iTfh50N42SUgHNa4+SYXqj3llkaJWwtGGwcFILw/a9TbMzfXWRtcbMkAM3811X+K5yLjlxyWtesihqyIhbiweAfcoPFOyEGaUCGQmAbw3rDg2id3DYhGZo4SHKl5+uLN9XHOYgvn33Kc1sb4twf7CG0HX69Fw7pqkbrlP418m/pZzYn4TWq4GDq6JZZq7lWmQQ5SdQ1TBe4WsIYY1vPG9DiaUJRX8Y3U9Tj9Q40XVnsgHazJvrO+wx4cGlXvjahanyCzCRhaWemCgJpOGUPjf8IE6jvrbrh13fsTP0X6n88bXaewEKoNVApXGUlCo7B0n6LDLim71utF0ayNa1d3vGXrb86YDcJwhN4rhRgHz5hPiBxOAsKRTNstSBl6nM1Pj6TIBCsM6PdPrEkX27duqjt+/M32dV5w41a3vGXEfoM3I5RcW/EY1p9p3fvyIMLl4mBwDke3Bh6o6quoYtJUZCGz1BrVBgQjkqrG/pDUKC/V1B0RZieB+jN1dctXujnrj/cRM7QuJ6qYoUr3uANSuZfE7Rgz6xYCpfGV43g1rbmynzs6+9Rb5K+/eNd4CEgb/EfmXZR9IqzvsptsIkse+99zzqkO1e7gNJF/F3T3u4/T2N/ZD2YzHZil2tdO0ZvCGLj7ShfAL1XSLH7T71tfGq/WWmaA9WXcCNFuPgcaTeU81hRowLlbPN+hiKQNAUR5WBmqw9FGukePS6ddA+psXk7FZnc9KIcADpndeKjoXTNmH5vhIdl8NY2sNvjAEOoSzern9Ana23nM5BIV702naJMx1eQHvl0J+Zs9Qjcnt9ObTx/eXdz9ffr51/3ANo92LQeN3k2p8fC3DWJ6rZVuebx/oF9+K29CfpOVnTWbhqCHlc3qPq1Z+x4irCQVZ/sU2us+crsZ1G38DxspaT1KkELqaedP9X/V+mzb7r/f2pJbir5lay4sV6GCyJ00lZa4eE3jYUS4taK/b7qfAyOXNpP/DKVvDezVQCfZd+M12/7Bq6bpg6vTOepB7Nt1Tp4kz/TcP2mU5ny08fJRnT/tZ1Y2/bNPXWKGz6mwb1Rj7X7oV6d/Cydfffvp/yvOC/rt5MvrtJ8X6J2KJirx9hzWrvW139D0mqS/OJP1pRp8l0cP63F3yTRQ6dNsuTOLaxnjbmc8NrUJHzAbv3HCtFE3dAHb3xv1axiW+W6BI4la04dDc4oEwvl9OWg4bAoJmf9fw/GwpAQ/S1j+2x+gq3M1t+0bPFRfRaro+dNwfWov13z3X3y33h07TfWXN2Rpt19+tmVXj35w6MG0g2Cei3+J+cN/9YPdb2l33h/WepcZg5i6vUxaP+b6qXL5q3AMU1m2IHEWbrxw9s2FyIbmh/XM2m8k3v9rg93kFkHv9N9//P4On2tnk2wUA + H4sIAAAAAAAC/+19WXPjyLH1e/+KMm3fac1o4b44wg8SJXn0uRe5pZ4JX3uCAYJFCRYIcLBIre7b/u1fVmEhdpAECAJgPsy0SIBAVdZ2zsmsrG9vCGlMJpKyNA298RfyL/hMyDf+f7iiCAsK3zYu7ya3nz6+v7r/+erzXePYuSwLUyqz67eauqDGIzX11cUZ1UVNWhqSqrBbVheM1yV/6EwwBF01NZGuri1l80FSbmbs+jLiodb1D3axPK/lN3yH//92bFVJo7+bkkYjKuW8fykoUHr30RJ/6YNgPnjK4xjgb/6vn6mmO/Wy3nwc/Y4HTZgLihB6S/Br9z3BC543DU47p4Pk10VXSROWj5Gv8n69dpWimk2KbTAl3FRRr2ydNk+bW9RNl5QHmeqGYIRfeRdxLVxLt8sIiqLCvXCV9Rnr9Q1Z0g23B60KBVf+9KdHQX/8O31lz1Kn/6Gi8ZdWszd03wX3TE1JNm7Y21qebz0GjDYM3EMVYSqz64ZmUs/3j9Is4ltJVJWxKqsae6D2MBXeNo9Ju9WC//V6x6R15H20Y57zVX3J/5BzmWqGrwir9tYfp6qgzRr2te/839/e2E3VoDPJCJS28aBQg4/i1rDftL5h3e1eVWVDWsL31pe8ERVTlq1PBtUEe75o9butYbMzglr0hvyqLClPEaOZ1Z0Vk36BXyuCTNh9nslGeOA/+m31jWTI1gCQjJ/NaXhe8j/A1PgM92gYS/0vZ2cPkvFoTk9FdXGmzTVBkjtn9mA+cQ2lR3fkDCUNzgrrFdX6FS+r/ffZqoxnrGWCY4CPsQgji6osC0udsuaaC7JOV7N8WmeGhpdmt+pqTFn9ODAmXuBzu+v54ovTR+zPbJg1nQ7onwTa/dZqfXArsDKjRpdUMDwdzW/cf5iS+ETGt5/JGXlPF/D/S0l/CttaU1+i21UUxEd6Ly2oaobekm6ewDJ5YeqvhM1ZlKhzIsgyL5mowkpGDPWB/Vpb/XguUXk2VpW59OC374zOBVM2/Fa3GpLPEt4v4euFyqeVhvEI73lUZacLB+xtPcLUDXXBnuH/fiEslzDj6p7pMjhpeluteRz8WmXzQuOvjdAFA0YMu/Th7DziotU8rdCFZ0E2eaVYgzR8V797Pv0WqATreK1m0/+tpARLzJ/6C3vFe9t2/DW+WzzWjLW4MNVV2TRooGIN3aDLsCnDxvS0qTXz92Dqbw3ax6QLM3/zdDQ4CtnMYxxW6MDV78ebvrLdGcA72yN4Z5O9czhKfOewl/2NrHK9rvUfVDLxfaPQ+3yff4vt6aYi8X63pDB+FWPVhzy3NVSAFDDDWUDzt9AMFTv7dQOzXydl8mvFTH7NFRp3FsjV1Kcu/XjG+lKToDbOQtt4hM9fVfjC23th0pyZIv0Y8Xs+48liuHMCHdCND6rxITDe/COMT1m6jxGsWkt31pdIU8NYerl3xtQ7xj304HoUvOu9oD0B5LOhSciEFp34JYixPWuyBlOuv6o+S9AvS94r3759K6qmYtj/V2B0T8SlOdEpLPozfWIA1JK/SQrM7IpI//rvxp/YLf9uHP9HnbJP8M+/G9+PyPSVvIXfHR0dkRMiPD+81c0F/5LNF0dvAR7RuIezO/76gzST6Q/HaS/615+AIMGzgPgBFgHT/3bEXvkjm/mOYAXMty7+hrYhbKDZWH+2y3ItiAYf6745vSHTB6rMrlVtwRfzQPfR6Nxijf4lgs+jHFs03wSHfwTQYmstW4HDq7/FC8ux/pO3PQJLEvQPwMsP9AjxAOIBxAM1xwOd9fBACwg/AoL9AwK2dvOVU1aFWW+N1fKM5LzqWqu536Zzd/U0YL2CZ0PL6/ksz/kswHevOnkHFoMlbsHwz1HZV+IWLsW4FONSfFBLcX/NpXiEK3HZVuJWaZbisqyzrb0vtB9Utn4KksycSOTT+XuyoAtVey3nWjqjorQQePduxi6yuHIVvnI1C165mlVcuUapKxebltjA/miXIOTws5a2Pi5tpVCd+dpkTZeT93Rxz1amyfTVoHrq+kROSODX1xqla/6YLZKZXp6Vn/ojIPJdNo+TrQ6lBtuFbH/uLGHrmsBV2zPY8ah4fn+RBXew1f0ziyPYG9pgbyd3L8ISqTpS9QoDnlbBgKddSareaq/pRm8hoCkdoGGzdAZEw36eBdJs/PpoTFMMn//1/LYMC+snVTXI9R2urbi2ophQczGh1VvXJd3FxbUEi6uPt84lmeqvMIYXE669rrfGHS+YIr5UYU2D787gi7nOprc/wCcNpv65Hia2njfp0lea44s2J7+FrMX2GpjvcsxnlwtBfHrQwDLhOHx+/Rd7Pom45O8f6XPUOvNm7HQess3GYILjLrDAYko15n53Xe9ZkMUKFmwzT666F4xazbOnyGpRXxFgkf7F5WneSF5Yp50LzcB8EDFNGKkTibHmJBI7ybeDmzCCk/xwzUl+Ncc7gyzYhSPmfhuR3QcQkvf7hInN2cnEp2ViqITDsPCM7Sl1vLrnPEwTlIeUh7UTZgFo+Uvo7Lds5tL9zc+u3VLtE2/qlYDObMRvj4Rp/NO9BS991lyqujGXvvi3VtpfXsNSeQdzLbvYa/7Zcx1muNBv+HexP+H2eC8sE9phrvFhFYUvY3Bxw1DDuDdsTH0JvRq6TWB0wYoi+7e7dVowB0FXhe46YpNQa+ibp+bsPeHBw57sfY71mDabz0Yd3wO84zPU//nmN3iOuQhudF13Zd5luLVnNDYaZYm39tIOr8G8cdhj/3TvrJ9ROz75wIztwfxqcg+OInbxnTeGt4XryW90NiuLpqa5GB2Xfu/S73rUW3F44I4DSGIuGbQrBQao6fof3IPZbm7jMvZ65xwSWDw+iN6iPer6R3OuKCLmlb3oV+4Qa2RHFTrCiorDCg4lbC7MscTaCvoUKOxmvyyQZ6eBh8+BRWKnyCFmwA/8FTs4fJGo8a4PMMIwJRu+aCbrDUy5yea8yAtecJHsUCWGToyPtoMSA0oMiAW2xwK7k+Irss3MBx0GzeNRsxGt31/fEb4koP5wUPpDGj44f4/goFhwENIfYsDBoIfgoJTgYIDgoBrgYLsA9BLxfhZbjos2LtqBRZuFRuKqXfCq3V5v1V5Rf1y1cdXGVXvrVXvTWPUSLds8dP2w1u1SZNPtxWXT7WTJpnsh6JLoy6b7gRqb59QVZEnQxw4Q8VSVZ+eDX/7x6vxi2Bl5hxtP2HOjvghWTPQfh6PmdbMdukPlKXTZDa3rwXDc8t5wM7Nq8cdmr33RC10iJ+RXeDp0SgLrOtF587BPMPE/wuJDlZgfP5hUtwo1Oh93h94u1pDi3ymtKhOurqT9zq9cXLcufHtOGopk9Z8/jlu9cWvgm5LUueH88Krdvez67AMD2goU+uP1+Kp9eeW7xnVDfrHXHPb9PzR126S91qg3vmqEutZU0PSI3q4/vqPKg8H7b9P3PdW3GBweUNiOTevkdlBJmav5AEN/3JQLWLbCixJfdLq+L/6mCTMWD+8Du7EzwSAwE7TaKTNBPyqkY0aVO0u3jcwAMBisEBoPFPOXA8bvg3Ku39t54YNrqPD8EPGtM2mGr1ibXkLf8k0vwW816eER1oYoedlZkQPp83W4+Vdpxrthu+dD2CzQ1z/V2d9eQq8KX+GxehHvjdw48N2LcakeDJthX77YpWolw+EtAGLkrguBHQLg7kuI3ZBgbTYRHkKRPmlbFbgTgfVlk7245/8+3NGgM8yoZs0sc1n1YADLo/DRO5Si13q+jrjZ3OzF4Thqv5m7XqRifveZfLWIedjg6qLdv9zgYbyAVphb3DPtNWDDZ37WPRnjA09snnc7g/GmT/zoS0IfeGT/snXdb6fgc5FGzfuA88SnYJ9iIHNJZ+8sPO/vIGtDZSeLsgOL18ik/G97xQPkvE025YKzLwaDdWN7U455MLYw6g8MKPxQWYOGh9ImWTF2YE4LIBZq0G3sFjXvupYb78Ny//134/RHAMLFju6tbPfpH3qM5S5zsVyqsf7gZJW3P1hj2P5gTZLuR6dDOh+131d3WtSj9J01YnVzLX6VW1/dfT7/YkwWxkCusa6zqkW+o5AW9NoSDAO6w4J+og82lA384O5RmscrFTzXPyODHlnCPZ7qm5czCBqdRbAGixs0gzB/4ogbkjKTnqWZKchhPuw5H857CNsX4YsUwORTU3yyME6Aa7gZEbzBritJOshZIpmPy0simOir8GXj4Pp2J66/+fdM2lux7dMDA1+rD9AsoYwJNgdshPq0xQIbgW899d0uaL/d3aIqfqOnViZ0u1WX0NcRQnLU4HkN9x7Ow6MIOPv+HX12S+07221tNe58yYVWQP3t0ZVfu7ow53PLtQRXW93LkbdrNMbMdenlC8FrM1tOag26PtVrrC4WkmE410NKFEv44qU2nks/C9rsBcYy21mmmUt4Bjkh5zy6jW0xZu5j41Ew4H+UPFFNoTIB8KgY0lyCWwWdbUC2f8e2thvkRdWeJOXBktqur8+vr32CoQIzqPRslaY37HYHPiXuFvgzl0d0u7i95nkreMMk8Q4o8cSt71XzenTpe8MqJ0y0THgHvdYivs3+1cUgeGmyaqOr5qDXvvTdwBKoxTwWLiW+Fq57nh3WKPkNbr3a171Bz6dEflZM5+l+RlxeudHy0hBTFx7oQUqOoTxcW2mOw3pqjh2f5nh4GqK/qUssIoYjjIIKXXASXg2vZgAMW1JXbCKZ+ALwqZP8RKwFNlbZs5xJmz2aT7p5CZrnz+zPabxEenl1eX6dwVhlEBZzi5wrUMqK6cU5KoP7ykrM0ht5f20hyXV//JPvzTZ+3e7Hd5+oKAsADtZPB3y01/7gy1WUt7JZ1SbZtXaSNpvnKJFuP6aKMEJ43clR49xnStFdGy+c9nNjvXKfEtx7ixahCre1ChcXXry9wBajSWXR11A6i5DOYDl6tqaRCTUe216MfRx5l6zGyUDsnpmmLlcP6l9dNq+a0TfZzwnrRfwWIEOeAoXVm9VN9nPG4/OO/5Z7TVD09Kp5b4utm3VTWuU8d8XWzronrXqeu2Lrp63RJpq3TZpW5N+o222eR99kP2fUH3T8EYuat02aMYXWgm0Sjnw0/G3SjGkTY502MXxtElc5I9Qm4doZvjaJq54RahN/pE7xsl+EwqdQg8nBPKiQLKlG+Eo+B0KKIYbshs42el+3SL2P0amrxdJ4jbn2v1RTi5cJLXjjFjs6CjHwo70oiKgV+uWvs9Mf+cR1+uNZYKcB+5ZBIm4k+iAwJ9HJPzNFzPmrkYuytQpNsKe2CSwzFIpqLw/rpaWMCEz4cbhrVsTWQ/Lt24w+SyL9/n1nMlfYRLxtF5JRfhvxknqMRBo5nvZUHIP8YC+7gNvmc1iGkUrmQSWX+j442oZccrncId0tO4/co6u7EweE2RYvwpcqwhz0zqnpq6RDOuHBFt6s+IfsAo/CxJQhRP9GyXSc3Oq1a+QYXwvxhna0o9N8b0B4vzg1j1NNLBD0h//+YOUz+yGfI0xCT925O+Dbt1Wutiy4d5+Ijq8id3wV4SeKIaLLAdE550Plgep2EYFbAg+Bv5zFATtvMgDfMlRkLoB2My4ZQC8iGUCC8hHh6ci8ez5tB33KLvrUnfTJu+lTdtSn76pP21mfurs+aYd9YFDF8IEkTpDAC9brhrGhsFEcoRE+Zy0C/ScxgCQWEMcEQlaKYwRhcwaZQSI7iB2Vq5HZDhTjJerLL+HTJ31ZlMLFTCIJq+HciUAP4XIG6ELwpHWXMURcWJGGiIvWtBx1gU/MERcS2EP8ipm8pT9+W3/y1v4kluFfp30LSbCdoghHPOmIIR7x5GMNAhJLQlKJSLg2sZkA1uEkSbwkgZsk8xP7akSKmlT2kshgYsNI04hMLJkJrp9Zdgnmv1d9TYLiIykWNky4IZrFRAc2Wef9nBBYcESq61Qn9AsVTY4eJMXZn8MBduhhMVQnju5ErA55NUy++923aZQNbM52t4PFP7AvZLKMNjyrUJrZL/ZudobXqmL2D1BWZnb4Z5bF6uO9Wz3Xvco7tvqahCTe3Jf7N3e+yR92bXBe2oDJb84+MmOL6mIpUyOpd1/t39w55i7Yta2138HQwAaeJdGaQeARbOuqnmDh671bOO8METu28p1VXJ+l7Sok2flv+7czEyMqY2VWWLAxy3NNdJiTDbYg8iSORAUCIvDJxHEwvjxShWimotgLp0CeJc0wgdJ+hdWVKvBJVRZBKdTfQD/vvYG42FWVBvobK6y/gZwGcK3P8y2yCV8gvG6hlktojpv1m8PzyU/rYvwXST6MRD9Goi/D78+AmgeTGIc9GElejBhPxlrejDBlj/ZqxHg2Er0bSR6OeC9Him4T9naEqxD2ekQOviSfgj9exCNjhG+K9jEk+UPifSKx/oZ1JpO16xM2+Fr1iPxZtM8kPiImeRi+RvewGEdKojPF/6LEzchxPon45CX8cmICE35HQhKT1fWYRCbWDcnJTPg98QlN+OVik5pYvpzExCb8lrTkJu5Nk9S7kpOc8FviE5m4l5OSmVg3xSY0cS8zoMf+8SXzSP5JUp6T1U3xuU74PeF8J5E3QPHYg9ziKYKuw9ixfxnMB4Buo3q6jUJf5uE3ancP0nHUQcdREY6jcKfbm+coaZtHJCjzbveIXIp/PD2L4i9RCU2SsVNJPFN7TO2RIRWE/8eb5aAIZABgYGKrX64w0fa5BzYq+dYiwGPsxLmdQsCANlTmvYObYGBM+b/aiRUuLiyXsiQK1tRSpDsvU/uUxr6rkgesbKhkAYB1So0XShVXjBGUGVk+vupgctmBi8JspnHHU6F+vf30721USKdwKwtzWvVEKfRtWIfEJ0a3ltASunXlUXimYHkw+5warFaEHXhFdMYgpqbBudYrNaxbFuqME7JC3XtbTmul6faszOFpxUN0mWeJtxkjI0SHxVE0TOjlXImUDJ2oL9y1St7y23QiS0+USKzSx4SRZO31mEDjHRXqBtx6pSpNwzgD5ZadLjjjG4CsJnjL/xRVeKRiHFmNU6jPb3sIURrj2sUG817IKsw61i4D8paePpzCpKPNZpL+lG7b653aNkOKqX3aNk1IKdKpV5q0VXud4j2y22pnYaG+O68dHYbnEryqdfCdqcW1dd/Z6OKOE2z04+3VjxdOceb348Vdz+L6Qg9eiT14TkYSWBlY107IbBV5u5MrLHLHjpvKY82Hh+73P71kjhd0oWy286bVy+hCGR6OC6VMzg50aWzs0og6Dac3HLTCy5XX+XEfletqNTEm5bvKwe8RNkrujo+i8mHtWMj0ZHw6IZ+sGhQquReWNKtIQ97bVchHPC8jDwok2kIqtA8qFD01D7tJ/T6YzCtAmbgMbhrk7QnLdiIp5O1PR43iuMfGHCrGBL0kEyDb2pJtlSxErIMhYjsLEUsiOOGkZBsQn1avfzjBY7G5y4IxYlHPxSCxyjKqclKWPHOXOR6lvDKsFQvVY9Oibc12yojRA6nTEKOjuwLdFQig64GT26XwA7QHWbdStEd7hcOekyzQTYBwd9d7Ij5RYZbBK3C8/pv0mTCJfFP8Ebabv2Ka9gr/geTbvEJMe4V9/FWGV8xSa3E97HbGWV5BU18Rcotv3NytlFfYW1OzvKKdVo2L827nPFN7C520d1x3L3ujYaZum2Yqe+ttlle00yxlbZIt+Ss6ad3W2jycZXyntcWo374cZhoZYpqh7O3sWV6RZig7S2+WWSrNUP3euHd5kekVaYa6Ho063atMr0jtUef9bj9Tp6Wt1E7Ld83v1FDjq/NxpuamndS2uGpftqsbNsDi8ica4CF94uQ7m63n7LbVs7/+F75gT7E+6tFe8DUUs+6mUQJQZjdH22xP0QLcfC+axEIE9my/DYMDfuWFXsuAFzUQIG8+Bg9mSpceI5kjao/ba4+Smuj2v/lI2Exke/35qCq74x/VyJyDpyWV8C4cPlA33P/Q9V9j139mTbPbPjQPPyaAQbEzTGLYkhqZ72UXYufutc7dS527Vzp3L3Ruo3NWkbVuEp2dkXDtaXOsj7TdmSJLxD+HCfDVwsox7vpiya9BlUNuCNsEqW1RcR7N0uJ/ZuklufYCDI1LCBjPEw3TEjHVboLuR/3EgPOLZWrcT+X4d7QZ+klmWCRZYa91RYqOFL1K5/6NMjL0VnuAFB0pOobf75FESerERtHeAz+2BvCkcAQfvWpnwvf1jcXnEN6QZOkrT2uK2L002L3V7LaSBoo975mKlHRmyJ9NaNwSB+/H1LyduE+2HHC9HC610AnwkafK+s75ASpnpws7Ix+oAf9nDvHVee3OwNOgmiU+C74VdxZ8P5+z4PHcFTx3ZXVT+rkrOzpEpSpHoeS9/6WZhYkOmlmZaKePZ4mUmm+2c+ObrOO/i1JlNiOa5SWRxR634cz/JUyP7xRtlY795VESHwlgUCvBPaw1TPJm7SxbudpPCbkxiKQTuJ0SCsNdgqHOcrZPKbtTFqCMM56q3Trsc2lqS1Wn+zsg4rys1j8P2N4+f8A2PU9pzY3s2p8dAGHqQBLhb4Y9VuY2Fd5SwlRXZdOg7Dpl7jdBe62vu802m23GM+ICrYJIu2guTJnHhGAQK26gx5BVZI7IHJE5Vpg5RoYZ50Ade0gdD8RVWS3quH7AqLt+TM7v7NMhtw4+tZ71ToL/xR00GdobHWYl1hzRrFiAqu+QLI9NS8jOAk2+Wunt84WWwHtsWibLqiiwhV1V+PJvJVrbG+H1dLDS2tXp/iuzOna153JmV5aoTlipC5vbuR4E17JYcYnhkNcir0Vei7wWeS3y2tp6RIeZae2ohbQWaS16RDN4RFlW6qqdxekrPDtzFv45mbLD42YOhQF6Ijme1XefPhNZ0o29sUG3vIKiKpU1Nis8wyXwzytgE537Qvmh7NZR4VEmP4avRNmcARghxmI518lb/RGscpQPY9zeFV3Fbn++Rqdfv8uPizBzFTv8+drd3Ta2TAXdOHFCBE541MBbaIGj6CGQ0CaX9YwGIJfUEAIx3yieoHiytXiSUpNixZPK7pE9XykAo1G32zxPUk96w1a3Od6LenIpacarN61JnLpyMbi8OL9IlDrsnKGBW94LDGlbAkavfdFr7VALCSoZloCwAyXj3uaEcabn15m4lFBv9x7yE3HLlHSre1O02vbLgnsQPq/kk2AKV5RP1g4LyK6fdJoHqZ9UaAszhpSX1XfOE8sw/lVCB69btmBUuaTbpEV+JVPKKImbF4jdbahk5t13V7RQ4pb7frEss1mheCvLcr43fSXXn++ueNC9QRdLVRM0K+mS1Q5TG0rtS/bgGKqEJuXliuilL4JkcMasEhiioV7KPGEpPXVcCy69GslMfrCQMG6IRxqNMQgYg4AxCBiDgDEI2WIQRtk59BBjEDAGoXKh9VFZbrqtTkR+Hzdw/o55bn82HyibcPUwAbIQp+OMtB29HLdart9V8DB5kYxH8giPIkv2rHWD64+3qtOw1HWq1IYBSygvIY+0Cgbt91lfNZekkIX1PW8Rwk9gd/fXP1Nrl/cC7qGzY6Kb8DUAGVmaAnmX9rhdnnfKEhqZl8uxcWhk8I9Tarwwo+qAtDVBJktNZZvgqX4MyNHUPb5whigZhdf3FwviH/xFGTy3ZIBFzl0FhY3wKt0uZsXOM/m2iFt8TqBcBmW3htUMZ7wFjpy2scN4oM3WaoN6hIncWQZh/dH2v2J8yB6ErWjQ1ExMT151CSymzoPEFI8olmHQCQadYNDJYQed5CCYtTDoBINOcNfOFvLL3WfFTjhXRnXALRwQn1tB46yHLTzHludIFBTmF/LlKFQVby4H3dTo/kSXT1ap2HxTRut6ihdt3wWbpnzmXWlafMuCXvs0DdyBh+ERGB6B4RHI+JDx1ZHxIZnLL/ih1cxK5ga4gQCjH3ADwVZkzp7Jxo+m8lS1PdLesgMbecfsoAPFUxVDAiBv6mQKl58YQ3m2biUAt4VVfPecLab7onp26fmqU1HLWyvmCbH+1aWv1Gtrm1Mzk+/Nue5ZqCtqYx7m4A2Cje7KPA6z7vsQbJMguUZyjeQa9x7g3gPce4B7D7L6UrPz71ZvhLsPkH8j/96Cf1/A+iKW0dVnFSwQz8s2lluc2joBnEytu9I3lp/XgoBYRsEQUeQfyD/QuYfOPXTuIbtIdu61MpOL9gi9e8guqn1smJuDNvKULyirojNQwK1FHzjkO/lno8JbdFnC4sL3NG6SXtlbvhXJkRQiuKmW19oFt+Mjq6EwZTZh2HxAEzV7ozOPs3X3FPJtz/U/g9rpPeijQo6IHBF9VOijQh8V+qgy+6iy08h+E31UpWaRuN+vrGTu73wluTPKmWbaUzpYDK1PTuwhN9gpuX+UdBYkx1Y8bbWDbW+0Dvq8uDQrl1gGij2+/cz2/QHHY385EZ5uCp/ZKxAASYQvXhnnk1VhxrcKAmUwD4H+2b3vjFkHvYTIAJEBopcQvYRl9BJGk8dIL2H0reglzNFL2M5M77r50ruoqQndhOgmrCfBW3nq2NpSKnbnLxpwj59drxxzNzGhcwnQ2pI9mTLKON4rNVakZG8sb1X0T/rzrKRWZUXzW5XFeFrbygQicriygEHEzrtxTErmsNqsTP8o6HYOY2FGj8nUNKARnJthAPMbWJO4N+1tO+Cq3nemtixpk7CixXd0Yao+U/6Rz+Xs2hnLb3ymv+pnz4szRZswx3WOGXRLzLdXLvwx84HApIxu172S7hSq2ojpltUj3Y0GUm6k3Ei5kXJX1aWamXMPWki50aeKlDsj5eYTYDmpSCi1i8tC4O8ShMf+7BAdKF4JPdPe4jmEjtmVm7P2vtBVL7oLVheJGXpDkZghMUNihjsm0Rca4GWd7DsmCzwMNmreYpnwrhZLPnDjrv8v1dSYy8jr0JVaWV4HCxYVDVhUWn8rISPxlM63ccRyM9nHlAo68DxJT2Mpu6V2bknb7ysXJ+spe2YrXxRj5e5Tda3c/XtmK9fDG+paBMk2km0MPcbNpxuSbdx8iptP0VO6C0beGeDmUyTUSKi3INSfFZZp1CjpgYie0vkWQ9Pzvb1L0jl+8gd+PKL+YhEU1TR4xK1AngUolfHKfq1RmD2V/flV37MUr7s/MWMLe79/x0sGtr6zXdIW3bMKzIKUF066WkAWHIos2LW3R0R/BQstCNucWnvfq7dbCsqM2GbDLanIC5EXohsW3bC1dMNat7o3RSsJ6KvN0Vfbzc4M25jdFqkhUsNtqOGH6zugh3pZuSEUjzjF8+VIZRcs2qLbOysZT4E3g/md/ZPWJlZr/6VFbPg98G7WL+p+zgZYCH1YyFWQq+yIq7wJvA7mjyUVgkMzNCzhH0mZq43VDfbA0KBubzyt4pS4IaqyLCz10AhNh4GRuK0RWME5XGt3PV8EoBqHae32m0CfscBXu+/K8o2loFDZP0CSaFwZMHKpoHC7kxUJD/oHg4QR6pYL6q5/kINqGvU9wEETDPqWA+znBbzWmCwflg+Skgqq//WnyYT9duKA5N+OisTZfJMRwOoTa3MV/KVCvxKsnlmkCyXSftBjKmFA5oiyLcj+XMuE9XCWWJvUbhRyRj6aBvKOCmQQ4V307ckRNBkM97c/HRW46QuZSPFek4OG25FBSZnxdruNcBvhNsLtMsFt/WVZAbjNS2ljRd0NZIJhu1+sDcWqAta2ihkyn2oaBwK0mX8a0TaibUTbFYlRwr0ruHclz6go3LtSmFemk90r08OdK0giK0ci//SnRxjhf6c8aFWd/oeKxl/6rdYwDAdcunn7MBdM2RXBF8J/2LYMZUagq8Ff1sUkWdwe+c0IqO7hlHmS1FBPKsIlxA1RepqVtTGLdRNB+SpiVqekLKCNG5UxhE3seVGEPbc2JoF6Vbt9JMVtH97VN22fcW2kBnLNaq3jpqd9SAzRC3CnkzQa0sSIebg9yxVzGFPpboZKo0qxpkqBO6nsW3AnFe6kKrVQEenPzqxUdJoD3EmFUgXupNqa2qvqYvIE438vJCe3tIBQC8JqQTUiKc/2EXQ6qe+WqY8f35O/8wqjM7UsTKfXGWRiOmLEaXblpzpQa6Q6m1GdLTdi/cJn7Mrtw+rE7MMadaq8DwtWeVGTOFQJrWIl26TVzRo02m9WDGRXIyoU3Xbh2M9fBE3i4O3H04gYUJFNAVY0w6h53WxXLfiTA2+GZr5MqA7/spOqJ2AnVQOoLKrKbIe5DB5jOecuAPmVUz3Cq8dCyOwaFp/fzjK4Op/r1Kibne+hKMSqGrPxlBov7Lxtdgq37CS/Yx5QsDGMWKgvEVlmvOJzz1uNAFN5Lbv7e8CKC3OxSWevuMuNd7y7V0XUVEX6CuP8UmM/QTK638jeBHoZ2yUrGMa7I66MO+fqTpCit9VlZkijFjIkZEi18xZ4kJusqkv+5wRWEYbZjAqdGnT7CEvCiZ0umtWEsPcTYfYfUzfq6yTgGO323TtyHq4ogrPSbbsqHtPgPivEZuUWrztZoVmrP0RohtAMxes8waD+qogT5ow0q3R25I1uiZ+EFf/R0a0MlQhEo9CMLGOvlY+XvG2Rv5JXqh+TJvyhqEf7Uq3nGv3dpIr4OrHA6gIGxITH81fI8O+4Gm3Z3q0QWVWovjkQQirpHR80iMQRiSMSRyReKZW0k10l7SIURyheZ5XUkMSn3fu2cwdod1aJ3fABC6qxyuwtXMMQpMJCNvJnGuwnCqeCAHzPDXUhiYRjwbf35zdHdsBGzWHve0kXEehiPACC3IKiyN3hd2fFfbFRyKSOykWMd6MjxjsrWIond2TU1QcZsXy/jVB+B1AeN06WGuQvNVXUJ1MrkKBUx87BpCNSXaeA4e0whxdBMthp2Oyc8Zuzj0znFtXFUqYGLR7TW4bTTEWBIpXUcJJCWAEtN4AhJJqp4vh8VWnUo0uWLafXbB7eHlKodQv3kNZU4i6Xip0V+HZaPUS+iHzLgnx3nwzUxW8cMPCoD1ohKfbbNwvMke/f6xvo68dzFOEcwjmEcwjn6i9jtrLCuVYXw4MRzh2SkLnKAAcT8pM+4b2m0Oxva23N30rC4zWy994fAtoj16zCCPdKA/f63Uxwz+q/Z6wHVw3y9bsI+RDyFaHg5YD5Roj5MA41A7Jbf0vYe+HL6Y9n8ccGVfH0WlsMnDxLmgFgYLLg6Skn01eD6vkAyWKTN63wlF0jYtWI6NJXyrNqsZoV6jR3TAwYk59I6NgYpo417VyWjFiCe9xiwLjCsyDJ3Nm+hoV3dH4Q9uSAnceF2nn93lxCW+fbwS/rQkmtXMVISfcb9z2jYkyPc7lm3PWysEzkjsgdc0+FnT36o4PUEd0FeyCVUaLboDlKOtr25lInsrSQjKRcJNft7qg3rg1BDUSrLKVK7Rv8YC6mVGNYkjXefsK0LcMxaO5+qlKmv4ROX5vobVZHu6uw3N7vQtVFxI0xPxjzgyC+ng6g7Ci+iTHciOLLguK7o34nAcafnf5ob+WLdCRBnRSdTbPcqvSBn0V+8s9GFR1LHIPq7LxlfjSlvV3Qyf+xh0il3OHp+PYz+faNiEuTfP9OTpxjQoi+hK5L7AoTgdjQu1AGENUEds87nCaYvjq2Zw3h3UNrPEo6gR/Xflco4c1v2rtgdfLJ7pVn5FfLHEg1ykI1usN+MtWodvqXmCojz0CeUQTP6GT3FmDCQ+QZ1fAW3APuEGYH7zAwLDNUSPE+l9lhj+ykTbvs+3YbVM+EKX2/NlkZ7Wqi6wBdB+g6qHfuRzcgr3LJH3vRyR/b/REmf8wpDGqYkdfglmlkNTmwGo0uqRAJWSq4mVpUYeX9Ykz0F8kQH+k+pfJNxHCr1MQp9Z4cDqxz1WAjOk/7rplLQ68viXB6zJ3dY8gZiak2MokDOdcIo/5RyC+1kJ8V7w4Q7yLeRbzryL2yKsxaBSbx7q5xwKQwI61F8co3M0WvfKboLQrdS73qFSW0RSvRGONqo3Fb7WMVRexdGhW/nSlEpaoqfrvfRhUf8fzu5etWVjzf7mECqF0geoy+CQTVnJ3+ONYkQxIF+cfTs6Somqt297LbXjeq5nijBFRpr74edjvjeqWoclRlW5esQ9z6t2+EwSoraB0+SMpcrXVS+5WsTC6pIUgyIlxUl1FdRjRatnykw8zbUfsIR1Fg3glQrc5uT7b06bIk0vrtMqwvRrtztgquWo/QL1Q0WTz69JVQQXxkRkDoVpktg3sBebhtEPFgXXIMZoWDrV4L4SDCwQOCgxwJgnU0dfk6cdMmT6aSoZfqrPArq4iezM6GSjRBmakLmEf5pir4AQVQyibH2oI+2wyI6cqzbaw/PMRtY/0RQjqEdAVIfO3sEh/umUJMd3ASn3Puh7g065RC7B7eZuULqy/IYwKm4daTHWFi6nbSAN0KOrR3vOGmH0wHhroegsC663qdrBiwj8nAMOhwd0GHkRS5005JLZweHjgcNa+b7dqEBzqQlGWqmm+VpWpnJ8mpgLQImJUSaHBRk5b5yYgbHjvJSlIW63zc0CoV319/zWp6GVNTRNf7VFk7nUNUWTsdVFk3A9ih5FyRG39Dm/jeS7pYuWxd/ehsXZ1mF7N15RVQ0M8cUIDiMzKPgpnHoN1PZh772BgVU9JBVo5Usy1UPB7j8WWhKhNYmJYTEeZv6GIVyujLQm8fpSXbLAV/6lQBDMw+sOoUnyXCa0ro9RNBFrTF7q1qpzcLz6JFWtwZ5uScVbr4tBRB29enLzuWzSe7RRabPgKAPbTu/LOkw7NS7H+5c/szGeswLA+rcIKlr6qt+vwsaLMXQaN8gWIBhCb8DY0sMauhAlQWBWjQS/Sv2gMxQQPyNG/VdKBBDx2t6GgtJNqunznaromCBwoehQoerX6vhzJCGKuKgFfYCaIz+iyJ8NHUJmy7L905Wk3O6p3fRl9rImBYlSEWhlQlhayStRSvNgQszhjCji2+C4/4Ngat/vlMqgF/E7sFidWCiP/Lgv9b/cHwAPdOQ7XRBYzQv4gYy+zIv43IH5F/BuRf5lNK1RcK6NlcLuXXiaqwRqwLigZ4560dA31WBeu7weaW1ZdY9UWIhxAPIV7Fo/zCzpxKRvQNYs7fHPUxoi8nmNtqZk1gPsIjiRDm1g/mevJC8pjo2URXGbaYCKJIlwadsWM2FViwWBPUJKm3I3KOVzWrcb5Iq1nJnWpBRsS9mNE7E5hFiIoqZP4ZvbPjM9xxgfhsdwEIbljBtSDJdJYUU3Dd7o5648Z258fcKABlpGea+ILr0bDT3PIF56mPH3QurvujLR9/SXn5BUNSHpLrML4YXGapQ+orxsPr9rjdyDOEIzSediM7O1DcVNbIx3nMQw3g48rwpYosSGqsXQdmZDAkLZ8RafF7ObYz4IyWtC8mz0673sSxnTHnfMEplRnj1sBd78LYzoCSUsIBHb/O12WDhSM8fGaNRe5YY6D4cIjiQ66uNFQftnKQ2YOxcq6xYYxrbNCssmvMSasE06sVhUqdZPZvhTkMQLKgQBv0I6LOyc3ZR6LR32EygFlUVBdLmbKTfpYskoEnkCQwJAi8JDJ2dS5ReTZWlbn0EMHP6VwwZSPcF7nYoRt8afj2PTyKojh9aOg2VJ/3I3YWK5efsJVRh9rvMccshOh/qabmpVFp0sOjcRcbl3TQKasFmWrGvYOv0mqTJnLJ5oOk/EI13Z4TBqedU3+SjN0IYZGJkraTyKKiXNrNTko6xE9UmJ3+GLFJB6qq6Gy55+1CHwQGl0/+2cgceQNl6iaXSZ8Jk8gyeUSyq4t2/zKXwvTSCjNNK8zV+cWwM8qlMP20wohphelfXTavmrkUZpBWmFmqZaz9XXkUZphWGJpaGCuxTh6FGaV24FZKYXrDbncwyKMw3WZqYdppprk473bOc+nB3VZqaTpppbnuXvZGw1xK004d3GkN1TzvNc9buRSmk1qYdlo7XbcumrkM7m7qHDztpI2n5qDXzmUO7qbOwWJaM4367cthLoO7mz4HpzVTv9W9HHVyKUzqHCymNdPofNwdnudSmNQ5eJbWTP3euHd5kUthUufgWVozXY9Gne5VHoXppc7Bs9TRdN7v9nMZ2r3UKZi2Uof29egyl9Wp187eTOOr83EuHbiXOgPTTmqfuWpf5uvVLDTMcCbpTxMNQL8+cdWMPOMKQ1p8d42QQEs6+f6dnBDGRzxCS6E+y4CVXjTJoEWZqbWZmX7lZVvLThXfI34JTUFuPi51Mo6s7Bp+jEjpBB0ZmzoyIufU1rCf5NWT1GVSdqgbJq4KM/L25IicET7gyNufjqq2mQisMMDNRDWN1Iz3FqhzMn1l0zDvw3P4PVE13osNqrCTclduAY/DAD0E2UJVs7oIOh30EaCPoEI+gu2DZXeu7xeg2hegxRegsBegmxeghhegcRehXBehRxegMhegHRegCBeg8xag3hagyRagtBagnxagihagdRagYB6qLjnhlGqPW52zKJcWH9y3aAnsszxW3ELYZOw5zZB1UDU/nf1KLgVDQD2zNHpmp5+YFegiUc30iEEVFzQ7gybqmYejZwrPVBMeKGGDlMczuxHPkq6bdBbQMuHTlBJYAZ7p7JTcP0pwnyLKJjyZ38Yfoy9ZkujpK/9m9TyFwF8mvFKZBe9lD5RElgYYLixOq6qTdspxhmJmmXSAMql3ZXWLG3GZnV6MGitqrKixosaKGitqrKixosaKGitqrKXVWG0Jj4WjZBAJyRkpLrA0z/Ts2yi7L4JkWGyVEZp9h6XurAV3GfRadIb9YLzs2m1YB3H53NZ0fmV1vg+qr6gy71Nl7naTTxZNOk8VanhaC425G7U5DzXmumvMlvwr8zezCFqfPGw8CgZ5oRqNFJ4xcjbbKbOZJeEeSsIoCaMkjEItCrUo1KJQi0ItCrUo1KJQuxuhVlItke+FMpxKZ9nVvkaRuml9j/RxlLV/cCZ7J31Faa080lqv1UlSlRQ16oRVV10TfjdP9K8lPv0nuspdFNIOR0hbbT7nKjALpLQ0YFdF4+lrfTlqV7oal99QV8sx0rKTVVZr4Y501NVQV8NQS1TwUMFDBQ8VPFTwUMFDBa8uaTYtNlam5JE8ntEqVgnya5bPPlawYKqB6rIF/X24oqhh7lXDHPay5NQ8+6hXLRawl5xFFCXM+kiYt64swSRMah1yZgVlz0yN7f5+eZTER//BW7EhgeTtVFBmXJshpiHJ0leBvShwKNfRKbm0btcFw9SsW1RRNDV49CPL1cn2sPNRDW8hoqzqfJd7q9n8M3+U9Rjd2qIORXQLxsQMQZZfTwm5MA3fveGbJYUsBQ1up/Ix0U2opKCTT+c3l0TQNOFV51oum6s0hdzdXerHVrlswXemwkMV1YDnzWUYNKx+ksZUXj5QYNEksrSQDB13zmeLk8ws6Hb6qOeinot6LqqsqLKiyooqK6qsqLKiyooq627jJKsVHklOyM3Hfaqv8D9R0GYVtZ1d+vqqs2zHNrnjefEuVaYihGQ91Gj3qdH2e4kn/thkzVSkBN2y8WcmWVUs2rTfG6JUe4jRpqppsGVh5pM0BS5DEmvFsP7WBZY6hLwIOjGEJ6qckhtF1OgChgOdMcVz9WONkgfp2TorSVguNXWpSbCuEB16qmg4N05WGUNn1PckeN0rmUuKpD+eYhBrliDWbmbNs4uaJ2qeqHmi5omaJ2qeqHmi5omaJ2qeqHnmpHn65E5FfUnV6IoXMwkUq76bvm8shisoVDV13PpdRklu0B5l2Pr90S9vnFZMlht08MSeA5HlUOnKFN6XXepqtlDrqrpcxeTjr8yIKFttuNK0m52IfCMFCFwxhemlFWYnUlhMYfpphdmJaBZTmEFaYXYir8UUZphWmJ0IcTGFGaV24F1IdtGF6TZTC7MTcS+mNK3U0uxEBowpTTt1cO9CMIwpTCe1MLuQFmMKkzoH70SEjClM6hy8E7kypjDpc/AuhM2YwqTOwTuRQGMKkzoH70QsjSlM6hy8E1k1ujC91Dl4JwJsTGFSp+CdSLUxhWlnb6YtRN2YwqTOwIcR8mpHP+7omJ7dCsWXdtmJW/YSRMHuMgvBbg40cs14EGkKbj4u9YieQ86iqo8q/D5V+HYreS9/WvaC6iUvgBoPUHvfSHt/E3gdzFZLKgQngtUkcAdzJMsYwCaDxuqyPdA0qNkbT+s55WXLvpVfIDAS0nX3SAW6ERAjufDc9qx/wfBK69id0ZtA37Jk5PbAfVRjKShU9g+48jslRGkh8EJ30Fuxq7jcVjujs6K739yy6I3A4Nk1vBAlDrqBkU/1V3j2YiI8C5I8mb4aVE+jCMcWVP/Df3/QVNWY6z98T0II9h48PpK3OzV1IxqxUE3F4M3HqcQ5qxYf+GE8QQ1NEqMZzA5JWtDwc43Sfdk9PNHkZPZrqFQ+tG1Lq+rS1/pZNRST5bfquNpk+NptPcInSSJEjt01GHD8zhUkwNkIcGfYTiTAfMQlMOC462WOPoM6IwXG8DMkdOnhZ9kZHUaflY/vIaurLKvjf3KKUSdSx9gFYVUjrCJ6fTdmMEhMPrA6hhkV4uC94uBRlnPpGkmdt+RgeNRDMIxgGMFwmnejnRULt4cV826g9wJxbtE4dz6bQINKC3Oxxa7hNfHsFlFh74UvRF1ShaNUfT/+BTCNIMuqKBh0VqBx0vH8x3UMc1ED5H5pr6dgF4zh2gd03wyQlxmLI8JGhI1yc+4QuzVCvRn1ZsThuerNtZKaXaGOWEPkELTmbXL/oNZcPGC1mgtRa51Rq//RZ/x824+KzDX5Pw5HzetmO37g7D16PjTdIADOUWLuZA632H9i66vF0nhFdIz6dXlwc6iNdg6b2WHqKkzqBSLnPMCxuxbtM37eMskEmlzVNrbf8Vxn4BK+MBbLXOyZ447ogLXtU355RestnNtR4JLi9jByRq5C9UZWss8ImP4gS1LOhsg6N9W2IS+NiBwiBUW/9ActjH7ZiOVsuxt6NQ9UbU90pxm3J7q97Z7oAI6lIpWe6QSwDJtOJlCVpjfx4HHy7bLqzTgXRLqaoOgLyVj34aH7/U8vGTMtF3trZmRv/ebBbH/W4Ue/2iyp02zW1HnBOvG7KLliM/ZVukOB7tkkEZm7ik8fbP3klqIPggEz1ck/q5vCSqHGi6o9TYKTbo65l/ZHSDwpmj5Z9dtTeivHyqHVp25mvrcrWF++98FqSlbT+VwSyfSV3FqNiVxvvx6oZWIuK3vEsZMzyduTIyDoQNff/nRU5gRXSNRqEURVriCkzDC+1W2XxgsT1X88bpioy+WhAbU5nRQ5BHIID4cAe+0T2WakCJbDQi8LU6iqMe3yp1uzZnzgKlxf5ALIBZALIBconaLfzU4FOkgFkAogFUAqEE0FZpq6rDAVYMUvCxGoqCkdj0CaLWtGAy6DtUUSgCQASQCSgPI5BHJgAS1kAcgCkAUgC4hmAeyMMEBmer5nuhXMBVaVKAsjqLRZXV6wll1rxg7G0XVGjoAcATkCcoTSOQp62SlCHykCUgSkCEgRoinCwgTgIwq6UWGG4NahvkmXgkD2fWSVEccijkUcizi2fFp3DkC2i0AWgSwCWQSy0UB2Ls3VCmNYVvyy6NsVNaWrbKfYsmaa9nWwtsgCkAUgC0AWUDo1u5+dBPSQBCAJQBKwVhK63iDiCHZkCxZb0AA67AHj5n8OQRSXYJU7HC38OlRdRMB7TPjZG46Shkf9oHKkEUZNzP+JmLoAZT0zqG53MIocQfVhguqqwNZV6LEAhadaBdXZOwNIhG4AXgtW5nCQ6jiiwohVS3dkVoaU8yjPIpSsqjw7yI4kMUEhIkmUZzFGI37nnCzr1d40BxU4IMAaqi7CVYSrCFcRrpZA+cyMV/utgzkXB0FhNXz2/X4nwWf/4RoAiMKKLj4RWYo870NkM4b30OcwZrJGcbNisNJClPOJa4EJtDAzeSqSLBI2+prILmDxx55GmWnCO0x5jRXTn2sTtnsNaNqqKgLqssQq9AfDLGeTxo7vTIh114eT9gcjjE1AhL57QXmYXVBuIkJHhH5wAQAcwwnaspQI99s3YimjhEuj559u88W5ZQRvrJJXEZVE8FY6NfSq9JgMoRZCrdzF0OxYazhErIVYK1DoA8FajqN6YZiT6atRbsR1wQpYX6z1/v4zYqx9YKwZFaUFDKiw2Xy9exrd/VwAFncd4RfCr7oqXaPs6AuVLkRfB46+9CWls/LjrztWzPrir3D1EIEhAkMEhgis1AJYDhBshBAMIdhhQzB3p8bvJoVlWubFKzMYI+SGPWMOtnQ3FhNeeGIVvr5A7R+8lu/CtUS8Vi68pqgKTc95g4ANAdsBSWatZmbANmriduP15jzcblyzuLXNo37bnXZyNkh2cjoAy8PYmayrcwMw72SpqeK+T/JkN+YCjce3n8m3b+LS5Mj41qnantLKOyaeWf2qfga2B0x9t6zcWS1IbiMAKnKMPW5baTcHWbatWISD6oQNTTvNpj0RVjHXZrs5xP0sSFkK0Jizc5ZOCzkLchbca1NmTsBAgD7Rfzcp/VpH3Hpn16zGIQw2cP1oGkSdk3+Y0ISIXmuCXiuQ7ghRKqLU/QnrrcwgtTlCkIogFUFqOWM0zOU3FawFrzcYHDWXgEXLFaEhOSEZHG9+hMIKrHMIMuGFJp9v95ASybaenXR+lzEt7MFG7ClPiYlGbx9fdUkEQ7GBaFkrUV+uTkpRby9gZwGYqCdjXlEMTNkhfn4TeB3MG0sqBEdrbPLfxuoOe3BoULk3nmZxisxyDsrCUg+N0nT4HAmEGwFcwvFvu+v5IoB9Oe7ttN4EOo0dJjLouF8sBYXK/kFSfm5B5dlYVebSQwS2onPBlI1w/+EQWzf4rP7te7jnR+Gx0HBrqD5kFDvzlIr+dNpZs8J2kPzkQX7qwl8EmWrGvQOg0sqcRmNk80FSfgH0Di9jBRmcdk79chhSHSbFi08M/E7ux7cTQZZVcfcx6Lmp7W6ZWXYs9i+A+BlhVYqM8t018fHZUlJMnVbMlrzM7BAGcelYkdhTqPzKzgeFy/kE4Wxt1gVd7M6o9pnB4Ql4VwaH2oC5PwOeI/Cnqr0SKCkxxGWClccFWFnVlo+CUrHeaxWaySDWH+nzwGUBpjReKmZG44V5Lu3R/yIYkvJARFlNHPlXVfdiWg1GoP6olRyiVoJ7eNDViHJAtpi9zHpAy6s8oSCAggAKAnsUBD5f3r67ub+qHJH1lZuxq9lSlgy6DaHdtU4ARa2ieb2mLaNOwIq4U51gJ0aN1gLMWU5aQKl5F9QfeRfyLuRdyLuQd23ohu1mp109pF1Iu5B2lYJ2XX86/1vlSMGq0ABhrzXhoYxs69P5r5UzrFtmsOsn4WUbs9aFI7A+Rs4IWASpQlk2mLV6g3b5dpjt+lhPqHUHd5gh/UC3Ty78o93GTA3IP5B/lCcOdEEXRR0OkRtSdsvM4hfHty5SBnxsPAJ8TNvhdV6MU6LSdoU6bGzXXTt7OPW0HCZbWHU3TNj234S9OXOgxolUrUJ87L1VrTvpK0VCtl/fDR5TgtQJqVPlPDe97J6bPjInZE7InErBnGxkPDEB9FUI4N95iu3Z+bEbx025o5B0PI6v7GFIsZu6EMzWOVOGM0YrlyqjHZcqo4upMg4H6Xc68UifMkwbWs3X9p20kAHkwQBqk0AQTHpLtU9quH8is8jnfJ/WcNBPPuDno2kcxuE+CvyOUZ+b5dUX+L/yUTTgm2qn8HZqwSKuqEihhWZE5d/s6Xgfv5Ghb+3RyhvY0S0oI5UwGaQbseKH+Hyw2onc3JIbhZzxnOgfw3VGUrnPgLXhcJQlYI31YRG6tAptax3nIylVPMmnNRw1MYQN/TAYwrZTejZEdobsDClYxY8rWgHwCcDbF0GbScpDtUnOqh6AzgGwzt3P9fXzePD5dXR1EZrvE5qPeplOK2LYCCy30Ku2m2TUw/OK6grFZ1SUFoIcNUIRpm+60X1XKL3V2v8G+LLAcHSDoBskPOMctnNDXCwnN8p7/aHyng1WBwD85D3VdeiVOnl5lMRHAgsXgX4qGa9Es10ep+SDalBri4fxKOnE3qpLJEWUTeg0RJBluKLq1L40I9NXIoGtbpQrTWNr8J7dJazZoLNWv93sSkDDhdoNGsZuOMEw6GLJmsFQiQ6/z9CC8MLUJqy4s+Zm/H7lpkESWO6gv4XT7avkdkEGhwwOHS0FUbheExkcMjhkcMjgEhichWmrzuGsWkSRgRCJI1PASzMKL4GJBj4KOnkUnpnTh8HfE30Ja9tcEgm1Hvl2Ksz4FSI+UvFJNxf6MWHfybxRjwk1xNOj+jqIeNUjiA8yAmQEyAiQESAjyH8L/M4IAbp0kBAgISiCEByv/6I7ZXZYzOPzjBGPSyc0percw60I0I/V3w7d2LPDh9kaenJNjO2tic/aengSr5FfhqWYQ7dMNUiYG3GHLAxZGG5yKbnvZWdUq9UcIddCrnXQXKtafKRmjhCGGVf4mAc8iaopz4iiGmRKyYzKQE80KzRKUIiwXMqSKLA+WgLK8kG9hWmh4q1hVyLUGK4nSgWzkyXcQ14k4xFahsgSmFOJOvxjh7mjI5vgnQRWzXdUJBs4ti+zklg25H/l36nHjRx77ifxeWrO6zCZeGti92H4PKea4xpdQ+K4LGi+uFNmNbG6tyaRVmdhrAkWv6q+1IHe5nJtQuy2O+0ybkLceUIQqDieaYUea9RR0jzW/Z2lchyhjIIyCrqs0WW9X4noXmQS0R0tfNcce5cRPXB3IyCxOvLkhA8LGB8rkndsb5FjgbLW/jhXxpAUi5ucknt3S91qQ50efBKTPVaJ9CnUcCpLsJjP4JeKAriLj+/9alCsvWE07aHB8/eYB1uU+crDrblqFG8z8GBp+sV/K1w3BElhX6iKzCKr+bhfSIaxTlrQijvj2Tl66IxHZzw645FgphBMXdQkDlfD9AcZ5oae+v7uPPVdzEeJFBQpaA08+QDbWTr4d9yLymoAtXspFsE/xg6sXcD7QE3ZMc9QOJ3ve7R8yeR3k5oA7+dEsI8pJqp9+96Dk33Ndampy/o3Fa8lY2P//KAzJ/W7m7v7qw+rA6QfFFVLbJeL4toFmM7dq/LJYnfVpsG+qljmP4H/zs7Hf/fQV94iU40KT2SmviirK7rOyTBQB/jRMfQY3ThjFWCswvPzxBiDYkQLu4r5CRebBW14Xu8XkbwKwYkVuSHpx3yeUszFlGpshuJH1Du/8f7CpzjAZKaRBQwTstTos6SauvxKNlIgLhs5WpzHqewvQiZSrHMlOfKWnj6cWhvWmXXdPexHiSEE+apon3RjL/axXx3Uvqxgq09392QuJ548f92ohhqF8RJV0KLsTHN4SCfqTrgJpEzBC4OdHXSC2+1RGTro4ITIoLpeN/mgyPfCF5i9FPLjaUSEgcimWnbjH4ej5nUzKi7RmluaFYs5CAHnMYztK+YcT4XOFRFA3AoxlYpxEY+bGQrsydjlOKKZLShh6XwBsQNFvLq7P794d3P389Ul44Djdx/vrk7Ir+c394UKWaGmsntsXRrKGYAn5J0ElJqFBrBW4dN0gK1729CTa00UFKKbS7615e2lBBhdElk7/rtx0vp346jewQHjmCgW5GT7jGHvZzriMjEyqdQn6XQHeKYlRhggDUyPMNgZDxzhgZcYYIA0Mp8Yd3Zq+sHFnjNv7N2rIqrqE1juWpBkOquxqzxYVXYcpwKPkGbM/0rsa2VJsRZqn09UfD6I1mEVtZzpm7TJxT7ahE0bB9EmrKKBNklJjjeuPuWG6o95ZZFwV8IJiuH4SJbRLVput+hwZ27RNrpFkc9iwHxJuOXkXGRM+eOSVj3I11ORCBcjjzl9FJ4pWQgzSgQyk4CsGFbkqMTuYdG/zInFg4OvPtzb/sc5LCj8e+5vvLS+LcH+5FtB1+vRcN6aZG65T+NfLoMt50TZp7RcDZyPl5Zp7l3TICMqOyOqgmcROVACB3oTeB1MKEsqBMfqapx+oMaLqj2RD9Zk3ljdYY8PDSr3xtMsTpFZ9JMsLPXQQE2nQJFMoRHAf5wgtD3bYYPkgBODjkMM3E5j52oauLmaGktBobJ/nFTcmVqm86G68cxsLQbW3e/BsJtTrAPwGGahUayDv4tSIjbjT+XlRiHz7ib8EiYAYUknbJalDLxOZqbGkwcDFIZ1eqbXJcLv2ze3jt+/M28Brzhxqlvf00w/QJuRqy8sMJFqTrXvg/gRYXLxMDkBItuDD30E6CM4SAwYfXJNVhDY6g1qgwIRyFVj709rFBWG7Q3vshJJ/pi48+e63R31xvuJANsXEtVNUaR63QGoXcvid/MY9IsBU7lreN4Na25sp85OpogW+Svv3jXenhMF/xH5l2WPTqs77GbbpLOLDAs7P2UCqt3DLTq7Vdy94z5Jb39jP5TNeGyWYlc7TWsGb+jiI10IqwCDdt/62ni13jITtCfrToBmqzHQeDKnVFOoAePCfb5BF0sZAIqyCkZqsIRtnpHj0elXQPqbH5OxWZ3PShHAA6Z3XiortMk/m/CR7L0axdYafGEIdQhn9fL6Bexs3+dyBAr3p+O1SZjn8gLeL0X8zJ6hGpd3k9tPH99f3f989fnO+8MVjPYuBo3fTarx8bWMYnmelm35vn2gXwIrbkN/kpafNZkF10aUz+k9nloFHSOeJhRk+Rfb6AFzehrXa/w1GCtrPUmRIuhq7k33/9TpJm32H//taS3FXzOxlhcr0MFkT5pIylw9JvCwo520oL1ue58CI5c3k/4Pp2wN/9VQJdh30Tfb/cOqpeeCqdN760He2XRPnSbJ9N98aJfljLfw8NEue9rPqm78ZZO+xgqdV2dbq8bY/7KtSP8STr7+9tP/Kc8L+q/myei3nxTrn/9bLMSp/GR9iFmv1vvtFqtZe9OOGnhMWi+dSfrTjD5Loo8Pejvrmzjc6LVqlPi1tVkzGNYLxyJH2aalWXOxtYOx33jLwMjIdwtVSdzYNp6aW0QSJoiXk5ZDp4Dh2d81fD9bSkCktNWP7UHuxst5m6jR8wRWtJqeDx3vh9Zi9XfP83fL+6HT9F5Zkb5G2/N3a2bV+DenDkxcCHed+Ld4H9z3Ptj7lnbX+2G1hasxmHnL65TFZ76vKte/GlPA0rqNseN497UjiDZMrkQ3tH/OZjP59lcbPT+7CLvPJeqgLB9+GlmtdOQSCNZUFTQo5vf/DyVqOEu06gUA apiserver.json.gz: - H4sIAAAAAAAC/+2d62/bthbAv+evINQNSAa39StpXKD3Is2jK5C2WetuH9rCoCXa5iJLKkk58bLcv/0eUpJNSVTitE7sJBywNuJLFHkePx2eqBcbCDm9Hg2iWHDnJfryrZaUMPI9poxoZTgIQoEFDQNZeAFFUOhTLlQTuLxUzYhHBe77BEoH2OdEFQ4DIt56UBTEvp+UMByNumHoCxpBeV0VjqhH9sNAsNDnuf4019enwak2sQgHRLX/ouaUzAzKXRiJBHJ6TndE0KfjD2iTEzahLkE+mRAfhf2/iSvohGwhHHgoFCPC0JgIRl2OPMojH08JlAdIjCiUYD7qh5h5CDOCBiFDNIA/x2pRsI+imEUhJxw6+NNnTi2biIcF5mHMXKI9RVJDuMtoJPuvfppDRr2TcL65qnAEl83a/PpMXre1gvNs99LrqbxOLy9nY6sNnI3jjENProUzxuzUC8+C+SQ47CbUNOZtBRVKmpz3oYA1mTcV00iVC3IuHFV4CX8mMsHIAIRXTt5p1bmTlIVnRinxfRzxnLzmKrxSTUHg9OGSntgdkS4dkzAWhR3PRg7Za+yeDlkYB6Xx523+xH5MqqvzU1DlT5qdjtvecWr5Yjbs481m60UNFrZTQ+16DdWf7Xa2iu2eeO02bmFHK/2Wu29OlJ0TFoIYjkjMnXwr4tIxVkvUKlTk5P338AyNcTBFEYERA4HCAZKGh3DB0WYfBB0usaek/oxRAQpAA9Sqg4hPORrhCUF9QgKo52eEgQrw2HUJ5wNY8anqNcBcIBKE8XD03/wcE42Qs0hvHgdU5JsMcTwkOXVIhBefZxvTqNcLKzimQVZZrOKj8My0mVKUR1JaQ987xn2St32mVu9AbYjafsFiojW6zM9/rtD5CqWN+Z2hYCvZBPsmec3bW20doogGw26iho2qurKU5tcTWgd4rORpIlcOiRApla4Vm02yVc9VXNYWG5zhYLjA4M384JWaAFJwAMpwEsLK8bIoOHIVVeW71NaBOwrAjIM9KbXryhkZFh7MtBhQaWEdx1RxBB7uE/1HDb9d/7XQBmygsa8qv7KrWqp3YPyu37oBmAA5hpx7eVFF8mDO++d7hspw1nHBNU+dQ7tYyE5BRA2aOqC+vy8tpdp/aQNbDTCBjV34Y7cjbWBjt2QDpfGoUEB5F328ZLgmWNNGp1UaSNf3Sh1VrARjxuOgvFUCsyERC+wCOY/UnHBEJToQ1kvN6Es8wdTHfepTMW3VvQuo67/66mDf/+rUkOvHHPQeCn5Jf/zqXJZ3am4rBXi1HtyCEl5ultmQI+wKtUbNUhOfDEngHc3GKw8C0qlI0dlbVCpmZpEbVjCDhz1tHdAmrMQW+g/qdJ7V60XhFzMovSjuJ/CUd/2OpljCwf75sAW44FWUodHVb7c4A9ViMfUL5TydVzfUvMzW3Uj5VKf3qTHFk+G8n/b8OQ7CPsV8PyOVgg/qY2b0c47E12MSDMVIWdVSHanqtnQyid0RIowBQPdjDxQRAVz7ZCCQH4ansLsIA7HEDMxIp1P/FemqhoYxBisqCOFF7qDKvBQeTJa+YdijyQtL/QZ+PG8NEwUrCy/sxTDY493iq5neBPbUXOPGjCUTM9WCJ6yqoUFFDaPDESiAVzWX1HZKvCnWQKe/qKeko+gwlfYKXGW+lQTzaxVYWnluuLmqOEtv3ViEkEoMUPKSGXzioZnzI9lbykUsh98u15mVAXbLAyBWPmHghwULxEhEsBE4Esv+AZwEiBwxPBE4W5dUaWfqnnfzhQJedIyTBHcTRcQ7Tlx3uf7G3g8QDP2GNpfiBNHTRK3hv627dIjK3iTmZum+sbCXcupHCb6VBEHWfRrRgTBXpk71UE72dWIbl+FTi6oeMmF4j1Jq3MvcLA08OqFejHOOzOySVdApP7NzfE55eV792D1NZM9gY7LohVykIvSlbxuGXtUmbWaXVBjN/AhTfL7Im1SVf1vwlTcRSfkOanwGWRkOX2NeeuHTHIGxW+IJjFXautzove4hPaquvxvF0pnWmg13WvoWTA1O8cVQ/TE8ywVgZ/PppsqcD3HNVPxgFkyEEZx8fQawo51yUI5B6w1tG228bW3jbdjTgm7Hbz91a28OuzbStrxI27aNtNlI272MtLVspO1nI23Svj6uUNtH6VHK8TYbZXtEUTYzdlSyBoAB4gTstoc8cBsjgmb6xZF8u+xPkQvW/bZjaTs2lmZjabcXS7vayihFlnd53nz27DktGxQ3c4hPtnf2dtqvnRvR2Xz41nXDHzX39+v7Pzp8+7rhW80X7YPOjw6/fd3wh/XmUbt9A9K5NqLZMkY0SzK99IAmj8fS8m1K07eV/AW4kdjflyUC6Sl9fcmwINvjhfDjTiObFxfKhKPLy/UPbCqE+XT8Fj1FH1N3ZQOaKwlozuURpDzi9zZaeW+ewxyKvP/suUjIS8NQmTrJiIhZQDx0RkV6EgzNt8/Ptyo4dFkY+sJiqMXQdTrSba3qSDcjoIx6fo6CZNdX//vqAEJWMxF6jpZ71/Vgr2zu942/1Bmzpa9V09eKjlHrtwBg6/8oD5XBErbiMo9Oxvg6HcCqdDeon/yqirSZMrVOUdm/isowGtIJCWYW7Jbha9fCl4UvC1/yHDWBld73GCsFNWCPFydJJ71UtV+OKBchOMbxrNfaHYTdYxg6SJfb4tCKceghBKK4zYez+XA2H64QHFS/Z6pFB08+AIeefIb/97r7v/97cHh82D202XHLy47r2Ow4mx1ns+MeZ3acsraPKz3uL+VgbH6czY8rYciC7LEu2XKFkWyozIbKbLqcTZe7z+lyV/OIzZfzr2IamzBnE+ZswtwDTJi7OZeuNH2u0bBcarnUHuEuL38uw6I7TqBbLxq7V4fGOpHZFDqbQmdT6B5NCp20miqFrgLT7jifrtG0NGZpzNLY0hPq1ufs7D7DkU2psyl1NqXOptRdnVK3hoS4dE5rPWBOM1ZZUFtzUNtZYdhsU4aiNs9CdgpgBu4bex5PglQXf4fq68EZu0kEowHMLnBVcOyX7KKSzb5sj79tbakIWda2hqQjvuOQVnbzy0t0cSHvfy/gDbYE/SH3BO154DNgmyy7rZjdwrs9oLuNkNb6PsJ9D2UtHVTaFlQsqFhQqQAVj0RiZBllXRjlQG6HBZRVB5dGcvXuOaKs80NYSClAyvaPQIpBHRZhFEO3n0UU08HWrR55XfkbhJZPfo5PCmewdwgo5eOrTfnvKtWQgVxSfimcgPUS73OLRFNDPtmyWHMjrDmGrQvcqQUbe2pmT83sqdnj5Tz7TwbYWNRasV57VagXsVB+9EQmecPsA9Ebk3HIpr3+VBBuwLeb0duq8Gz9qeydWmZLYismMSXmD4DG1vw5bKCpACD2Y/EWQCyAJF/WEmQzoxA3imcRpKq0nR+IH1kMqcKQ/ZPPKOZSkC2J2MMue9j1mBjEfjPdMohlEOVnwt4wZGEs5AbaoMet0cab2SJb3HhsuHFbx1Dr/RwP+ShqI30Ch7sjMsZ/EsaTWSa5vmCmp8mtPMxO1UBgmYdzKXVO4z5hAZGRq9lYgowjH8tfhZ9pHThJLjTZnkuDxhc5esu+YOqRAY5900dU9VoTDDqjBDE041IhU7NPRGsUp9WG6msARWvqfI8Jm6bx9zLtScMt7XBeQqF0SIrfZJ7ti3b7jcLjaAuGfT/7wnjhMbSl1NdhATbN1krzVmB2wR96ZM/0NeTZQma/Wq2PNYYdoYY+2Sobuly3xOp2vcSKblZ8w2bmyrfM29C8bhtSZ6RvFwi7Wmv+RzYTJ19bmrIsMzdONzl5Jq0i5qSbDKSj6V3vfL1q5/PerEqDrtvzDHGWvemm7CQj0c2Tlh6GeGxk/ugysbl0PP/++fyj7OHZ00aGQ9n31hPjP+8WUQAhNu+cLkkvQ1mdSZztuew4jbp20dIvGuP5z9vazw39olXXa+bQ5jS1nxteYgq/Zc8g4VsTnGvvog+8ow+s36XZ1i/mH+h3Xnj6fLO55Jbvn1C9wjifu/upg8zc897JW5RIZVIRq3d2p94h7i7GDdLp7Hg7g4Hr7ew2XvT7eDBoe/1GP2k8mbni+sbl/wFvRsHeLqIAAA== + H4sIAAAAAAAC/+2dbVPbuBaAv/MrNG53BnbSNm9Q0pneO5RCtzO0ZUu6+6HtZBRbSbQ4livJgSyb+9vvkWwn8ktI2AJJQDuzLdabZZ2jcx4fHdyrLYScTocGYSSF8wp9/V6JSzj5EVFOjDIcBExiSVmgCq+gCAp9KqRuApcT3Yx4VOKuT6C0h31BdGE/IPK9B0VB5PtxCcfhoM2YL2kI5VVdOKAeOWSB5MwXmf4009enwbkxsRAHRLf/qucUzwzKXRiJBGp6TntA0NnJJ7QtCB9RlyCfjIiPWPcv4ko6IjsIBx5ickA4GhLJqSuQR0Xo4zGB8gDJAYUSLAZdhrmHMCeoxziiAfw51IuCfRRGPGSCCOjgj587lXQiHpZYsIi7xHiKuIYIl9NQ9V/9NPuceqdsJlxdOIDLemV2faGum0bBZSq95HqsrpPLyXRsLcDpOM6QeWotnCHm5x67CGaTECBNqKnN2koqtTY5H5mENZk1leNQl0tyKR1dOIE/Y53gpAfKqybvNKrCicvYRamW+D4ORUZfMxVeoSancOZwcU/sDkibDgmLZE7i6ciMv8HueZ+zKCiMP2vzB/YjMr86OwVd/qTearnNPaeSLeb9Lt6uN15WYGFbFdSsVlD1+X5rJ9/uidds4gZ2jNLvmftmVNk55QzUcEAi4WRbEZcOsV6iRq4io++/sQs0xMEYhQRGDCRiPaQMDxFSoO0uKDpcYk9r/QWnEjYADVCjCio+FmiARwR1CQmgXlwQDltARK5LhOjBio91rx4WEpGARf3Bf7NzjHeEmkVy8yigMtukj6M+yWyHWHnxZSqYWrWaW8EhDdLKfJUYsIsyYSpVHihtZb53grska/vKWn2AbUO0+CWPiNFokp3/bENnK/RuzEqGgq3kI+yrBakNs+vgkz7RappbCOzTfnAg2om5V3PJ6x3tD+QZ9ciiqWZNurHUYUiDfjve6bV5dcWNkJ0ptA7wUKvsSAkHSYa01ajkm41SwWYqJpXlBuc46C8xeD07+NzNBor2FvbbKQPhiKK2Ocqw6MoPiTkFjxeApwCTVWjXVjMqsUXgCWSPKiPuOGUVx+BEz+jfevjd6i+5NmBmS/vq8mu76qX6APZ1seh6YGXUGGruxUWV8YM5H18clFSyaccl1zzxP818IT8HFS0xBj3q+4fKGGv5KzPbqIGVre3DH/stZWZr+wUzq+zTnD2u7mKOFw9XB4NdazUKA5kmZe7e0jgGY0bDoCgqiXmfyCWkQC5DPSccUkUnhHcSS/0KjzD1cZf6VI4bVe8K6rqvv4F18L85FeT6kQDTAgVPkx+/OZOipGbmWILj7MAtKBHFZqmZOsau1GtULzSJrdXxdLziIKCdGkadg2W1Ymp5RckKpnxyYKwD2oaV2EH/Qa3W82o1r/xyyr1XeXkCsnmLJZqQjwD754MIcM5xaUNjbr/9/Ax0i+W2H1PzdF7fcOeltu5Gm093+pgYUzzqz/oZz59BLXBBWBymMJRzc13MS12powj5hAR9OdBWtVBH5nW7dfiJ3AEinAOjdyMPNiICfvdJTyKfsXOQLsIARREHM9JqVX9B5lZD/QiDFZWEiDzaUG1ecg+mSt9x7NH4nah6A1Ro3gMqKHGXm0Q34jyec1ktOMl5NTSYU5PHknKrWlYDff6kntabvCvV+1rieYZd67ZYuLWV/RclN9cVF8mta8uwU4EOCv4zJV/cL3/JCFVvpTGRGn63WFe+TUBYHtC49hY9n+VsEychwaUoEtv8T+A+QBlJyROBG3bJvH2bOO79bKGEt6zSSYIjCkPincROvVh/Y78IcIZ+Rdu34h7Rs3jDw3879+kqtSWKDdGte82cLNXUj2OwKyiCqjsb0J4sr0zc7ZGa7JvYat6Gt81vdcZlyUuc3sad1AHTwKMj6kU44+LKnbWOeGVndokvqSjOqxu557HuldiYNHSiFimPg8l7SEmv+SZtapd0DK/8Ecb4cpl3rHmeb8n37Vgl1Qtw6TOoStZ/g0XhVdDwA6XdYkdQWmWsy43e+B7So5r7dytfOt215YY7KX0PpgYnYFNS/ZldZKK/0/m0k82cja9Nt/jbaSQTRnCy9SnaDvaKEUEOrbcMMdpg39oG+7BnRPxO3p+1K++O2jbMd3thvl0b5rNhPhvmKw/zNWyY72fDfMqEP64432fltIrBPhvie0QhvnKymYszwB5IELDbHvLAbQwImu4vgdQLbHeMXLDudx3I27OBPBvIW0Ug73r7o7e4usuL+vPnL2jR1Lipq3yyu3ew13zj3IjbZsM3Fg1/XD88rB7+2+Gbi4Zv1F8237b+7fC7i4Y/qtaPm80bMNDCcGqjNJxa0Olbj6aKaKhs4rYyijvxXwAisWV+VWCTjt6vrziWZHe4FJjca1j16kobdzSZrH9UVcPN2cl79Ax9ThyZjaauJJo600fQ8lBsbKh0Y56jPA66+VS6TLzNAFSVNMqJjHhAPHRBZXJADc13Ly935hDqbQHqSwuoFlA346S5saqT5pSNUh76OT5SXV//75sDcDmfltALdLt3XQ8qS+e+aWSmj74tl62ay1Z0ulu9AzRb/0d5qHQWU5dQiX8qLthqAXAl0qB+/Os7ymaqXEDNa/9oXsOoT0ckmFqwO8ayfYtlFsssll2LZQnGdH5EWG/dEiASPut4UZwp00k2/qsBFZKB2xxOe67d0doGo9LbZLktLK0Ylh5CAEvYJD6bxGeT+HJBRf2buUZU8fQTUOrpF/j/oH342z9vj06O2kc2pe/2UvpaNqXPpvTZlD6b0ndHKX3aoD+unL4/tQ+zSX02qa9AOkvizbqk+OVGsrE6G6uzOX42x+9h5vhdTyo2yc+/jnZslp/N8rNZfg8wy+/mxLrSnL9azRKrJVZ7unwfSX8pMN1z1t96cdpGnWWbrGbz/mzen837ezR5f8pq6ry/OQB3z0mAtbrlNMtpltPuOQtwfU7jNhmdbB6gzQO0eYA2D/D6PMA15Mdbp7jGo6S4UjtuMW7NMW5vheG2bRXC2r5g/BywDRw79jwRB7eu/mL6C84p2Sk4owHMLnB1UO1pejGX2r4+7XTU8J10+33f2dFxtrRnBSmHfc+BsfTmkwm6ulL33wjIAwGh35WE0IEHvgVW1TLeihmP3e8B4F0Extb3ETY9IHbrQNO0QGOBxgLNjYHGI6EcWJZZT5Z5q4RjQWbVwaqBWr0NR5l1fggLMzmY2V0TmCnptnFHbIt+b9GSzE+QTO40+B5RpnhUtq3+tawKKmGchHRyp22d2DPdG/tUkE92LADdCIBOYFkDd2wRyJ7X2fM6e173eInQ/psNNrq1IUzYXBUShpypr8eotHSYfSA7QzJkfNzpjiURJZh3M8pbFbitP6990MtsGW3FjKbV/AFw2po/hw1W5dDEfq3foolFk+vRRAekUj5xw2gag5qXSvTTESiLK/Nw5fD0C4qEUmtLLPZgzR6sPSZWsZ+wt6xiWeV6VumzTp9xFkklQBs2uTMOeTddZAsijw1E7uqIa72f4yEfc20lT+AId0CG+A/CRTzLODMZzPQ4vpWH+bkeCCxzf6alznnUJTwgKvY1HUuSYehj9ev/010HTlJIQ7dn2mDgRYbr0i+9eqSHI7/sY7NmbRkmOoOYMAzjMtUpR33iGp3FKGfUp9/UNkDPqGX6Uwh5s+r8iAgfJ6H8IhAqC64MclZVobRP8h+xngrIuP1W7rmMlcO+n371PavR5pqaC7IEvqaLZrgtsL/gGD1yUPb56NmKJs7THGsIoqElfdJVLumyaIn17TqxOd2OwqKjn0w9+k65EOqLhJD4JFNYoPN6pcXv6TycbG1hwqqsvHEi4viJjIpIkHY8kEmo9y336jy5Z53aHOO8UOIp6dyqyOcw3Cwh6mFowlbqgSaxlaXD2ZfhZ5+rZxfPaikApV+ij839rFtIAX34rHOyJNM4nEkhzu5MTZxa1bhomBfGi6iza/xcMy8aVbNmhmlO3fi55sU273v6DAq3DR1ZeBdz4D1zYPMu9aZ5MfunC5yXnjnfdC6Z5fub6ZcW50v7MHGJqUM+OH2PYq2MKyL9/u5UW8Tdx7hGWq09b6/Xc729/drLbhf3ek2vW+vGjUdT51vdmvwfuahlJt2kAAA= cluster-total.json.gz: - H4sIAAAAAAAC/+1d62/bthb/nr9C0HqHFDeeLT/yKFAMeTYF1q63STfstoVBS7StRZY0ksqjQe7ffknqRUmkbCdO7MTchy4+lI/I8/yRPKRvNwzD7PddP4wINt8YX79vxRQE/4lcBAUa8P2AAOIGPiPeUhIlei4m7BH+yUiovGUQuR5579NGayunOoAAHETIhrTB/ISCCSRjGGFTeAb6YOCxdoIiKNDHriOhunbgHwZegBhDNBqAzdaW0bYs+k+vt2VYr0XWPpjwF+/nYzF+NvY9iEihC+Qm5M85AI8HAUCOmbTd8f9/p//ecaFAxyWl3pojH5L3DqX4kefFFATC8XkQeMQNKb3FiWw4h4FPUOAxgQ6Bh+Pvu4Xveq5/IWghBD7kz8ciTwVu2oHngRBDkVOhwam0jJDrfApyZcZCLinsin5udwXCdTqA5PMN+5xKJ+PNx9DOPua9/p7REAwhIMJQBep7AhFXj7z5c3BVEDBvwuPg6twlFcsxSUI0DyOEoE+MA+A7V65DxmbxmTP3B39uvC00JJaAgqvYBpIxZoIHngswN0AuyVwEA8Apha4we/oN+iPCxdwq0CGuaGiqt5hD1/MKkmaEdwg4Lh2oqCm1tvdK2rbaU7RtybXdyT56cAR9p/gmKqaRv4/P5a4NLkflwTPTjfUljwTHk5DcyJv+C1FQbZmAa8k7Jq4voSJ3NCZn0njDzExCpc/+yU2qaJWsKUBsCNloSk1HENtVfoSGJ0/SsUvgRTA1q4oeaKyQWBGjXiV9s0TqRckjqSzOqKcWPN5ko/kUuD75EHBxcIIpeLYNZRYdQmq1PgGjakQKGTdmohF7f69Ir3afCs2BCPL4PvSCXICKCIIhciH+/RIiavGwNESs6i+uDBwTYF9UeoMJDEPo/EZlWmkjAI0gwUI+FDMiyxbXIR8G03uf9s3exNFk06WhDm7SNEYA5Yr6NH1cBeiCpmAbupewP7ghEPe5RdzaXkR7gN5+M18lf34zt1hW4wN7+79v5i///mbefX1FRQkRtZY3r2gSD7yIBdPvr18bgxtjM3v+tZgcWfAI0IQLlMbDCezHkiw+kvI9ATbhOdcqNMeuf5Lxub3NXnZ3V+SE4JAHcXO/SGcS5rrJiHfJX7kayZiOahx4Tkm9rNsnNEyWbYLRz8bukFQbSrnhM9WFEQyNAyZ043OsAkfIB1kWvxVjAkA8wZajQuz67bID99Os4vqOe+k6EfX1ijOnz3DwkHfgGly7pSg+iOyL2O5KoWeSuGxFkSkQKoeqOLYpg85XUU9pPMto3ytDuAHXsMYdcnM7CEtm5oEB9CodZA3B6ABwmFM0vDi6Vx6Pw3uraGB5BM9NbGv1u7ghinn9YEiBUINDuhqHaByiccjicAiVkY8nLtFAZHlA5DzRAdFYRGORVcciNnXziV8jbAKv+TjOqTcUB8LVmzVNFUnK6A/2NeOnfQWzrHlehgf1DA/mZnhYz/BwboZH9QyP5mZ4XM/weG6GJ/UMT+Zm+K6e4bu5GZ7WMzydnWGWYxQc8/aanDIz3M5B1JAm7XT1cq/1L3N+uD11jddSLPL2KqivMNV4BNBnAGz8YNB6BmiGbRRwWRU6xULaKQQO/0axJc6SgrBoMCtJw4mRMs9MFZnMBexuvLqUxKd08jBps4leIpNKELfTWWAu3dSuYA6I/qL/NT58aBwdGaenbyaTN7iUUxxouxPAV+3bxZxClVfNzElDvsthHiFqpoYTXPlm5bEviEnVLDaEgOIb5MuHrEBaBXhCJ10OLL0t8l0+XvpdRKZ7cib0ymZBdTFoPVQhz62zqMOPJgPqYlJ1MAjzEGXIAPFa6eNgmfrYp7NaOqXWziGHdctWxto7x9Ey9ZHO3FOXMD6BeE68lqo4XqwqwnupQnCI9dbGySpoo+wYxhEK2GLmeqrk3Yo6yHpr5XSZWvkoX0ZYBdkXV5XvJXqnuTvYAbuDTnvb2QHbQ2vY6ra73db2dmd7197eAcPmRUQl6UMCcSPZhXD9USNbPWmEgYN/DdDovfPW+hnBIVPO204L/3wJUP7Y21f9vg2FXaKyrhXLNQ9VdnmqWd01eEGFInwjtbQzgwmQbJfKNm3atZs2996n2Xr++15LkOvB4uQKLkfaXBOxHj6NWNfOXI+eJgyk5hrGsGgNBHv8xPF1fSR7shyTdWIkvwYCfrck010fCZ/eo0jmwcUwZwQQcQM0ReGxAFRFCOmBmMIOn3geptCwuOMwqq3Sbcl5mJqZoKSaU1XRWVfVWVPZOeuRLEmFZ22Vp1Kesu1nacWnTLa5fC2BVpQNl/OOxK6r3aitBFVXg9ZWhE6pCq2vDFVXh6orRKdUiaorRadUi06pGK2vGq2rHFVWj1Z1Ka0iVRcVKAoLlMUFM1SV1lWWTqkuraswrakyrS9nUFebTq04rS1OkBcoqKtPp1WgKhc5yoHuOcweZyxInaEodc7C1LpFD0XuFfNvIQdPX9eS5+jaPF3M1ek25JQTNMrK1brqVUUF60xVrNXAIq9mVVS01la11le2qqtbaypc1VWu8kpXsdq1OtRq1avUBdWlpbXlpXUlpjVlpqpSU2W5aWVcz3QIGzKV1e4GaAwoI84CAnc1CNQgUIPAlwkCn36tW6PAh6BAZUGaBoIaCGogqFo4fforciq1pIu4ImepdxNZimPieyt2OVFevXvqYpo2bp7icqLyOf/VvxZgur7bcn0L47nvvQAl4mNcC1Am8sBSIi7yUoCHne9XnvRqP/ykFwV9PrQL551f/hn/4lk0fdXQizzhnyyJSnPsczzRT4pn9erP85fGkONbfWL/cbv4oNuDXiBOaCtwoWVpnKBxgsYJ+iqgpQOFdNVMIwWNFFbobp9lldV1FGV1VnuhdXWy3dHnvala3M2pKazrTNlTtToL3VSVNDz2nqqsQYKdHnVHdXGborLDeFVstZg90SrO0hujUjz22PuiK3ZWRW+MzrYxOtutEC9yP5SUL3OaZTdUMs4qBFzAPmf4/Pc5w/UpeFtjcNZtTQFnXQ3ONDjT4GxlwdkyzrtqdDYfOpt6UZQGaBqgvXiAtgKFaGXnW0z52ZMvWao2GXt6yXIxqLg3DRVva1SsUbFGxc9myfIJb9LQ4PhhS5fyGyI1QtYIWS9harAmA2t7U8DajgZrGqxpsPZ8ljA1WntOS5kasGnApgGbBmyLA2y7GrA9d8BW7/ExkDjwgPT2/2Lm+XMMiOFi4/zwk/EZpljh12qciOIfAhgTEuI3zSawbei4wP/FDiZNyFJ3iFwMcXNAY0YzwR6NGHI0vADjBkq5Y0yRA24A32k4Uei5Nk1+DfqgH1x50BnBCWWHm+aMiVmj1WeJVjlY9almGE6lnSD9czvsJxZ4BkdYCkyVMNRoGnKOv0dkfm4xqI2v5X16TJu+94VA2kJgwUYQEUYGnmdgdk0whiPu8BrfLhnfxqGO/x7Kc8e5Kz8UjXefEO/uabyr8W6Od2+MSUAjQ4CMs78+GihPTdNB78gl42jAIW+YeUaTYw6Kcmj2gahJ4S0VRtNqtTs/8b/p0yzBNbqtXau929nuaWS7fsj2+Jr+7/DT2Y2fYKEFwdt7A2YNcRcLcVksUcBcIcRokKtB7pqD3BWoUz1GiGHp+5SpbiQ9N5MfsGStnVbMiz2WxxkT22M4AX9AhOP+x+uuNITdxL1wALqIv0ezcG7FZv47mmb2NgInoUcF4Y8yr6TgCBPB9gv33Xj8F1Gr1zSAiASSG2EYuW8HEcewnVa5Jda8abXE+CWg3sJshcBrbp+9STGmXCY9Yg2yKc8sMw+Olkslu65Pk50D9z3ZVTdyPzAnEY27kseTOGXmiVJ8eRAySjnelEM39GIkJofKqXQ6rcpds5mAWJsq5KnfJrvJW64KlTJmflXdwKyx8l20aUMWHoS0bP4TQXSTCGGrN9kqsBOcrl2gjuB16devTHzhhl+QRwGXLVF0kkTF2nDqhNxp8H/SLpjF1jKE4DT5w1kWjqGR2BZheB7z4n0ql5qvkxe3H9mLZeKf14drvKqrtvTuPJbeXV8Tn2J8DhwCqmOFBaatUjOUJAqFFaXWIhiu3F6kygulBp4r0ZqmxFSQwuvvFRMEUa6oOyZz0nmky9/Uj2cCm1F4+3cwoLNbGziXLjVvOrvdMhKur+/rQ7GXtB7XS+IBTXeRjRQr38Woz+WSS/DeMJ6rmn5w1cjSoskzAaOZha+FLp2qofzLiUj6qb+KEdDsCVhYTA8FlGJaeSoQ04JpiR86LbFFuHqsLfxtObGNf0/HwNYXqpFZ/RaR8bbIWHxLuyt+EK4a3HHE/qZ9KYjvR8DXhswv54cJRE/nDh+zn7Y3msahYNJmxFebzeFwu9MDrXZvYA8h2BnYHcfpDnut3b29FujAvfjhy2xa0Nq4+z91+P/u0acAAA== + H4sIAAAAAAAC/+1d62/bthb/nr9C0HqHFDeeLT/yKFAMeTYF1q63STfstoVBS7StRZY0ksqjQe7ffknqRUmkbCdO7MTchy4+lI/I8/yRPKRvNwzD7PddP4wINt8YX79vxRQE/4lcBAUa8P2AAOIGPiPeUhIlei4m7BH+yUiovGUQuR5579NGayunOoAAHETIhrTB/ISCCSRjGGFTeAb6YOCxdoIiKNDHriOhunbgHwZegBhDNBqAzdaW0bYs+k+vt2VYr0XWPpjwF+/nYzF+NvY9iEihC+Qm5M85AI8HAUCOmbTd8f9/p//ecaFAxyWl3pojH5L3DqX4kefFFATC8XkQeMQNKb3FiWw4h4FPUOAxgQ6Bh+Pvu4Xveq5/IWghBD7kz8ciTwVu2oHngRBDkVOhwam0jJDrfApyZcZCLinsin5udwXCdTqA5PMN+5xKJ+PNx9DOPua9/p7REAwhIMJQBep7AhFXj7z5c3BVEDBvwuPg6twlFcsxSUI0DyOEoE+MA+A7V65DxmbxmTP3B39uvC00JJaAgqvYBpIxZoIHngswN0AuyVwEA8Apha4we/oN+iPCxdwq0CGuaGiqt5hD1/MKkmaEdwg4Lh2oqCm1tvdK2rbaU7RtybXdyT56cAR9p/gmKqaRv4/P5a4NLkflwTPTjfUljwTHk5DcyJv+C1FQbZmAa8k7Jq4voSJ3NCZn0njDzExCpc/+yU2qaJWsKUBsCNloSk1HENtVfoSGJ0/SsUvgRTA1q4oeaKyQWBGjXiV9s0TqRckjqSzOqKcWPN5ko/kUuD75EHBxcIIpeLYNZRYdQmq1PgGjakQKGTdmohF7f69Ir3afCs2BCPL4PvSCXICKCIIhciH+/RIiavGwNESs6i+uDBwTYF9UeoMJDEPo/EZlWmkjAI0gwUI+FDMiyxbXIR8G03uf9s3exNFk06WhDm7SNEYA5Yr6NH1cBeiCpmAbupewP7ghEPe5RdzaXkR7gN5+M18lf34zt1hW4wN7+79v5i///mbefX1FRQkRtZY3r2gSD7yIBdPvr18bgxtjM3v+tZgcWfAI0IQLlMbDCezHkiw+kvI9ATbhOdcqNMeuf5Lxub3NXnZ3V+SE4JAHcXO/SGcS5rrJiHfJX7kayZiOahx4Tkm9rNsnNEyWbYLRz8bukFQbSrnhM9WFEQyNAyZ043OsAkfIB1kWvxVjAkA8wZajQuz67bID99Os4vqOe+k6EfX1ijOnz3DwkHfgGly7pSg+iOyL2O5KoWeSuGxFkSkQKoeqOLYpg85XUU9pPMto3ytDuAHXsMYdcnM7CEtm5oEB9CodZA3B6ABwmFM0vDi6Vx6Pw3uraGB5BM9NbGv1u7ghinn9YEiBUINDuhqHaByiccjicAiVkY8nLtFAZHlA5DzRAdFYRGORVcciNnXziV8jbAKv+TjOqTcUB8LVmzVNFUnK6A/2NeOnfQWzrHlehgf1DA/mZnhYz/BwboZH9QyP5mZ4XM/weG6GJ/UMT+Zm+K6e4bu5GZ7WMzydnWGWYxQc8/aanDIz3M5B1JAm7XT1cq/1L3N+uD11jddSLPL2KqivMNV4BNBnAGz8YNB6BmiGbRRwWRU6xULaKQQO/0axJc6SgrBoMCtJw4mRMs9MFZnMBexuvLqUxKd08jBps4leIpNKELfTWWAu3dSuYA6I/qL/NT58aBwdGaenbyaTN7iUUxxouxPAV+3bxZxClVfNzElDvsthHiFqpoYTXPlm5bEviEnVLDaEgOIb5MuHrEBaBXhCJ10OLL0t8l0+XvpdRKZ7cib0ymZBdTFoPVQhz62zqMOPJgPqYlJ1MAjzEGXIAPFa6eNgmfrYp7NaOqXWziGHdctWxto7x9Ey9ZHO3FOXMD6BeE68lqo4XqwqwnupQnCI9dbGySpoo+wYxhEK2GLmeqrk3Yo6yHpr5XSZWvkoX0ZYBdkXV5XvJXqnuTvYAbuDTnvb2QHbQ2vY6ra73db2dmd7197eAcPmRUQl6UMCcSPZhXD9USNbPWmEgYN/DdDovfPW+hnBIVPO204L/3wJUP7Y21f9vg2FXaKyrhXLNQ9VdnmqWd01eEGFInwjtbQzgwmQbJfKNm3atZs2996n2Xr++15LkOvB4uQKLkfaXBOxHj6NWNfOXI+eJgyk5hrGsGgNBHv8xPF1fSR7shyTdWIkvwYCfrck010fCZ/eo0jmwcUwZwQQcQM0ReGxAFRFCOmBmMIOn3geptCwuOMwqq3Sbcl5mJqZoKSaU1XRWVfVWVPZOeuRLEmFZ22Vp1Kesu1nacWnTLa5fC2BVpQNl/OOxK6r3aitBFVXg9ZWhE6pCq2vDFVXh6orRKdUiaorRadUi06pGK2vGq2rHFVWj1Z1Ka0iVRcVKAoLlMUFM1SV1lWWTqkuraswrakyrS9nUFebTq04rS1OkBcoqKtPp1WgKhc5yoHuOcweZyxInaEodc7C1LpFD0XuFfNvIQdPX9eS5+jaPF3M1ek25JQTNMrK1brqVUUF60xVrNXAIq9mVVS01la11le2qqtbaypc1VWu8kpXsdq1OtRq1avUBdWlpbXlpXUlpjVlpqpSU2W5aWVcz3QIGzKV1e4GaAwoI84CAnc1CNQgUIPAlwkCn36tW6PAh6BAZUGaBoIaCGogqFo4fforciq1pIu4ImepdxNZimPieyt2OVFevXvqYpo2bp7icqLyOf/VvxZgur7bcn0L47nvvQAl4mNcC1Am8sBSIi7yUoCHne9XnvRqP/ykFwV9PrQL551f/hn/4lk0fdXQizzhnyyJSnPsczzRT4pn9erP85fGkONbfWL/cbv4oNuDXiBOaCtwoWVpnKBxgsYJ+iqgpQOFdNVMIwWNFFbobp9lldV1FGV1VnuhdXWy3dHnvala3M2pKazrTNlTtToL3VSVNDz2nqqsQYKdHnVHdXGborLDeFVstZg90SrO0hujUjz22PuiK3ZWRW+MzrYxOtutEC9yP5SUL3OaZTdUMs4qBFzAPmf4/Pc5w/UpeFtjcNZtTQFnXQ3ONDjT4GxlwdkyzrtqdDYfOpt6UZQGaBqgvXiAtgKFaGXnW0z52ZMvWao2GXt6yXIxqLg3DRVva1SsUbFGxc9myfIJb9LQ4PhhS5fyGyI1QtYIWS9harAmA2t7U8DajgZrGqxpsPZ8ljA1WntOS5kasGnApgGbBmyLA2y7GrA9d8BW7/ExkDjwgPT2/2Lm+XMMiOFi4/zwk/EZpljh12qciOIfAhgTEuI3zSawbei4wP/FDiZNyFJ3iFwMcXNAY0YzwR6NGHI0vADjBkq5Y0yRA24A32k4Uei5Nk1+DfqgH1x50BnBCWWHm+aMiVmj1WeJVjlY9almGE6lnSD9czvsJxZ4BkdYCkyVMNRoGnKOv0dkfm4xqI2v5X16TJu+94VA2kJgwUYQEUYGnmdgdk0whiPu8BrfLhnfxqGO/x7Kc8e5Kz8UjXefEO/uabyr8W6Od2+MSUAjQ4CMs78+GihPTdNB78gl42jAIW+YeUaTYw6Kcmj2gahJ4S0VRtNqtTs/8b/p0yzBNbqtXau929nuaWS7fsj2+Jr+7/DT2Y2fYKEFwdt7A2YNcRcLcVksUcBcIcRokKtB7pqD3BWoUz1GiGHp+5SpbiQ9N5MfsGStnVbMiz2WxxkT22M4AX9AhOP+x+uuNITdxL1wALqIv0ezcG7FZv47mmb2NgInoUcF4Y8yr6TgCBPB9gv33Xj8F1Gr1zSAiASSG2EYuW8HEcewnVa5Jda8abXE+CWg3sJshcBrbp+9STGmXCY9Yg2yKc8sMw+Olkslu65Pk50D9z3ZVTdyPzAnEY27kseTOGXmiVJ8eRAySjnelEM39GIkJofKqXQ6rcpds5mAWJsq5KnfJrvJW64KlTJmflXdwKyx8l20aUMWHoS0bP4TQXSTCGGrN9kqsBOcrl2gjuB16devTHzhhl+QRwGXLVF0kkTF2nDqhNxp8H/SLpjF1jKE4DT5w1kWjqGR2BZheB7z4n0ql5qvkxe3H9mLZeKf14drvKqrtvTuPJbeXV8Tn2J8DhwCqmOFBaatUjOUJIrUiswjaqLGWWyjErMRLFhuOFIthlJLz7VpTdNmKlHh9fcKDoJMV9Qvk8npPNLlb+rHU4LNKLz9OxjQaa4NnEuX2jmd5m4ZCdfX93Wm2F1aj+su8YCm+8pGCprvYvjncsklwG8YT1pNP7hqZPnR5CmB0czC10KXztlQ/uVEJP3UccVQaPYEUCzmiQJcMa08J4j5wbTED52W2CLcQdYW/rac2Ma/p2NgCw3VEK1+i8h4W2QsvqXdFT8Idw7uOGJ/074UxPcj4ItE5pfzwwSrp5OIj9lv3BtN41AwaTPiy87mcLjd6YFWuzewhxDsDOyO43SHvdbu3l4LdOBe/PBlNj9obdz9Hx+Hf4PapwAA controller-manager.json.gz: - H4sIAAAAAAAC/+1d3W/bNhB/z18haBuQAG4nf7XxgD6k6VoUa7q09baHrTBoiZa1SKJKUomzwPvbd6S+KIlKXHdNnIYvhnXHj+Pd8e6nMy1d7VmWPZsFcZJyZv9k/fmxl1Eo/pQGFCs0FMeEIx6QWBCvgATEMGBcNoHLtWyGvYCjeYiBukAhw5Lox5i/9oAUp2GYUShKllNCQh4kQHckcRl4+JjEnJKQ1foHtb5hEJ8pglG8AEGXQLCHDrMzGrmQLaSUmaxAdUkYooTVZKsxvBYnQTEOq6HU4bKeyF3iaRBhknJFxopPQkKfI/fMpySNW+NXbX5HYYq72XURJP27wWTijp7YvTqZ+nO0Pxg+7Vn9waRnjZye5Tw+nBw0233njUZoiGyF+rE2r4c4YiSlrhDLPqUkwnyJU1YbyF4QGiGxdDsmMa7zfJT6uHSWkhyhVbHavuM0xIqCuGA2WWxJLnQaAhZfChcgofcGzXHdeXStThA9w1KnnKZYabSuy08D75RId68zpD8O6qSYY3qOQp0T1B1W0UOSBLE/vUykKrp4bdPX9QmtYxRJI50LzVmcWByveMPgVs4WU9UY695mg1MU+xsMPqgP3ule4AUvwMNOCWiOtV3BFlqUzBPiyfldEsfY5bBJW+2mQiKN4hPC+CJYid62jvESgs2H4B85/Nj5odEGAou2r6Rf21Wq6gQiys2mW8C+krsHZG8rlWcLs9/+eKRhkrLjhjpnEM5angtEegYuqtmpiyAMj0X4kfYXgWXYh7jSP4SPw4kILP3DVmBZCIH0G1DMoo6XDTeAENWfDFsDqfu9c4/KZANjplHcNhVH1Md8AyvgVSJlYmm0nyZXbpgy2M7P/rK/z7/+Zfesv8kcKGfpHD9ysywVYvooQjHyRYP1QdtEVXTkkCNmDNMAs3azIni8RC6Xyhm0moTYx7H3shyvPQi4pcyx9tGm7lDGQ6ZRXcBlErd/SxqMMmlfNc2FaJVBuw2WhTubQXgLMQNIUR9fxhF1dx02d5dssdnuIkJO+9lnbqwilH3W3pKd3uaxErJY1U9Zfw07oDBA7LjI7o0UM0dUm8YgLbPlGxz7fCmDZouHu7ptlM0DuXf7LeIrirwAx7yZlm/IkcN6HpQ+3PYcUIQfH7FpjhtFTm6EAnTud4QUN6U0k0vTDXJMRzdhID2HBv4SfM/rkCQPSjoO9Pkj8KRdmplI7huOuuKi9J0b4YiInkwzt2Rc5DP3N0Eerdzayj52gsFVYo58PShNRG/hE6kYftzm6b0QTOVhimWsXYSksfUpTjDSJvIscP56jim4G9asCJKYi7u2RZ72mlQOsFwrJQT8JMHemywntvlbpRWKON6/IPTsU4pTPEOex2bSJ7bLNj0riGEFsYuf/QsdiwtIQ3+Oo48HB9b80trPh6va9iyB5m41UV0V61uvraurQhB5IWRZr//3TNZwDbGglxnKavmV4H1YBguuZ+Yp8A+wmvVOmM068jzrPVhyq5zYDBiEcs1tjgwGsyJNBrEXnAdeimqJSJ9S5U11XbIVWgWsLdc8dc8yD9ZEqigPCkI5TUyW3wxoenUHxjK6yTKBfgmXaLXJjU7lpiTRuGco7v604gkm8Z8j1rrVUhKFtluWKbQsZcmfdUd1T1ah7ra9JrXcY/qonVNfw76XNSM9+z25qJWFSnmm+darF2PKDfkCcM6cIAp7EWSv8wvYuHyiMPINQqH1nmKhr1kZegDobmTQnUF3O4PuBjuE7jyc8KUBdvcI2L0QFjOo7o5RHUxH+W0iIucrgLpdXoTBdAbTXYvpxgbTGUxnMN0yYJxA5o1mn1IU8yDE+87jyaRnacBeDvnSLCoCxAJY57FZlo1vHwT2rBAfGCS4NRJ8A9aN3UuDBe8aC34D9T1T3TNI8L4iwSfbIEFdFNwECmoR3xdiQW1E/qpg8PqTDwYNfhkaHN15gQ8QCJ+5odhM8oQsXBY/4m4L6HoumFDQBo8fl/DuNsHbYLX6DHjW2yGNDe9IY8NrNfZ8hzU2uiONja7V2PEOa2x8RxobX6uxF7tx0/QL6NU6On1tvc+0bk5GmJMRD/hkxMM95vnU3CqYW4VdulU43N26sQat3FLtuGeBUkXX018/TOulZMHoWSkN76J4LCaXxWKY/z7UiiEO8hLy5NViazLhS+tdbnODgR5S7di574Vjc37AVI3/Lyh4aM4PmPMD5vzAvcCBr342MHB7GPgKGxRoUKBBgQYFGhTYRIETUxA0BUFzduAn8bQY4mLGAOIJJwGsF+GI0MvZ/JJjtiWw6zobert4rTrnuftY7UQq3QCyOwZk0um/gR8nd3wd5ufJOhppyGfgiIEjDxWOyNJTgUncJC0rTV/yKJpr/6tiQEkXKDk+/c1KmXBrg0vM344f4N+OHzAi6RtEYhCJQSQiz5CZTyhJuTCgKYjc1o9XpcoN+Hho4ONr/dd1t9fxLf9mtZevwGbuEkfod0xZJmV/lJH5ZTaVh+hZ9oYESF6Vl8rQSmMsqlrlWBxHSQjLjf32+x72mt6goI0aliues+zhBUpD3aOeVa4OGsq3QtSDS4dPlU+qVzCdwiVJ8fIKNZran1JML/NKfRv7KW+X6NeoPm4+Gr60izL9XmM5isJQGBYvOmgsQ1GlqocNkGqhKyVbQdiFROrhI91D2UtF2nmyVceKwCKBpk+hZU2Xm1Qsp5tlUVQ8Z/2mJ6r3rHySA71JBjeZJE9MqunA8aXe2btCKrvObYkvaPrGucGz9SmMlOFpNpAKWm/bC5wuL6hntq7ddJP9C7jzJQ6w3YP2q5NN34Zf7BVJKXuFjkQrZcgtXxBBLh71C0xUvPshywBVtyQANESrzrlKZgWeVYGJPa6cxu47ysVQvehH1fex8r2vXgwdlVMhN3ugfO97WTz8WKxBIHDFY26cRR34iTqwOstgpF5ULwuxn3qqvIUsNfX9Q+LsnQfT4zxLlrW60g+tk8wPswapvJ23nw6wg53xHC/GzmSCxwtnNJk744XnjgYT7I2yxudlXnb21v8BquF9oXtpAAA= + H4sIAAAAAAAC/+1dW2/bNhR+z68QtA1IALeTb208oA9puhbFmi1tvO1hKwxaomUtkqiSVOIs8H77DqkbJdGOk7aJ3fAliM/h5fBcP9K0dL1nWfZkEsRJypn9k/XXx05GofhTGlCs0FAcE454QGJBvAYSEMOAcdkEPi5lM+wFHE1DDNQZChmWRD/G/K0HpDgNw4xCUTIfExLyIAG6I4nzwMPHJOaUhKzWP6j1DYP4XBGM4hkIOgeC3XeYndHIpWwhpcxkBapLwhAlrCZbjeG1OAmKcVgNpQ6X9UTuHI+DCJOUKzJWfBIS+hK55z4ladwav2rzBwpTvJpdF0HSv+uNRu7gmd2pk6k/Rfu9/vOO1e2NOtbA6VjO08PRQbPdd95ggPrIVqgfa/N6iCNGUuoKsexTSiLM5zhltYHsGaEREku3YxLjOs9HqY9LZynJEVoUq+06TkOsKIgLZpPF5uRSpyFg8blwARJ679AU151H1+oE0XMsdcppipVGy7r8NPBOiXT3OkP6Y69OijmmFygUmuhGdT2E2MfS9g1FoDDw4yM2zuNFyNI0ZuDP+RmExU2i1mNCUXWSBLE/vkqktlfx2t5VlxRaxyiSfnAhjGNxYnG84A2fsnK2mKrGWHY2G5yi2N9g8F598JUeDI72Cpz4lIBxWNvbbBGtknlCpIYh1OIYuxzyQKvdWEikCfCEMD4LFqK3rWO8hnx2Fvwrhx86PzTaQO7S9pX0tV2lqk4gad1suhmErgxQkL2tVJ4tzP71xyMNk5QdN9Q5g4zZCg4g0nNwUU0ymAVheCwynLS/yF39LqSu7iH8ORyJ3NU9bOWumRBIH+NiFnW8bLgeZMHuqN8aSE0pK2NL1jMYM43itqk4oj7mG1gBLxIpE0uj/TS5dsOUQcZ48bf9ff7v33bH+odMgXKeTvETNyuEIaZPIhQjXzRYHrRNVCVgDmVowjANMGs3K/LTa+RyqZxeq0mWpl6X47UHAbeUZdw+2tQdypTLNKoLuMx79u9Jg1HiguumuRCtivRqg2XpzmaQ3kLMALXUx5d5RI2uw2Z0yRabRRcRctovbhlYRSq7VWzJTr/muRIKZdVPWX8NnkCFQey4ABCNKjZFVFspofKz+Tsc+3wuk2aLh1d12wgwBDJ2uy3iG4q8AMe8WflvKMP9eyjD6MJfkW7clNJMZk03qD8rugnj6TnNiq9PWDoO9Pkz8KTNmlVKxhRHq3Km9Cu2AcQoG7UYl/nM3U1QSavutiqTnWBwo5gjX4+JE9Fb+Esqhh+2eXoPBVN5mGKZh2chaaQFihOMtEU+S6q/XWAKrog1K4IC5+JVIZOXxCaVw65AKyUUgyTB3rusXrb5dyo5FHG8f0no+acUp3iCPI9NpE/crRJ1rCCGFcQufvEfdCw+QIn663vYPcJkkyIUPx4cWNMraz8fvOrZsQTuu9eSdl2sdrm0rq8LQeQHIcty+cVrXsNRxIJeZ3is5WWCdzYPZlzPzIvln2BD670wonXkedYHUPWdqmczfRDKNXsumRomRUENYi+4CLwU1UqWvvjKHX5dsgVaBKwt1zR1zzN/1uStKE8RQjlN9JZvGzS9VqfJMtfJMwv9Eq7QYpMtUeWmJNG4Zyi2olrxBJP4LxFrbcqUsqHtltUNLUtZ8q32XjuyCjXa9prUMsb0OTynvoW4lwdYevYHclk7oyrlGeehVz8ZKgPyFSCiKUEUYhFkr/MLgDl/pjDyAKHQek+x0Nc8pnoEOHBgcKDBgTuBA3tbhAM9nPC5gYA7CwFfCfsZ/PfA+A+mo/w+sZPzFeDfNi/CoD+D/taiv6FBfwb9GfS3Dv3NA8YJVOVo8ilFMQ9CvO88HY06lgYW5uAwzTImwC8AgB6bZJX6oeFixwrxgcGMd8aM70DXsXtlUONDo8Zv4MzQnBgazLirmPHZlmBGLTTcNdC4/kaGQY2fhxoHD35kCNiET9xQhJm8HAwfiy+Q7wr8Oi6YUNB6T5+ugIH3CfJ6i8UtYFxni/TX3wr99dfq7+UW62+wFfobrNXf8Rbrb7gV+huu1d+r7diG/QJato5O31ofMhuY+xvm/sYjvr/xeK+tPjebD7P52I3Nx+H2nlhrUM09nVp3LFCq6Hr629l43SG2aNaxUho+xLG1mFweU8P8u3BKDfmSl9AoP6e2RiM+t97nHmCw0mM6tXZ2/cja3HEw59VfCjIemjsO5o6DueOw84jxzc8GMH4pwPgGG7xo8KLBiwYvGrzYxIsjc8RojhjN/Ya1gDGhxMWMARQUTgKYMMIRoVeT6RXH7I4AcNU91/tFctUt1e1HcSdS6QaqPTBUk07/DXwRuuXrMF+F1nFKQz4DVAxQMUClCVTk4VWBVtwkLc+qPuchPrf4RY4BL6vAy/Hp71bKhJMb/GJ+hv0If4b9iJFL1yAXg1wMclmLXHwy8QklKRcGNEcq9/XFWKlyA0seGyz5Wr/13e51fMvfh+3lK7CZO8cR+gNTlknZHWRkfpVN5SF6nr0CA4pX5aUytdIYi3OxciyOoySE5cZ++4Uee01vUMBGDeUVT7n28Ayloe5B2ypXBxrlaz/qyaX0KVs8v986y4Cdwi9eGKDAPoVLkuI1JWpatT+lmF7lh/5teKi8R6Rbo/q4+YT+0kDK9HuNdSmaQ2FYvNKi7tGqTlWFbABmC6UpZQvyL1RUDx/pno1faTSvuupYEZgm0PQptKzpcpOK5XSTLJ2Kx93f9GD7jpVPcqA3Se8mk+QVSjUdRIDUO3tfSGXXuS3xBU3fODd4tj6FkTI8zgZS0et9e4GzygvqJW5Fqr7R/gXu+RwHuNv7DqrLVN+GX+wV1Sl7WZKELWXuLd/TQS6fdAtwVLyCIysFVbckAFhEq865SsoTOxWh2MPKaeyuo3zoqx+ULas9VP7vqh/6jsqpIJzdU/7velk+/FisQUBxxWNunEUd+Jk6sDpLb6B+qN7ZYj/3VHkLWWrq+5fE2asnxsd5uSyP80o/tE4yP8wapHLHbz/vYQc7wymeDZ3RCA9nzmA0dYYzzx30RtgbZI0vygLt7C3/B8u4cmNlawAA k8s-resources-cluster.json.gz: - H4sIAAAAAAAC/+1daW/bOBr+nl+h1XYXydaZ2E7sHECxaJJmWqCdZtq0g0VbCLRE29roGorK0SDz24eHDuqw49hOLNtvPsWkRFLvxUcvH5F3G5qmI8/zKaK274X6kXbHilihY4eU/fr2g/28b/DLsGVT1HMwK6UkwqJs4GH6zmIlXuQ4soSgYHjh+w61A1beFIVD28InvkeJ7/Au+sgJ5f2O7V2Gohvxk+A+weGQFei7zVCXZf61uEIMSw6OlZq+46AgxGpromKI7cGQj1xvNZvBjZ5WBMjDTtaS2pqoR46NwhPf8YmQwn1DrewhEha7EhUWCofvsTegfNCtZqkOj7qNotCPiMkfQD8nvovpEEehnruqbzsObzZf6BMXiQcMMLvfo5Fn0/x9tlW6y/YoJlfIEYJx85c7eIA9K1V9JpGrQdXgufQjQljPI2pddDOqxvZG1IRD/1oxLKWGMtt0RtzFHijKJKzU5bXHrExcVWxdVFzbllResUYxzLSYm/m5z4T5wbeE6niBhkLtJyZ+XqqxdtAAV1pAwFshyLIj3k2nXFdtOEzsFiaY8L77jl9QfIiJjcOPV5gQ5nEV4w8DZOJR9soquXra+UKKzMvKoYQUBwG23jMZVtZTRAaY5j2u6HXiQnwTiOdpadsas7lNgije9JiADTOIjBCbvmeFhrCDO5cVv/rOLNzB3/WGZjoRGwZhJS/if7/r999eGAZvw0iM/sfWlt4o9pr5EbVdbEjRlS+zPSYBYepl21T86gyZ1CdF6cUq64sQqb/Wc1X3yq+8luiQR0Hfsbjw9P1m46CZ1zMf8RkLGkrgzdV9Htp9Wl1JRQDXT86/aF+ozaK8CPyF5tPwfVf0UkSwNcqFfULLApA+atDbQPRre5Z9ZVsRc+mR7ppcG9rewMFM/AUrv0E3dlgeXC8yL6W9FR+bR57YX7l0CkrWPeTi6rtGR6U08ogJsvo5btENnsD8M0Nk3RFaNkEH9bBTOUBe6Q+OkZgHW6U6GYgrb5ORuFkqV544b6yN2j5EZVX8HMV5oeB1GxWPt9KgoA0TP0z8tZv4w8jd5EFYDJJP+keXUQ8bgW8ZbO6niHVEDDYlCt9g//zJdE/DI3bbXSUA2NJ2NN6maEUACT6JRKGBHMc3kXiLuUvaY7eyHhmYqG4KgEMZOHyKNaCd+K5rU5dHBAAQACAAQKw8gNgFAAEAYskBhGOzSQvgw8Lgw3shfwAPAB4APKwVeNgD8ADgoZbLDny6PxJTvYtdn9waH7D7+grZDp/rjd4txRPghcL9F9yC5L13AA4mAQcfhOxgXQKQASCD9UIGHUAGgAxqnlaQM/szL03ITiG98DgEAQsUgCQASawlkugCkgAksfxIYt5rFIAjpsERsFIBKAJQhIoiNgoOydw3wKjoTHHpO+b8Mo1XWf3Jv859O5IO6CJ2w/yXHalzvsXIkvNevu6z/VPUD7vSZ2JdPOabkXZnVb4ZaZah0T5AoxI0agI0ktCoVY2NSsJ7GmjEAUyGj3KY6ChDR/xzkChkss1/FMJRkmHzLz5GYaXerZbBr1nAzoNoRnrVWdre3V3a7/29PuJyJszLkQGayVsobRpwVLCDOX02wjVQSyAkvj0EDAQYaIUwkGpPOT8E8DNh5khgnwPAPpAWeiT2eXxeKKS3ziRhXLgOf6CL8hQjjR1nEOJ/7G/7w4ft01Pt7dsj1z0KK6BJgCgLNN7oNpM5cmhbFvb0RwXsdLjnvlXRt8ljQGwelQHWTIJEXjfTPquFTdtFIjSVJyVHYqnKbBSvuxD49NhB3mW1iyXXpZBGPyUsrGiWf+1p1NeCShnwW74Q8Xn9jrVz0EGdbru5f2Ba/X32d4gRaiHL2ts1TWy28M7lQbidZOfC7RQi/vcKke0s1L16kf3/b16VwNsE3IrC9O5XLwzDxI5jtMbax1ceQrR/vq6wkdHYMWdDXuT2mEOW6kVKPp3WpzOxP3xy6fgI7Ox6tCDyxoYO9vu9pnXYOsRmE6NOv9vZ222193Cra+61UadgbGm7izG743qaXfWLzaLNrj3K7MbY0yx2N9bUJtHuSX21m3ACQMGzKPi0/grW/gUqnkXFb55OxerS/fSKlqtyoORZlHxWXz+OF13Bi2dS8K919uLfEuy4HDpe55fJsbbmjdZjHaJJza1LEeTOL//ZmV6GISW2N3icDEcvIE2zgJdSmZhdYzKHZTix0eQTsI2K63P6OD7SpItxjckEZfqRRzf5DnfZUmfyNs6Fd5SX4hHBYiFknDST2xtaJtdlk/PxvOVcsxXlGoj45ElEPNcdm5ZLoKfLb7PJR6trrMU3i3WLhzm+yyXOszV0ilXT4a/6EjCQfo+YbuvFQCLIC7kmq/WYIvWKKmAoAUNpNRlKBT8FntIkJO1DICoBSbuWJO0M/cR73JCwemObhpZe+g9WAQzseeKfZI8cjnW1zesdXzOROSyIFAjZzw93xEZPAHfWFO5ItwSs8xhOdqEnwDpAygZSNpCygZQNpGwgZQMpO/+uA2ywmvKy5XvPTAoGanbNqdlz1DHwOleYnZ3bNgn0XFOC9hy8GTjawNEGjjZwtIGjDRxt4GgDR3sqat9zLSivPgF7TlvTAwd7njZZ4pKupZbeLNzygWZdf7sHGvUz04jSdByTa/WbHlCqgVINHKP6cYzKqyVANpqEbFQoi8O9OADS1YGIBEQkICLNSEQ6kV7AApSJ7SusHSPPktYEqwQzrBI8ISfoOAhn1PUFR2kMu4Oy56Ps4zoq+xPTgeb3E8e2tHMkkTSoup6cnGBmVSd+TUHbtWfnBHN3bO2U+BzKgMrrStV5CgcHrdecuDO91oHMAWQOIHOsEJlD7GmjrNp4mHKegUHkTG4Ijp/cA2fEMk5meH991395+V2//8YskDdrJMmxH1trzwcZJ2saT6HrKuzj5xR2YtiBxCrrJ+2ThZj22or7dJHGbUkgvn5Sf7NQI19bsZ/Vfte3ON38m1RdHc+gBLoC0BWArjCxzwJpoURaqNgNrrD2DMwE2A5OYSZ0F7gbXB0TELBz3HjK5zhaBmwZ9/z4iK/IAzpaADpqrAQ22AVsANhgmbDBQhPmAA7GgYMHeHyADgAdQO7k+XInZS+EfMlEmGgPMBFgorphIr6dBORLlg0SvWbGxjfZP0l0lmGj3FezRyl9FiATQCZIqCwveOgAeADwsEzgARIqK4AelC8xAEAAgICcy4JyLpN6LKRkHoequoCqAFUtI4VlMdx3gFVjSSyTbEUAyOn5kVMAyAlSL9ODhH0ACQASlpLLAiihvijhoV1sACgAUIAUy/OlWBK/LPoiZFImAkkHAJIAJC1zJmUxH1gDVpomo1K5GxgAJgBMkFlZKtBwCKABQMNSZ1YANSxVhgWAAwAHyLTUK9NS8knIuGRXZVuGbrcmhVXtJsAqgFV1g1XsDd7ZZNiK4Rz1QL7NAtLqhwbByErXrPIH+jW0yiP4vnXcH1vaS63c1jWxFfbx5I1tAQCbHoC9+3j+efMT1+LLP4T8twBvwXZ3kKpZ4lRNuwWYAjBFDVM1k8KJ3FdI8wEVUzYJyGKGrWCGxI8Gw/OICnwh4QWgC/g0CbI5i8rmfGahiH+a9O4jJHAec+hvG7bPLSOq9TjYV041BUUzd2CFewXhsgbMcmC7h4OCn+qg4Cx3sVWrE6O2W3A28FzOARMKrkpLgYZrdCDwzBoWLsze3kDTNT8PeBZNx6+DQfw6uAWng86i5NM6nu+tqLjiZR90XJ9zf+eiY+7GSdQGVc+k6rM6qhoO+4XDfuGw39U57PfJ2BxreZzvw9Kcms+yjgf2LgnVaB1P933iZdu1PMJ34uixqkJ9s3yG+uz8AjjRd/ITfSuWNmvBMIDjfIHfCByESg6Ctq2d2vytqBeJXh6gJGzE49ZDc4hd9BWTUI5NHtYiVzv5HRYil6Ix5lWDzDL1y4i9w3qYBWo9bYtiN3DYQ3qD1NPYi31IFXvOjEBZxs9xGSi+ETZh4T6KnMISs3AhtbaKbcrXQXHeUlNT0k8RRdpnSV9Q6mN/VskNSq0fcJkWQ67+Z4TJrfioq4oEweM+D9Z5C2WlA3xTmCjSyKV0v1F4LkVyyHG+xoLIG/JDMh0hzGopTsDzSCSthHYWyNm0bOHXTgVtYoRH6y5TpV1xeaKVeKJ/jEpET4YMuZtRcPd/v8cgg4msK5vNRwwypEhiq1pn7Yd0Fk9rqm6ZiwjFhL8nw9DztaXx8rLqi2OLkA+kVEQhvpANqYFKBiMenO6lK9pCcrET9iUq0D3/eruVcPPYbB6X6bnbApvNiSS7ORZJ+smuOj3pHWW/oFZT+bGr/mi52f8d5f+W+mO3qdYoh2y1lf9bljTWH8kzcNqoYgsP9qI23FUbVntp76k/lA/09i11vMlYcuL76QsCif7l4iSOmym68t0golj7lCQutR3tRLFsPRL8KJ2FtoNu32pZTbPVbqF2d29vr9fd7aLdfqd5iA7kxVdpyG5u3P8NqZWh+zMsAQA= + H4sIAAAAAAAC/+2da3PaOhrH3+dTeL3dnWQbGiCBXGY6OwlpTjvTnuY0ac/snHYYYQvwiW9HtknSbPazry62kS+AuQUDT1/QIPki63ke6Yf0t/S0oygqsm3HR77h2J56pjzRJJpoGp5Pv/3xg3593meHYd3wUcfENNUnAeZpPRv7H3SaYgemKVIIcvu3jmP6hkvTqzyxb+i45dg+cUx2iy4yPXG+adh3Hr8N/0pwl2CvTxPUw6qnijTnnh/BiyUKR1M1xzSR62H5ajyjj41en5VcrVWr7oMaZ7jIxubwSvLVeD4yDeS1HNMhvBae9+XMDiJe+lY8Q0de/yO2ez4rdK2aycOjTvOR5wREYw+gXhPHwn4fB56aOKprmCa7bDLRIRbiD+hier7tB7bhJ88z9MxZhu1jMkAmrxgrebiJe9jWY9PLNdKzz73blNHlIwa9vMdj9gkIoWUbkWuhh1E5hj0ihzC73lBHyi+K13fu83N86tzmiIvSGgmGJpLykuanbsqPSl+dZ9wburB+Okfy7DiZxcm1Q63xyeFPwhMU5Ck/MXGSZgnNi3o414VcdhWCdCNgt2lk8/I9j1pFxwQTdu+u6aQ8x8PEwN7nASaE1nRO+T0XaXiUw9NMZr16MtFH2l1uUTwfuy7WP9I6zM33EelhPxmy6bDlB+IHlz+PZgb0muTMpnXb1tzgjLBWrU0/ccN6CnPffldfhX9+V5/V/fTFhvHlGxZuixrJHmbY9MG4g2ddToq3K6T5DklXSmiJLm861XM1kfUsfUtWvt9nraNj6qxO1OPq/kk1aT5W4ivamEgNciLvpm90/fxMn8e42rr+qnz1Ddr68w4hdfm4WX9KBx8iWB8RY55D/GwFiNBr+48uv69h68bA0AMaqSOjMDrWM+yeiWn1p5z3AT0YXrZwnUC7E26UfmzW3oRhyGonZWTVRhbOP2t0YxM3KLzjzH+OR/SAC3j10BHp7YifdUETdbCZW0CW6fQuEO8fa5k80fzmniba32omXXripLPul/YhcrPC50g396mo28l5vI2GhTrAAsDCdsGCF1i7rIXnheS0cBd0cNt19LZGf6ogeiPSpv0tDzz6x1/U9r53Rk/LJ4k95UBh1+RX4QTCeqjAayPTdDTEfzo9/el06FnskArLxRUaysTQvO/qfnQrmk8LQxPy7wLAkgWWL6FxlJZjWYZvsbYEwAXABcBl48HlEMAFwAXAZTS4mAbtEQFbyogtH7lpAFoAWgBatgpajgBaAFq2C1pqSoVjhpibsbDlkMf2J2ydD5BhMi9udx59XIBTUuffMg8S54aUwg6o0NvS9o+dCDxSiEc+8RqFqR+AEYCR7YKRBsAIwMg2j6AImCjP7I8oD5DLdOQCc0BAMEAwW0kwTSAYIBggmLJMAwG/zMIvMBkE9AL0ItPLTiogafi6GKWDKUz9QINfDFvmZn9x7hOvKcUFug3DMPkSURyc7zHSRZeYzLsxfvL8flPETGiLaV5Pqjc25fWkahbJjgHJXhbJqoBkAslq+UyWqbzlIBkDpyGXJVjsbEhlmhu0A4/WLWUdmqp7be4mjM7aBntbaRSjdR6VIfbNQ1ITUUmE5VV8vaen+L7POS9NicNpZd6NbP1pfXOjzUJeKT9Y0GtOzAKlpCz+Di0AFgDWBgGW7E+JOASyKjgcxsHqBMAKxrrKBFbTD3Z5/qNZpI/gccke6Dbbf4lIwkM++Q/9V/n0qXJ5qbx/f2ZZZ14O97jIp/Fij75m1AH3DV3HtjpVbxAX99rRc+6tsQYmdI/c1luLWqCkbWZ9Vh1rhoV4u5ft8UwBarnjaCzvlsPvhYnsu/wQi46LeUm9JLTNUnTn3lZ8R3Fz64Cd8pXw1upAPzhpoEazXj0+0fTuMf13ihGqIV0/OtQ0rNXwwd2JV4nGFb1KzJ//HiBSGbajb18N//4ny4rYOSJnnhif/fZVu61h02zXxvrHN9aEKH8/z/GR0WCa8CE7sDo0IDP5fBIjZobZXOx3h9yZDgI/ux9dEUlnQyfH3U5VP62dYq2KUaPbbBwd1upHuNbUjuqokXK2+LqrcbuLcrpd/q+mVbtdfZTbjfGnefxurKsVsW6rvNaNVBRg4HkMfFl+Ayv/ABPPY+J3yzOxLHaY3dBiPhGMPI+Rr8obx+F0MUTxXAb+pcxR/GvEjuth423+MTnW1+zRdixDa1Jy75Iq8uDNvw5mr0PPJ4bdm64OR89OzTI7GOuzqF9jMk5Npcw//8eHd5egoUpPDKrjVFZFZwH3C67v6AS2v4sGPUn7Fv1SZxV7lqzhM4L5DMy42dTo9H1lWK/rVs8Xi67nkk1ll6CKW0up4oWuPrZeFXq5/j4bvZC8xVZ8t9qwmCxqXq/qvNrCoNg0G/6iroH06beA2rZc0ieCbI9ZMt+OMcXnZIE0CqRRmymNSsUpCKSKSM9PQSEF0vPtk54P0SpcHIl40YpIGtIHBu2pRw0s0dTo5L/RDFCaLxK3ooWVGForu/cHjqIhrZ+qUhCevzxd8TXDgK62lK5EWAJaTaM9T9MWoBWIz0F8DuJzEJ+D+BzE5yA+X67byT+kQPVWUv25+FE1l4FBgl5yCfoCbQz61Q1WoScWtgI7l1SIvoBoBi06aNFBiw5adNCigxYdtOigRV+4hHF1c9ubLz1f0O4HoD5frpdmdLVbabd3K48FkJyvYySAyPyFVU/xAB+t1/zfjiA4B8E5SKLKJ4nKzr+ANqqINqoG2ijQRoE2qszaqJYIEtrCadgYYOUC2brwJpi4mGPiYokypQvXm9PWtwzzKPyDsRdj7IsyGvsLtYHidKPA1pVrJFAcTF1OmZA7t6mjuPbB2qUXDLkLD2zlkjgMZcDkZVUPLSPAweol1xLNbnXQl4C+BPQlG6Qv4UsGSRNBNvaZvKFNRE/e5rJDscRQ4ZmhoSv+77v65vV39fkP6pPsRu1ovO3H3tYLU8bVvh92qlD9S9WrTHJ+V/AM1P+S1C0T3R8MsFwpTNEA0AXQgx2WJG0pHAhgiBkUMatawC8c2v5VGLOM+5iCtgK0FaCtKByzoLDIKCxyFvZLzXODxAJW9nspiUVzhQv7rcdICiwCOF4OO05xAqv/vTyOMbEBwNgKYGx/I1DkEFAEUARQpJzTCsAi41hkgiISYARgBEaGXm5kKBuFMBpUCMGOAMEAwbYKwdhiIzAatP4Edk7dj+0G0YqsOESxxPvSZ7HuGQgNCA2Gi9aXVRrAKsAqwCowXLTJsCK9sQO8ArwCI0orGlEqGrEw4DQdxDUB4gDiYM6vpO+yAMWNFSAVWSEDQO3lQc0FUIOBpdmZ5BiYBJgEmKSs73cClBSBkklrOQGXAJfAANLLDSBFcZmORRgnKsRkJ8BkwGTAZCVf8gHQbJbxotwl+IDPgM9g3GitGOUUGAUYBRil7MshAaTMOH4EnAKcAuNI5RpHysQkjCcNjxouC1ypFaW4ehUoDihuqyhOw4a5S1GOQpS8JehuCuy6XptgpI+ZAExuMrqv6HhgCHCzLK1j3r1x37z+rz2wMP2PdHT66bGPAft44J+6VWGfiKUXYMJwN9McIlReK9ni3xNjrDY+Xf6Z774HBDo7gX74fH2z+4V52uvfucH2ADhhnU0YGlvjobE6bOQJULV1Q2NFeWrCG3trQlVTPsXsbAVoNfsyVX3iBL3+deBzwBJ8BXgFLxbCeN6qxvNuaFPEXiz88BmG8KbZG74OC5e/MFJux97woh9LeQqNNZp4lKpcegEt22o+w17zy9prfjgytFeqTQcrNdhefiFbSXID5w36gYVLtKf83BbmIUx/S4KlS76l/DyWDn9ruuFvzT3YYHoeIy9xJ/mLmTeYlkycM5IANi7P1vELsTEL46jVBlPPZeqrMpoa9ouH/eJhv/jN2S9+Xj3PAiaaXmjWZ0O3nJ9swAmKJrDginet36QQnEVSBw44hQO2VuiA42fvwYxTmPFylT0B2HFhdnwH4bhUSRA44xTOeKWWXFoUbcaeo2cohayIvc7ITJpv0PhHW04WqLpBeLTRwiOlolwabLSiE/C7TNAh7YTlVj2tjy30DRNPlE3sHidUCOwMHZE7fjEaVb2hZ6p3QQcTG9O+QY2v5WPLNelD2r040lSTFkry56ETSOqbhIDJxw/cJ3TcRYGZkn7wEJJz8zT2TJ+Ak54au5J6iXyk3AjNkpQfxrOsaJJyHZfVabrJVf8KMHnk7/LmKZ9Yu88a66SH0tQefkh1FHHLJd1+J/VcUs0h0/wWVkTSkSfV6YjKzK/FAuKuqKalpp025LTX1vG5mSNnGhHRqkVNaeQcHlkl5IBpTMLv1BZN7m7gZtDlOQaNvXyb1SfZLOzWZNvSEOGG8X6LiqEmczPlZWn5B4ceIR5Iygg8fCsuJDdUojFijdOzCEWD11wYhF1BBart3FdqkSCX9uZhmpo4zTVon0iGJ4dVEoOW3D2pDWkRvFpV+nIof5HEfGpD+rsmfzmsyjnSrp916e+aLpz1R/QMTCsu+cLEu8gXbsoXlu9SP5K/SO9lH+tyeaOyJKrvp8OFXerX21bYbsZ05Vhu4GPlSzShoBwoLcmz1YCLIlXatJ00u3pNr2q1eg3Vm0dHR53mYRMddhvVU3QiDh7ETXZ15/n/3X7JRa46AQA= k8s-resources-namespace.json.gz: - H4sIAAAAAAAC/+1da1PbuBr+zq/w8facIbuwJIEAZaZzpkDZdqbdskC7c2bbySi2knhwbFe2CZThvx9dbEe25dwvTvLyiUi2LL83PXr1yHre0TQdOY4boMByHV8/055pES20LT+gv/75Tn++7LHLsGkFqGVjWhqQEPOyjoODDyYtcULbFiUEed0717UDy6PlVV7YtUx84ToBcW32iDayfXG/bTn3Pn8M/0lwm2C/Swv0w6qvizK3z6/g3RKdo6WGa9vI87HcGq/oYqvTZT3Xa9Wq96gnFR5ysD1oSW6N1yPbQv6Fa7uES+FlT65sIeJnH8UrTOR3P2KnE7BO16q5Olx0W4B8NyQGewH9mrg9HHRx6Oupq9qWbbNm04Uu6SH+gh6m9ztB6FhB+j7LzN1l4w52zES9g7d+6Kg6yCQcEkJbL6jtoceiGsspqPG7bl8yHqkmoPZnF9z1gOxwIEWpLq0hakn8qmzrvKJvmUJB2RrJ+JJiZsrXruUEn1yTq4cVaMjXfmLipgUdaQB1sFLLHmuFINMK2WMa+Tq1cVCxm5hgwp7dtt2Mcn1MLOx/fsCEUK9S9N/3kIGLbJJWMvUcpgsDZNwru+IH2POw+ZHKUFkfINLBQdqrsp7FL8SPHn8fP+ztOlSuTQf1MO9p03PNpkGDA6IPIWfJf03DC5uhT2Xb9DEtNf0mN5Mz2kTTIijAz4Yd0g6SN9/0V9G/3/Q9LWmZlSc/vukvFe1AY8+/D1uZpzZp1OHeSP/5Qa0t8Cdse0+LW6AVtOPsafpeVggD1w2sHnstpsn8ZZZDFcI9L+8qvJp2hnrFFTIClwm1nruExlEelfW3eqrqRfqVNpqgywKva5tMl/pJde+0mjY71uMrGqekWJ+qu+1a7UBdGfAxQ7+4/qJ9CSw6sPCxRttt0+a0WOCVzOOSEeQ5G0QQwWZRhHFJkBeICCHN4Mnj/bAc03qwzJBGnMJoEl/rW07Hpt1DGSd8RI+Wn+9cKzTuhTtkxcACYxROmLQySteZKanvKg6aSWDkY7T6PZ7QIx7DOweGSR9HgrxJ2qiFbWUHWaXbOUd8KK7l6sQ4obxNDBTVXLn0xmnj3SvtSyirovfIDlsZL9xRvN5G45I64BLAJYBLJsQlttWzAJUsDZUIcQMmAUwCmGQLMMkhYBLAJKXEJAMo0MM9lzw1+y65pwMAHaaDZuspwBNjgqTFf9EadqXVozoRP5aWIhEvA3gkj0c+cclAogRACYCS7QYlRwBKAJQAKFlWfgQgycSQBLIkAEgAkCSliWNSN/YwyjpVVPqBBgHuROrqG7efItQkHbqL3DFNd0mc9D1GphhC03W31k9e3z0WPhPpYhIiTb2xKUSaah5lNQBl5VBWdTNR1vAgxM2WtfUjpGLW9pMJdz4cGcy42aW/XNWPXjcu8lckFqsEApE55gNo1zJN7NwKeFFwM+PSDfh1yksSK1bXSprOI5AUlJwoiucEKODBUPFdvT49rG6J+Aoh3Dh4v1ZXAv6cG2/YwmjrSdulj5sFko/E3ELdV0l7z8/0iS8vesGFVLT3hcCBSp+rcCLHSURtIBsRMc2JJybclyae1jAQTAsp4DbpT2mSE8e032dfDZ5YrqPj6gYIWMS8VYq3KOpOLdzxpp4ZdDCnhXkWTUo5veQ0d5hZwsxyg2aWsj0N80OYWA5P8PN55THMKyF7PyGanzx97wdP9kST27v8QCOMHQ+QxP/o3/6nT/uXl9r792e93pmvgBMeCmi4cYrbjEdKMSnTp5tKqkHAYBYZ2Ygy1hpxpEgraNoXNrFh9RCPT3XVpPBe7UBR5R2fd53baNR1CbzRLwkNLprp9h1deekXwnxWH6qar8x7tV/eKtRTDN5S6nPCXov6Qq6eL1om4+r02r0ZnmUBBY+l4PPyK1j7N6h4FhVfLE7FMg9hekV/HJbuAyWPpeTL8vqxUC948WwKfldmL752zfXQblFifoHKPTAPjhunNXx0jI9wwzhpoaMqqp4cHx0fvm7g9kmrfnhwf+rvx7lAf99zzf8+ILI/mEm+eTX4/z+sKk47xklHXjjIOg5yjryCNvjmVbNpYDrnHGZqnkqNZQgjJTcuSYQHv/96ML0M/YBYTmcyGRZnWTdwxYZ/RmQB9KlsTlwfRrCa9+ICI8YJEZ0lIjnjkvfCM9pn6wGfLYBXv0YyP1+EzFdv6hE9EZQ/VPkXpXO4qTijayTxS3C3bVX9O30NVnf/YivW5VrdJcjxmSbVekwAnqIKVn9h9XdDV3/Tfgqrv+PQik9g+RdoxUArBlox0IqLJ29z30aojdhHCHTiBdGJ57J5EhjFQxjFq5XwppCK462sLDOh7fYPXM1ARhdXgGO84lkmD/cwy9zSWabsljDRnIRmfArzTKAZA814RpqxMv4Av6lcTGMBEWZSMJCNS042nqOOgam4wXzjSNdAOS415XgO3gys441nHadzMje3txVQ9izKviq1Q0dqvsin3UDREyr6j3VQ9G0feaDnmfT8vpx6ht0ixWqF3SKwW2R9doss/Xu+sEkkR1wWkoetAgvbJ7IyIx/KVwe1r2SHyPhih00iW+hrsD1kHkon/qxqXiORXi1FpJyttD1C/WMpQvX7yNsemb7X14OjB7vDYHcY8PbWgrcHG8Qm5u29ThdFEZ61VevpwOkDTh9w+mb9dKjwAu0GG5jO97Rz5JjCmmAxqpwEv3PPn1HXdwyh0ak7KLv0TL/plX1DdaC57dixTe0aCRQNqi4p3W9mVcd+HYC2S0/68+bu2NolcRmUAZWXlQO4CAcHrZecDDi91recOaQFrua5pg8MImAQbQiDiH/ST1pZcXDAlnybRAzlYs1XfAJw0lXVf6gJstabcZLse2XLCURFwg6iQXRrpX2+TGnHpu0JuLKF4r5YiXFvr7wvV2nepkDjWyj2dys18+2V+1Xpv2EbZZ3/FLor42mlwFgAxgIwFsb2WaAu5KgLim/bZsqAoLDBH7ednKBwvMIvipYyDVH674muCD4NJ2bARxiXD43YmjwAoxUAo72NgAU1gAUAC9YJFqw2YQ64QI0LRpD4ABgAMICMyfIyJnkvhCzJWHCoDnAI4NA6ZklWtMIKeKggTzIO3x1Q0fJRkQeoCNIl0+ODQ8AHgA/WMl0CAKGUAGHULinACIARIHOyvMxJ7JdZX4T8yVj46AjwEeCjdc6frIi6CzBpsjyKcqMpYCXASpBPWSu80AC8AHhhrfMpABjWJa8CmAEwA+RXypVfyfkk5FkGVw2+Q7FfGxtRHQOiAkRVNkRFJ+/2LoVVFNuIIx92M/Cq7TcJRmayPJU9OUKJq4phVaP3vaL9puUf0ieWxBqew1MqANOmgWkfPl/f7t4wjf/2N1dJBVAZ7LGGXM4653JOAHkA8ihhLmc06EhtJVow9Jj3swB/TLVfqUvcsNO9DgOOQgQIAQwCe5YgM7SqzNAtDULs9PEPnyEZNMmpNLVTwF053LUdR8+IoSajaOoOtPAoI1zagJEPbC9wlM2ijrIZZDgqpfqk8X4NTq+Zy4equYJVySvQcImOrJlZw9yF6YQONF3yE2tm0XQ0HfSi6WAFzq+YRcmXZTyBSlKxYrIPOi7PyTRz0TFz4zhqg6pnUvVVGVUNx9HAcTRwHM0GHUezTGrIVh5BM0zA86fFbOOxMxvIbtrG42yWv168lafYjBGPtk7O7zbCnMtDf4ATccY/EUex/loKGgQchwNUTSBKKIkS2r52abGZWyvkTxnBm9iJ+q37Rhf30FdMfNE38dkOsSTL7jARueeNUa/qDCxTvw/pPNvBNILrSVsB7nk2fUmnk3iabtNOSfY8MAKJa5AiXAT4kduEidsotDPr4NyF5FoVcZYt1uK0pSampF+iAGm3gmMh1Uf+LDMwpFrXYzLNhlz9R4jJE09ZqJgaLO6zYJ22UFrawY+ZgSKJXNLjdzLvJUkO2fbXSBBpQx4l0wJhqqU4BhkllrQU2mkgp+O1id/aCm5HgUfrPapKS3F5rJUIAUyiEv6kpgi5u8xYmxT4NC2n7SbYoqJWVn2UsqLxTFYq9Q2uEf+v+Pl6ujbXUVamvjgyBfEmUkXo4zvRkByh1sI+qgu2jwQKzsdClFu3X6S925thOTvxePYiorfFhRnF7bYAkrrj9vdrMeeUAsCoTE/d5lkURpHBzZFIkq3sMqLRG9KXtGpV6ceh/KPWG/zfkP6vyT8Oq3KN9IXzuvR/zRT2+z1+B0aEloxj5FPkho/lhuWn1I/kH9Im1hNT7m/cl5T4frqcGKV/ubuIhtoEkLs9LwywdhOn5bUD7c/YCrXdazemFOkhJwDqpw3UOK5XT04Ns31C/15jhGrINI8ODQMbNeEg+kMy3Fd3Xv4P5hxjp0QWAQA= + H4sIAAAAAAAC/+1da2/bOBb9nl+h1XQXyTZubCdO0gDFok0m0wLtNJOkXSymhUFLtK2JXqWoPJrN/vblQ5Kpl9+OZfv2Q2qRelD33Esekofi45am6ch1PYqo5bmBfqI9siSWaFsBZUd/fmeHT7v8NGxaFHVszFIpCbFI67mYfjBZihvatkwhyO9fe55NLZ+l10Vi3zLxqedS4tn8EV1kB/J623JvAvEYcUhwl+CgzxL0/XqgyzTvTpwhiiULx1INz7aRH2D1biKjj61en5dcb9Tr/r2eZPjIxfbgTurdRD6yLRScerZHhBWedtXMDiJB9lEiw0RB/yN2e5QXulHP5eGyyygKvJAY/AX0C+I5mPZxGOips7qWbfPbphM94iDxgj5m17s0dC2avs4yc1dZLsXkFtnCME76dBv3sGsm0KsW6blvg+sM6OoZt72i1+P4hISwspXkOui+LMdyS3IIx/WKOVJxUYK+d1ecQ5lz2yU3ZRYJBxApeWn4mZuKs7J3Fxl3linRz+Yonp0k8zi58BganzzxJiJBQ4H2ExMvDUsEL+rhQhfy+V0IMq2QP6aVzyv2PIaKiQkm/Nld28t4ToCJhYPPt5gQZumC8gc+MnCZw7NMjt5+OpEi46awKAHFvo/Nj8yGhfkUkR6m6ZDNhq04Ed/74n2C0Nl2mV3bLnKwKGnb98y2wWoexB5CTpJfbcMP22HAbNsOMEs1g7ZwkxN2i7ZFEMWPhh2yApI33/QX0c9v+q6W3JmnJwff9KcdbU/jz78JO5mntlmVJkKd/fjBvI0Gj395HXY9P7XGzENxjYU/sYyAP2Gy5+5q8d1ZBnspXhJ9N2ugQZ1BLYe/Mkc5f5rlstKIoM2HkVKHnCODetzgzdwprAIXzYH+Vk9lPSlHaYeifV7je7bJcdaP6rvH9bRL8hKfswpSaWRSeVd9q0uLM6mot/TTiy/aF2qxFk00ctp2l91Oi8HYyTwuaboesxUMItgsqUcCj9C8QWT10qYPviiH5ZrWrWWGrDYqrWnicwPL7dmYO0e6ePfo3gryheuExo0MlawZeJ0aVTXcWhnQde5KxVeVV6hJpSnIQfF7PKB7PEbkDhyTPY7QvEvaqIPtwgLyTK/3DgkO0MjlySam8DLZxtRz6cobp513t7IvUZgVvUe2SctE4VbB6601IWoCIQJCBISoSoTIthwL6FAl6JCEAsgQkCEgQxtAhvaBDAEZ2jwyNOAgDnY88tC+88gNa10YB6DtzgPFMRkxkHlrsYZsCgqSPONvLIefaTkMJXlQiWEi+epAjfLU6JOwDAwWAT8CfrTZ/OgA+BHwI+BHleRHix01AnY0MTuCsSPgRsCNktQkMFkY+xhlgypK/cAqARFExdmX3l1KWJUU6DoKx7TsKQnS9xiZssFN511ZP0V+/1DGTITFJIKqZmtdBFX1POFrAeF7XsJXX0/CN7yGEzHB7/UjZGbWasnAQr6uM3jk8FN/OW8evG6d5s9IwqGQZUS+nq+d+5ZpYvdKcpeSi7lgcyDiLDwlCYPiXAXpPL1JsdqJmoicASX3GGq+89fH+/UNMV8pPxyn69FoFvY9cmG8ZvPUnQdtmz1uFr4/ktBLuM+T+z0+sic+PeklJzLT3pSyEmZ9AeFEgZOY2kA2IrJ/Ffd6RCw9Ttpn4gybJTI2b7JDpQcV12mvZp+An9iuo+vVNTCwrPOWad6yWndq447Xr82wgzlpIXhtUsm+q1hLAd1W6LauUbdV9adhcQi91uETGaLTegidVpilWMosRUlXYfJpioA+2BP1nK/zrZiMJDygKf9h/2qfPtXOzrT3708c5yQo4Co+oixe3PJ7xs2w7PHp0/VTixnGoIsa+UhhRW7E1VAaoGlf2MSG5SBR+TWLepw3xQEUZV6LTt07G406L+FO+hlhNZdmeneuXnjqFyLqqqHQfOXRq/3ytgCecmaYgs8NnQ6LhVy+mPlNGu3p0b0cPoQDAI8F8LvqA6z9HSCeBeLTxUGsijmmB/rjsLFEAHkskM+qG8cSXoji2QD+tcpRfOGZq4Fu2aj/AsHdM/cOW8cNfHCID3DLOOqggzqqHx0eHO6/buHuUae5v3dzHNTigcag5nvmv24RqQ26qW9eDH7/g2fFY5rxiKZIHAxpDgY0RQa74ZsX7baBWYd2mKv5RTBWoRqpuHMpJtx79c+96W0YUGK5vclsWD6Eu4bTQWK8ZAHCr+yAuz5MGjbvmQsuAJQmOklMciIs74cnrMzWLT4Za+HC2tr83SJsvnxXj0SXAP5Q8E8rF3CREnZtLX4G4bap0P+qr8DU8R98OrxaU8cEuQFHshjHhOAVZMHUMkwtr+nUcjpOYWp5HEH0EcwtgyAaBNEgiAZB9HIE0c+xFlMbsRgTpNELkkbPZZUpqKOHqKOXa+F1EUjHa375QIi2fbfnaQYy+ngH9NJL7tSKBgA6tRvaqVXDEvq1k0imj6FbC5JpkExXWTJdWLmBVqtaqmnJP2YCGITTFRdOzxFjUF2usXY6whrk05WWT88hmkFBvfYK6vSAz+XV1Q6APQvY55UO6Ajm0/yYHgA9IdC/rQLQV3fIB5xnwvl9NXGGlS/lsMLKF1j5sjorXyrwDWZYApOTZUssYCHEwlbBVMjth+rzwRGWsiJmfLPDohiIPlggMyc3IMH8gV8hI58/i5GFpGqTzfzbs5g5uEP+Jlv5vb4aYkNYVQer6kCAuBICRFhYN7EA8TUIEEGACALESn+zVQaJdokNzLqZ2jvkmtKbYOasmmrEd34wI9bXnOI5FgWwKy9LnB7sS4aB5nXjwDa1CyRpOEBdUW3izFDHcU0B7corFP25B7Z2RjxOZQDyqgoWFxHggHrFlYvTo77hMieNeprvmQHInUDutCZyJ/EtRWWyxsWUzz23iWzK5eSz/PbizJM3T38yp+TPa8fjbt93NlztVGZ+GjWrYP/Fyp5Gub8vKQ0AsCi50cgAAAQWrD0aNwRMyeoBiEWpf8YOBUBiGonQsj5UHI1w/y7RrOJ+tyCvAHkFyCvGjlnQWeR0FgUfMM6mgdICvmD8TEqLwyV+NnaS8ZRn42mV/0TskrjZcIUJfFfz+XkXFxcA61oC69pdC87RAM4BnAM4x7BJBCAdSyYdI6SOwDqAdcBYz/ON9eSjEMZ3xuJaTeBawLWAaw0VDADZWvYIzzhLDoByPT/l8oFywUDP9ORjH8gHkA8gH8PFcsA+KsI+Rq2CAwICBATGfJ5vzCeOy2wswsjPWOTrAMgXkC8gX2Pp5IGDVXQEqHCVMhAxIGIwErRSZKQFZATICJCR8daKARup7ogQEBIgJDAyVK2RoVxMwgjR4KzBF1JqjbHp2iHQNaBrG0XXDGzZ24yzMeIkN1jZznC3btAmGJnJrF12nxYT31qMm/3vm+44Rse+eeW/evlf99bB7D/SMdnfgP+55X/uxV/TqfG/iKfP54MA2kstX+o7Yim68ioWewdY6jQs9cPni6vtS+6TL/8tMN4BUgofJIBxslUeJzsC4gXEa9PGyUZzrtTSuFVjXpUvPNCvqVYI9okX9voXIRUkTHIwoGCwShDGBZc1LnjFKiHW8mofPsNQ4CTbbTWOgXY+L+3cjP22ZDuW8RQWayzxIGNcdgMjX2s+wf5di9q/azB6tFOp77jXGrBl11y+zi8ALhoYBIQrtE/XzAiLEGbdT0C64tt0zYJ01Nf0o77mDmzaMwvIZ1Xcdk+BuGAkATCuznZcc8GYh3FcawPUM0F9XkWoYQ8u2IML9uBaoz24xhUG5Tf6mMMUT3bWaNFzPmu6kdcwDFMyKQCxuruBrVkgjtbsgTNWd2e0yaQLgGR1d1gbo20AKLWV2KNtLYNyXIUROOaGbVlXIHqohPYI9qsDeTiokwrVSVpNO7P4iEYnFE8ZIVbaisqtB0YfO+grJoEsm/zGk5Qq8CtMRG7EzVhU9Qaeqd+EHUxczBoJPbkXxY5vs5d0e0mk6TYrlOLPAydQJDoplRPF98InTNxFoZ3Rh4gQUnOLxPpcxIDTnpq4kn6GKNKupLBJyY/iWZU9Kbmez22arXL1HyEmD2Ior0gexet9XlmnPZSl9vB9pqFIai7l8VuZ91Ish2z7a2SItCOPsmmJMYutOIYCLLa0UrWzipw13yZ+axdonkoiWncYlFbB6TEqESGYBBLxpLascrdDP+Iw3GtrrMWnuMbehlhGwKhEwj12itFrjkIvauBUlFmwCIiCP+IC6encXMl5WvHJkW/IV1MywgBfyxupVdZKOEx9wQ6TMMWpXYZ7Sju5TZv7TBi0/T5rcob4UjGNfVJ47Hr42FbcFD7Jit8SZo+q/K7koLrr3dUasUaccccoTU9d5luMgZHBxZFJElqvkiG9pXyxsVFXDvbVA0VfqreU3w31YL+u5ih7gDSV3w1Tevr3+B34wgXFjUY+Rb3xoXpj9SnNA/VA+eTAkamWNy5Lynw/PaE11L9cn0atdMLlPccPKdYu45kubU/7PfZCbfvCi1V6eigEu/pxC7UOm/WjY8PsHrF/rzFCDWSaB/uGgY2GDCX9NmEK9a2n/wMIsR9wxyIBAA== k8s-resources-node.json.gz: - H4sIAAAAAAAC/+1cW2/bNhR+z6/QuA1IBnexnaTNDPShTZamQLOluRQY1sKgJdomIokaScVOA++3jxdZpiQqceJkcRz6SeKhePm+cw6PySNdr3kegHFMOOSYxAx0vGtRJApDzLi4+/ubuJ00ZDUUYA57IRKlnKZIlQ1ixD8GoiROw1CXUJgMzwgJOU5EeVMVDnGA9kjMKQllF30YMv18iOMLprpRtxT1KWJDUQC2mgzoMjJSNdSw9OBEqU/CECYMma0pwRDhwVCOHLR3mskY5IIExiictWS2puQwxJDtkZBQhcKkYQp7kLJyV0oQQDb8hOIBl4NuNSsyVPcYh4yk1JcTAMeURIgPUcpAoVYfh2G1WSzxbhWKQjRAcZCTN5vT5cDWvcQvpRTFvEYawXGdBMc1EjYkI0M1DAkX2hXWPHUJw3SGkSEr4i/0RNUqt64EIxwo+JtliaFaebFU1GOCY35EAgW+LPAg874jSorwJ0jwE3M4QFYOE9kKhQFOZTc7VZmdegF7gCiisu9+SHixT4YoRuzPS0SpsBnL+FkCfVSncUIo6Wm1i6Uc+hc28BhHSYKCTwJD61A5pAPEizZTthtVEY0TNR+WRuuxwLUbwwipkXYTEnR9YfpQdEI7+VXXT9JuygS2XYZEacC6Sk06ookuppCjaz9MxQDp26/gp+zyK2h4svm3/4oyefEVTDa83pW3LnrZAI3ysPqERlD5Ao4j2ZHEtlpNUIWoUMQD6HMi59GuVNEGdpC3d30tepxMQE1FgeiF4RSLdiJAV8wVBBPjrkg4H0qXSMLAogxyWgfCeVj6UrLTIe5zu5ArRw72js+9c0lDUQ157sCvy1YOKQrqXAChvAqftvEuv0pUjzgO8CUOUuESas19WlctJsWRjeEYs+q4eql/oVW1CjqIMlOXkJQoA1JT7U/VO7TcaanV0T6FKzhGc1jOTEVFd5RbFAr2UFinSyEZvIdqEWxVZNqHWx/TTrxZVc3ZjIvK2VjaSdhNTM+jvKSUrGytXJrblvDRCYJlw8lKPwpnoYIlu/iEjAoBUT6gs8zkTH26yQ617BR/V/Lha20tGRUvMw6qhkFtFwZVwqCWC4NuDIOqY7ktDmL8KpzHmSvTkRM6qy40WtnRLIT4S/xeHR292t/3Dg87UdRhltAkgVy4m7i+zelKKf5hBSgGd3Lb+XDtQcDUvxCa6YjV1/pTT1Ek6L4TDpCPI6j8kyUK02GVxYAy4ZkKWN+H8LZ6eXgD9qlwLl5ARjGwVj2n0mbBjdR8kdbr/fjOQk998FagL06jnrCFijyNsbGu3p/dE/SP8C+cOYIXIfj98hPs/ewoXoTivcejOFtH1e39if6EI+zseDGS95fXjjW9zooXI/j3ZbbiYxI4du/FbmJDbhksd8n5NCDc/PWXzftjyDjF8eBuGNZvbK7O7rI6FrPsK4u/nGpzobobcL9t52ofFPXVdhOw/Pmo2WluzI92hkwnB7yjAE/SjhgzvkSdi7RXIqErNElt6IgLHZKuHNTvHwPqJ1Nsb9NzVNdRvbd0VhWqAHHlgN53NvUyiP4dPIMT0c+pUIDlOhGlMGaSSTuPeYRmEbkTU3diuqInpkU7dSem82SObbkjU5c59pwzxyIUEXrVHRF6geOBiNl4t3fF0ZyhUsPLW/pB1HQZZIvGS0eKDn1+7K2PNonnQ3+INlxC2ROHR8ooXHj0QsMj0yxdhHSXnLJtFyC5nDKXU7ZgTpnV/7jjzuVKK9MhwkIEu8yyJc8se0COXVrKCieXZVy7/LKlzi97AGt2KWYrn2JW3JM5OT3dcGQvQvbBUht0RvNeddvNEX1Hoj88B6JPRzBxPC/E8+Fy8uxSg11qsEsNXurjw3ueHr6sDGGNsUsSfqKExkdT8RvzGh3p/2e68Pxou4zhF2dgLz53+NEop2z1A4WDp4dZpRCtPNAfnh5oNoLJyuN8CJ5HDp177cC9duDy6p5FXt1cbx6sZUMFTKxmEfyCKNPDaW3rYpkDJJ8IIL3QX7vlcDBTRiBjPBqjbENRt8VRlIRiXvGg+rHetTLvRjJaISGPo7FSgwD1YRqWEqWU1ZjSNYt+qW/6FpUz1x6wDzn0TnUOniHPTNjM0DOkJJl+gtj0r0D8VaRXaovPlslnfDK4VSgdoHFpVcidldH9WmleBnIwDL9kQBR19zZMa8C0ozhHsuIUacObC98tFuoAvQstyX81RgwiQSW2VJ+yki39d6FE9dTVXnY9/0OC4z5peFlzG3ay2reRlS1hJqnCNhQj7PO0f1CUVgYqy+yVM1XQMzEEKUNnuiHTKT0L/Wg+kH4U19FcPWQU+DC6YY03JzrgXA11WZuuW/rL6SqcyZ11X4eKAtDRq9b0ZQQR6GVloPBYgkW4RGcPZ5B0pwGyGbmAnZlagFbTuNkyb1rR7HrHuG6ZN1tNUzJ7ZQK0jetWoJX223QO8h0ZQy9u7cVs+LXZsNlLe9u8mR3zgDeBOd7pWArwfScq/Racn+1l62v++iCJkpQj7yTbM2HepveHUEBv/ZgETKshSFVCOGg3m9Df7Qe9fq/3Zru39Rvs93d3W61dtB20/LavK1/my3tzbfIf9DiwZVlfAAA= + H4sIAAAAAAAC/+1d/1PbNhT/nb/C07Y72MFIAmxd7vZDC6PdXbuxQnu3W3c5xVYSDdvyJJlAOfa370lyHDmWS0hoCUH9KX5P1pf3ed8kPdPrjSBAOE2ZxJKyVKBucA0kIMZUSHj66294vNlWzUhEJe7HBKiS50TThimRv0ZASfM4NhSOs9EZY7GkGdBbmjiiETlkqeQsVkMMcCzM+zFNz4UeRj9yMuBEjICA9loCGRob6xZ6WmZyQA1ZHONMELs3zRgROhypmaPOQSu7RCUjwymJpz3ZvWk+jikWhyxmXEvhZttm9jEXs0NpRoTF6DVJh1JNut2q8UjTaxILlvNQLQCdcJYQOSK5QJVWAxrH9W6pkne7Skol4RdYNUbtpNpJTIYkjUpg7fUO0+fibAZSu8XF0DV5Jf2cc5LKBm6CL5s4NG3gcIXaKaiJeypixMZujgTVjRs6BYnkUwAsXhVcUELdarZ3zRjTSGPbmuVYeluSlRWcMEDjDdMr0YQAi+Aj4awKS0YA/FTiIXEqSKZ64TiiuRrmoM5z6xWgEhFOuBp7EDNZHVMQTon4/YJwDpKu2sKsPUxtQvUFkAYhznBI5RXanm0WKqtRzb4+7uz/dHBYb1GaQh3BqZ63agzwGxFJT/W0m15WvmXqb5xNShNwcy2UOzWukDg8d6gQKJH1VFUDAYIiTV4BmMoI2p0qtRimpoNCkiwj0WuYpBNxifmQyDmwJJeZxkjkyeZ53ie9FHS0B+PKXPQm2F6HcQ4j8p8/oG+Knx/QdqCa/vwf0NQPRQAvrb0XtAuz/AO62apjPmA8wdoTS5rASAbFWrOJ7zrGodRa1GnA77js79PaaFqDwM6tuDSDKck0MFU8t+cXoJZdihOioe5lLOqFEN8woMS75a8eyKaXC7BxWD1QI9HT7qoLXfQox5LMJ++braB/FWzCKF9UzNfXMOLNzT1KuNFi5EjFfRZHDqeqlnUMEdIxluadjuhAuplShzZ0ePIueKdgqLpDWXqN69logzmJGoKKYFzWxWdiTU9eZXpEmkb0gkY5hKbGsDNpqzOm6swu8SUV9Xn18/Dc2Hpd6CgpQo4SyQxkSGmq+63mwFoGT50CupdwhS/nCSNTFYXhuHQoFO6TuEmXYjZ8gXWm167xTKrhfM3kGvWYYq34Tub/gItwm5hZxy1xaWOWWtoW5AoZwbOGU1B/BWehdwRu9ls2rmT95YTOCpOz9elTdmh4p/Sj5o9+MNZSQPE0k/16rt/xuf6XzfXb65nr31uSWp/LbVmqkFfxnTYcZ/UoZiyJTPOTP+Hfzps3O0dHwatX3STpCkfek2EJ9pI29zkJw2avge4UE8rpujOM6eao0BGnIw8nbqgK0KILjkhIE6ydX8e11zl3G1DBPNPbiRcxvq1dmTuhIw6eK4jYOEXOpu+49lWfhOa9st7g6+cOeJozwwp8aZ70wRZq/DylVtBeHN235F/wL1J4gJcB+MXqAxx86yFeBuLDzwdxEUf14+JAv6YJ9Xa8HMhHq2vHBl5vxcsB/MsqW/EJizy6C6GbuSS3Cpa74nhaItz9/rvdxWUoJKfp8G4ybD41XeTsfzWPrvXJhOPQGrac+uzBeXOzwJl2fQxOBvosCzk2H8tfFBSS6ZYC72qBZ3kX5kwvSFdfxVRA6E0uV+CHSUnXTtQv0DrdyQS7gYe6CerDlbOqWCeIayfoI29TTwPoX9AjuG79IwcFWK3rVo5ToZB041hmaA6Wv47117Freh1btVN/HTtP7eWev4/1tZe+9tLXXj547WVCEsavfPllw7bEiKc3ZvycpkMQgez1ryQR80q67OkraOnLMJfdF7zRcJg6iWBzvMtAi8IR2fJVmQ+8DdBG4bcBT3QbYJul3wncpTBz328EfGGmL8xc5cJMp3PzNQOrVZtp8o+lAPblmStennmPGPvarjWu0Cyw9kWaK12keQ/W7Os0175Os3rg8/b0dMuDvQzYxytt0AXMh/UzPQ/0HYF++RiAPh3jzOO8FM6vVhNnX1/v6+t9ff3nra9f8m5ywavJp1Vmb2TsK+0f5/V7s4p/sjjYg/4la+7nl7Yvu39yBvbkC/A/G+RcrH+icPzwYtb1SWsv6JcPL2gxxtnay/kVehwFev7bHf/tji/aexRFe3N9vrNRTBUJiGYJfk+4MNNp7xuyqgFSb0SYn5s/ui7xcKqMSOV4PCXFgaLpS5Iki2Fd6bD+N+M3ZnG3atUq1X6SXGo1iMgA5/FMoZS2Gpu74dAv/QlCVTlL7UFHWOLg1BT4WfzChO3yP4vLsslfwrf9K4KtIr/SR3yuMkHrL9e3K9QhuZyJCqWzsobfmFmXJTkcx+8LQVR19zaZNgjTLcU5KiEnkra8OfhuCNQReR47iv8ajBglACV1NJ+gUoT+u0CiR+oZL7uZZ9f/sD7kBkprd9Q3FmQHVsNpKCBb2A6K/rfc6HVuQ6+IaTbKYCwaIvHHZEKoyq3NXNHcjQvdMEuzGLkgZ6Yj20s9CoVp3ZPCVANrqS8qLVxYWaYf49B0wJwZ6I1JQddDXzYmkcz8lx46wSnd98AkjyDR8U578u0DpH4FDVVeyygkUHz6ciGS3iRltnMZdDDVC9RuWQ979oNVNI0OrN9t+2GvZXOmX2igjvW7HRmt/XuyBvVJjqUYt45id/yD3bE9Smfffphe/KAfI3u+k7lUxPeR6YJc9O7ssIi45Ve5LMlySYK3xSmKCHaD30ABg80TFgmjhijX9eeo02rh8Nkg6g/6/R/3+3s/4cHg2bN2+xnZj9phJzSNL8qA39q4+R94i1XE8mUAAA== k8s-resources-pod.json.gz: - H4sIAAAAAAAC/+1daXPbNhP+7l/Bsn07dhvHkmw5iWf6IZbjJjPJxLWddN6JMxqYhCSOeRUEfcTj/vYC4CGQBHUfpLz5EhOgcOyzCzxYLMDHLU3Tket6FFHLcwP9SHtkSSzRtgLKnr59Z49PL/hr2LQourYxS6UkxCKt72L6wWQpbmjbUQpB/uDS82xq+Sy9IRIHlok7nkuJZ/MqesgOot/blnsTiGrEI8E9goMBS9D3G4EepXl34g3RrKhxLNXwbBv5AZZLExkDbPUHvOV6q93w7/U0w0cutoclyaWJfGRbKOh4tkeEFJ5eyJnXiAT5qkSGiYLBR+z2KW90s1HIw2U/oyjwQmLwDuhnxHMwHeAw0DNv9SzbLhZrcXk3M0k27mPXTMEb9um2r6qeyy8kBLu0JNdB92U5lluSEwy8O0k1pBzKtMsu+dUtssOhjKS8rPyZnoi38qWLjDvLFOJv5HMk1UqTuaKeeZZLP3mmED5P0FCg/cDEy4rfxwwfl6I+VmLo81IIMq2QV9Mu5qmhZ2I3McGE192zPZqtM8DEwsHnW0wIs5msuuZVdqi2vCyC/2GSpFkVSmzFE9X9fNo6eNPuFN+IFa1RyOCGOzTmIray7qlzJXxahdyAIuNGAX4O/lHdti3HGtPp0zev9xu16LT0lNXawEcGLhtnWCY3ymYrmxpXUzCZgGLfx+ZH1kilglJE+phOoHr43hcSDkJn22XW1HWRg0VLu75ndg024CNWCTlK/+oaftgNA2ZR3QCzVDPoisHhiBXRtQii+DEt448r/Zf04Up/obEyeRr7jz8Zdsg6QnhK/OeV/rSjXT9o22l1OwrQPeIgMT1Qy+Gt4OZWfI1ZLyZsbDpFBhVq1CpRgdO0vMfHtN6nJ73kdSb0G2m2zOkF9gW4UxmCDMKVy5NuwuscAF02qYrJppsMEo8q8b3QJhV+Uh5LYogywV+5O1fuKqVdPtpVQ9DRsFR7MZeNrjMLuXSEowPO/DzbVMzZvFenjCMp6hJ5FwOrR9WZVPBVvXP2RfvCx53sbEvTUf4xT2YQwWYZ0/EILUovojJd+uCLGi3XtG4tM2TMp5TVJO8Kzpxt2T26t4Jiu65D4yYam4tC152Y0XCR5CDTuZapf1XO21JuJhYB6i48oPtJWMpQQ1l1hCoUCl1ju0yXbK9/jATXbxbyIqqq/FnEVYvzu9TjqUaANXZCbWJRP8bwiK18ampbbBD1McobTpz6gY0VYk2ozj737jLrvrRBl7HJyfo0yg6jvAvrh8gfHEbWEkMBy714udda2HJPYeeR9qkyYLG36sVezWi/5RoEs1FtO0vxjR4j9gPiUWbaZpeJ1vISqj85wU9K/IklljL+b23n+06e9mt7Y9q28hZtwkJkQuo2WnnEsjyxRoNY1DLYcFG2Ni9dXZfleb7gU4q5WYw9fNx42WoXMh9itsWEw4jkpN1eCCu9jMzEcvtATddMTeMpIHStxXO7JrDTerBThT0CRS1Q1CJD3YcNiQJHbQJHHclRi20ZR1ID+mBPtStyWZxwImXHQ8L0f/Zv99On3ZMT7f37I8c5ChREzEeUjTlueZnJjDmwTBO7+my7GWo/VYE5KQdcIxkpsgDN2mETG5aDxPjUUnGwG7UBxZmXYjVxbKNx76U0Rz8hbHDRTO9O4eHkr34h3Gb1kdB85dar/fxWAU+5fzEDnxs618wWCvmCFaST6+zono/epgOAJwL4uPoAa/8DiOeBuLM8iGWiPzvQH0ftPAPIE4F8Ul07juAFK54P4HeVtuLETwYYz4SxUS6/KlhxxVGVBLn38re92WUYUJJ3H46VYblXc63RP/PES0wX/SMiWRXudrY2Ldknm80bX6yD4J5wTumKVcr8oSmx2I5SMR0JNPzwiLXZusVHSw8Pqh0Ox8vAoZomoe1poCQzKUmncsa6gBCz2qFwAqYKKjJaRd7pNYhF/CtkqlOtDV+C3IAjqcYxJZuKLIhVhFjFDd0NztopbARPEqt4ADvBzyVacQVH01JNVVKHZ3lwrXYiqe+xtiGndrDjkYfunUduLLfPlgC0e/1A8bxHfAoxpZbDjDp6gONsPGm1x9kimOFE2/JPtK1P0ptyqO2TkGAUL6Rt/31xsQMhxGteMYopAVaMz3TFKBskLBqniR5uw5oRoochenjO6OExhAACW6oTRhwRhblghkjiikcSLxBjCEPc4GDiGGuIJ650PPECrBlCijc+pDhLwc6Bgs0J9mmlDTqGuYOMAQag5wL6zzoAfXGHfMB5LpzfVxNnOAoCR0HWh+oGHQWp0Y758zoCEsEBAf5lUCzlFMjSrWGU8o+MIAd1qMx5j8mhgCMfdbHLqWapBRkqnPZYlnaQYGH6oP30hxbfHplNOvt8UkP5n65E/gZ3sAACSgT+XAkCwR3yAQAlAO/1esQJwpkzOHMGEYS1iCCEY2fTHzs7zCbFQzwvrOnoEF5YCC+EC/Tj8MLDdd6fz+8NkNiWiylfibI1nYHZci9aisbX1M/Cvv5NF3bfful2eWXdxDKSS+pZ9ooPFrEa13Ex/aJp1XmEkXaMXDMyKjh9sV7udOzD2Yu1MKcXm0AgXgGBAAKxKQRCLK0diwKDqDCDuIxBAgoBFAKcL2t3vhStEDwvkxCn10CcgDhtmueFNZ9PZcCcqul7YdLUvJ4W+2BM7SyCC/jTur+hB/wJXDAzM4k3wCSASWycCwaoRB2oROKMocAmgE2AN2bt3pjELvO2CD6ZSZhUPg2oFFCpzXHKmMTjlQOjqpVzRjuJYANaBbQKnDT1pRZNoBZALTbWSwPcoqbeGqAXQC/Aa1Mtr03BJsF7M3xreCPQbnNi8tUCggUEq2oEi63z7W3Gshi3EdRG286xrV7QJRiZ6R5Y/nYMJc2amGW1HcarVsqnznlnVv+lnknkfEcsKeK77oL+W/Sm/rz1w+ezCyCmcLYePF919nztA/kC8lVB79Z43pU5BrYEUvAMyNdkvGuTBL0p5OtyQLywPzgLKVAwOF8HvsE1+QYv2LjDr9H/8Fnb1U4sflf1dchr2T7zTJbEx3XtVy0adXbAZbgol+EBsFZgrVV2GUo34a7AcZglVNrv2gp8aOt0oS3rS+5r9KRtiyXA7/mZAkgd+NXAr1ZDvxp8/hkYSlX9apOQk0U7fSanKMutGXjKIpxOgq1EZAW4CjigwAFVPQdU+vk/8DuN4m4K6nYI1K1A3ZrPgrpFs1AOaGYOLPEgJ1xWgFEc856mpoK56MeECxb7No4MBvTBnmTeSD8geVmc02b7DqT0qUd1mcmkPLBME7szfu9y6CSp1udqd5t1+17t22p+0FQArPJ/AcLTInxcYYSFCbP1HyC9CKQ71UQ6Xin68UoRPjA+F8gnywOZrx3nhljhBwCMp8T4XcUxFpEM8agNUM8F9WkVoe6Uf+0esJ0AW6NcflWYkCuOqiTIvZe/7c0uQ+4Ac/vTybDcm77E/ZX17G9U4Nuib/XlBDGPFvqyw1+qLvXjtUh9A+Ocqg50Z41Ar3PntgKiP1nnyPbMZf9uY9W+qqESFQD9tOqfKe9EO4HacJcWPlYOgZ8QTlG/cIpxIRRbcbv1wBhgB33FJIjaFh1ViXZn+S9MRG5EYcyq+kPN1G9CtvZ3MT+cmJZFsePbrJNuP7U03WaNkvR5qARS2EEm9oLie6ETJu6h0M5tiQsTknNVYbh83xZnNTVVJf0EUaRdROEWUn5sz3IwhpTr+Vym+SFX/yfE5EFct6UK2uDjPh+ssxrKUvv4PjdRpCOXVP1Wrl+S5JBtf40FkVXkcTItEaZaihPEpSSSloZ2NpCz2dvEb21FmEeJResOg9JSvJ6gEvOBaSARNXWjIXebK2uXMYWu5fa8lGnsqMFqjQMrns9kUJltCESCv5L69WxuoaE8Tf1yrApRT6SMMMCXUUHyCFUL/WgsWT9SYrgYDVFeOfkk3TkJmrMpmsMgX6LOlF9T+iQWMJuhR1sJL3qKWIAlRBvP/71oQaK73t1uM4lwZguJOE3P/My3GB0nwx/HIknvbpWZsd6WPkjRbEgP+/KDdB+y3pb+bsoP+w05R/r4aEv6u2lG2vw96QMPvpcUZmwtcsGHcsFyLa0D+UG6tfGVKbc3aUtGfD88EWunf7nsxJQtXdh5jh9SrJ3jyO4CbU87i1VfD0UYqX7Yft3EB4f4ALeNV9fooIEarw4PDvfftHHv1XVrP3r5NmWKja2n/wCbu7mVvPQAAA== + H4sIAAAAAAAC/+1dW3PbthJ+z69g2Z6O3cSxJF+SeKYPsR03mWkmbuykc6bJaCASknjMW0HQl/r4/PazACkKJEFdbYu0Ng+KCVC47LcLfLtciLfPDMMkvh9wwp3Aj8wD4xaKoNB1Ig5Xf32Hy7sX4jZqO5z0XAqlnMVUlg18yj/YUOLHrpuUMBIOz4PA5U4I5S1ZOHRsehT4nAWu6KJP3Cj5vuv4F5HsRl4y2mc0GkKBudOKzKQsuJJ3yGElg4NSK3BdEkZUbU1WDKkzGIqRm529VnhtZhUh8ak7bkltTdYT1yHRUeAGTErh7oVa2SMsKnYlK2wSDX+n/oCLQbdbpTpa9TVOoiBmlpiAecoCj/IhjSMzd1ffcd1ys46Qdztf5HPKLom42Wx7+UZcOqC+nQGrznfgv43OC5Cqd1wOdIMX0o8Zoz6vqPXIdVWN41fUMIHaGaiJfijRMLjS13BQXbeiUZBIPAZAqcuDC0oo7yq2LiuuHFti2yrWKHqbFQsrOA0AjY+BnIksMEhk/ENZkIclpAC+z8mAahUkFK0wYjux6GavXKfXK0DFpowy0XffDXi+z4gyh0afLiljIOm8LRTtYWwToi1G/wZJ8rx+jgwxkN39eNLZfbN3VL4j1eJWqUKsCuOVooytqrz6WgWfTqk24sS60IBfgH/StF3Hc6ZM+uTN651WIyatXOW1NgqJRasWMagUNtvu5EvTbkomE3EahtT+HQapVVBO2IDyGVSPXodSwlHsbfhgTV2feFSOtBsGdteC3YRAJ+wg+6trhXE3jsCiuhGFUjvqysXhAJroOoxwepu18es386fs4pv5woA2RRn8J64sN4aJMFGS/vnNvNs0ejfGRtbdpgb0gHlE7j3c8cQohLmVbxut1ifE4lKNOhUqcJK1d3ub9Xt3Z1bcDkK/ULbigl7QUII7lyGoIHzzRdFF3CsA0IUdW+5k3dEicfufoAeiE7dugaJwugW7G3OsqEq0L4xZgRn1BUWANoDyzd/85j8mEtUrYT1ASJasJw1B1aq8MACVKyMfCjoauLZmrxezOgHipulL1p0NnT7XV3LJuMyj0y/GF7Fe5Xdpnu0Ot0USRBi1K7hOFDBell5Cgbr8JpQ9Or7tXDp2DIypkg2N7pVEPj+ya3LtROVx9WLrIlnTy0I3vZQJCZEUIDOFlum/Vc33Mk4nPRP9FG7I9SzsZqyh0B3jGoUiPepW6ZIbDA6JdEDapbqEAWu/llDgMi9QZjzX6rDCSehNLJnHFP7xrFia2RYssCElRcNJSz/AWiEdVX315+Aq54xmAzpPTU7Vp0l2mNSdOf/I+uF+Yi0pFOiDpj5opyY+qOZrie7qKtADrZkH2jBfxPEtRmHJ3Mj7HVYfvI0hCzisG3YXROsEI/8j5WIWsS8d2KXn4VpZHz9AYaVj8tdP3a5wb7ojE/y+WfRVjO0pY6/diJ+CdzUjr5ysfDLWMLJmizncsWC5qQo4VIYMquqCUJI9DXGQa5dYd1529kqVNykVBOEAy5112vdCmc8TM3P8AfLmFfPmdAuJfef+iWcbqXMzqLPGHpE/l/hzmT7v4COcxyXQbSTQEwl0eSzTGHTEb9y5niOdl3ezxJLomI39G/5tffy4dXxsvH9/4HkHkYblhYSDvfjVbY6246Fj29Q3F3v+o4/QlWiZdjW3RstQHqBFJ2xTy/GIXPw6OoJ3oTegtPJcujqHLpl2X8ahzGMGK5dhB1ea2K649QuTa9VEaL4K6zV+fKuBpzqymoPPj70e2EKpXlKObOdeHN3Pkx9sIsAzAXxYf4CNfyHEy0B89HAQq17E4kD/PulZPYI8E8jH9bXjBF604uUAfldrKx4F4RDjhTC2quVXByuuOaqKILdf/rK9uAwjzoqxyakyrA6ZrjRfaplMkfnypWQIQxPLB9+04hnfYqH+ch+M9mXky9R4Kcsn7KRiO8jEdCDRCOMDGLNzSQ9mSqhaKxwOHwKHepqEsW2gkiykJEe1M9Y08W6tUDhGU0UVmawi78wGZGH+EYPq1OtpMiN+JJDU45iRTU0VZmliluYTfdSct1N8yjxLluYuPmbGPM1mnBTMzEDLS9byHGHjRNLcU4Zjwu5RL2A33auAXTj+APwL3u3dcBpp8mKXO0tVypZ1PDDz5ALPG4qi+pw3TJQCjxyu9sjh6lB4KqcOP0oJJmlNxsafZ2ebmEa9YsdWbi7o2K6pY6saJPq282RQ76FrixnUmEFd5wzqKWwDk3vqk0qdsJClYMZs6ppnU98jxpiK+YQTqlOsMae61jnV92DNmFb95NOq8xTsM1KwJcE+qbVBpzAfEWtIEeilgP6tCUCfXZEQcV4K5/f1xBmPw+BxmNWh+oSOwzT6wf56HYxJAMJjD1VQPMjZmBXYxyRzmJhpjwpSm3Mxs0OBR2Oaa6lz7WT3ZLp4Tuah9IVFD6ghxg+/GumPfuaLTj8dNxCRk0dBxBKBGsRkRkx+exRMoisSIiQzQvLebEbCI57xwzN+mArZiFRIPOY3/zG/fcyFfNxcSHwdQ5oLub/KtzGIH3pQCJxPuXCJwZW0KHiZiU9c+VKDRQjd/zIPs/IlBlD9yIe1oMdVvLjgvpna5wQ145D4dmJmeDJltXTsMMRzKSshYy+eAid5hZwEOQlykpSTSHffcziSkkaRkvMUNmQlyEowRLTyEFHZCjE+NAsXe41cDLkYcrFCfAiGL/ZCJGNNiRCBNI2gb6SRIts4TQBESrbqN0EiJcNA0cLk5A2SEyQnSE6KgSJkJ81kJ6OQEUeCggQFY0YrjxmN7LJoixg5moWcFcuQnSE7w9BRRs5sFojOkaQ1PIRkHCdAIlNDpoahpOaylTayFWQryFaqYklIV55MTAkZCzIWjC3VK7ZUskmMMY3vGv/e1FZ7Zj7XQT6HfG6t+JxFHXcDSB0QJ8mbjI0CuetHXUaJXf1g0KaXDnA14GWeZ/Xci5fhy+f/9S89Cv+xng2fkfi4FB/X8tP2tsQnEeXan325f174qHzws5DX479AaxYor5gz6TzAY2HZGCj/lPJqPrP/8On0DKk7/t4EhhubHG7cQXqK9HTdwo3TmenkM46N4KdrQE9nY6aI5Vry0/MhC+LB8DTmyFLxwCsGmFcUYD6DdUe86ePDJ2PLOHbEz+n3YtHLxmlgQ5HYOoyfjWTV2cS4833FnXeR2COxX9u4s/Kb3nNHn++Fz02ic8ZzY+4w6oMPavOR8xuyCT2FLAcRC92QTtDz4kaGnBMjoxgZbWBkdA8JFBKodYyMzsKdpvwG3CoZ1IqHhjTqPkJ2kkwlXAqpFIbvMHxXv/Bd9n5XjNpNopYaZonvOnhkZtleC2aZbHEFTQFbg8LdgnChAau8oN7NzVQL2c0jqlqe2zSuGvEbd5ZNKXv98Hl5w1zsLcLKi4L1bY52/KFj29Rf8G3J4wBRvV52vtVu2tvO39bzddgSYF3sDxGeF+HDGiMsTRi8T0T6PpA+qifSqRsapm7oZs1fdV9zkI8fDmThmC4NsSbIgBjPifG7mmMsk0zSVRuhXgrqkzpCncUiENuFsLWq5VeHDbnmqCqC3H75y/biMhTRNX8wnwyrQ/UP+Pin5snUiz+fqcELqd+aD5MlPxnXyelHiOvyuB6uBNfG2+siyXKorsur69EK1bURJ3Yaje7xKjcZhPeh4X2HxruK5CJU3eVV98SseZLSUfL83BhnRtQrQUn80J6AVA9o5mtqqjAXHFOY1imFaVra0rN03GZkDalHvlIWJWNLDtclSQviGzZhF7IxsKrBWDPNi7hHmU/FceqsLU690IVJ+oPM0kwXBqXo81gJlGSdXL4Tp9dSJ2zaJ7FbyBSRJqTW6jLzRToDzWtqpkrmMeHEOEtSnJT61J7VBCilNgiFTItLrvl3TNmN/JVJXaKUWPfFYp3XUCgd0OvCRpGtXEr3zwrzUiRHXPdrKoi8Ik+TaYUw9VKcIRdsJGllaYeFHDZzm751NdlPFRZtegClo7l9hEpKD+aBRPbUTZbcjThMGY3Q2i3Y8Tndgtkwx4qASmRMZFOPXmcaeukGp6IMxiIhiv4YDcjM15ZGLsr0N6e6kUxNqYgjep40pC5ZjVCY1gMrTMYbF1YZoSndrJmu0Jk46oZD2HIm6JKe1N4prBZ17KnoGPghy2kXNADeSz+YW5+qnaQ76SU9DR17NqJbdwm5cKTYU1rRT/wc0w+uttqjwwrgn6RlZu5roQMsn42/nIokcx1Vwm3uKS+hareUix31QslmNveUv9vqxU5LrVFei95R/m7biaZ/H81BnKNRlGlqL2rD+2rDai+dXfVC+Q3kV7Y63tFYcuL7J5CZreaX86OUCWb+YuCFMafGZ5rYZGRsG6epWZixzAg39/det+nuPt2le9arHtltkdar/d39nTd7tP+q19lJbr7MCGjr2d3/ARnJjsv7AAEA k8s-resources-workload.json.gz: - H4sIAAAAAAAC/+1da2/bOBb9nl+h1XQXycCZ2Hk4TYBi0CQNZoB2N5OmXSwmhUFbtC1EEjUUlUcD728fPvSgJDqxkziW3NsvtUiZpHjPvTy8OqHv1yzLRkFAGGIuCSL70LrnRbzQcyPGr/78xi8nLXEbdlyG+h7mpYzGWJaNAsx+d3hJEHueKqEoHF8Q4jE35OVtWTh2HXxMAkaJJ7oYIi9S3/fc4CqS3chLiocUR2NeYO+0I1uVkRt5hxyWGhwvHRDPQ2GE9dZkxRi7o7EYub291w5v7awiRAH28pb01mQ98lwUHROPUDkLk5Ze2Uc0KnclKxwUjT/iYMTEoDvtSh2e9jWGIhLTgXgA+4wSH7MxjiO7cNfQ9bxqs66Y706hyMMjHDiZ8fJnuh6ZuhfzF1OKAzal1ke302rcYEpNNCY3GjS0GsbR5U351jXy4nyOtLri/HOcyLvKrcuKG9eR098u12jQyooFUM+IG7BPxJGTLwosFFnfMSXF6Q8xt0/A0AgbbRiKVihy3Fh0s1etM5ueT7uDKaai76FHWLHPCFMXR/+5xpRynzGMPwrRAE9DHK8U5ulsF0sZGlyZJi9iOAyx85HPoXGoDNERZkWfKfuNvBHfhvJ5othfvwxEUcBntxcgH8vx9kLi9AY8ACDeFT3MPvUGYdyLIz7DvQjzUifqSbAc8oZ6LkUM3w+8mA+Tvru03yQfL+2WlbUsyrOLS3siev/ZIsF6VtjifW+oQY0oicOeh4ds/YbQK48gp2Wln3rsLsQbecu9rII3cHgV99VTkBvxBBR7qI+9OUeXdyYq0s96uRyEqBT/y8fZsPp31rp6BrtVnvghoT6S0Y65vphEgZ7qbRyMmHJXO0UDRoSltiu3qBBymrV3f8/7nEzsKTdyzFxpYb8YCTisJDYLFRPtqghpNhZBn3iOAe7isU55eDT0Jes+j90hM1cyuVTZx2dfrC8CYkVHY9kSdV+OY4hiZ1qQI5RVp09FMWk70aMbOO6168Q86E0NaOm9crksjuwW3bpRdVz9eHClnLE66bafBDMxJSWT2QKA5m9ND9lZWJbrv/kR7tAtniE25BDl3VFmAJTwpGlY8sjoCMllvlOpU6uU8WtqmWpXoZk/cRGcrdo+hNnF1HOUF82Sl62VSzPf4qtQiFHZcZLS33mwkHTQXH1ObgqULxvQReJyOp4e8kNV99n9LuvHXeUtiSl+TKZXJXrbQPQqRK8DRO9Bolcdy2NML2J33izBXLqOeKCL6kKjwI5zCvE//m/z06fNkxPrt98Off8wMlCTEDEeboLpbaYrJd9DOjiw5wrb2XDNJCCNL4QmGDHG2kEaKYoGeuoDO3jg+kjGJwMLU7TK4EBJ5YWk5Eceeuy+jN7YJ5QHF8vhxNU23vqFCp+1HzTNV+G91k/vDeaZTt4K5gtiv899oVIfB662rj7duuf4Lx5fWAQGfo6Bj+pvYOufYOLnmPh4cSZO1lF5+XRDf3R9F/z4eUY+qa8fK/OCFz/PwB/q7MVnxGmGdat7mIUbd8vZ6u697eDdLt7Fe4P9Ptpto/Z+d7e7c7CHh/v97Z2tq7fRJrel3DVGmyFxfr1GdDPfSb57k3/+l6hK05BpElIW5lnIPAcpK3iD7970egPM95wPQS00mbEOYaTm4NKmcOuXn7eePocRo24wmm8Op2dZIZnfnGS+fM9qSOPzHb7M5RgD1xOy/NU+KB7K7J5t2OtNSey35sVTNv85dtJwxz8omj+3NdIWeAUHIqBnueg5Whx6IBotAE9b4J0/kHceLye2e3LrB9hpNHZOILKvYmQH31wB3/xgN0CM80fMvbVeYhyKgkhY0mzHbD9uqAKxDoh1VlSsU/RTEOvMIsveAbUOyLJrLMvOiZ+PfULvJMdygxFn0qzXv2N4fgaYNfkPXiUKXJ+bRl2AHBvk2KWV5ZPEHSiya0HypMcDyftBSZ7RFYHnzSDK3gWaB6JsEGU/U5Q9lQqAGqw+umxFEZ5lYJBm11ya/YI2Bl3nCquzE1uDQLvWAu0X8GbQaINGGzTaoNEGjXZjNNqQ2QdtdhO02QqdAKCVlWdDIGqmLBscE5TZAB8QZ0NcXyVRNrgl6LJnV+WANBuk2aDaaYRqB9TZc6t2iiKRNMKLtjq+DYoeUPSAoue5xywqL7DO8QC719g6QoGj0AQvEusp7zkKo2fa+kIwNL5NAWPXXufzdGOfcxtYZJg6tmOdIcWiwdQ1Ffs829SpXzOwdu0lP+GLO7Z1QomgMmDyuiqAFuHgYPWXsPppHa0Oei/Qe4Heq+l6r3Xx3kiecbSevy4KMBOvSXpUreHqZZw6E2nek5D+5BgUrffS7Ni3jctAvDayKu+NTO+MlvPK6P8zvjPK3xj9iBKvB6HDEi4A2FkF7By9KnbSsBMqDgngaTZ4jpcTeAA9K4Gek6WGHkdtXwFEzQbRh+WGIEDRSqDotPaHYSZv0P6tgFjHM5FAfQXqK1BfzeyzIMOqyLAMh2R2QWtV0VrBIZmJ1qq7pDMyIbP6KrwQjsU0c8GHFXNwNubr8zwhlgKWtwSW11oFjrMPHAc4TqM4DrwCBJKzYJLziFQcWA6wHMhlvV4uq+qFkL+ahdu9BW4H3K523I7jDfJXQO2WRO3ec6/hzmodp9jLOZ6Y2zNu3OxvTID0AemD1FZj6c8B0B+gP42iP5DaAv5TA/6j/cElUCCgQJD3WlLe63FfhYTYXIywVAaUEChhUxRd8GdHwAkXpuma5dAkIH+vT/5CIH+Q/3o62+kA2wG200xtF9AdoDuLpjuPHRwIjAcYD6S7Xi/dlfpl2RchtzUT29sGtgdsr9G5LTjPAkjfa+e4jCfJAvMD5ge5rkaxnx1gP8B+mp3rAvoD9GcJOS9gQMCAIPdVr9xXxSen5MDWklHb0WCMffQV00iNrLOrisXvuIlvOIheycb4IjbKUWmLiE75ioTlzzWothj2Q48/YjDKvIyTh4hpWM4hoBGnAn1k+FYiwsFDFHulRV26j15rYqPiF95wEacZkOwTThitz4oxavWJL+t8UqslobBbOb7af8WY3kkHNPFOcU6jiMxFfPLSEb4tHeyYRS2t+7XSc2kzhzzvazIRRRg/NqdTJtM8izNQ63SmtbDNgzRfyh383jMQ1Sn+bPvclK7h9tQqCTmYxySyp54Kt+sZ/XCDIWlZSXMbZmNtP2asZK3Sjcp9Q1ok+iPt3y7WVgYqysw3J1BQT6JVxBG+UA3p8akR+GgvGB8ZSXwZhBh56UQjpoCcVUFOunF4MnAWueGZ5BsbQNyqIE7OWR3Rpm2vC7vrSWl7vRpQXEup/ERRV1daJyGtQ7VltgNys9lJd5F8Z5uU2YWvhS5n3jT/cjIlWT5F38rZe9rb6U5bu9jRL7Rf67b3tM8d/WKnrddoR5xsa587jnKIb+kziDyHBrlHe9Eb7uoN671s7+oX2uZj39HHm46lMH3fiUya2V8ujpN9RnYSMfHDmO9uztMfJbO2rP/q8dqOZQrXRp3uLtoftnd2DoYHB/jtwQCjvUF/dx8f9HG3s69uvs72OO21yd/0NFeRF9oAAA== + H4sIAAAAAAAC/+1d+2/bOBL+PX+FTtdbJIvkYufhNAGKQ5M02AXau2ya9nDYFAZt0bYukqglqTwaeP/25UMPSqITO2li2Zn+Uosj8zXfDD8OJ/TdiuO4KIoIR9wnEXMPnDtRJAoDn3Hx9Ps38Thel69hz+eoF2BRymmCVdkwwvxXT5RESRDoEori0TkhAfdjUd5ShSPfw0ck4pQEsokBCpj+fuBHl0w1ox4pHlDMRqLA3W4xV5eRa/WG6pbunCjtkyBAMcNmbUowwv5wJHvubu224hs3F8QowkFRk1mbkqPAR+yIBISqWRivm8IeoqzalBJ4iI0+4mjIZafbrZoMT/oaR4wktC8H4J5SEmI+wglzS28N/CCoV+vL+W6XiyKO6RWSL7vtsFxJgIc48nLFmuMdRu/ZeUWl5htXQ1vn5ewnlOKIT5CG6GaSxI8mSKjU2mcBE3tX2Ihc2yVcQDeYUKmYkaRQgCErK1eAUL1VrV0Jrn1P6bZVlRi4zYulFZwSoY1PRI1EFTiIOd8xJWW1xFgoP+JoiK0AiWUtFHl+IpvZrcvsuBJa8TDFVLY9CAgvt8kw9TH7zxWmVMy0pf8sRn08Cc5CKLXX3iqXctS/tE0e4ziOsfdRzKG1qxzRIeZlg6wapXoR38RqPCwJVy8iWRSJ2e1GKMSqv92YeN2+8C5INEUP8k/dfpx0EyZmuMuwKPVYV4HlQFTU9Sni+K4fJKKb9N2F+yb9eOGuO3nNsjx/uHDHsvWfHRKt5oXrou013akhJUncDfCAr14TehkQ5K072acuv43xWlFzNxeICg4uk54eBbmWI6A4QD0czNi7ojEpyD6b5aoTUij/V8NZc3q3zqoeg7tenfgBoSFSrpT7oZxEiZ76a5nzOUF9TqSmtmqvaB90ktd3dyfaHI/dCS8KzFwaa0rZEwhYKWyWBGPjqQxpPpIrCgk8C9zlsE6E77W0pWSfR/6A24VcOU336PSL80VCrGxoPF//7qp+DFHsTXBXjFBenz7txZTuZIt+5PlXvpcIpzfRoWXvqrW43LMbdOOzer96Sf9SG2N90t0wdWZySioqcyUA7d+a7LJzt6zIhX0It+gGT+EbCoiK5ii3AEpa0iQsBWR4iBSHaNdkehGzfk2vYq06NIsRl8G53thB2E1Mj6O6aFasbKVamtuWWIVijKqGk5b+KpyF4pp28Rm5LvHJvEPnqcmZeLrPDrXss/9dyUcdbS2pKl4njayzyC1gkS/LItvAIu9lkfW+PEQjGb8NplkplF3KAZ3XVzFtSbjgJ/8T/zY+fdo4PnZ++eUgDA+YhffEiAt7iSbXmS3DYvfr4cidaU3Iu2tnGJnzIjTFiNWR9zM3VFbQYwfs4b4fIuX8LBRPczaLAaXCc8X3DwP00Hs5d3KPqfBcjidYsWt99QtVvupe1XyV1uv8/b1FPZOZYUl9URL2hC3U5EnkG4v247V7hv8Q/oUzUPBTFHzYfAU7/wAVP0XFR8+n4nQdVY+PV/RHP/TBjp+m5OPm2rFWL1jx0xT8oclWfEq8xdBufQ/z7Mrd9DY7u2/beKeDd/Buf6+HdlqotdfZ6Wzv7+LBXm9re/PyLdsQulRbUrYRE+9fV4huFNvUd2+Kzz9JURbjzCKcqrAIcRYBTiUQFb570+32sdjQ3ge12KbGJriRhoPLmMLNf/68+fg5ZJz60XC2OZwcwoWTgsU5KVCBIMsZgdjhq1CP1XE94gih3gbFAxU6dC17vQmnBuuz4imf/wI7mbsTHzTNv/s/6Ym5k69uiFFzvBFiYQ59Jmd4Vk1ltQuBACkga77IOnw+ZIGnegY8bYLlguUqyz2az5oQqC0j4GppcXUMK8Iyrghgt0tutx/cBcgs+i0RltyszCKKIiY1addjvv+3iCDzCDKPljTzqGynkHk0TQL7NqQeQQL7a01gLxhniENCbxWB86OhoPC827vlmM1M1/Iq/yZEssAPhWr0AySuQ+J6Zdn6pHAHueuNYJDK4oFBvlIGaTVFIJFTpK/vAIeE9HVIX29y+vpEngF5c83JYNf840kKhiT2hiex/0AdQwbsEuexp7qGVPZGp7L/AGuGbHbIZodsdshmh2z2hclmh2MDyGJf9Cx2jVwA19ImsoOTWr4EdjBayGEHaEEaO6wHkL4OJgsZ7LOkGEESOySxQwrSQqQgQR77zClIu5CCBClIkILU6Bs0tZE4Z7iP/SvsHKLI02iCk89m5iMdxuyJuj6XFE/sgUDZjU9Meryyz4QOHDLIDNtzTpGm4aDqhmYnPVnVmV1z0Hbjc5TiH27YzjElksqAypuasvQcBg5a/xFaP2mi1iFBDRLUIEFt0RPUVuWhlLpqarU4i4owl+csXarXcH3Sp6+mSs+l+si78hmhs59GjX8XqJTtdbOA27e1i0ieRDm1oyjbMdR8TqH+nPIYqjiEeo1ZaveCiafsANC0nGg6fFE0Za4p1jwT4LRscDqaj3MCPC0pno7n6p48vQ0GWC0brD7M100BrpYUVyeNv+M0Pa37t4ZmE2+jglQxSBWDVLGpbRZyxmo5Y5a7TzuQNPaySWNw92maNNaZ09WnECKeEw2F+0/t1PP+ZEC4BPXlaaXMAwNSOQdSub4MlGoPKBVQKqBUcFAKnGpunOqBpHsgVUCqIFL3cpG6uhVCdG4aKvkWqCRQyddFJQUcIToHTLIxTPK9sCNhvs5RhsaCUsq5PRXKzf84CDgmcEwI3C0s29oHtgVsC9gWBO6AbjWabhl/mAuMCxgXRPXmFNV72FYh3DcTAa2WAQMFBgrZePBXkUBBXzQfb5q7vIBrvjzXjIFrQnTv8eSqDeQKyBWQK7gjANhVA9jVQ9dnAsECggXBvJcL5mV2WbVFiNxNRS63gFwCuQRyCRcGAcdsbgTPen0zEE0gmhDJWyiytQ1kC8gWkC24Rg/YVrMjekC4gHBBZK9Zkb2aTU6I8K2kvXZZf4RD9BVTpnvW3tHF8rca5Tc8RC9VZWJZGxaoVD/2TcUahdVPsui6OA7jQAwxGuZWJugE4waWCwgYTKvEVjm+UYjw8AAlQWWZV+ZjSm3kV/6KIy7jNAeSeyz4qfNZE1RDntqySV8NKYml3qr+1f0jwfRWGaCN5sr7UaVnLuNTlA7xTeVC1dxrGc2vVMZlzBwKgq/pRJRh/NCcTphM+yxOweSzmTbctnDSYin38PvAQl0n2LMbClX6ltczraTkYBaVqJa62t2uJvE9P1E/zinTml17Ww9pL128TC0LY1EqYr9lHXLL0lrPZZn95RQbemiGIGH4XFdkOqyFAEzrmQGTs8ZHQ0Yx1oLNSswkrBuPxIJzD5bs7HtscFrA2LJgTM3ZY+H1nPukcWU/BJBbFshlam0k7CbvwgvJckBxJdsGjDXt9ZV2UsI70NttNyLXG+1sByp2xWmZW/pa7AvWTosvp1OSx2LMbaC7a5zbt1vGw7b5YARJ3V3jc9t82G6ZEuPini3jc9vTBvEtG4OMkRhwe7AVs+KOWbHZytaO+WBsXPY8s79ZX0rT952oEJz75fwo3aPkt4eTME7Ezugs+9FCZ9P5r2k9bqKizS5qd3bQ3qC1vb0/2N/Hb/f7GO32ezt7eL+HO+09/fJVvj9qrYz/AtFMtIzM4AAA k8s-resources-workloads-namespace.json.gz: - H4sIAAAAAAAC/+1d62/bOBL/nr9Cp+0dkr2ksZ1XE6A4NE6DXaDdy6ZpD4dNYdASbQuRRZWi8mjg+9uPD0mmLCp+JLGlZPqlESmR1Lz0m+EMfb9mWTYKAsIQ80gQ2UfWPW/ijb4XMX7113d+OdoUt2HXY6jrY97KaIxlWz/A7HeXtwSx76sWisLBBSE+80Le3pCNA8/FbRIwSnwxRQ/5kXre94KrSE4jLynuURwNeIO904hs1UZu5B1yWWpxvNUhvo/CCOujyY4B9voDsXK7tdcIb+2sI0QB9scj6aPJfuR7KGoTn1BJhdGm3tlFNJqcSna4KBp8wkGfiUU3G4U+XPYYQxGJqSNewD6jZIjZAMeRnbur5/l+cVhP0LuZa/JxHwduxrzxO133TdML+sWU4oCV9A7RbVmPF5T0RANyo4mG1sO4dPklT10jPx7TSOvL05/LibxrcnTZceO5kvyNyR5NtLJmIahnxAvYZ+JK4osGC0XWT0xJnvwh5vwJGOpjIw9DMQpFrheLafaKfWbWc7K7mGIq5u75hOXnjDD1cPTva0wp15m8uE6K7FhsxVg/Yk5ma8ui+AenKMuLUqozRE77y2lr93CvXbwjk9giF8fi2Ch0cPV2cfBFLr3sYWECxmbBeEsmxeZejdOtQm/EkHNlEKMJQZqFgL439KaQ7/Tw3U7jlZBPu8prUhQiB5fZPt4pDEWzlW9NpimoccRwGGL3E1+kUWkYon3MZlAHfBtKHkXxcP0ysKyAa3knQEMsV9sJidtx+IcI8YnoUfZXxwnjThxxTe9EmLe6UUcarSM+TMejiOF7x4/5Iun7S/tN8uelvWllI4v27OLSHl0Gv1okWM+aNvnMG2JBfUrisOPjHlu/IfTKJ8jdtNK/OuwuxBvjUTtZB3/86CruqjcgN2L1FPuoi/05VzYxmegV/8slb1jdO6t0WZdBUeJ7hA6R/NoybyiIJ2W4cBs3hphyU3+KHCZ1qFUivafZePf36eyjEVfK8aVczGhkl4zARehKwyMTUo5DKapzGYhMpBzkI7ouucBRivx6S5sxNwcSwg8QdfllOhZvSW33Wy6PnCMbyyT49O/HCyCwsu2rJG/Z12Vh4pbaZzYQWJr4rgEFibc75ajTMJfs+zLweszcyaQHYLfPvlpfhcXM4xeWfaPuJ+Ehotgtw46EsiIVFTiU+i5m9ALXu/bcmGPJUpyY3iu9kPzKbtGtFxXX1Y2dK/VlKRLdHiYYUZBkgmW2EDjzU+VIOEO70q0yv8Idup0F940llU9HmUGgxMehTJZ80j9G0ntqFvoU+Dc+ptB/EcFobzyX5q/wJcwqpt5jCgpam2zNdIsb8BCjScVJWn/nNkN62ebuc3KT86SzBV0kKqfL00N6qPq+eD9l/2BfaUvCitfpQBf95xb4zwW3oPky/ecn81qKa5nmtkTszp/Lib8ofmiUsOMxkvgv/7f1+fPWyYn1229Hw+FRZIATIWLc3ATlY6ZfSuV82ou5zOdxEHhB3zojbpnDnIiJ0dw6qbHI82jRd3ax4w2RNFENk/97ZdahpPNCupjHPpp2X4Zw7BPK7YvlcnfMNt76lQq1tR/kzjehwNYvHwwcKsdvOQ4G8bDL1aHQHwee9mldjMFmlLdq7rbqxt3j6nL3/OFwITB4Jga3q89g6+/A4sew+OT5WJwAJXm5OKM/PRS3BibPxOSP1dVjxV7Q4scx+LTKWvyfJMRdDxaXbTM9I4e33W3U3N9FB73Gzs5h7/AQvzt0MNpzursH+LCL95sH21fvoq004httpbsG/7pGdGscNHj/Zvz3P0RXGmFO48uycRxgHoeXZUc66vs3nY6DfV82yqBz0tBpPSiMN6WMroK1ScXQuhCTgblZxNzkdquqyeaKc1Yj5vbbX7cXp2HEqBf056Nh+abKvBvRDokDtr6aHd2H93Mf3P2SiV6Gfa+IIRn1NBr/BbbFinNQ3JNxcNsQEnnk/iLkBCw5J6ACUnT8XFKUUX4sMSns6KR76HPzQduqVnvUIDMrkJk2WJ4aSNE2aOKL18STVVhvlaIDElNLifkItvul2G7Qwxrr4aldgwTGP0VSZrUSGCkKIsFJMx+zoIahCxIcIcHxhSY45vUUEhwLCY6GCsEdyHAsZDhChSBUCEKFIFQI6n6dZY1djyEeEnonkb4X9LkHxzrdO4bn90GyIf/Gu0SDN+RGQl2MxKwlHgnUB0J94Cz1gUpQoUTw+UoEV0vhl1Il+FlSEQoFK+FHy08Z+NGv1I82qiK40gVXuuhJ74InXXAQoFYQagXnc56hVvCF1wqWYj3IGq5OuaDCgI9iMFQMVrxi8Al5DOVGj+NypYsGE15D3WCl6wafQJuhdBBKB6F0EEoHoXSwDuYGSgehdBBKB6tZOgjJAlA4WPXCwWznGvLeVyA2bTA9UDkIqgjFgyA0UD8I9rvO1YOgiKtUxMoXECbBdaghhBpCyH2sRe4jlBHOnfuYT7VLLbwYqzm0IS8S8iIhL/KReZFtpQXWOXawd42tYxS4Sppg+6yaSZLHYfRIXl8IhMbdE2B25RMmF2f2OeeBRXqpYrvWGVIoGlhdzbzJ8NGsTvWaAbernz/55IptnVAioAywvKKplM+i4MD1qidXLsx1SKq8CSxGrNBYslWv5EpzMh0kVkJiJSRWvrbEynWxfygPY10fbxwGmAkB6VCF6tSGrDq8dd4jW//i5keM3knjpd831BaiVdhDNO0gVmIDMb99uFGDDcMnT/p8UExYggRBTuomJ8dLlZPUnITKWwBBqY+gtFdjUEBSaicpJys1Ka4KQIDA1EdgPq7WtIDE1E5iqn+efrK3+YcSuiqe+Qd5cZAXB3lxM+ssJMgVEuQM5+zvQxZcIQuuAVlwKgtuf0WHm0OEs4gB3//v0n77zydEhc94onn9T35+OJ0Rjn9ePtQTmWwA9FYA9DZfAsw5AJgDMKdWMOe17tABzlkmzpmSyg9AB4AORLSWF9EqaiFEsWaBd+8A3gG8qxy84/IGUSxAd6tDdx+44nB9tdqp+I1hnqBympV9lNUCAfgD8AdRrtrCoEOAQQCDagWDIMoFOKhCOEgrkAUoBFAI4mArioPNqLAQJZsLHk60AT4EfFiXZK9XV1UEAHGp6V6zHHYFIHD5IDAEEAjxsMUBTxMADwCeeqZ9AeIBxLMExDPtzEcAPQB6IPK1vMhXqpeTuggRrpkAXwsAHwC+Wke4Xt3hFoD7qhDpMp4DDOAPwB9EvGoFgHYAAAEAqnfECxAQIKDVRL4ABAEIgghYtSJgBZ0siYStJau2I2eAh+gbppFaWXNXNYsf4hNPuIheycH4d6w/lkpbGHXKP0pY/t6GGovhYejzVwz6mZZx/BAxTZbHIqBhpxyCZPhWSoSLeyj2J77rUn30XhMgFT/Rh/NymgmSfcIxo/VFgUatP9FlHVJqvSQUfJu0r/aPGNM7qYAm6CmOcxSWOS+fvLWPbyfOf8ysljb92sR7aZRDvv8tIURejKfRtISYZirOgK5TSmtmmxtp/jV38QffgFVL9NkeclZ6httTriT4YB6WyJk6ytyuZwjEC3pk00qG2zAzqzWNWcm3Smcq1w3JkejPdH4731tYqGgz35yIgnoTrSOO8IUaSLdP88kHihkx0Fk0dxwSS8nZaUz2KItpNxs53k/V39And0NxR5kKZzcsLH/cDHiBlxjVPM+XC0ZHk7+5bj9sj55DSyZ+6WQ+FakQueZRxSsv/Er9L3eBYyBMoqiNGirqigz5c4toJhRPY8qNcjjSBPFlmPi1FHaOFMzyJDETgNVT7p0dkJutZurx2NLMizY791jocZRIxw8nJMncf93tsPe0/VTd9ts7+oX20+D2nvZ3U7/Yaeg92nkdLe3vpqvk93v6DsIx14Rj6iz6wPv6wPosrV39QgPKB66+3nQtOfL9JDLGY3+9aCeYODtclwzDmCPx8/RHz6xt649UCq31tAQjUgJpxzL8aKN3B71uwz1sHmKngdFeb39vd6fZ2sXNfWe3hfbUzdcZOG+sjf4P5J8blcXvAAA= + H4sIAAAAAAAC/+1d62/bOBL/nr9Cp+0dkr14YzuvJkBxaJwGu0C7103SHg6bwqAl2tZGFlWKyqOB728/PiSZsqj4kcSWnemXRhyJpDgP/mY4Iz9sWJaNgoAwxDwSRPax9cCbeKPvRYxf/fmNXw63xW3Y9Rjq+Ji3Mhpj2dYLMPvN5S1B7PuqhaKwf0mIz7yQt9dlY99zcYsEjBJfDNFFfqSe973gOpLDyEuKuxRHfd5g79YjW7WRW3mHnJaaHG91iO+jMMJ6b5LQx16vL2ZuN/fr4Z2dEUIUYH/Uk96bpCPfQ1GL+ITKVRhu68QOotH4UJLgoqj/EQc9JibdqBdouOwxhiISU0e8gP2ZkgFmfRxHdu6uruf7xW49sd6NfFPAML1B4ma7Mch34uMeDtyMsfr79oL30eUYS/U7bnqmyYvVjynFASuhDtBdGcULSihUcO2Ci4l5KlGf3JopjIuuX9IpX5F4xACNlmcuF0J513jvknDruZK39XGKJrdZs9CCz4Rz4xORbyIbLBRZPzAlebaEmDM/YKiHjQISil4ocr1YDLNfpJnlinPFxRRTMXbXJyw/ZoSph6N/32BK+UrndWFcH0Y6Ifr6HvNltmoWxd/5irK8nKYKSeSwP5019472W8U7MnUocnEk6/UCgdsOFwcXcuplDwv7MrI5xlsyNTBTNU43C9SIIefaIEZjgjTNAvrewJuwfGdHb3frr2T5tKu8JkUhcnCZYeVEYUcazXxrMkxBjSOGwxC7H/kkjUrDEO1hNoU64LtQ8iiKB5tXgWUFXMvbARpgOdt2SNy2w3c5xAeix9lfbSeM23HENb0dYd7qRm1ptI55N22PIoYfHD/mk6Tvruw3yZ9X9raV9Szas4sre3gV/GyRYDNr2uYjb4kJ9SiJw7aPu2zzltBrnyB320r/arP7EG+Nem1nBP748XXcUW9AbsXsKfZRB/szzmxsMEEV/8spb1mde6t0WldBUeK7hA6Q3MqZNxCLJ2W4cFu6+Z0hh0kdapZI71nW38NDOvpwyJVydCknMxzaJT1wEbrWwM6YlONQiupMBiITKQf5iG5KLnAIJKGBtBkzcyBZ+D6iLr9M++Itqe3+hcsj58jWIhd88v6xBgusbPsyl7dsd5l7cUvtM+sLoE5814CCxNudcUhrGEvSLvpel5mJTGJRu/X5i/VFWMw8fmHZHvUwDg8RxW4JCowIZcVVVOBQ6rsY0Qtc78ZzY44lS3Fieq90cfIzu0N3XlScVyd2rtXOUlx0e5BgRLEkYyyzhcCZnypHwhnalT6b+RXu0d00uG8kqXw4ygwCJTaHMlnySe8ESdesUaAp38D4mHIOighGe+OZNH+JL2FWMfUeE1DQxnhrplvcgIcYjStO0vobtxnShTeTz8ltzk3PJnSZqJwuT4/poaJdeD8kvX+gtCVhxev0zovOeROc88U65431dM6fzSUqzmWSTxSxe3+mCMFlcRdTmoRHMOW//F/t06fa6an166/Hg8FxZMAqIWJcX4LyPtNtWHm29nz++HkcBF7Qsz4Tt8wbT8TEaMud1BLleTTvO7vY8QZI2r+6ybm+NutQQryU/uuJjybdl8En+5Ry42W53Nezjbd+odJcPcqdr0KBrZ/eGzhUDg5zHAziQYerQ4EeB562b8/HYDOEXDZ3m6vG3ZPqcvf88VgkMHgqBreqz2Dr78Dip7D49OVYnAAleTk/oz8+FhQHJk/F5A/V1WPFXtDipzH4rMpa/J8kfr4aLC47w3pBDu+4O6hxsIcOu/Xd3aPu0RF+e+RgtO909g7xUQcfNA53rt9GtTScHNXSI4l/3SBaG0Uk3r0Z/f0PQUrD12nwWjaOotej2LUkpL2+e9NuO9j3ZaOMaCcN7eajwnhbyugqWJtUDK1LMRiYm3nMTe4orJpsrjhntcXc+eXnnfnXMGLUC3qzrWH5ic2sp9wOiQO2uZzj4scPix89WpMhU8OhWsSQDIoajf8cZ27FMSjuyiC7bQiJPPHwEhIOFpxwUAEpOnkpKcpWfiQxKexopwf0D3+RDl9NcWuNvzPDtQHm1siJxJrPyiPtjFwdjoM8LUGeWmCVVkCKdkBLX7WWni7D6qu8IZCmtZOmD2Dz18Xmg46uqY6e2SuQjfmHyDCtVjYmRUEkOGnmYxZEMZAgWxOyNdc0WzOvp5CtWcjWNNRS7kK65mLTNaGWEmopoZYSail1h9KyRj7PAA8IvZduhBf0uOvI2p17hlPnx0Hujcdx5DwuTzbI3zhJNHgDbjbUxVDMo8QBgtpKqK2cprZSiS6UV75ceeVyV3hdKiw/yVWEIstKuO1ycwO3/ZW67UZVBM+94LkXHfc9cNwX67hDnSXUWc7mq0Od5ZrXWZYCSci4rk6ppQKYT2IwVFtWvNryGXkMpVpP43KlCy4TXkPNZaVrLp9Bm6HsEsouoewSyi6h7HIVzA2UXULZJZRdVrPsEnIToAxzsQU5L13glR2iQ8b/EkSqBYYKKjNBcaE4swLFmSBQyxSoD7ATQL0mqC2UbL5MeidUbULVJqR/rkT6JxRuzpz+uQ/pn5D+CemfVU7/bCklsc6xg70bbJ2gwFXSBKeE1cwFPQmjJ/L6UkA87vsAsyufFzo/s885DyzSTRXbtT4jBcOB1dVMDw2fzOpUrxlwu/ppos+u2NYpJQLKAMsrmjH6IgoOXK96DuncXIfc0dvAYsQKjZVpq5VDas4ZhPxRyB+F/NHXlj+6KQ4n5bd1N0enkgFmQkDaVKE6df6rvsX75FPg4Z/cIInx2mkI9tuWOpW0CseSpkPJSpxJ5k8kt1bgDPLZs10fFRyWYEOQnNWXnJOFSk5qckLlUYDorLLotJZjdEB21kB2TpdqdlwVyAARWmUR+rBc8wMytAYyVP2fW0jOUX9XYljFbzRCEh8k8UES39Q6C9l8hWw+w88wHEA632LT+eqQzqfS+Q6W9O17CNVOAznf/e/K/uWfzwhCX/CD96v/YfDHMzXh6+CLR5YiSQ9w5RJw5fY6oKpDQFWAqgBVwTkmwKplwqoJRRGAqwBXQbxucfG6ohZCjG4aNPkW0CSgydeFJrk4QowOwGSVwOR7rkpcg61WKpAjVClWOU2nP86KuABrAtaEGN7Koq4jQF2AugB1QQwPYNdKwC6tkBqQFyAviPItKco3pcJCDHAmNDreBnAU4Cgk6kGBI+DRRafqTfMNNsCci8ecIWBOiPbNj68agK8AXwG+gq8AAMCqBsCa9OVTwFiAsSCut7i4XqqX47oI8bup8GUT8CXgS8CX8KUggJmVjuMZP74NWBOwJsTzVgpv7QLeArwFeAs+qweAq/JxPcBcgLkgvlet+F5BJ0vifBvJrO3I6eMB+opppGbW2FPN4sc2xRMuoteyM76z9UZSKX+lnfJtCsvf1FF9MTwIff6KQS/TMo4oIqbJ8kgENLCVA6wM30mJcHEXxf7YTi/VR6ea8K/4GU6cl9NMkOxTDlGtC4VRNXqiyzqC1agkFHwbt6/29xjTe6mAJqQrPqMqLHNePnlrD9+NfXc1s1ra8Btj76WtHPL9r8lC5MV40pqWLKZ5FacA8+lKa2abG2m+m7v4vW9AryX6bA84Kz3D7SlXEnwwC0vkSG1lbjfjMIFKQmprHNsxXONvQz0n4nAhQ01bZu41J3Ev2bx0LnNlkSyK/kgnZOephZmLNvPNiWyoV9MIcYQvVUe6wVoJgam/sMBkwHFukclAqxd0ySPSY4bcQw3IvkKpQjEjBu6I5rZDYilvu/VxitqY7UY9JzETt4nQJ/cDcUfZTpHdMLfU8t3GC7xk787LyWJ9nuGY07O1BN0a+9Gs2dSqQss1iypee+EX6l/cB45hYRJFrVdDUTdSjDpUmMyTXEvQWFf5gnZAbmuN1D2ypbKKNjv3WOhxSElHDycLlsUKdB/F3teOlnUNtnf1Cy2IZ+9rfzf0i926TtE+O9PU/m64Sp2/pe8gvHhNFCeOond8oHesj9Lc0y80VH3o6vNN55Jbvh9EhojsL5etBEBnX8AmgzDmsP08/RVEa8f6PZVjazOttYmUuNqxDI3a6O1ht1N3jxpH2KljtN892N/bbTT3cOPA2WuifXXzTYbk6xvD/wPkH1LMs/YAAA== kubelet.json.gz: - H4sIAAAAAAAC/+1dW2/bOBZ+z68QhH1oAW8R23ESLzAPbXrBYjrTTNrpAtsWBiPRNieyqFJUYm+Q+e17SOpCWVKcqy0nJ0/RIcXLOYffd3gRfbnjOO5oxMIokbH7L+fbj46RCPozYYJaMhKGXBLJeKiElyACYcBiqbPA45XORn0myWlAQTomQUy1cBJS+W8fRGESBEYiSDT9wnkgWQTyXS2cMp8e8VAKHsSl91np3YCFZ1bDIhJSnf+bbpNpGch9IknME+GptrjHgs+onNIkdjtZjjGjgQ8Vjtkk75J5lY5JEsi4JK3UnItnJIpYOKlJkVPQ4ZQH/nJR6i3u65aR05gHiaRup5weSxrFmXKzv6tS8UnIlPrdkIfUzROudpbygraZf8zLbXCn8HhQFOdewPOe9TzP7JI+L9RzpWhtml7+WNWQy6Oy02ihxwMufks1cE6CxO6+8Y4slQhK7MS/kliy8SJPTiS3k7lgNDSOWpcsqJ949FNNm1SrSOAVnlSYncTydy5/Bwd0rZSyqbUzqZfdkiFN53J33qkxpCvpXJa6U9FyFCQTFn6lIk67dVB4sSRiQmW52aVe0Xkk1DtxMntxlpzSgMpRCNWNQjKjl14A+qTil+/uP9J/v7sd5y9+CpI093f36mW5U2MuZkS7nmQzOoopKD0uZ2EhFAWdf088yYXtJEandEJD/31eTvllQccaL9zXVbcuPEsyqYHGPUnCEAag86tprzXEpSBhHIELhdJGFJO2iPTbMXiLqSZVOELIA0LI3s0gpI8QsmUQIsygG0Xcj2+GIh2HhTDYQo/+8jdkzB4UvDifTpyGwkceT0J57xoeG8AuL7Parq4eBMyOQa8IZO0BssObAdkeAtmWApkHUw/CQij40eAsr+K5gtpRpgBEtvYgW7d3M2gbILRtCbSdg68BAsxISCYANpJLEoyM8L7g1nHUYKSQFyAkgWL144iPRxdcBP5WodJr3QPnq1YMYBNAMuJSi3Bp/2a4tI+4hLjUyWDJpzEDNW8zLr01XUBgaikw9W64Ln6AwLQlwCQAK8qL454eACMqBBf3nad9G8x+vNwe+DFj33mnuv6w4EMCRuIj5ePaxwpDnhIRV8rzSTz9SMOJVGOyu1uS07rsq5ENPBWKKgk+COIz053d20NDae5Ut2V2UA8NhwU0aDOWawI9TcLX8Zd0J1WKhFrFkvPJcufVIE1EapWl7DMyr8k+Y2GNVLDJVH5mfk2tANgXNVLI+x/maxNlO7SZOyh2r6kjH/mqqIp2ACnzxJLwIq2lex2oqiYccxg4GW4oQeEAEQXvABSc0Ir3ROot5QpJXJrqannV2UDVPhVUA8g44NYIETSiekDa6nDN+P50DngCGltqNQwoj9Z5Oowk76xSt6LAiPofQSnVMXhn1BMwzhUScVCS2ec38Vk9+C1j3/XQ55wunBd5wSMFFAVarhUYncvLcjvuiJV2BPPNxtAZfQ/Is2x+Jf88ZWNZTUhB91PWKueEWFEPjKLskMSlPRYBgf2a0ciFXMIgPdpGGTSz0GfnzIdZb5VMszw6yigaMCdztgSEp4l3ZpxsachnwZvqrh1dqJ3mau56RMnhIY/wiiYuyJxe49uF6/BoyWUCAp5aaYJK4JM3JKY2sFioWcluYLMitrpSF8q0uZWZbz9Xwq6sfjYw9hAZGxm77YytpysPTdyZtOOUqROJu0zcZsKE9I30jfT9WPTdW3VEtbtXT99Wf5C/kb83wN9TFksO6Dwb/UwIMHdAX+y+Gg47zk2I3U9S6oupx0M/HhkQf3SO7zgBfYlEXyb6zBjOcCinTmZNJP37kf42UD4SfusW2HvdBsLvIuEj4bdqwq5Os0N9Qlbp/JrzoPdg8/USN3TvBkzduYPSLrg4o+Jpas30rUFxb9Yb4hxz3/msHBRXMXAVA4OaDW5CNEY1PYxqMKrZjmWMtYc7G1iseJCY53YqbQqGHm1FaO1KbWdI9DZVOIZFuM6DIdH613kOGyKi/voiInVDzbtZJBf1Sf+lgmMYhWHUrRaHgPoFKKTY7FlftFRUqUig46Sf55hT9Zvb9FGN0aJSe9qyDfTZ2MvBA5u4VoKBQSvWSpoigz2MDDAyeEqRwdqOd2I8cI94AM+BYlSAUcGGz4H2Bw1BwQCDAgwKnuCuyw3WEa7bIbjt1+y3CR02fIh02+KHt3i8FLcdMI5ox7bDXq8hjtjHgxgYErTqeKk3ETyJ8tu1brON8GDsv6bvQtv5hciRNoCTGqDQiiNwLQDXApDDN7ZD0EjiB0jiSOLbcZpyFbuvZXKPn4ReR/g4X8f5OnJ94XeAsJ5gUXbnqDpKG7Ax9RZeQB16DhTgwKBXA5uLDc/yhw0BAt7TiAFCyz4iBUccCap+enF9U/z2X1778Af/P7774Bg94/wd5+/I6ZucvzfRM17KiPS8LV9DWrydkePT+3avfdyddQb5G+fjyN1rP4c32G/4zXC8jxGpewup++l+dt8+6vbxi3ukbqTuTVH3fr+Bum91s+Iy+a3i7gpN35G8l9Menr3tXy5D+t7YwjgwkBx5gRoIQNI/wTbyVp/INf6ipAcmULLeq1dr+xG33nz+gHcoPrpq+mtUTb9RNW/aqJq9Napmr1E1R21UzWCNqhk0qubtegPbk+Mj/CYU95EwoN1cQHvQ8Du9PbxUE9eiWr4WVUPK91uLut0RUFDVacdJRLCR05+qdv0PNKAtC1Unxgj4YyC4ZIUMvyaGP1xF8A2/3dnr44oVrlhtmuQjwT0ax0DfyobA4zM642IxOl1IGt9vA+n57RP9pnWHNHs/mtWutwVU25J2PnO6Pbwp3e4h3SLdbppu9bQ541wvSvJZ8kNdn/j8SPfo+E8niZVzIe/eb3o7VZrYgiluO9r5zHm3uGtoFfEOkHiReDdNvBM+mnDBE6n0jfPa21Hsh1xzyLHIsRvj2J3Uy9U4UC6uD+DsGp90Bb8o/N2NvSmdka9UxOY7f3PtPkDJwni0T8SZeQ/QsbCcHv0ipGpin9cm6SwKCLj/JPc2Vx0Ftux9WUdNJdVKOteK9emYJMGSarVr2al1ltLX8y4NonqbZG5tBwpWKteXH5TBAcQ/EyoW6ZLgckBR0nm3JJ1QZWXXzpoNWKv65RFTCgCCr2n/l7phqdLWw8rwp9CV/c0mCwHqffo6qGPpTJFuSgd2WTOwCKt5J9NyzSurVKyrGxlEeZFEl0vUc9Vx0kJf1pugt8oEKeDapgJH13qO/8ha4ZZTK81VsvrMqYFNf6yEJKZfTEG10c6arL7bZPUlRG8YPavsnbHznQ2efcIhklDHDPk9QtcuAzhVJ6m7CWF7vWQng/orA7tMqzsF3LEJXNyQX/yzm0UBEISkMrf0WsS8Mx2rpi+nKsk/cbWp2h1YQU1313ro2w/dWfH/wPq/az/0d+0UK1bpWf93fYOGP7I+GBfI/WdlLXbB+3bBdi29Pfuh+D1Q98C325u1paS+/3Edabt/fjlKOTILBH81zmekiZ7euf1u/3BMuoOBPxh2Bwf7w/GpT/3D4SHxdodjf2gyn+dUvLtz9X/j7g7C47UAAA== + H4sIAAAAAAAC/+1dW2/bOBZ+z68QhHloAU8R23ESLzAPbdIWi+lMs0mnC2xbCIxE25rIokpRib1B9rfvIakLZUm51IklN8dP1iHFyzmH33d4kXS9Y1m24/hhlIjY/of15VtPSzj9nvicGjIShkwQ4bNQCq9BBMLAj4XKApc3Khv1fEHOAwrSCQliqoTTkIp/eiAKkyDQEk6i2SfGAuFHIN9Vwpnv0SMWCs6CuHS/X7o38MMLo2ERCanK/0W1SbcM5B4RJGYJd2Vb7BPO5lTMaBLbvSzHxKeBBxVO/GneJX0rnZAkEHFJWqk5F89JFPnhtCZFzECHMxZ4q0XJu5inWkbOYxYkgtq9cnosaBRnys1+N6Xik9CX6rdDFlI7T7jZWckL2va9E1Zugz2Dy4OiOPsKrveM60Vml/R6Ka8rRSvTDPLLqoZsFpWdRgldFjD+R6qBSxIkZve1d2SphFNiJv6dxMKfLPPkRDAzmXGfhtpR65I59RKXfqxpk2wVCdzCkwqzk1j8ycSf4IC2kVI2tXImebNdMqTuXO7OOzWGtAVdiFJ3KlqOgmTqh58pj9NuHRReLAifUlFudqlXdBFxeU+czF9cJOc0oMIJoTonJHN67QagT8p/+2r/kv79avesv9k5SNLcX+2bl+VOTRifE+V6wp9TJ6ag9LicxQ+hKOj8O+IKxk0n0TqlUxp67/JyyjdzOlF4Yb+uunXhWcIXCmjs0yQMYQBav+v2GkNccBLGEbhQKExE0WnLSN0dg7foalKFI4Q8IoTs3Q9ChgghWwYhXA86J2JefD8U6Vl+CIMtdOlv/4OM2YWEF+vjqdVQuOOyJBRr1/DUAHZ9ndV2c/MoYHYCekUg6w6QHd4PyPYQyLYUyFyYehA/hIKfDM7yKp4rqB3lOkZo6w609Qf3w7YRYtuWYNsl+BpAwJyEZApoI5gggaOF66Jbz5KDkUJewJAEilWXDps4V4wH3lbB0mvVA+uzUgyAE2Ay4lKHcGn/fri0j7iEuNTLYMmjsQ9q3mZcOtZdQGDqKDAN7rkwfoDAtCXAxAEryqvjrhoADuWc8XUnal9+cRxZg5MByreX2wNGGgmst1IRjwtFJPBJfCQ9XnlcYdZzwuNKeR6JZx9oOBVyhPZ3S3Jal/1unAO/haJKgveceL7uzu7DgaI0k6rbQTuoB4rDAiiUGcs1gZ6m4ev4U7qxKnhCjWLJ5XS183LIJjy1ykr2OVnUZJ/7YY2U+9OZOPO9mloBvq9qpJD3376nTJRt2GbuILm+po4cB2RRFe0AbuaJJeFVWkv/NoiVTThhMHAyFJGCwgEiCt4BmDilFe+J5F3SFZK4NPFV8qqzgao9yqmCk0nAjBHCaUTVgDTVYevx/fES0AU0ttJqGFAurfN0GEnuRaVuSYgR9T6AUqpj8IcxkMM4l0jEQEl6219Ha/VQuIqEDwFC63xpvcircSRsFEi6UZi0rq/L7fhB5DSjmy8mos7pO8ChVWeQ8rOZPxHVhBSCP2atsk6JERHBmMpOUFybIxPw2KsZm4yLFURSY8/JgNoPPf/S92BGXCXaLI+KQIoGLMjCX4HF88S90C63AgBZYCe7a0Yechu6mrseX3KwyKO/oolLsqC3eHrhOixacZmAgN9WmiAT2PQNiakJMwaGVrJrEK2Ija7UhTldbmXm28+Vvisrow38PUb+Rv7uOn+rqczT0niWp2eViRRpvEzjejKFZI5kjmT+VGQ+uOs0a3+vnsyN/iCbI5u3wOYzPxYM0HnufE8I8HhAX+y+Go971n1o3ktS6oupy0IvdjSIb5jxe1ZAXyLtl2k/M401HouZldkWQ4D1QoBtCACQ/ju3FD/oN9B/H+kf6b9Tk3l5DB7q46JK7rccJH00bt8sjUNn78HbvR9Q4RXjF5Q/Bx3qnjao8c1mw58T5lln0nlxvQPXOzDgaXHzojHiGWDEgxHPdix4tBwKtbCs8Sjx0MMU3BQobWglaeMq7ma4dJyqH0MmXB/CcGnz60OHDdHScHPRknwlztt5JJb1Sf+hnGGIhSHWgxaVIBDgoJBiy6itSKpogKSEnpU+HKTP9Le3dSQbo0Sl9nRlM+lMW8/CI6G4qoJhQidWVZrihD2MEzBO+JnihJYOkGJ0sEZ0gCdNMUbAGKHlk6bDUUOIMMIQAUOEn3C35h5rDLftJaz3ZP1DAomWj6luWzRxjAdYcYMCo4pubFDsDRqiin08zoEBQqcOsLpTzpIof+/XQzYcnigW2NBTqd18IuVImcNKzVFoxeK4ToDrBMjore0lNFL6AVI6Uvp2nNC8i+tbmPjjA6m30T/O5XEuj8xf+B3grcv9KHtTqjyQG/gT6i7dgFr0EgjBgkEvBzbjLa8AjBvCBXyfJIYLHXuEFRzR4VR+MbKt6X/3X7n7+A8TfHj73tJax7k9zu2R4duc2zeRNb48Esl6W56+NFg8I8ef/enA7jF51hlkc5yrI5Nv/DTfaL/hM+j43kgk8i0k8ufymH/3iNzDJ/yRyJHI2yLy/WEDkT/oDZCrVHgXk1dI+wepfDXt8bnc/DIbknlrS+jAQMJxAzkQgLK/g23Egx7Ca/xipgsmkLLBq1ctfaRusFg84rsen1xRw9YUNWxU1JsuKmqvNUXtNSrqqIuKGrWmqFGjoo43GxCfnhzh86m4N4WBcHuB8EHD94sH+GJQXNHq+IpWDUWvt6K1zpFTUNx5z0p40MppU1m7+gMN6Mpy16k2CX76BBe+kO83xPeHd9F9w1dMB0Nc98J1r7YpP+LMpXEMZC5tCKw+p3PGl875UtB4vU2p57fb9IfSHdLsejSrXG8LqLYj7XzmdHt4X7rdQ7pFum2bbtUkOuNcN0ryOfPTvObx+VHw0clfVhJLV0MWXm+yO5Oa2IIJbzfa+cxZuHgL0l00PEIaRhpum4anzJkyzhIh9Y2z3IdR7Ptcc8ixyLGtcexO6uVyHEgXVwd3drVP2pxdFf5ux+6MzslnymP9lgH9sQCAkqX2aI/wC30foGNhOTX6eUjlND+vTdB5FBBw/2nubbY8XmzY+7qOmkqqFXShFOvRCUmCFdUq1zJT6yylXiO8Mogym9jHEBJYZzomqPq3GTEYqUy9g6GMEiD+nlC+TFcKVyOLkvL7JemUSnPbZtZs5BrVrw6dUiQQfE4VsTIgDZ2aCrkzDiqUZj4s6oeA+R59HdTRda7RlBfMsuZgGr/mnkzLNbfcpWJVnaOh5UUSXa9w0E3PSgt9WW+CwV0mSJHXNBV4vNJz/K+sFXY5tdJcKavPnBpY98dISGL6SRdUG/ZsyOq7TVZfgfbc6BnxPsTqdfesafZeXXxyU+yc/xzOsJNB+42GWV/pMwXYiQ5U7JBd/drPWB+CjlRml26LfPdCxabpzalK8mURk5rtkRHE9HeNi6F50Z8X/0fG/755Mdw1U4zYZGD873sa9L5lfZBhouEgd9ZiFrxvFmzWMtgzL4ovmtoHntnerC0l9f2Xqcja/uvTUcqJWeD3u/ZILU3UdM4e9oeHE9IfjbzRuD862B9Pzj3qHY4Pibs7nnhjnfkyp97dnZv/A+pcQKEItwAA namespace-by-pod.json.gz: - H4sIAAAAAAAC/+1dW2/bOBZ+768Q1G6RYpOpJV+SFAgWuW4LtJ1sk3ax2wYGLdG2JrKkoahcGmR/+5LUjaRIyU7SxBmrD21NUofkuX48JKWbF4ZhDodeECU4Nt8Z38/W0xIE/0w8BLkyEAQhBtgLA1p4Q4pIoe/FmDZhv4yslNWMEs/HHwJSaa2XpS7AIA4T5EBSYR6jcAbxFCaxybWBARj5tB6jBHLlU89VlHpOGOyHfogoQTQZgbXOumFbFvmr3183rDc86QDMWMe75VyM18auDxEWhoCvI9bOBfF0FALkmlndLfv3jPx9y5gCXQ9LozUnAcQfXFISJL6fliAQTU/D0MdeRMo7rJBOZz8MMAp9ytAx8OP0eU941veCc04KEQgga5+yPGe46YS+D6IY8pSECrdSM0GeexyWwkyZLAnskvy2e1zBVT6B7Pc1/Z1zp6DN5mAXP8tRnxVlCEYQYG6qXOkHDBETj7r6S3gpMJhVxdPw8tTDFc0xcVZo7icIwQAbeyBwLz0XT02xzYn3k7WbDriKTBNQeJnqQDbHkvHAmcJTbwbDpDIZh+rlHnDOJyhMAlclmxB9A34C1VUxZ1qk7KW9ve30BryiMoW3u5tE0e3tdaNHdL/z29a2oPQv3V4PdEGuwqUEGo3RdKHjzQCTXCFzcxyiGRMc4doMDmOIPMg9MwHJBIpKNQNX+SytDq88My/IK/hiKkqZI1QWU+KQpqHvfgQjKBqN3OITQOeQ8Y+qQkU9taq/Lam+ZTeovlWhPYXeZIp5Wqk1dMufAVHvC+DL6iKaesa5KPKCyWmqhZaqXFSScj6cu7ugPDZwaGB4hTnVMLIqbh7cTDTEEAgmDcTsklhF6YguHBC9Ow4JF2JRIag6nBBXIfDdpCxirT+FzP8T0wgC6GDi0YQ2p3Q4EkfDSIxXqf560Hd/V9Qwa/YdkaGpZAAJc1zRmTBtF45B4mOZWDrZdIr5n3Wp3gtEnRId1ivK8zgCxEClJkngMQvci2J+XLfCuDIlEVUq5csFRCgNpzfiM4UFVbkgzi33UWmYROcbEwRhII2TKbsLr6qT5LSFSkyou11fpN9r6PvEXWg7trQd9wux3KN7xOlhpW9b2/empm+tkrEHC59XMbDS/0RhjMce7d005cIjgjfyKNfv/I2rR7D6DCvTPsL8wCcCLfT+Z0xCCn2OSlj0FDg1VvPz212pIiweqHEiccVLkBJ0ThwolG3d90V02CXI0LK2yF9b2zRYWltCsBzTnquBhVLm6aRkbBJure2uQICPXRXZMKxI6CSzQOQ1BmgCcQ0r4VXE+o6T2ZpHsBFcI24QAzIsNCR48zJE5wSzO9C7gMPRNYbxEBOE6984fhKTaLPzw3yV/feHuV64lZ3//eCczA/z9vurPDi9e0UcARkpdZNnb96IUqqHAJkJxBgEskcW4t8RcDDjqGCipg8nMHCPii5EukQpGfYzd+sURHBjPJ/JcI9SrRQCBS0/mXpjXK2QwOMXwnsjHBt7lMnGl5TlbhUwpjCoKGbmy9vSFm9LrLbelkK6cjB35jSj3M00WhJr+DkL7U46xxbotkBXLFgM6fZapNsi3Rbptki3Rbot0v0lSBcTkcQzD7dQ9/Gg7mnGc7waaJepbs0I88FQRKwZDatqRBs5IYYPjZe6qRXVixLcqye4tzDB/XqC+wsTPKgneLAwwcN6gocLEzyqJ3g0P8EodDW0aE2N6javYWgEEKDzmDNAgsw4C5x7WdC49WNp9n76PNaHsbwnQwvTLRh+wKplQQGUy6FUgTItMEBs/IQo5OI28a0QQRZNxn5YQngzdlDImCUMiga09xC47AmxJkRYZBbxDxI3XBg72nDIgs5HGEzSGXfk+M5NLsbXft1SB/geiNWeh+G1jCeVUFQutCVMT7x8GYH+Q/5sfPq0cXBgvH//bjZ7F0vBjlsp22IsI8Kr4omsotz8NA8Q0VPDDS9FCM2afUV+NQxGgMQcFKinLKJ4cWZ5cJp6risB9mJZQZ5FuNl8C6YXe4fV1M9qiEAdpuYRQ5DMRsS0lGIQVncLCEEFSlZKDntPKYccIOamYBwD5xzieDVFsf+woojuJArOIFZbGgfLIA3ZMIwDFEbRqvqqwyU1kNWWytFTSuU4XEKuiycM78R09+0msLY6g00Hbve6oNcB0O73+r3BZt+xtsfjvvP2PCE8DCCG8UaW1vKCyQZZBf4jRJMP7o71GsExlcVOtxO/vgBoo8xllYksVkEe2nk1HDpQTj5y0q4sPO8raBk5VxNTy76nbYyujTXCF33Gjx3t1OT6KlpSyfXZd8v1CeVkdhG/zNbb1JPnTZeSnXu/kp25ckZpHFkFfu4/inquEEMPHlNB3RTorAJfDx9VUVeIsUeNjK0igXtvRZ1ggPlcdw5QUgbodnCe7kaEta1Oiw+W7ErEg16FYKh+P8fo5XELcwRQ5RATu1ujyojTcqhqPuf+hy0U/BMB14PMHjq/4Pi/rdn+2Cz3NJhViV0RRk2C3fg0uzkkb8GDi4miNN+urNbQm0SHswhfq+9M/ZduilRq0gM7Mqn0mI5ciug5r5P0FI1cl50CkKjHpPG/s/0dcRlnMhepIKQ89nE7/zaSPd820oLnrXT7NhEkmkjCwqTqXyJKkqpdEgsbYVF+EExsrt+lUruD9ATC79m5JmmeTftM/DkSTGJXZaOLeHMSxz6mx0vEgf4F1nP6ExwLHdO4uSG93N7eeQVXf4bj+0MEzizzqLzrVmQybngzBohFRNmQ051PWzbUYR4NvMD1Ljw3AcXBi9vq6RB6/7EcwBW48iTnO0rSVHXFW8wy08Tizl9+RlNqrXZGhWf5flYZ4jW4qttsLXWH7oyISkHP5qqyVH442QMxrGhR6nErzRUnI/l5NOLVZRpirtirCg8qh6Z1+GCrxQctPmjxwdJlKFcHIOT7YC1CaBHCsiAELmUjeB0+YyNUPFzCxtYkbLYVCZuaPU0FvNFCnDqYUwN15oM7SshTC3u0DC2ZKl/fkTGQirsFh7sd7fUdxmlp8gpcNAc20uOjeozUgJPqsZIeL+kxUxNu0jurBvxUh6HqrtfIMlHhKT2m0uAqLbaaF1/VYawGnFWHtWrwVj3m0uOuRuxVi7/UGEyLw5qwmBaPKa95LetW55zgbA6ANjdIqwNqGrBWf5mt9pSFGrzVAjgpyzPXAUwVnKuDdBpYNxe0qzoSNcTTwLxaqFcH9/SQr8GTVqFfdQpVCKi0o1JVo0ihojqsVYe3ajCX/lqtBnvNc+PzOUyBt7X5Tpq1qKySnZofllktLGthWQvLlhOWPcmJqRaXzYvLGm9jtNCshWZ/eWhWyWY//jkn2fju88LXp0tSdjW7iZbdZikfKEvZa4LD3RYOt3C4hcPPI0v5mMeyW1R812yl+v5lC41baNxmLVuUpsxaNsK0XgvTWpjWwrRnkrVscdozyV62UK2Fam0W86mymIcIUUR4lyTmi2zkZvZSEVrb7aS0aDPuE2OxM4Uz8A2iOB2/tZUW0/cS5i+DTp8jkbLUWrN8oYlZ9IbhLPIJI4LJPN+MKxGV8t2Z2dvHNe/PzGtV4Dr/dhyfUFWrWvHydw4Vc7Xla9Z5r2v+mUB0zZRa+UW7kueWUDqBV9JFa+7Lc0X3cs5XuCjif+PeLa68DcKvHub6/l7GK/5AtxeQQOzCXfUbo9WMnBFheIrmOYOz0L4Id1lPw9ShriXRzR/hiIAEB7gXHokmBAqsGxnVN2r2203sz4ISrydEyRmP43/lwzDF2sp4aZm6cSbcdEJcRRLD05SQ8nKLWuLmb3/naYCEvVG8clmIFA+dMGG60O3INalzNK2OoAINhkgtfSO+JoyeaYyRb6E0yHkUkVi0F3iZFxWFf58DikRLCvj4xqx3EoLiS5cG7qj3qi8dLKb5v2TyixjJuRd9Rf7JdeCoLnylJmQ9BxOSRPeIJtTXWU7/HgbTpMAP5LnLRZZGhV/oACBZ0/rpmlydTsi50+3I2LBkEK3TAWl9bwpgrhGFThhzd1U3MWuq7YtUvVCBUKU/IExY78/WBXLPzYSLjwO1VvwU+EvF/kVtuMaqenpN7y2i6b0VUXHpw9Is+1AsmYrPy4SXG4XN51+QSdd45WORR0AAKh/OeDbMB8OL1+xzy0le9wUXbFqlnvM6b1r8j26Hr+Fuc9rc/63sIwdn+Rxo+q2qdvpeeMIDnjDfi93jf3DXtTddfrz5WAT2/QxZCtT8erqfrXLz5ffn4jWdxlvjc46ijLXj0I1TLGUmbPfB3Bptgq1R1x64m2Awtsadnt3rdQaD7mDLGWyCcdr4olhid17c/h8c4vp8in0AAA== + H4sIAAAAAAAC/+1dWW/bOhZ+768Q1E6RYpJbS16SFAgGWacF2t5Mk3Yw0wYBLdG2bmRJl6KyNMj89uGihaRI2U7SLNfqQ1uT1CF51o+HpHT9wrLs09MgSjKc2u+s7yervATBP7MAQaEMRFGMAQ7iiBZekyJSGAYppk3YLysvZTXDLAjxh4hUOqtVqQ8wSOMMeZBU2IconkI8gVlqC21gBIYhrccog0L5JPA1pYEXR7txGCNKEI2HYKWzarmOQ/7q91ct541IOgJT1vF2NRfrtbUdQoSlIeCrhLXzQToZxgD5dl53w/49IX/fMKZAP8DKaO1xBPEHn5REWRjyEgSSyXEchzhISHmHFdLp7MYRRnFIGToCYcqfD6RnwyA6E6SQgAiy9pzlBcNtLw5DkKRQpCRV+LWaMQr8w7gSJmeyIrAL8tvtCQWXxQTy31f0d8Gdkjabg1v+rEZ9UpYhmECAhakKpR8wREw8+uov8YXEYFaVTuKL4wDXNMfGeaG9myEEI2ztgMi/CHw8seU2R8FP1m4yECpyTUDxBdeBfI4V44E3gcfBFMZZbTIe1csd4J2NUZxFvk42MfoGwgzqq1LBtEjZS3dz0+sNREVlCu9214miu5urVo/ofue3jU1J6V/6vR7ogkKFKwnMNEbbh14wBUxypcztUYymTHCEa1N4mkIUQOGZMcjGUFaqKbgsZul0ROWZBlFRIRZTUaocobKYEIc0iUP/IxhC2WjUFp8AOoOMf1QVauppVP1NRfUdd4bqOzXaExiMJ1ikxa2hW/2MiHqfg1BVF9nUc84lSRCNj7kWOrpyWUmq+Qju7pzy2MKxheElFlTDyquEeQgzMRBDIBrPIOZWxGpKR3Rhj+jdYUy4kMoKQdXhiLgKie82ZRFr/Slm/p+YRhRBDxOPJrU5psNROBoncrzi+hvA0P9dU8OsOfRkhnLJABLmhKITado+HIEsxCoxPlk+xeLPqlIfRLJOyQ7rFeV5mgBioEqTLAqYBe4kqTiuG2lcuZLIKsX5cg4R4uH0Wn6mtKA6F+S5FT6Kh0l0tjZGEEbKOJmy+/CyPklBW6jEpLqb1UX6vYJhSNyFsWPH2HG/FMsdukeCHtb6do19rxv6NioZe7D0eTUDq/xPEqd4FNDebVstPCB4o4hy/c7fhHoE68+wMuMjzA98ItDC7H9GJKTQ56iEZU+BubHan99uKxVx+UCDE0lrXoKUoDPiQKFq62Eoo8MuQYaOs0H+2tikwdLZkILliPZcDyyUskiHk3FJuHU2uxIBMXbVZMOwIqGTTSOZ1xigMcQNrISXCes7zaYrAcFGcIW4QQzIsNApwZsXMTojmN2DwTk8HV5hmJ5ignDDay/MUhJttn7Yr/L//rBXS7ey9b8fgpP5Yd98f1UEp3eviCMgI6Vu8uTNG1lKzRAgN4EUg0j1yFL8OwAeZhyVTNQO4RhG/kHZhUyXKCXDfvZ2k4JIbkzkMxnuAddKKVDQ8qNJMML1CgU8fiG8t+KRtUOZbH3hLPfrgJHDoLKYma9oSxuiLbHaZluK6crB3prTjAo3M9OSWMPPeWj3+BxboNsCXblgMaTba5Fui3RbpNsi3Rbptkj3lyBdTESSTgPcQt2Hg7rHOc/xcqBdproNIywGQxGxYTSsaibaKAgxfGi9NE2trF6U4E4zwZ2FCe42E9xdmOBeM8G9hQnuNxPcX5jgQTPBg/kJJrFvoEVrGlR39hqGRgAJOo8EAyTITLDAuZcFM7d+HMPeT1/E+jBV92RoId+CEQesWxaUQLkaSh0o0wILpNZPiGIhbhPfChFk0WQUxhWEt1MPxYxZ0qBoQHsPgc+ekGtihGVmEf+gcMOHqWcMhyzofITRmM+4o8Z3YXIpvgqbljogDECq9zwMr+U8qYWiaqGtYHri5asI9B/yZ+3Tp7W9Pev9+3fT6btUCXbCStmVYxkRXh1P5BXV5qe9h4ieWn58IUNo1uwrCuthMAEk5qBIP2UZxcszK4LTJPB9BbCXywryLMKzzbdkerl3WE/9LIcI9GFqHjFE2XRITEsrBml1t4AQdKBkqeSw85hyKABiYQrWIfDOIE6XUxS79yuK5FaiEAxiuaWx9xSkoRqGtYfiJFlWX7X/RA1kuaVy8JhSOYyfINflE4a3Yrr/dh04G53Bugc3e13Q6wDo9nv93mC97zmbo1Hfe3uWER5GEMN0LU9rBdF4jawC/xGj8Qd/y3mN4IjKYqvbSV+fA7RW5bKqRBarIA9tvTo99aCafBSkXVt43lXQKnKuJ6ae+p62NbyyVghfzBk/drTTkOuraUkt1+feLtcnlZPZJeIy22xTj543fZLs3PmV7CyUM+FxZBn4ufsg6rlEDN17SAX1OdBZBr7uP6iiLhFjD2Yyto4E7rwVdYQBFnPdBUDhDDDt4DzejQhnU58WHzyxKxH3ehWCofrdAqNXxy3sIUC1Q0zsbo0uI07Loa75nPsfrlTwTwT8ADJ76PyC4/+uYftjvdrTYFYld0UYNY620+P85pC6BQ/Ox5rSYruyXkNvEu1PE3ylvzP1X7opUqvhB3ZUUvyYjlqK6DmvI36KRq3LTwEo1FPS+N/5/o68jLOZi9QQ0h77uJl/G8mdbxtpwfNWpn2bBBJNJGFhXPcvCSVJ1S5LpY2wpDgIJjc371Lp3QE/gfB7fq5JmeesfSbxHAkmsau20UW8OYljH/nxEnmgf4H1nPkEx0LHNK6vSS83N7dewTWf4fh+H4Ezzzxq77qVmYxr0YwBYhFRNWS+8+mqhnpaRIMg8oPzwM9AefDipn46hN5/rAZwCS4DxfkOM56qrnmLaW6aWN75K85oKq31zqj0LN9PakO8ApdNm62V7tCdEVkp6NlcXZYqjMc7IIU1LeIet9ZcczJSnMdMvPqUhlgo9rLCg9qhaRM+2GjxQYsPWnzw5DKUywMQin2wFiG0COGpIAQhZSN5HTFjI1XcX8LGNSRsNjUJm4Y9TQ28MUKcJpjTAHXmgztayNMIe4wMrZiqXt9RMZCOuyWHux3j9R3GaWXyGlw0BzYy46NmjDQDJzVjJTNeMmOmWbjJ7Kxm4KcmDNV0vUaViQ5PmTGVAVcZsdW8+KoJY83AWU1YqwFvNWMuM+6aib0a8Zcegxlx2CwsZsRj2mteT3Wrc05wNgdAmxukNQE1A1hrvszWeMpCD94aAZyS5ZnrAKYOzjVBOgOsmwva1R2JHuIZYF4j1GuCe2bIN8OT1qFffQp1CKi1o0pVk0Sjoias1YS3GjCX+VqtAXvNc+PzOUxBtLX5Tpq1qKyWnZofljktLGthWQvLniYse5QTUy0umxeXzbyN0UKzFpr95aFZLZv98OecVOO7ywtfHy9J2TXsJjpum6W8pyxlbxYc7rZwuIXDLRx+HlnKhzyW3aLi22Yr9fcvW2jcQuM2a9miNG3WciZM67UwrYVpLUx7JlnLFqc9k+xlC9VaqNZmMR8ri7mPEEWEt0livshHbucvFaG13Q6nRZsJnxhLvQmcgm8QpXz8zgYvpu8lLF4GzZ8jkbLSWrt6oYld9obhNAkJI6LxPN+MqxCV9t2Z+dvHDe/PLGp14Lr4dpyYUC1UzaYvabeOOADWfCtOgMdCbfW+ddH92n9mEF0x7dZ+2q5iviOVjuGlcuNa+ARd2b2a/JVujITfhJeMa6+FiMuIuT7ElzNNPNkdRCQi+3Bb/+porfHaUyKVQNO8YHAe4xfhLuvplHvWlSy5/iMeErTgAf88IGGFYIJVK6f6Rs9+dxb78+gkKgzRdsbj9F/FMGy5tjZeWqZvnAuXT0ioyFJ4zAlpb7noJW7/9neRBsjYq8Vrt4ZI8akXZ0wXuh21hntJ2+lIKjDDIqnJr6VXhNFTg1WKLbSWOY8iEtMOoiB3p7Lw73JSkWhJiSPf2M3eQlJ85fbALfVe98mDxTT/l0x+ESM5C5KvKDy6ijzdzS9uQs5zMCFFdA9oQn2T5fTvYDCzFPiePHe12jKo8AsTEiSL25AvzvV5hYI73Y4KEisG0ToTojb3pkHoBlGYhDF3V00TcybGvkjVCx0a1foDwoTV/nRVIvfcTLj8SlBrxY+Bv3TsX9SGG6yqZ9b03iKa3lsSFVe+MM3SEOXaqfzOTHyxVtp88SkZvtirHksCAgJQ9XDOs9NiMKJ47b6wrhR1X3LBtlPpuajztiP+6HbEGuFapyv838m/dnBSzIHm4epqZ+5FJDwQCYu9uD3xh3Bve90Xx1uMRWLfz5jlQu2vx7v5crdYh38u39dpvbU+FyjKWjmM/ZRjKTtj2xD2xnAdbAy77sBfB4ORM+r03F6vMxh0BxveYB2MeOPzcq3deXHzf4qvyjuTfQAA namespace-by-workload.json.gz: - H4sIAAAAAAAC/+1dbW/bOBL+nl8h6HpFcufcWrbjpAGKQ16bAtu9XpN2cdcGBi3RtjaypKWoJG6Q++1HUm+UREp24vgl5n7oxjMyRc4Mhw9nOPTDlqbpvZ7t+iEO9EPt+3UjoiD4Z2gjyNGA63oYYNtzKfGBkAjRsQNMH2GftJjKOP3QdvBHlzCNRka1AAaBFyITEob+GXljiEcwDHTuGeiCvkP5GIWQo49sS0C1Tc898RwP0QbRsA+2mw2tZRjkn729hmbs8E27YMxefJSNRXurHTkQ4VwX8MRnz1kgGPU9gCw95j2y/1+Tfx+ZUKBl40Jv9aEL8UeLUNzQcSIKAv7oyvMcbPuE3mREOpwTz8XIc6hAB8AJou/bue86tnvDacEHLmTPRyJPBK6bnuMAP4B8SzmGVeIMkW199jJlRkIuKOyOfG51OMJ9MoD484R+TqSTts3G0Eo/Zr2+TmkI+hBgbqgc9SOGiKlHzP7i3eUEzFjByLu7snHJcnQcE/WTECHoYu0YuNadbeGRnn/m0v7Jnht1OUZsCci7i2wgHmMqeODYIGAGyCSZiaAPGCXXFWpPv0J3iJmYmzk6DEoaqp0t+sB2nJykKeEDApZNBsprSq7tdwVtG60abRtibbfTjw4cQtfKv4mIaegeBVfiqQ1uh8XBU9ON9CX2BGdjH0/ErP9C5JU5Y3AveMfYdgVUZA9H+FLob6iZCajk2d+ZSeWtkrI8RIeQjqbAOoWBWW4PE/fkCDp2C5wQJmZV0gPxFQIrotS7uG8GT70pzEgii0syU3MzXqej+ezZLv7kMXEwgs7NbBOKLNqHxGpdDIZlj+TT1qiJhvT9e3l6uftEaBZEkPn3geNlApR4kAAiGwb/uoWIWDwsDDGQ9TcoDTzAwLwp9SbA0Peh9SuRaYmHARpCHHDrIb8i0tXi3mfDoHrvkb6Z20E43raJq4PbZBnDgLSKemT5uPPQDVmCTWjfwl5/gmHQYxbxYDoh6QF6/0N/E//5Q2/QVY0NjJLTDz/0x+9viEQhIkZz+Ias5Z4TUp96vfPD/Zvmudp2+mzD9yxCHSIv9HsOHOBt2gPHA1Yj+aNHPeGOln6llzLIdw9vwj6kf/S8OzKEQwQd0IczdrehJU2+/98P/R9/5yns7fR5+n8ysh2tP9HSTu6QvnPzirpBD42ZaRDPPoa9yCbyjySiOQcmZujByLEjJ3aetvPwkHZGe3zMN4XggK1H+lGeTo2FmVlKfIz/yiwSj4hmRp5jFSyV9vucePyieVP65cge4DKjsMx9IWaleQPtmNqP9iWyJotb2lJA8sC7N4AYVig6uMiLtYq+qJcskLZr2be2FRK3VfJLyTMMB2UduAf3dmFB6ofmTTSFCl50HHufkiYTTFf0upGblvrP77yeEtec0q5LQ5iAe1gxszN7O/YLdsZmQqmDlOENjwFDbHnLixaq0uPRStXMG1i2GGUm1lj9Lm7xYt48RJUjVECqjoJUClIpSDU/SEVk5AZjGytMpTDVHDDVVWxOWMEqBatWHVaZxFWN3QphY3jPxnFFZkN+IEy9KatWJElD3+jXtL8cSRpL2bM2eFzd4PHMDZ5UN3gyc4On1Q2eztzgWXWDZzM3eF7d4PnMDX6obvDDzA1eVDd4MX2DyRojaTBlV6woU+8bMjQ4IOgjiSi/a/5Vn33fUBt3NySB970SfM3tmV4AvWog0H7SPcIUGDMwkcdklesUdWgXEFjsG3lOtEZywiKurCANK4L8bF0qyWQmhDpxqhYktjcVO0mT7lhjmZRcuJlsZzPpJnYFMzz0H/Lf7qdPu6en2sXF4Xh8GBRWFAua9hiwTEorv6IQ5ZXX5ZiRZZ70U0TMVLMIotRLj31FVKp6nuEDgm6QKx6yBGflwAnZPVqw8LbQtdl4yXcRrp/HqdBLCZxyVGszVCFeWadRhxuO+2SKCdVBAcxzlCGCwxulj+Nl6uOIbM/BEKrJIQZ1y1bGxk+O02XqI9m3J1NC+wyiHfFGquJsvqrwn6QKbkJstjbOV0EbxYmhnSKPRmU3UyUfVnSCbLZWLpapld+FQYRVEH0+pPwkyVu/7LcO+oP9fdM0jG7Xag3aRrvdH7T2Djrdg/2uaf5CsxHIhRgGu3E2xXaHu0ns5J8eGn603htvERxQrbxvN4O3twDtZomJLC3BGCz5wFIP7GOapnjT65mQS4cVbUEczHmuLRR3ouWUgjrbs4J5KJb6LmSgAgwECW5RcqpVmZx6cjqqoTKVG20hx/OzEHA7VC7k1RnIyWIMRLmQtbWQ08UsMokL8aPdhTKRNTKRswXjEGUj62cj58txI1YUpFCmskam8mFJ7kTZyvrZysUTTmQ+++TlJQaYP2+TRHUiAchOvCU1sbkDJXxJbI4xv4pY4534ZE5XUBJbEXgUVEHIKiGqqiEqKiKmrcoWVEZUVkdI5Sk67SSslBDJNpVvq8nR8rJhct4X2HW5G5UVFPIqispKippqiuqKCnlVhbyyoqa6Ql5hUVNlUVNpUV1tUVVxIa26KOtSWH0hP8MmOccmPcs2RTVGVUVGTVVGVWVGRXVG9ek5eZVGbaVG5Vk48Xk4edVGXeWGNGhedHQq8rVI5DBlIccUxRyzFnRURdElMIKHEjk4UZ/yEcONSsiRhx3JAZ6aIlppxUdV1Yek8mOq6o+yjxRXgUgqQSqrQaorQuRVIRWVIfLqEHGFCF8lUh5quVpE6E3kJRmVZRlVpRkV5RmyEg1pmUZpXGs6hC2Ryirz6ArOiojT4NkDhWcVnlV49nXiWZWoU4B2SYBWeipdYVqFaRWmlYWzF393YamgZB53Fy710siWJET+bsVujcxKeC4+BmTdmCzi1sjirUWv4NrI9oFY39x4prjlqHjzRN01R0XOQu85KvJmvuio9s4iccFzbdV3a7qq70yn5V0FgbEuNHM3n7yyi4u40SdbhnxduroKclOvLUrettqXFsXRaiFmWMdLinD+AoLqK4oKY8jwurqE6GW7+Ky7HV8B7ild7igFPoYCPgr4KOCjyqAU8pkz8knCmgr6KOizQvcvLus0alsSajNacz2OKsrEb8h51E5N/t5oPyWBL0o8TJPBF3GXmsIX8Z+cw58qDc/jQ3keXnR1QhkrTpeGz9vKNLhxE3PxBSmJ8OVLp+JVReS65uIl4HPNMvHT3UX2KhPwuHiF6DTpd8E4y5B2Dol1f/0T6/7mHBbdELApPC1aizY7Cm0qtKnQ5qqjTXW5goKbC4ObtfetKsSpEOerR5wrcJSzOPnmc4Bz0THljuS3R4w9FVOeU0zZqEH5XYXyFcpXKH/dYsrqyiMF9hcdWxZfHK8Qv0L8KsasYsytp6DPfYU+FfpU6HPtYswKfir4uYRYs0KgCoGqmPOyYs5nCFGg+5SQ81bccz3+aSfKbTejtuhjmSPRA3MEx+AbREHUf+MgItMfVWY/MwXQTfQ9ggAyq9Wz35bS07dhOPYdIgh3mM5CglgCzNn6g6juSfgL4BYcgNDBkh8AT7iiPQODkYXwt9jUkinOg32O6/lUqUWvq/8ZQjRhRi3aFHAyN3LUIbwvXNCcao57fTFCn6suc77F4y8MgxMlL4cptjCJrPhqCdsl67QFjxxRNZdEkGOiDFvweCLgeOWfRbrsTb3IoW6H/sMfXp8s/iawbm2ympD1v6HFre6Ixd+qE3+8KPF2QoycyTj4d9INPc8t9ZfSxA/Hyo0GxDHCAF5FDQkr4qbSOAixJ6owJOSe6YXMFtrNIidyjrrRzJlAzUSkM303mBBBjyWTkX9COCGnMUQyo23Xjr1oXvnPOfNLrCSFlzt6tZN4CcNPX/5k03+R0c8yS25s/ytyLieuKSoTjeaQoeZQ9WLmO95kXLy+kF/P0gdeYgYtdMf2WNiyLWPaMcN56oxbHWnNe5qqpa5ymu7JVri9ZyxsC7L4LLYjsfst2UZND6ATxQTF4cxEOu1m6R6+VECUJ9vwyt8murBVrAqZMqZ+VdXAjJH0XYS1JdosCp0IEUJjb9zINbduK20SCVOzeCn7JJH4Z53DFbOqI7f0ziyW3tkQE99KwkKPUYDDZlqKQxuDKO6qu97dbjrndWbmlKbnvubbBKuj7MuxzHpJZ3j16ntc2Ie3/ZwL1o3Mznmb1w3+Q7vJc7grDVrc34YV6f46GQONlZfNTv4WvuEu3zD/llaH/8DdxbJv8f1N+pIT30+PpWD0r1cncTQqCZP9lv6yufaL9luCtLTt5MffIzylhywDqvf7/RbY7+6BbqsNYPvAaDdbza65b73rAKPbHEQP36bhsObW4/8BAYppRXCnAAA= + H4sIAAAAAAAC/+1dbW/bOBL+nl8h6HpFcufcWrbjpAGKQ16bAtu9XpN2cdcGBi3RtjaypKWoJG6Q++1HUm+UREp24vgl5n7oxjMyRc4Mhw9nOPTDlqbpvZ7t+iEO9EPt+3UjoiD4Z2gjyNGA63oYYNtzKfGBkAjRsQNMH2GftJjKOP3QdvBHlzCNRka1AAaBFyITEob+GXljiEcwDHTuGeiCvkP5GIWQo49sS0C1Tc898RwP0QbRsA+2mw2tZRjkn729hmbs8E27YMxefJSNRXurHTkQ4VwX8MRnz1kgGPU9gCw95j2y/1+Tfx+ZUKBl40Jv9aEL8UeLUNzQcSIKAv7oyvMcbPuE3mREOpwTz8XIc6hAB8AJou/bue86tnvDacEHLmTPRyJPBK6bnuMAP4B8SzmGVeIMkW199jJlRkIuKOyOfG51OMJ9MoD484R+TqSTts3G0Eo/Zr2+TmkI+hBgbqgc9SOGiKlHzP7i3eUEzFjByLu7snHJcnQcE/WTECHoYu0YuNadbeGRnn/m0v7Jnht1OUZsCci7i2wgHmMqeODYIGAGyCSZiaAPGCXXFWpPv0J3iJmYmzk6DEoaqp0t+sB2nJykKeEDApZNBsprSq7tdwVtG60abRtibbfTjw4cQtfKv4mIaegeBVfiqQ1uh8XBU9ON9CX2BGdjH0/ErP9C5JU5Y3AveMfYdgVUZA9H+FLob6iZCajk2d+ZSeWtkrI8RIeQjqbAOoWBWW4PE/fkCDp2C5wQJmZV0gPxFQIrotS7uG8GT70pzEgii0syU3MzXqej+ezZLv7kMXEwgs7NbBOKLNqHxGpdDIZlj+TT1qiJhvT9e3l6uftEaBZEkPn3geNlApR4kAAiGwb/uoWIWDwsDDGQ9TcoDTzAwLwp9SbA0Peh9SuRaYmHARpCHHDrIb8i0tXi3mfDoHrvkb6Z20E43raJq4PbZBnDgLSKemT5uPPQDVmCTWjfwl5/gmHQYxbxYDoh6QF6/0N/E//5Q2/QVY0NjJLTDz/0x+9viEQhIkZz+Ias5Z4TUp96vfPD/Zvmudp2+mzD9yxCHSIv9HsOHOBt2gPHA1Yj+aNHPeGOln6llzLIdw9vwj6kf/S8OzKEQwQd0IczdrehJU2+/98P/R9/5yns7fR5+n8ysh2tP9HSTu6QvnPzirpBD42ZaRDPPoa9yCbyjySiOQcmZujByLEjJ3aetvPwkHZGe3zMN4XggK1H+lGeTo2FmVlKfIz/yiwSj4hmRp5jFSyV9vucePyieVP65cge4DKjsMx9IWaleQPtmNqP9iWyJotb2lJA8sC7N4AYVig6uMiLtYq+qJcskLZr2be2FRK3VfJLyTMMB2UduAf3dmFB6ofmTTSFCl50HHufkiYTTFf0upGblvrP77yeEtec0q5LQ5iAe1gxszN7O/YLdsZmQqmDlOENjwFDbHnLixaq0uPRStXMG1i2GGUm1lj9Lm7xYt48RJUjVECqjoJUClIpSDU/SEVk5AZjGytMpTDVHDDVVWxOWMEqBatWHVaZxFWN3QphY3jPxnFFZkN+IEy9KatWJElD3+jXtL8cSRpL2bM2eFzd4PHMDZ5UN3gyc4On1Q2eztzgWXWDZzM3eF7d4PnMDX6obvDDzA1eVDd4MX2DyRojaTBlV6woU+8bMjQ4IOgjiSi/a/5Vn33fUBt3NySB970SfM3tmV4AvWog0H7SPcIUGDMwkcdklesUdWgXEFjsG3lOtEZywiKurCANK4L8bF0qyWQmhDpxqhYktjcVO0mT7lhjmZRcuJlsZzPpJnYFMzz0H/Lf7qdPu6en2sXF4Xh8GBRWFAua9hiwTEorv6IQ5ZXX5ZiRZZ70U0TMVLMIotRLj31FVKp6nuEDgm6QKx6yBGflwAnZPVqw8LbQtdl4yXcRrp/HqdBLCZxyVGszVCFeWadRhxuO+2SKCdVBAcxzlCGCwxulj+Nl6uOIbM/BEKrJIQZ1y1bGxk+O02XqI9m3J1NC+wyiHfFGquJsvqrwn6QKbkJstjbOV0EbxYmhnSKPRmU3UyUfVnSCbLZWLpapld+FQYRVEH0+pPwkyVu/7LcO+oP9fdM0jG7Xag3aRrvdH7T2Djrdg/2uaf5CsxHIhRgGu3E2xXaHu0ns5J8eGn603htvERxQrbxvN4O3twDtZomJLC3BGCz5wFIP7GOapnjT65mQS4cVbUEczHmuLRR3ouWUgjrbs4J5KJb6LmSgAgwECW5RcqpVmZx6cjqqoTKVG20hx/OzEHA7VC7k1RnIyWIMRLmQtbWQ08UsMokL8aPdhTKRNTKRswXjEGUj62cj58txI1YUpFCmskam8mFJ7kTZyvrZysUTTmQ+++TlJQaYP2+TRHUiAchOvCU1sbkDJXxJbI4xv4pY4534ZE5XUBJbEXgUVEHIKiGqqiEqKiKmrcoWVEZUVkdI5Sk67SSslBDJNpVvq8nR8rJhct4X2HW5G5UVFPIqispKippqiuqKCnlVhbyyoqa6Ql5hUVNlUVNpUV1tUVVxIa26KOtSWH0hP8MmOccmPcs2RTVGVUVGTVVGVWVGRXVG9ek5eZVGbaVG5Vk48Xk4edVGXeWGNGhedHQq8rVI5DBlIccUxRyzFnRURdElMIKHEjk4UZ/yEcONSsiRhx3JAZ6aIlppxUdV1Yek8mOq6o+yjxRXgUgqQSqrQaorQuRVIRWVIfLqEHGFCF8lUh5quVpE6E3kJRmVZRlVpRkV5RmyEg1pmUZpXGs6hC2Ryirz6ArOiojT4NkDhWcVnlV49nXiWZWoU4B2SYBWeipdYVqFaRWmlYWzF393YamgZB53Fy710siWJET+bsVujcxKeC4+BmTdmCzi1sjirUWv4NrI9oFY39x4prjlqHjzRN01R0XOQu85KvJmvuio9s4iccFzbdV3a7qq70yn5V0FgbEuNHM3n7yyi4u40SdbhnxduroKclOvLUrettqXFsXRaiFmWMdLinD+AoLqK4oKY8jwurqE6GW7+Ky7HV8B7ild7igFPoYCPgr4KOCjyqAU8pkz8knCmgr6KOizQvcvLus0alsSajNacz2OKsrEb8h51E5N/t5oPyWBL0o8TJPBF3GXmsIX8Z+cw58qDc/jQ3keXnR1QhkrTpeGz9vKNLhxE3PxBSmJ8OVLp+JVReS65uIl4HPNMvHT3UX2KhPwuHiF6DTpd8E4y5B2Dol1f/0T6/7mHBbdELApPC1aizY7Cm0qtKnQ5qqjTXW5goKbC4ObtfetKsSpEOerR5wrcJSzOPnmc4Bz0THljuS3R4w9FVOeU0zZqEH5XYXyFcpXKH/dYsrqyiMF9hcdWxZfHK8Qv0L8KsasYsytp6DPfYU+FfpU6HPtYswKfir4uYRYs0KgCoGqmPOyYs5nCFGg+5SQ81bccz3+aSfKbTejtuhjmSPRA3MEx+AbREHUf+MgItMfVWY/MwXQTfQ9ggAyq9Wz35bS07dhOPYdIgh3mM5CglgCzNn6g6juSfgL4BYcgNDBkh8AT7iiPQODkYXwd2Jq+imB9dplhOvLh/N51M9xPZ9qt+h+9T9DiCbMukW7A074Ro46hPeFm5pTFXKvL4bqc2VmzrdYEIVqAk6mvECm2MskQuPLJmyXLNgWPHJEZV3iyauPiVZsweOJgGMIMIt02Zt6kWfdDv2HP7w+QQEmsG5tsqwQINDQ4lZ3xOJv1Yk/Xp14gyHWzmQc/Dvphp7nlvpLaeKHY+VGA+IYYQCvooaEpXFTaRyE2BOVGhJyz/RCZgvtZpETeUndaOZMoGZG0im/G0yIoMeSWck/IZyZ0xgimdq2a8fuNK/85xz+JVaS4swdvdpbvIThpy9/sum/yOhnmSU3tv8VOZcT1xTVi0ZzyFBzqHpV8x1vMi7eY8gvbOkDLzGDFrp1eyzs3ZYx7ZjhPHXGrY605j1N1VJXOU33ZCvc3jMWtgVZfBbkkdj9lmzHpgfQiYKD4rhmIp12s3QhXyogypPtfOVvE93cKlaFTBlTv6pqYMZI+i7C2hLtGoVOhAihsTdu5Jpbt5U2CYmpWbyUfZJI/LPO4YpZ1ZFbemcWS+9siIlvJfGhxyjSYTMtxTGOQRSA1V3vbjed8zozc0rTc1/zbYLVUfblWGa9pDO8evU9Lv7D237OBetGZue8zesG/6Hd5Dnc3QYt7m/DinR/nYyBBs3LZid/C99wl2+Yf0urw3/gLmXZt/j+Jn3Jie+nx3Ix+terkzgslcTLfkt/4lz7RfstQVradvIr8BGe0kOWCtX7/X4L7Hf3QLfVBrB9YLSbrWbX3LfedYDRbQ6ih2/TuFhz6/H/xs1ceXmnAAA= persistentvolumesusage.json.gz: - H4sIAAAAAAAC/+1bX2/bOBJ/96cQiF0gOXh3JSdu7AX2oc0ihwXaXm6b9h4uC4OWaJkXmtRSVP5skO9+Q1KSKYmqbaSbpq2BtohmhqPhzHDmx6lyPwgCNJtRnhUqRz8H//1jaCmS/FlQSRwa5lworKjgmngPJCAymisjAo8PRowkVOE5I0BdYJYTQ0w5Ub8lQOIFY5Yicba8EIIpmgE9NMQlTcip4EoKljfW08ZaRvmVY5gkCzB0CQR0FObI0sSNkTBWWluBGgvGcJY3bGswkg4nw5ywtSpXneFjRnF+KpiQxisPQ5c5xzJvazSMBOfL14SnSlsdhR0e6VumcC4KGesdoHMpVkQtSWH3XEstKGNabYf4T4kTSriq/F0zU0mTc+HZgPH7qEFiJCU8qTPAdUTKX+YXZeyVLBq2a4nr1M+ICymtWR7mCt/2MCj3MyRNl+odpJLPh8DPl+LGvzKHRf+hiYlKlW0OW0H+sx6l15gVJmparcNq+hMytxbqMG7KN0dtjpPsNVmbdy4oV2+E2aghNPMgI5AoXOHU6wiU6dU6Iwqtftzl+XMQIpUQSaR+54IJhVrcjGDlcR/KiaQk/9c1kZBsxLOjPMMx6TsUwNTRnjaJCsdXPm/mimQZSV6DU717UFimRDVPdftkG0Fym5mNHlzyIMiLVXBD1VIU6oByeDmPyTDg4P/D4ECJ7OogGgYHV8WcMKJm14IVKzIDMZXPYgybo+puNr9TJL+PWQEmyl8u0Xflj5doGPxPzIFSrtcEjlfEeEUL1g+aA6HNofJCdO1rYobpSkvZx0v0cHh4qG3+4ZGG42tMmT7ST2w5/EHDdjQWQq5MciFFtYUmo7pikLpEwnk8w7ESsn2gnCJ2Vut7D6U/eKdt7KqD9mJaF3qJGqyH4Xa58wV6/yldfyYJ2ej6Vy3XO0/NIqKWGgoIlngKjLb8DHqmrzpp3rslXSg/U5mehj4YL1lrg/e5rqxNwRrP3LdbDpZdcFExhey05KqjzNRdZl5NeUKvaVJAA+rtLpWswVZNy27xLc27ds2L+MoWQk+7W5WdRfumFRuks8q/qr+71i3SoEX/Fu7wLdmiLq/T0RyMbubAqSHMa6BmivQVNiiwm54WbniXWcARdujOjncqEF/GJtyjNvBsrYGGYxwvyQXkCxQ730mKNVB+BV07laLg3gNhZT7oXOlnd1MEyXSOD8bhMIhORsPgeDwMwh+nJ4ftxDVio6MTkBtNQS7UcpNpj5xWMz62f0HdIeotPFtC8zrkJThrslNcGLB234HAlT+iMOzC4IoZbn0U6zL5WqdY3lOYaqk3WF4RuRHdfvwacdQklZ3ClyU9kHeFs4zy9MJWuaiPt0X5KKuXrUiBEoEit6p7/q4rp+90rCvlEvN0C+WjLTsbJMGvkGPnFTJvZUL3QhALzkmsoO105C60RR7HZyJXC6prB0I+xhnczd/Rv4z6cfh9Swa6tXetoX90qXHVG7iAb1H5bQPvXndsxtqNobc/vfQwRb1wS5+Xt46jNlFeMXu1aB1Ufck2swATf11BjgDbRdEE/plMdQmJJp1KsyhY36VSv8XVZ9WNdJGbHnUUlcfdKOrHCBpOnmocw7uh2vleBEnZxbYltL3k5tZUIdznci+qb0UfNexzIG9z6/npkj8/j/1DV5tPei8YbbwX7HAR2+42gCZwbqYh+luh/UZsnkOXYkTHsqnftAO3SE7aRdJIbFckhbYT/bJjfaw60k4l0ix6W7a8apA3aK+r1/gHRCX1N0gSM2D2s38XN40Zcl30LsogNie3dWh/xflyLrBMAtCAmvzK28sXDqMMlQTpgRPJ/Rj5UWPk4/0YeT9G/rbHyI8YBVItlc8KKC5f5RjQTGDtJj/9CPax43tr1xc0tv8c2fLECROYwfHGjHlek2Nr7n50/CxGx1xw8qVPjp/xHvaD4/3g+JMMjsf7wfF+cLwfHH9Dg+OPTkCfGttunsk+4fXgW5rFPhos74exX+MwdlDuAOUAGVf4gz40xsro2JLVnX1VAuXWfg2scLoOlDmLkpP1Jxcozimq1SqyyhjsnKfdz5wH7Rg7Q8cGnq1im5AFLpgvvVyuDx6bj6Gb2KwH19fAwYGSDldk1TfbLixCfxZE3hlA6YOczkfVUYOaknanrkPkvH7Q2o7jMMxYBTtb23Bc6fphC4Bc+cqpZFDmoAgn5KWvR9aORGWhdnWtICLUs6bysmfJJheb183sJXabD0SHQfmSQ39IRptCUs4F3NDBGTB+z/9dWYWa3I75muYXLgNu9+cwoCdfWEVu/X3qLAh3zoK3VRfeJQ+4b9GnzoTtQMWDgyr2KfMkKXNeYzWLVk41VtslfWzQP3/u9APShx5E+nVk2KAaTNlfWzKTyrrf15dFcfNDVM1Dq3ugRSLrZRmNr8x/ZJWLS5fMKqDtYkQ0XqcfikLn4ch9iNaphMbOz5H7cBS6nPXUFo2cn6PENuM/qj3oq4GTaRvf4ip+4Sp23zI6dh/WgwN0krj2VrY03PeXMDdi9P7itERrFVZcn7HAHrLyl7sKMxZC02g6n47whEzC42iejF+c4MWUJHgejeLJcRhb4esaH4aDh/8D0iZvTe82AAA= + H4sIAAAAAAAC/+1bX2/bNhB/96cQiA1IBneV7LixB/QhTZFhQNtlS9o9LIVBS7TMRSZVisqfBvnuO5KSTEl0bCNZmnQG2iK6O1LHu+Pd7y7qTcfz0HhMWZrLDP3i/f25ayiCfMmpIBYNM8YllpQzRbwBEhATmkktAo+3WoxEVOJJQoA6xUlGNDFmRP4WAYnlSWIoAqezU84TSVOg+5o4oxE55EwKnmS19bS2NqHs3FJMkCkoOgMC6vsZMjR+qSW0lkZXoIY8SXCa1XSrMaIWJ8WMJIut7O00HycUZ4c84UJb5bZrMydYZM0dNSPC2ewdYbFUWgd+i0eWLZM447kI1QnQseBzImckN2eupKY0SdS2LeKvAkeUMFnau2LGgkbH3HEAbfdencQkERdYvQEF8/qbExITFlXRYRspZgfZaREXUuS1cymJi9jNCHMhjMoO5hxfLWFQ5mYIGs/kCYSZm53N+OUSDqz5i0baYWUgWmwJVyNx+Qx4YKxcO1Rta7HqpoagroRajMvizUGTY92DiqzUO+bgp/dcn1MT6o5KCcQQkzgmzjhL1WoVLLnaftDmucMTHBURQYR65zThEjW4KcHSYT6UEUFJ9vsFERCHxHGiLMUhWXZfgKmcPaoTJQ7PXdbMJElTEr0DozrPILGIiaxf+Oal14LkKtUH3Tljnpflc++SyhnP5Q5l8HIWkq7HwP673o7k6flO0PV2zvMJSYgcX/Akn5MxiMlsHGI4HJXX48m1JNlNmOSgonh9hn4ofjxDXe8fPgFKsV4RGJ4TbRUlWD0oDrg2g6QM3jWvCRNM50rKPJ6h293dXaXzi3sqji8wTdSNfmTN4Q/qNr0x5WKugwtJqjTUEdUWK5PXEQ4lF80LZeWwo2q/j1AVvBOlY3s7qDy6qqEDVGPddteLnWdo/cc0/ZEgZKXp3zRMbz3Vk4icKZTAk8iRYJTmR1BOXdlJ8U5mdCrdTKlLGvqkrWS09T5mKrPWBSuoc9MsOVi0cUfJ5KJVrcuKMpbXqX41ZRG9oFEOBWhpdSllNeyqa3aFr2jW1muSh+cmETrK3byoLMo2Dd8gFVXuVcura1UiNZB0H+EaX5E18vIiHPXFaEcO3BqSOBVUTB6/wRogtsPToA3nMoM3/BbdOvFGCeJ5HMK+ah3H0WpAOcThjJxCvECyc92kUGHoN1C1Y8Fz5rwQRuaTipXl7HaIIBFP8M7A73rBfq/r7Q26nv/zaH+3GbharNffB7neCOR8JTccLZFT2wz2zF/YbhctTTxrovbK5QU4q7NjnGuwdtNCwKU9At9vo+CS6a99Fas0+U6FWLYkMVVS77E4J2Ilur27w+g/QofRRP53AXEXqp7jNKUsPjWJNFjGWyNDFQnSJD1Pck+SK9m+4helXzfKHOXmArN4jc17axZPiLO3EMbHJfhvBFu75wg5YySUUNlacqdKI0cGSHkmp1SlJ4RcjCPOwIFf9fYD/8eGDAAC51pNv3OpNtV7aP/XKC4GI7Q7KnMpzMHQh5cHDiavFq5p86Kx6TeJ4jwx3UvjDqgWX08itP9VkuoDfAyCIfwzHKksFQxbyWyaJ8v6VvUWez+zXU/l0VG/tVGRUfRGy2GIup+HCiqxtqs2br0gKNvwuUDPZ0w3ZiWIfiqtV9V43anYtwD3urF6ecaensV+UtnmQVuP3srWY4Neb72GAw3h3ox89J92DyvhfwZVKiHKl/X9dTmwk+SwmSS1xHpJkis90esN82NZkTZKkXrRh6LklaPCTnNdtcY9gyqov0GQ6PG2m/0nv6xNsKukd1o4sT43rlz7FmezCcci8mAHVOeX1p69shiFqwRIdyxPbofY9xpi722H2Nsh9naI/fCDSKqksnEOeee7HELq+a855MMPgO/7ywOj1zP6pcG3iJZHDhhPj61XRszTmlsbdbeD6ycxuGackec+t37CZ9iOrbdj6wcZWw+2Y+vt2Ho7tt6OrR9ybH3n/PWx4fPqifAjdiD/p0nwvfH4dhT8PY6CO8UJUAaodI4/qUujtQz2DFlem1dFkG7Nl9ASxwtH6bsoGFl8U4LCjKJqW0nmaQInZ3H7E+9O08fWWLMGmUvfRmSK88QVXjbXhcD1h+B1+Fe1DkgVdO/EAFOLXyIIC7ZaXJ6WH67b+Ah9yYm41uDVBW+tL8uDGjUmzZJd+cp6fadxLstyOElKiFvHE7ZNbYOsAcZLo1kpDfIdZOOIHLiK5cKiRca295qDa6hjTWllx5JVJtavG5uGea1PYRvV47brFW/ddfuot8pHxVDC9iXcDu2I7I9STVTnts6jaG7hIgLMgS0GVOtTs5GdmR87LPyNw+JDWZ83CQzmWvTgobEW3Li18MY2ZB4lZI4rFGdwzKFCcZuEj3H6t4+d5VD1dglW/T4irFNOxcx/5tJj0goJVG0kv3wRlMPYskM0GGWxLKXhuf4tWrG4MMm4hOA2ekSDRfihwLce+vaDNVVBA+vnwH7o+zZnMTJGPevnIDLV+XN5BtU0WJG28i32xq/sje239Pbsh8VIAe1Htr6lLjXzfeW6V0YfTw8LHFeiyMUd88wlK/7LW65nUmgUjCajHh6Sob8XTKLBq308HZEIT4JeONzzQyN8USFHv3P7L0EqTdcFOAAA pod-total.json.gz: - H4sIAAAAAAAC/+2daW/bOBqAv/dXCJrsIsG6M5aPHAX6IclMFwXaTrbJdIFtg4CWaJsbWdRQVBJP4P3ty0MHSZGykx5JEPVDGpMUj5fv8fCVpdy+8Dz/4gIlaU4z/5X3+bwnSwj8M0cEKmUgSTAFFOGEF96yIlYYo4zyJuKTV5SKmkmOYvo2YZVBry6NAAUZzkkIWYV/QvAC0jnMM19pAxMwiXk9JTlUyucospSiECfHOMaEd0hmE7Dd73mDIGA/xuOeF+yoXSdgIQY+rNfi/d07jCGh2hToMhXtIpDNJxiQyC/qVuL/c/ZzJYQCI0SN2fqzBNK3EStJ8jiWJQSk8zOMY4pSVt4XhXw5xzihBMdcoFMQZ/J6pF0bo+RS2YUUJFC0lyIvBe6HOI5BmkG1J60iatTMCIpOcL2ZUsjGhl2zz4ORUnBTLqD4vOSfS+lUfYs1DKqP9azPqzICUwioslSl9C2FRGyPvfojvtYELKqyOb4+Q7ShOT4tCv3jnBCYUO8IJNE1iujc19ucor9Eu/muUlFoAsHXUgeKNdaCB+EcnqEFxHljMSHXyyMQXs4IzpPItjeYfAJxDu1VmWJarOynwcFBONpVFVUo/GC4xxR9cNDzRkz3+z/vH2hK/1M0GoEhKFW43oG1xuhHMEQLIHau2nN/islCbByT2gJeZJAgqFwzA/kM6kq1ADflKoO+qjwLlJQVajHfSlMifC/mzCHNcRy9AxOoG43Z4j0gl1DIj6tCQz2dqn9gqH4wWKP6QaPvOUSzOVX7ktYwrD8mTL2vQGyqi27qheTSFCWzM6mFga1cV5J6PYq7u+Iy9ij2KLyhimp4RZWyDmUljs4ISGZrOhvUnTWUjunCr0zvTjCTQqYrBFeHU+YqNLn7XESi9Xss/D8zjSSBIWUeTWtzxqdjSBSneryS+otgHP1uqRHWHIe6QOXOABbmlKJzbdkRnII8pmZncrFyieW/nlGPEl2ndIe1xWWepSCEr7ytFEe+0TBPkLDDozRTZ7fSZleoiq5YUjpXkBAZVG/1ayo7aspCX2HpqWSwJJcvZwTCxJinUPkI3jSXqugM3zetbtW7y7hLGMfMaTgHDpwDj6vN+YrhCYzcYw+cY+85xnaqmriw8nwNM6u9UIozOkV8dN83C98w6ihj3bj/N6WewOY1osx5ifAG7xlguL3QlAUWfh3fYd1fUGmy/odfDo0KXF3Q4kqyhq9gJeSSuVFoWnwc64w4ZHwYBPvsx/4BD5nBvhYyp3zkZnjhPav9yG4GLOgGB0OtAzWCNfZGECPrJ18kuqwpIDNIW0QJb1IxdpYvthEjJLjNnCEFbFrkglHnNSaXjNxDiK7gxWRJYXZBGefGt2GcZyzmvP7ibxW/fvF7lXN5/b8viqthNR7zNaKQ/f/FX33eKiPWqy3mF9jEue8839nRN62dCwqLyChITDetBcU3IKRCwJrF+jGcwSR6Uw2h98t0VAChf9imL5pXU8XOpvtGKqkWPXj56RxNabPCIMqPbCs8PPWOuMy9j3IHoiZFSjaqioU1q6a1r5qWqG03LcyPE/7rDa2q9DprDUs0/FDE+1CusaPfjn71grvh76jD3w5/O/zt8LfD3w5/vyP+UrYl2QLRjn8fjH/Pii2gzwOBHzLzHjhS7+NHlnr/pil3ECOQHZdHhDp2+xNAGlws7uG8g8mMCgH3tXJoa77uWMDdqXZ3gxf8k4AIQWHa/e+RZg7s+7xb07NwEPpQTFCz5DA7K+5Qmf4cXM0spaWyN2v4HavfFild2u/N/QcS3KyRDGh2JcnPLCX86HAqkcysK0KK0XvGGv9b6JXpUn3h+S0dWRlipZ5CYGbqMS+8LoYZtB5Y7ovwPPLYtDSFTBNZjJs1/UvKu+Rql2e6xZdnC70529QIEigC6DTG1F/jDmT8+r2AZGOdmWu+FiihILw0BcpCcJrC6J1kFX2iTy0L5k2W3jZrcx8cuFPMv71lo6xWztCvlXMJqwFiLRAou3t/IChSX9ZbrNX971vVqgERAdK0a0yofnSQdntRBgd2uEBXKMpBFcVXTdTgt93rCdyAG2T44kkeXkpVM5zHorBUvmLLFwiM1nbfVDmaz+eNKS7BTVvmotYdfrbVlYJnf2wAGePZEchgQ4ukA240t5y91XWszYc8pimWiv1caaGZlnPgwl6HCx0udLjwsFmDjhdqXihTBR0wdMDwWIBBSehoTkjN52gV3y6dM3Ckc/Yt6Ry7lF204ySeNuppIZ9Nv81rIaBWCnIKtBaqeb/IRCKbdGsJB847RULSBxaP2pxHOym5aamdmNZQUzs5uenJTVDrKMrtq9bQVBtRtd3IMbfERlduwnJQlpO0NqWtNuJaQ11t5NVCX+0E5qawtSTWSmN2InNS2Toyc9KZ9YbiXZI6bP48Cj8Epm2Iahvg2sbI1oZtDnRrv4vqwLg2lGvFOSMFVNwLKr8F5Z3I3TJvcTbhrg3wHJC3Eeg1/Yod+BzQ1wp+bfDnBsA1jrUJgs0lNIHQala1qqapRUVd5NVGXy0E5v5Wh4PENvmqwVNYgmpra0m4YzR76mpjSDNW31FaR2kdpT0aSqtyaR2mPQFMU76s05FaR2rPlNQaqe4f/xUp0/i+5pnkB8xgOu48BkGXwvxGKczhYA0dDzo67ui4o+OnkcOMCOaDd5D8lHKZ3q9y1zpS7ki5y2l20LZBTnMttQ07auuoraO2J5LT7LDtaeY2O3LryK3LcT5UjvM3Qjgg3ifF+aKYOXdD3Hfw2mFf9sWbKe/IzMI5XIBPkGRy/sG+LKbLGJavLJDXscBZa61/mU8gYR4fitdJyL4oXKQxE0Qy2+SlpzVg3doeRy7elOF4JLmstbF2+fJTNd1qV7XqRSUKJCu19StBVK/r/5lDshRKbX0lay3zQCudwRvjYXTl1anV8GZGWHvkJP6kvAHD+lyJepjY6AWyhazU74KjhMXlCB7a32tgF+SCbQayNC8FXET6u0hXjHQhHep2nt7+F08YM4QgukIsmjAU6HlFrzt28Q/Wib8ISqqeMCUXMs7+VU7D12sb8+Vl9sbF5soFKRV5Bs9kR9bHZOw77v/8D7UPkIv3XjQeO2LFFyHOhS4M+2aNdI5+0NdUYI0hckt/mS2ZoBcOY1RbWA1yE0VkFo0SVHhRffO/5suNTEsqmtzx252EpvjG8wb31Ptq7Htr/ndZ/F2M5BKlf5D4dJmEtkfHpAkFnQm1mZDDbh6dsbiPYStxDruTAX2jyKG/y+rH2c6dZPEs7cnYyB9oT2NXJBp/RQD6QfpcJy0cav3CdaDyMxjLlJc9W1dKZ9g3z1q1gHid62DqHs1y0HVshWszNh6qbWHB3DkWq3phO9RZfQQTQm+86GndPTUTrl4M2VnxQ5xnbOK/qw23WNXIremju2j66JmouPGXRkQ2r0pBVC8VxNcvK5sv3xsocyb1ZSliYEDqiwuZXZSTUbfXHyvpGVX3NRfsB7WeqzrvB+qHYV+tUR6sHii/B8XfVjkv18DT2U21c4+idryrdqyOMhipH5QXKexF6nzLuWji+wuLOwz+H2fHRdaoTGd9kASGkpn3i3dSQJ2fi/t5/h4I9vu7eyE8GA3BqA/gYDwaj3b3xmFwMJ2OQ9n4qspS9V+s/g+n66MejmcAAA== + H4sIAAAAAAAC/+2dW2/bOBaA3/srBE530WLdGcuXXAr0Ie1MFwXaTrfJdIFtg4CWaJsbWdRQVBJP4P3ty4suJEXKTnpJg6oPaUxSvByec/jx0GKuHwQBODvDaVawHDwNPp4OVApFfxaYIi0NpilhkGGSisRrnsQTE5wzUUR+CspUmTMrcMJepTwzHDSpMWQwJwWNEM8A7yhZIbZERQ60MiiFs0TkM1ogLX2JY0cqjkj6giSEigrpYgYfDQfBKAz5j+l0EISP9apTuJINHzVjCf4eHCWIMqMLbJ3JcjHMlzMCaQzKvI38/5T/3EihoBgzq7dgkSL2KuYpaZEkKoXCbHlCSMJwxtOHMlEM5wVJGSWJEOgcJrl6HhvPJjg912YhgymS5ZXIK4GDiCQJzHKk12RkxK2cBcXxO9JMphKyNWGX/PNooiVcVQMoP6/F50o6dd1yDKP6Y9Pr0zqNogxBpg1VS33FEJXT485+Ty4NAcusfEkuTzBraQ5gZSJ4UVCKUhY8h2l8iWO2BGaZY/yXLLfc0zJKTaDkUulAOcZG8DBaohO8QqRoDSYSevkcRucLSoo0ds0NoR9gUiB3Vq6ZFk/7aXR4GE32dEWVCj8a73NFHx0OggnX/eHPB4eG0v8UTyZwDCsVbmZgqzGCGEV4BeXM1XMO5oSu5MRxqa3QWY4oRtozC1gskKlUK3hVjTIc6sqzwmmVoSeLqbQlIuZiyR3SkiTxazhDptHYJd5Aeo6k/IQqtNTTq/qHluqHoy2qH7bqXiK8WDK9LmUN4+ZjytX7Aia2upimXkouy3C6OFFaGLrSTSVpxqO5uwsh44CRgKErpqlGUGZp49BG4qmMwnSxpbJRU1lL6bgu/Mr17h3hUshNhRDqcMxdhSF3IEQkS78h0v9z00hTFDHu0YwyJ6I7lkRJZq5XSn8xSuLfHTnSmpPIFKiaGciXOS3p1Bh2jOawSJhdmRqsGmL1b2Dl49TUKdNhPRQyzzMYoafBw4zEwCpYpFja4fMs13u3MXpXqoqpWEo6F4hStahem8/UdtSWhTnCylOpxZKeP1lQhFKrn1LlY3TVHqqmM2LejLzN4CbtrlGScKfhbTj0NjytJ+czmqco9rc98ra972nbq2rywdrztcys8UIZydkci9YBsBNfcuqo1rrp8G9aPkXtZ2Sa9xHpDd5wwPB7oTlfWMRzYoZNf8GUyYK3vxxZGaR+oMOV5C1fwVPoOXejyLb4JDEZccz5MAwP+I+DQ7FkhgfGkjkXLbeXF1GzXo+qZsQX3fBwbFSgr2CtuZHEyOspVqkpawbpArEOUaKrTLadF6tHmBMSesSdIYO8W/SMU+cloeec3COEL9DZbM1QfsY45ybXUVLkfM159gk8LH/9BAa1c3n2v0+aq+E5Afc1MpH//wlsPj6sVqynD7lf4B0XvvP08WNz0rq5oLSInMHUdtPGovgSRkwK2LBYkKAFSuOXdRNmvVxHJRCCoy59MbyaLnbe3ZdKSY3VQ6QfL/GctTMsonzPpyIg8+C5kHnwXs1A3KZIxUZ1srRm3bQOdNOSud2mRcR2Ajzb0aoqr7PVsGTBt+V6H6kx9vTb06+ZcDP8nfT42+Nvj789/vb42+PvV8RfxqckX2HW8++d8e9JOQXsx0Dgu4y8h57Q+/Q7C71/0ZA7TDDMX1RbhGbtBjNIW1wsz3Beo3TBpICHRjpyFd+2LRDu1DjdEAn/pDDGSJr28GuEmUP3PO819CwdhNkUF9QiPcpPyhMq25/Di4UjtVL2do44sfptlbG1+2zuP4iSdo5iQLsqRX52KhVbh2OFZHZeuaRYtee88L+lXtkuFUjP76jIyRAbfReCcluPReJl2cyoc8NyW4QXK49LSzPENZGvcYu2f8lElULtity0+GpvYRbnkxojiuQCOk8IA1vcgVq/fi8h2Rpn7uuvA0oYjM5tgfIlOMtQ/FqxitnR+xYFC2br4BEvcxscuNGaf33NW9lsvEu/kS4krC8QW4FAm93bA0EZ+nIesdbn39e6VUMqF0jbrgll5tZB2e1ZtTjwzQW+wHEB61V800YNcezedOAKXmHLF8+K6FypmuU8VqWlihE7vkBglXb7ptrRfDxtdXENr7oiF43uiL2tqRQi+uMCyIQsnsMctbRIOeBWccfeWx/H1njI99TFSrF/VFpoh+U8uLDf40KPCz0u3G3UoOeFhheqUEEPDD0wfC/AoAV0DCekx3OMjC8Xzhl5wjkHjnCOW8o+2vESTxf1dJDPrt/mdRBQJwV5BdoI1T4vspHIJd1GwqH3pEhK+tDhUdv96CYlPy11E9MWauomJz89+QlqG0X5fdUWmuoiqq6DHHtKXHTlJywPZXlJa1fa6iKuLdTVRV4d9NVNYH4K20pinTTmJjIvlW0jMy+dOQ8UbxLU4f0Xq/BdYNqOqLYDru2MbF3Y5kG37lNUD8Z1oVwnzlkhoPIsqPoWVPBOzZZ9xNmGuy7A80DeTqDX9itu4PNAXyf4dcGfHwC3ONY2CLaH0AZCp1k1qpplDhX1kVcXfXUQmP9bHR4S2+WrBvdhCLqtbSXhntHcoaudIc0afU9pPaX1lPbdUFodS+sx7R5gmvZlnZ7UelL7QUmtFer+9l+Rso3vc95JvsMIpufkMQz7EOYXCmGOR1voeNTTcU/HPR3fjxhmTIlovIfk+xTLDH5Vs9aTck/KfUyzh7YdYppbqW3cU1tPbT213ZOYZo9t9zO22ZNbT259jPOuYpy/USoA8TYhzgdlz4UbEr5D5I6Hqi5RTLsjM4+WaAU/IJqr/ocHKpmtE1RdWaCe4wtno7XgvJghyj0+ktdJqLoYWmUJF0S62OXS0wawrl2vI5c3ZXheSa5yXaxdXX6qh1srVQPiQpHgWPGw47JTjZa13OZuEN39gj8LRNdSu513szbCD43UBbqy3krX7lCtm7dDw8a7J8kH7SoM5wsm+q5ip5tkS6HpXwrHKV+gY3TkvuDAabxgxWcFO4pXAi6X/JtIV7Z0pjzroyK7/i+ZcXiIYHyB+bLCmWAQlLU+dot/tE385eqkKwzXdinj/F9VN4CZ2+qvSHMXLidXDUjLKHJ0oipyvi/jnnHw8z/0OmAhL8BovX/Ek88iUkhdGA/tHOUlQTg0VGCLRQqTf5KvuaBXHqvUSzgtcxdF5KaNU1y6U3PyP+dbjlxLaqx8DLq9haH41osHt9T7uu1ba/5XGfxNjOQcZ3/Q5HidRq53yJQJhb0JdZmQx26+O2Px78c2ckN2IwP6QiuHeanVt7OdG8nih7QnayK/oT1NfSvR9DMWoG+kz030wqPWD3w7K5CjRMW+3GG7Sjrjob3pagQk8nw7VH9rjh2vZyp8k7FzU10DC5fetnjWA9fuzukjuBAG09XAqO6+mXB9Q2RvxXexn3GJ/6Y23GFVE7+mT26i6ZMfRMWtPzkiw3p1LKK+XZBcPqltvrpAUAVPmscyzMGANg+XMjurOqNPL5hqcRpd9w0XDMJGz3WdB6H+YTzUc7Q3rEfa72H5R1ZOqzGIuHZb7fyt6BXv6RXrrYwm+gftRoX9WO9v1RdDfH8RedQA/jh5UYaPqrjWW0VgOF0EvwTvSqgDhTzYA/swPBju7UfocDKGkyFEo+lkOtnbn0bh4Xw+jVThizpcNXyw+T+Bl8Fjl2cAAA== proxy.json.gz: - H4sIAAAAAAAC/+1dW3PbthJ+96/gsD0zzoyao3uszuTBcU4ymSapm6jtQ5PRQCRE8ZgCGAC0pXp0fvtZgBeBJGjLcR1JMV4y4i4ui93F7oclHVwfOY47mYQkTgR3f3b++txKKQx/SUKGNRoihAokQkok8RpIQIxCLlQTeFyrZtgPBZpGGKgzFHGsiAHB4o0PJJJEUUphKJ6PKY1EGAO9rYjz0MdnlAhGI17qH5b6RiG50ARjeAaCzoHg9trcTWn0SrVQUqayAtWjUYRiXpKtxPBrnBgRHG2G0odLeyJvjsfhAtNEaDJu+DSi7AXyLgJGE1Ibf9PmDxQluJldFkHRf+iORl5/6LbKZBZM0XG396zldLqjltNvt5z205PRk2q7H/x+H/WQq1E/l+b1kUCcJsyTYrnnjC6wmOOElwZyZ5QtkFy6SyjBZV6AkgAXzlKQF2iZr7bTblfEWoQkZ1ZZfE6vTBoClphLF6CR/xZNcdl5TK3eIXaBlU4FS7DWaF2Wn4X+OVXuXmYof+yWSURgdokikxOUHVbTQxyHJBivYqWKJl7d9GV9QmuCFspIl1JzjqCOwEtRMbiTseVUJca6td3gDJFgi8G75cEb3Qu84CV42DkFzfG6K7hSi4r5jvpqfo8Sgj0Bm7TWbiwlMig+plzMwqXs7ZoYryDYfAz/VsMP2v+qtIHAYuyr6Dd2Vap6BxHldtPNYF+p3QOy15Uq0oW57/99amDSouOWOucQzmqeC0R2AS5q2KmzMIrOZPhR9peBpdeBuNI5gX9ORjKwdE5qgWUmBTJvQDmLPl46XBdCVGfUqw2k7/fGPaqSDYyZLEjdVAKxAIstrICXsZKJJ4vjJL72ooTDdn7+yf0x+/nJbTn/pVOgXCRT/FPM6HL1yV0/qVtlExAFpIUJxyzEvN4sjxevkCeUPrq1JhEOMPFfFePVBwFPVGnVPd3WA4oQyA3aCoXK2+7vcYVR5OnrqoUQ2yTNZhulEc7lENEizAFFlMdXoUPfUCfVDaVabLehqJTTfX7HvZRHrzttJ9XpfRYeIXFt+mnrL8EFFIWIn+UJvZJVpogZMxdkYj5/i0kg5ipO1ni4qdtWCTxU27VTI75myA8xEdVMfEta7JVTn/LhuueAIgJyysdVqKg3uQwaOF7CWCqYiQuJpYkTkgYOC4M5eJ/fJEsWiiRaqHKg05+hr0xTzT9q6wjUFA2V+/Bbd48MmtwwuWJcZVN3tgEctZRaSzpujMFdiECBGYvGsrf0i0QOP6jzzJ4I1vIxwyrEziJa2f4MxxgZ83caPH+9xAxcDhtWBLnLw01bI8t2ZSkh+ngXRiEhzMcx9t+mmbDO/6pkwpDAxzJlqIwx4SviTdKfLIFQOPETps5UkCcA3/h84sExQWydgFpOSGBFxMPP/wdt8wfITH8NFp+ffNP0JJf6j6eoir2lzK9SxFRzFsn7OA9nwszMctsHqXbnI9jB+VAVeNtEVw0BlAnDcUVt70me+0Lih5ehn6BSdjHnSXU4Lku2RMuQ1+WaJt5F6pOG2LPIdrlUTBVbZaDe0Ks51BXxSh33zUtYoeU2B5aNF9LY4H2RPMUZxZNMGrxAvHZk0mK/sVsa/Ns1urbeOx2LDmEJ+h6z2ORnp/812MSwE7aBJoZu90UmJvjxoMDkxuqIxSUHikvmIRcUcsxi8iVBRIQRPm4/HY1ad8YqafY5SLByfZ1Pv14fBmhJMctbsBHxVs5oJObOb5n5LIjZMYg5eAhzWADmqLIDzZE6o76B2KDClpn9gV6V3kwV8oyzzVd+H1RsyZeAbaYUMd+BEdwyPy9jzYcaI9sWDFofacZ5yJdTjwDRDWy1yVab9gnVDfeo2kSwuKLsQoI4CffAIQNbcPrW2O19agTnfGMEW3mylSdbeXpUOGVoK0+28vToMUpT5cn5WvDyD1egnOnKOc6pLSfCtih1N2Bjy1O2PGXLU7Y8ZctTKex7ZstTtjy1T9Cvv/PyFEASMfEiuZnUX5nAI58oz7g/jGt5YEtJ6z59upOSVHe5vANwa+2j6no7Ul3vRtW9OATV9Xekuv6Nqjs7BNUNdqS6wY2qe7kfJ61fQJXO6fkb50Oqfls/foT1YyPr4ErId1uFrSKXjxMn9jhhjxP7dJw42d9KsgG23L9+3ARjQIuy7fmvH8flYrJktJyERbuoJsvJ12vn+hrmP4SKMgQ+UWAcW0q2pWRbSralZFtKTrHfyH5BYL8g2Bfc1+k+MuDXWMByMuj3+j8W+X098nuNLfCzwM8CPwv8LPCrAr+KfLbqZ6t+j/UjAkBjHuYcYJ10EsB3C7ygbDWZrgTm967ire23nk347J3SswVhOwZhys+/g5eOe74O+9qxgkA6FoFYBGIRSPY3p8c5DPHipCgo3fGDqBv/DsXikCYccnb+u5Nw6ckWiuy4HjSX2jv0mtAeL8KCkAoI6VoQYkGIBSEyz9BJQBlNhDSgLXs84GupQssWbzw2vPFApY89X8f3/DbqKFuBy705XqA/MOOplJ1+ShardCofsYv0HiHIVxsvVdGUESxrV8VYAi/iCJZLgvqtSEdVb9AARgm+5VcT+HiGksh0O4LONaFBdXdSObg0+FRxn4sG4zQujfMrnvRo6n5JMFtlJfg63NPuYOqUqAGuXqBS2EWb/qiyHE1hKIry64Aqy9BUqethC3Ca60rLVhB2IXf6+NR0dUmhSDfLr/pYC7BIaOiTa9nQ5TYVq+kmaRRV/yHGJKb+JCQz2nKy4Z6Yld+9TflZCtKNBC6uNMx/y+d3y9yaoJJmbpyZNl2Jxkg4HqcD6Yj0W9u73WTvcg5r2je3WToHNvcy9cP93yfrzTdM34f3HOVJKr14TqGXIgQX1yrRq586OUbKb0xKM8KmWxyCJtmmc6aSSY5vdaDiDjau5Xba2kNPf+gsNr8H2u+O/tBr65wNknO72u+On8bHz/kaJCLX/OrWWfSBh/rA+izdvv6wuWLLfebr8uaylNT3NyXptUHjsyxrFh/0S9dLaYk6xLvDXhd3hwMft7uj4Unf67en3Y43fTaceqg/G/XTxpdFam4frf8PZcHkVKRwAAA= + H4sIAAAAAAAC/+1d3W/bthZ/z18haLtACni9luy48YA+pOlaFGu7rPF2H9bCoCVa1iKLKkkl9gLvb7+H1IcpiYrtfsVu+BLY55DU4fn88cgBb48syx6PwzhJObN/tv760MkoFH9MQ4oVGopjwhEPSSyIt0ACYhQyLofA15Uchv2Qo0mEgTpFEcOSGMSYv/KBFKdRlFEoSmYjQiIeJkDvSuIs9PE5iTklEavMDytzozC+UgSjeAqCzoBg97rMzmjkRo6QUmayAtUjUYQSVpGtwvAbnATFOFovpS6XzUTeDI/COSYpV2Rc80lE6DPkXQWUpHFj/fWYP1GU4nZ2VQRJ/8EdDr3+wO5UyTSYoGO396RjOe6wY/W7Hav7+HT4qD7uB7/fRz1kK9QPlef6iCNGUuoJsewLSuaYz3DKKgvZU0LnSGzdjkmMq7wApQEunaUkz9Gi2K3T7dbEmodxwayz2Izc6DQELD4TLkAi/zWa4Krz6Ea9QfQKS51ymmJl0KoqPw39CyLdvcqQ/uhWSTHH9BpFQhPOvKqHCAdY2r6mCBSFQXzGRnm8CFnqxgyDGb+EsNgkajUmFFUnSRgHo2Uitd3Ga3pXVVIYHaO59INrYRyLE4vjBa/5lJWzxaMqjFVnu8UpioMtFneri7d6MDjac3DiCwLGYU1vs0W0SuYbIjUMoRbH2OOQBxrjRkIiTYAnhPFpuBCzbR3jBeSzy/AfufxJ9z+1MZC7tHMl/c6pUlVvIGltNt0UQlcGKMjeVCrPNma//e+ZhknKiVvqnEHGbAQHEOkVuKgmGUzDKDoXGU7aX+SungOpyzmFP6dDkbuc00bumgqB9DEunqKuly3nQhZ0hr3GQmpKaY0tWc9gzXQeN03FEQ0w38IKeJFImVg6P06TWy9KGWSMp+/tH/OP7+2O9TeZAOUqneCfEkoWy/f26lHTKuucy6HyjBmmIWbNYUVKeoE8LvXhNoZkmelFuV5zEfBEWbnts209oMyyTKOtkMtUZ/+R1BglFLitWwjRdV1ut1GW4WwGGS3CDIBKdX2ZOtSAOq0HlByxXUARIaf9dMdYKrLXTuEkJ73N0yPUxvU8Zf8VRAJFBbHzAjPUCtcEUW1xhGLPZq9xHPCZzJMNHm6bthVGCGW4Og3iS4r8EMe8Xuw3VN7eN6i86DpoyTBeSmkms44LNaeNE8YtnHqV1ycpHQfm/C/0pdHqlUkGFUdteVI6FtsYVyKdMs3DJeMmf7SzDRRpFNtGObITDI4UcxTogXAiZguPScXyJ02e3kfBWD6mWCbfaURqiYHiBCNtZc/S6m/XmIIzYs2OoKp5uC1o8jpYlRLyknelFRIKQJJg/3VWI5v8TyozFHF8LIqJrCVjtoy9cfaRppAkx35K5YEOKgggH5+NPTij8K1LU8cKY9hR7OGn/8LY4gvUrL9+hBMkPHxcxOaHR9+0jIlnf/FSVrO+kPlFhqwariN4l7NwyvXMvAa+E0awLsEq1ru6wNsWxHpCIJRrTk4y2MdFjQxjP7wO/RRVqpC+nspzelWyBVqErCnXJPWuMg/VZKJ5HvNCMXUMloN/zaz2xFdmL9l50G9hiRbbHGzWXkgSjfdF4kCpFU8wSfAMscbRSikE2mlZJeg26Mp+dzo+HcIW1BgzGOZnq78nGEYz7eAgzKbOiEEwh4hgZiHjBOrPfPwxRTEPI3zcfTwcdnZGNVll+g5gze1tIcxqdRjwJkM3r0Fnsbe0hkM+s37PjWngzj3DnYMHO4cFdY5qEajP2zn1FeQGmcT07HfkpvI6rZRnlAdf9SVWGZLPAQVNCKK+BSvYVX7RGJsNFEYeFhRGHynG+Zpv1B4A9jsx/SvTvzoM9DfYo/5VjPkNoVcC7AlYCP4YmBbW/WK8t5lJrIu1SUwvy/SyTC/rQeGZgellmV6WQTOf0MuyPhXmfNWeljVZWsfFmI4VYdPm2g0CmYaXaXiZhpdpeJmGVwYQn5iGl2l4HQZE7N97wwvACh97kQgz+W848JWNpWd8PtzreGBLQXMfP96DJpe7WOwA8Dr7qMjeXiiyd6cinx2CIvt7ocj+nYo8PwRFnuyFIk/uVOTz/Ti7/QqKtc4uXlnvMmOY3vUD7F1rWQfXvt5tF6aDXT2gnJoDijmgHMYB5XR/e9gaePP5nes2uANaFGMvfrsc3dXGFsM6Vkqj++hji4evVtbtLTz/EHrZkCB5iYVME9s0sU0T2zSxTRM7w4hD8ysH8yuHQ0CIjvvAIGJrS8zKQeLLXwxG/FIY8SU2ENFARAMRDUQ0ELEOEWvymT6i6SOaHzrUQSKgNg8zBvBPOAngwDmeE7ocT5Ycs8/uC67ML1XbkNsbqWcDz+4Znkk//w5ecO75Pswrzho2cQw2MdjEYJM7sYnsURUAxUvSsiW144+0dvhvG4NX2vDK+cUfVsqEXxvIcs8dpZnQ3qF3lfZ4Ewas1MCKa8CKASsGrNwJVgIyDgglKRcGNI2Tr/jKq9SyQSIPDYl8pebJnu/je37TdZTvwGbeDM/Rn5iyTEqnn5H5MnuUj+hVdp0U1Ku1l8psSmMsul/lWhzPkwi2GwfNy7GO6t6g4IsKsCuuj/DxFKWR7gYLlavDifIKrWpyKX3KFnfhWJcZllP4xeU7CtJTuCQprvxS06r9McV0mXfzm4hQuZPLqVADXL/tpjSQ8vij2r4UzaEoKq6Hqnq0qlNVIVvg10JpStmC/AtF1Mdnuntm1hrNC6261hxME2rmFFrWTNmkYvm4cZZOxdUxmktiOla+7iO9FdxNVsiLkmotcHqpavZ7IYhd5TYkFjT94NzG2ZYURsrwKFtIxajf2vDdNsNXq1pLdt5o8gLqfFmblybffIXQ+gdS34d7HBV1KbtyUAKWMuuWt12Rm5+cAhYVF1llRWA9LQkBENH15FwlZXtOxSb2ydp3bKerfOmpX5TzqX2ifHbUL72uylmDN9tVPjt+lgk/FHsQIFxxnI1PURceqAurT3H76pf1zWf2E1+Vt5Clor5/SJzd5jQ6zwtl+Z8EwvUyWipP9Pag52J3cOLjrjscnPa9fnfiOt7kyWDiof502M8GX5fVuHu0+j/y+qz8nnIAAA== scheduler.json.gz: - H4sIAAAAAAAC/+1dW3PbthJ+96/gsD0z9oyaI+riWJnJg+OcZDJNWjdR24c2o4FIiOIxRTAAaMv16Pz2swBvIAnakqL4EuHFI+4C4GJ3sfstSIM3B5ZlTyZBFCec2S+svz53UgrFX5KAYoWGoohwxAMSCeINkIAYBozLJnC5ks2wF3A0DTFQZyhkWBL9CPN3HpCiJAxTCkXxfExIyIMY6F1JnAcePiMRpyRklf5BpW8YRBeKYBTPQNA5EOx+l9kpjVzJFlLKVFaguiQMUcwqslUYXoMTowiH5VDqcGlP5M7xOFhgknBFxpJPQkJfIffCpySJGuOXbf5AYYLb2VURJP2H3mjkDo7tTpVM/Sk67PWfdyynN+pYg27H6j47GR3V2/3gDQaoj2yF+rlyXw9xxEhCXSGWfU7JAvM5TlhlIHtG6AKJqdsRiXCV56PEx4WzFOQFWuazdbrdmliLIMqZdRabkyudhoDF58IFSOi9R1NcdR5dqw+IXmCpU04TrDRaVeWngXdOpLtXGdIfe1VSxDG9RKHOCaoOq+ghjoPIH1/HUhVtvKbpq/qE1hFaSCNdCs1ZnFgcL3nN4FbGFreqMFad9QanKPLXGLxXHbzVvcALXoOHnRPQHGu6gi20KJkfiCfv75Iowi6HRdpoNxYSaRQfE8ZnwVL0tnWMNxBsPgX/yOGH3X/V2kBg0faV9Fu7SlV9gIhyt+lmsK7k6gHZm0rl6cTsX/59qmGSouOaOmcQzhqeC0R6AS6qWamzIAzPRPiR9heBpe9AXHFO4M/JSAQW56QRWGZCIP0CFHdRx0uH60GIckb9xkDqem9dozLZwJjJImqaiiPqY76GFfAyljKxZHGYxDdumDBYzi//tn/Mfv5td6z/kilQLpIp/olB1PeSUNBXR03LlEGRQ2qYMEwDzJrN8pjxBrlc6qTXaBJiH0fem2K85iDgjTK12qfrekERBplGYwGXudv+Pa4xilx9U7cSomXibLdTGuVsBlEtxAyQRHV8GT7URXVSX1SyxXqLigg57Zcbrqc8gm20pGSnX7IQCcmr7KfMvwIZUBggdpYn9VpmmSKqzV6Qjdn8PY58PpexssHDbd3WSuKBXLJOg/iWIi/AEa9n4ztSY7+a/qQPNz0HFOFHp2ycwUWRimsRAF36LZHETShN5dJ0g9TS0k0YSM+hgT8H3/NaJMlikY4Dff4MPGmXegKS64ajtnAofedOFCKCJtPcWzKusjs76wCORkptJB07xuAqEUe+HovGorfwiUQMP2zy9F4IpvIwxTLEzkJSW/oUxxhp83caOH+9xBTcDWtmBLnLxW3LIst2VSkh8rgXWiEhzMcx9t6nmbDJ3yqZUMTxYZEtJrgH2SC9gjg48RIqKyrIEIBuPDZxoUjgG6WfjhVEMKfIxS//B+3zC8hLfw0Xn4+OrOm1dZiNUra915x1k09otbJubnIR4AK0sUE+62yt9WkQeUbdVqaGW1T+amcqV5wchT6hAZ8v9l3/pU6sQie3GONsZ8a4FPDYBJ7SEqlCbtH9662wdC05iem8Scu7RmYTvE/zYMb1zAyEfyo95iOYdCtEXocrhHLN3oqEIpMcpItAcRl4CarAYD2glzt5VcmWaBmwplzTxL1IE6gGJy0ySCIUUy8Esx0ITa92WFZgK7k3qZ/CNVqus7tSOiiJNY4Zii0nrXiCSfxXiDX2dxSYqu2W4tRug67Md6Pg8BSmoK4xU0S9sAamiDJF1N4XUfOAcQI5ZjH5kqCIByE+7D4bjTrWFtVVmoA2QznbgJyOFeKj77jE2sQmrbXXHhrjmxVgmxhkvcpsD61zP+XZJqZao27bQzs9weItBLtG7rU1GvG59Vtmd1PNPXA19+RruadVyR3UlqAesmbUd7DwZaTTsz+Sq8r7RIU842z1Vd/iKdbkayjypgRRz4IR7Co/f/A4P1YY2bKg0PpAMc63fKVoD0rb4TalrS7+rVPbakvYryxutbH4m1a3tz9dN+Xt15W3gwd/RghIhE/cUCwm+fIlXLKJ9Izd7Mp3XLCnoPWePSuw3n3iud5y+Q0f9t2b+voPpL7+rep79VTUN3gg9Q1uVd/ZU1Hf8IHUN7xVfY+k4PoZ1Gmdnr+zPqYmMM/MzDMz88xsrwqLY1NYmMLiMRUWJ4/3uZkGtOxmf7kVxVigS9Hj/NdP4+p+s2B0rISGD7HPLG4uN5nh/qvVzt+w3znOgfDHC4zz3mwtm61ls7VstpbN1rJEgM/NW1PmranHgv6cnoF/Tfj39j8G/W2P/t5iA/4M+DPgz4A/A/7q4O/EbP+Z7T/zXsELcUgJcTFjgOyEkwDEW+AFodeT6TXHbEf/53e/OK18A/TxY7QPUtkGiD0wEJPOfp9YRsv6ejz2yOdhHkJWUcjIoBCDQgwKkUfGcXyYQxE3Top9pR2+G5VuJBks0oZFzs5/txIm3NnAkT2DI93vAYuYt6G2BiI1+QwSMUhkX5GITyY+oSThwoC7+XdZAzlaH1EVmjaY46GfRc2F9p7+Fsgjn8f3/GTqIJuBLYLiAv2BKUuldAYpmV+nt/IQvUiP34ecVXqpjKg0wgI3FmNxvIhDmG7kNz8mcFD3BgVkVCBcfpqvh2coCXUHCqtcHSKUnxyoBpcWnyqOQVegnMIlcf5lBDWa2l8STK+z/fgm5FM+XeBUqD6unzte2EW5/UFtOorCUBjmp+jXpqGoUtXDGgA115WSrSDsQv708KnuxO9CkXaWY9WxFmCRQNMn17Kmy10qlrebpFFUHOLdclx3x8rGPtJboneXJbJ8pFoM/F2qm/2WC2NXuQ2pBU3fOLNzOi2FkTA8TgdSIep9G7/bZvxqQmtbRHeZPUc5W9t9N5tNK/2BlU/XSw7yzJR+pEVCliLuFp8gIFc/OTkwyr8ukKaBslscACSiZedMJZMc1KroxB6WLmQ7XeWir1445TE79lD57agX/a7KKeGb3VN+O14aFD/ncxAwXPGfO++iDnysDqzepTdQL8rPUdjPPVXeXJaK+v4hUXq8/vgsS5W142KymGcnsoK3e/h4eoz608HU82bOoPd82kfDIUJO33Hc42GauO3LIid3D1b/B+dQ2oPUZwAA + H4sIAAAAAAAC/+1d3W/bOBJ/z18haPeABPD2LH+k8QJ9SNNrUVy7m228ew+7hUFLtKyLLKoklTgb+P72G1JflEQ5duomccOXIJrhx3BmOPMbSpZuDyzLnkyCKE44s3+2/vzcSSkUf0kCihUaiiLCEQ9IJIi3QAJiGDAum8DlSjbDXsDRNMRAnaGQYUn0I8zfe0CKkjBMKRTF8zEhIQ9ioHclcR54+IxEnJKQVfoHlb5hEF0qglE8A0HnQLD7XWanNHItW0gpU1mB6pIwRDGryFZheA1OjCIclkOpw6U9kTvH42CBScIVGUs+CQl9jdxLn5IkaoxftvkDhQluZ1dFkPQfeqOROzi2O1Uy9afosNd/2bGc3qhjDbodq/viZHRUb/eDNxigPrIV6ufKvB7iiJGEukIs+5ySBeZznLDKQPaM0AUSS7cjEuEqz0eJjwtnKcgLtMxX63S7NbEWQZQz6yw2J9c6DQGLz4ULkND7gKa46jy6Vh8RvcRSp5wmWGm0qspPA++cSHevMqQ/9qqkiGN6hUKhCWdR1UOIfSxtX1MECgM/OmXjbL8IWerGDPw5v4BtcZeo1T2hqDqOg8gf38RS2228pndVJYXWEVpIP7gSxrE4sThe8ppPWRlbTFVhrDqbDU5R5G8weK86eKsHg6O9ASc+J2Ac1vQ2W+xWyfxIpIZhq0URdjnEgUa7sZBIs8FjwvgsWIreto7xFuLZRfC3HH7Y/UetDcQubV9JX9tVquojBK27TTeDrSs3KMjeVCpPF2b/8s9TDZMUHTfUOYOI2dgcQKSX4KKaYDALwvBMRDhpfxG7+g6ELucE/pyMROxyThqxayYE0u9xMYs6XjpcD6KgM+o3BlJDSuvekvkMxkwWUdNUHFEf8w2sgJexlIkli8MkvnXDhEHEePWX/WP27192x/ovmQLlMpninxgkFi8JBX111LRMGXc5ZJ8JwzTArNksD0tvkculTnqNJml0eluM1xwEvFFmb/t0Uy8oIi3TaCzgMtzZv8c1RgEHbutWQrTMze12SqOczSCqhZgBWKmOL8OHuqlO6ptKtthsUxEhp/1qy/2UR7CttpTs9EsWIiE/lv2U9VdQCSQWxM5y3FBLXlNEtQkSEj6bf8CRz+cyVjZ4uK3bRjghkFvWaRDfUeQFOOL1hH9H9u0/QPZFV35LlHETSlOZNd0g7bR0E8bTc+qJXh+ndBzo85/AkzarJye5pzhqC5XSr9gGyKJo1GBcZzM7m4CRRrptJCQ7xuBGEUe+HgrHorfwl0QMP2zy9B4KpvIwxTL8zkJSCwsUxxhpc3saVH+9whRcEWtWBHnNxW1bJsuEVSkhKrmXWiEhBcQx9j6kWbLJv1eioYjjwyKTTHAPMkV6BTFy4iVUFnSQPQD5eGziQo3Ct0pNHSuIYE2Ri1/9D9rnF5Cz/vwRqkiYfpLvzc9HR9b0xjrMxix7Pmh2u82Xt1pZt7e5CHAButki83XubYNpEHlG+TXlZ0pZY4DXOzOAsgFQ6BMa8PnCWEO1Rqkhq9DQGtOc7cw0VwJkmxDVZpdUPWss8eZe+LyW1MRy3qYlYyMjCt7FPJhxPTMD9hel/3wC9d4L5ddhDqFccyQkIcwkB/4iiFwFXoIq0FpfJMgDyKpkS7QMWFOuaeJepolXg68WGZQRiqkXl9mphqZXO5wrMJk8UtUv4QYtNzmxKR2UxBrHDMVJmVY8wST+a8QaZ0YKvNV2S/Ftt0FX1rtVqNiHJah7zBRmP1sDU5iZwswUZmuAzzxgnED+WUy+JCjiQYgPuy9Go451j4otTU7b4aGvh0MdK8RH33HZto2FWuu5Z2+ab1bUbWOezaq9Z2+rhyn5tjHcBrXgs7faHhaEIeg4cm+s0YjPrd8yLzAV4iNXiHtfH+5XdXhQ24J6qJtR38PGl3FPz/5EriuPVhXyjLPdV32gqdiTb6BwnBJEPQtGsKv8/Abp/FhhZNuCQusDxTjf8umqZ1AuD59IuaytivetXl7/fIApmL+uYB48+p1MwCh84oZim8knVOGSTaRn7OaOQMcFewpa78WLFkz4kLivt1x+w1uSD6bM/pNQZn+tMl/vizIHT0KZg7XKPNsXZQ6fhDKHa5X5REq4f4NyrdPz99an1CDmzp65s2fu7D2rUuXYlCqmVNmPUuXk6d7b04Cb3Zxst6IdC3Qpepz/ejFed9ItmnWshIaPccItJpfH2zD/arXz3yDsHA9BmOQFFvpgDrXNobY51DaH2uZQWyLFl+YZMPMM2D7gRKdngGITKL77l8GJu8KJ77CBiQYmGphoYKKBiXWYeGIOFM2Bonn2YS1OjClxMWOAAIWTABRc4AWhN5PpDcdsR7+DfFgEVz6/+vTR20epbAPRHhmiSWd/SJSjZX09Unvi6zA3PKv4ZGTwicEnBp+sxSfyqCoHKW6cFCdTO3xeS3cUZTBLG2Y5O//dSphwbgNbnhls6X4PmMU8oXVvwFKTzyAWg1gMYqkjFp9MfEJJwoUBd/PjYQNGWm9/FZo2aOSx73PNhfb2/xDlia/je77rdZCtwBZBcYH+wJSlUjqDlMxv0qk8RC/TLzFAziq9VEZUGmGBKIuxOF7EISw38pvflTioe4OCMSrgLn/rsodnKAl1L35WuTqsKL8+UQ0uhU/Z4jXy1kWK5xR+/t56Be0pXBLnX8tQw6r9JcH0Jjvab6JC5XMWToXq4/qL4gsDKdMf1NalaA6FYf5lhapHqzpVFbIBhs2VpqQtiL+QSD18qntFe6nRLNmqYy3ANIGmT65lTZe7VCynm6ThVLx1veX96h0rG/tIb4neXZbIEpNqMXB8qW72Wy6MXeU2pBY0fePMzumyFEbC8DgdSMWqD238bpvxq5mtJULfafYc7uze7oXZK+hvpX/t5/66w0Gei9Iv9EiQUkTa4uMQ5PonJ4dC+Xcf0sBfdosDAEG07JyppDiWU/GIPSx9xXa6ykVfvVDqUnuo/O+oF/2uyikBm91T/ne8NPp9ztcggLfiKHfOog58rA6sztIbqBflh0Lsl54qby5LRX1/kyj98MH4LEuOtRfkZMHNTmQ1b/fw8fQY9aeDqefNnEHv5bSPhkOEnL7juMfDNFXbV0UW7h6s/g/BmvPv0WkAAA== workload-total.json.gz: - H4sIAAAAAAAC/+2dX3PbNhLA3/0pOJxMx75TW1GybDczfXDS5i4zvbtc7LYzTTwaiIQkjCmCBUHLqkf32Q8A/wEkQFGOLNsxXhJzlwKBxe7ipwUk3R04jjseoyhOaeK+dj5d9TIJgX+miEBJBqIIU0ARjrjwjomYMEQJ5beIKyeXCs0kRSF9HzGl16ukAaAgwSnxIVO4HwheQDqHaeJK98AITEKupySFknyOAo0U+Th6i0NMeINkNgGH/Z4z8Dz2z2jUc7wjuekILMSDz6uxON845yEkVOkCXcXivgAk8wkGJHBz3Vr8f8X+XQujwADRWm/dWQTp+4BJojQMMwkB8fwS45CimMn7QsiH8xZHlOCQG3QKwiR7PVJeG6LoWpqFGERQ3J+ZvDC46+MwBHEC5ZYURdDQzAgKPuBqMjMj1yZsya4Hx5LgthhAfr3i14V1yrbFGAblZdXrq1JGYAwBlYYqSd9TSMT06NUf8VIxsFAlc7y8RLThOS7Nhe7blBAYUecNiIIlCujcVe+5QH+J++YnkiL3BIKXmQ/kYywND0IEEuGAwpKVCSZASJSucH/6BUYzKszcV+QwaczQxmhxpygMFUtzwT8ICBAbqDxT5tn+oTbb3mDDbHv62R6WlyGcwShQn8TMNIvOk0t9aIObWX3w3HWz+dJngp8XMV3pVX9AgpuaBbjVPGOBIo2UoNmcXmjzDXczjZTd+7twKdUruQoTPoRyNDXVTzDxm+1Rlp5CTcduQJjCwq0a88ByhcaLuHSZ982Tpde1iGS2uGCRqkS8y0fzAaOI/gsLcwiBK0W2D3UeHUPmtREFs2ZGinlr3EVT/vyRKm92nxktgASK/D4NcWVAQwZJIEEw+c8NJMzjYW2Iiam/SWPgCQX+daM3CYVxDINfmE0bOgrIDNJEWg/lFZGvFrexGAaf9zHrm3+YpItDxFIdPGTLGAWsVTJmy8cSk2u2BPsQ3cDxZEVhMhYeceeHKesB+fGz+yr/87Pb46uaGNiP/2Py8uqzu/70ipkUEuY1r1+xxRyHKU+qV0efo785OHIOy3t7MQ6YdEZwGo9DOKWHvAshBkGv+GPMU+GRU75kXCrYa19fpxPI/xjjJRvDawJDMIHb9rfnFG0KTXEhK0QveHP8fzbCI2eycg5571n3pdjiqRCThXAPlt0XcJz5hXpLYZ13wKeCIDxFnSWyd2U7d3cOe5KzXqutEDgVy5F7rsq5rwgvK4Xr/K/KIemczcsch0HNUXmX37GEX/duLr+YoyltKmqr3EfmVQ6eOm+4+zgfM2cKpJWt5JE7ObsBIlChnt+yJDaop6JxsT6iKEA3KEhZ1mqkpeIegUFVB27BLaqtR5PUv84iqJZEF3nyaUxigXT1pJtlaWP6/CTPU5GZS9lVYwgrcAtbArtytTdxzcVEHDQ6yBV49gYIYFOdLlunGrdnC1VfdbBqLapcrPf0u3ggm/nlAZUiaCGqY0tUlqgsUe2OqJiNomSBqEUqi1RfjlSXuTdRS1WWqp46VUn1QQWg5PKgothdddAzlAdHmvKg3somJDRhYRsatuBh1wq1BhNbUdFoTx0yarFRZ9vKvp4kU20j7HyiScDNbrTipBkpW7FyA1q246UZMc2YuQE1zbi5ATk3YGc7erbhpxFBm3OpRVEzjhqQ1IilHdC0DU83IGobpragajuumpF1I7a2oqseX80IuwljjShbT3QGpGWBZ4uED0O0Ham2A9luQbdthGugXHmBV2i3jXjbqLeVfFX6PWcRxEJ6U0HRiL9tCGzA4E4o3EyReiQ2YHErGrfjsRmRWzDZjMp6XJaRuTnUJjprk4mZT1sZtY1TW1jVxKtGZm2M65kO4UA3Zab3DJZmdYXQzjh7anHW4qzF2a8TZ22F1vLs4/CsrpprkdYirUVaGWkbmy37P8VZhO9OT3E+6vFZz7D/f/bEzs+WFnf++T5hS8ZqH+dn6wc4voYDtAP9fP+wzXmP+ibcpgMfdc1eT3zUdVsf+dh4ekMojMc3FH+W318Mup3eqKa0+Z6CQWwEfWUT+Cs7wiGNvnjDoBjUnol90Qc42IOe9vGNvFStJYbneFyDj7jzYY3aGCpat8cxnvAh16+AepqnXA3YIw3Ico/lHss9ti5qwWcn4FPUNC35WPKxB1GdgaHO5nk7PYmq24V/IUdRB4P2vXu1oa6b97pdhy679zrto27f6/T33r/vtAUv46F5D75x2kSLit224GtT3AEbX+I+fM1KOrx86G34LmU2NhK+IFvefHL78E32fGa78MXue3GO1PmQ+doL2HyvUWy3rXfNOJtEu4NN9fj5b6rHL+ec6AthTe1B0Y2wObSwaWHTwuZTh82ytmlp09LmA9OmdMrTAqcFzhcKnE/gFGc9+HZzdnPvFWXTlvaxrSjvqKI83AD5Iwv5FvIt5D+3inJAMO+GZX3L+vuqLDs/ZT5ngd8Cv60wW/bsUGHeBJ8nFj4tfFr4fHYVZkuflj73X2m2AGoB1FacH6vi/DMhnHPvU3A+yHvO0xDPHVw77Gdt8duk3+dK/DlcgN8gSbL+e2eZmK7C/BfFyHX2OgYAlde6PNUTtl4xTy6fRuEiDpkholmXH1yrQFFhfApvhccEcArSsIYFIrxkre4tQ/HDa3LxW+9q5e+rSawvaXFc/HqcnHXdP1NIVsKptT8HV9ncU6QzyJ3W1f9sW/n4en1e+WBZ+Fs+/towJFOut/zxutxW8iclUMRW6QCeh7oPchkMuWCTgTS3FwbO1/1trCueNM4S6mFJFiia4p6TN3ekt/tgk93z1Uh2EObdwrjJf4vnu6q20VEu09+cz2o2EkmRJvAya0j7KTj9VLvf/V1uA6QU6z5VyMRjH6fCCYb9uibLiq7XV+Z+QwRym3+brJihF4YolO/QRmIXD2ShjCKUp0911r/koO+6V1HqkdueHRSPr30M554OXz773i7/IIPfJkiuUfwrCS9Wka/7ZGgWQt5zCKHa1O0xhAxx8yDB8pDvydbVe6+tYmlHq0fx7HvH0mPYxobaXkMtgHGIV4v6N4PKyFje8NzCz1gSWddqIo8RnMKBnmJg3s9quw7bvg3btrAdmdhy9AVIuSfPr4qpBv8/MNVG3ASGWRVev4FQWGfYb3znZWkgrjPVmMxP0303sn4qTJPR+VFtA/Pmxmcx1YGuPqNNJswIvdGipzT33Fbeou5so/hRShM6828bwy1RdWz29ONtPP34hbj4QVGJXWc1RSRmKa8mTrOtDjfCy2/LmHeFm3OZq7wsRuxtMqlenNtsXHRGnl53JFVaZd9XUrDrVX4u+7zryRfDvqyRvkFkIP3tBdncXxVj4DtTTbczP0Vu+ERuWH7K4Fi+kL756DSQ+1v0RTHfX1hserq/Xr7NC8BFZfrfWT0CRTPne+d3+b2Zm4pzBu7p4GwyPT31fc87OQkG06E3HE6mg9HZ8cnZ6YnvZzfflFXn/sH6/+UubNfagwAA + H4sIAAAAAAAC/+2dX3PbuBHA3/0pOJxMx26VVpQs25eZe3CSS5uZ610a++5mmng0EAlJqCmCB4KWdR71sxcA/wEkQEmuLEsxXmxxQYHAYnfx4wIUH44cxx0OURSnNHHfOF9uOpmEwN9TRKAkA1GEKaAIR1z4wERMGKKE8lPEkZNLRckoRSH9GLFCr1NJA0BBglPiQ1bgfiJ4BukUpokrnQMjMAp5OSUplORTFGikyMfROxxiwiskkxE47nacnuexP4NBx/FO5KojMBMXvqz64vzJuQwhoUoT6CIW5wUgmY4wIIGbly3F/xv2dymUAgNEa611JxGkHwMmidIwzCQExNNrjEOKYibvCiHvzjscUYJDrtAxCJPs+0j5boiiW2kUYhBBcX6m8kLhro/DEMQJlGtSCoJGyYSg4BOuBjNTcm3A5uy4dyoJ7osO5McLflxop6xb9KFXHlatvillBMYQUKmrkvQjhUQMj774M54rChZFyRTPrxFtWI5Lc6H7LiUERtR5C6JgjgI6ddVzrtAf4rzpmVSQWwLB88wG8j6WigchAokwQKHJSgUjICRKU7g9/QijCRVq7ipymDRGaKW3uGMUhoqmueDvBASIdVQeKfNof1cbba+3YrQ9/Wj3y8MQTmAUqFdiappEl8m13rXB3aTeeW662XjpI8EPs5gu9EX/hgQ3S2bgXnONGYo0UoImU3qljTfczDRSdu5vwqRUq+RFmPAulL2pFb2Hid+sj7LwFGoadgfCFBZm1RgHFis0VsSl87xtniy9rXkk08UV81TF413em08YRfSfWKhDCFzJs32os+gYMquNKJg0I1LMa+MmmvLrD1R5s/lMaQEkUMT3cYgrBRoiSAIJgsnPd5Awi4e1Liam9iaNjicU+LeN1iQUxjEMfmQ6bZRRQCaQJtJ8KM+IfLa4j0U3+LgPWdv84ySdHSMW6uAxm8YoYLWSIZs+5pjcsinYh+gODkcLCpOhsIiH/+DR919dHwR3iFXy1e04fpiyNhEmfZV/ZFI+z4mufv9fJi+PvrrLL6+YkiFhdvTmFZvecZjyMHtz8jX6s4Mj57g8txPjgEknBKfxMIRjeswbFWIQdIoPQx4cT5zyK8OygH33zW06gvzDEM9Zr94QGIIRDB82a2/HKeoUJcWBXCBawavj/1kPT5zRwjnmrWfNl7yNB0dMZsJgWLyfwWFmKeophXY+AJ8KpvCU4iy0fSjreXhw2JWc5VKthcCxmKDcS1XOrUfYXSlc5p8qE6VTNi5THAY10+VN/sCmgLq9c/nVFI1ps6A2731mdubgsfOWG5TzOTOvQJrrSkJ5kOMdIAIe6hEvC2u9enAaFjMmigJ0h4KUxbFGoCrOEWBUNeAe3KPaDDVK/dvMp2phdZaHo8YgFpBXD8NZ3DYG1C/yOBWxupTdNLqwAPewxdUrU3sb10xM+EGjgbwAT94CgXCq0WUzV+P0bOrqqgZWzU6ViXX2v4lHsppfHmIpghbGOrWMZRnLMtb2GIvpKEpmiFrIspD1FJB1ndsXtZxlOWvfOUvKISpIJacQlYLtZRA9QwpxoEkh6rVsgkQTKLbBYgswrpvF1oBjKzwa9amDSC1I6nRb6deTZKpuhJ7PNAG42YxWwDRDZitoroDNduA0Q6cZPFfApxlAV0DoChBth9E2IDVCaXMstXBqBlQDpBpBdQ1YbQPWFdDaBq4t8NoOsGaIXQmyrTCrB1oz1K4CWyPc1gOdAXKZ49lE4q4Yd03OXYN1N+DdNuY1cK885Sv828bAbRzcysIqD18yn2JOvirpaATiNig2gPFacNwMmnpINoByKyy3A7MZmlvA2QzPeoCWIbrZ1SZMa8OLmVhbqbWNXFvo1USwRopt9OtAu3CkGzLTXYTlW12ydG3APbeAawHXAu63Cbg2i2sJd18IV5fxtZBrIddCrgy5jQWZ3e8GLdx3q7tBn3UbrmfYNXCxZ/twS407//iYsCljsYt9uPVtH9/CRtyefry/22SXSH2hbtU2kXrJTveJ1Ms23iiycs+HKDBu+lDsWb7j6K2356Ma0uZdBsPaCPrKQvE3tvFD6n1xC6Eo1O6ttds+lJsEdqH93vSRp7O1DHGImzx4j9fe4lHrQ8XvdhPHHm+W/QY4qLlb1gBCUocsCVkSsiRkc6cWhZ4IhYq8p2Uhy0J2Q6vTM+TiPG+rO1p1a/cvZEtrr9e+4q9WtO6Sv25lYp01f13psy7668ofveq/1sK9DIzmlfvGHhUtPK63cF8b4jVA8iWu3te0pAPOp168XycVx3rCJ2RLoAewet+k0QNbuy/W7Iv9qM6nzPpewJJ9jWvXW7DX9LPJuFtYio8Pfyk+fjn7TV8IfWo3nK7Ez77FT4ufFj/3HT/L/KflT8ufO+dPabeoRVCLoC8UQfdgN2jd+bazB3TnWWfTQvipzTpvKevcX4H9A4v9Fvst9h9a1jkgmDfD0r+l/+fLPjvvMyu0twD2FsBmoS2NrpGFXoWjZxZHLY5aHD24LLTlUcuj+5CNtkhqkdRmpZ8rK/0DIZx8H5OUPspbzsMQjx28tN/N6uKnSe8US/wpnIFfIUmy9nsXmZguwvwtaOQ2+x5DgspqXR7qCZvBmCWXV6NwFodMEdFknZfEVeioUD+F98JiAjgGaVgDBeFecqnuJqJ4WZycIC9MzX3PON+5ykBf83I46TZAKsVx8eo7Ofy6v6eQLIR1a99lVynfU6QTyK3X1b9zrrx8PZmvPLsW/porovZEgqTT5YZv3suVJj96gSI2XQfwMtQ9K6Z3XnfGRgVpTi8UnAPAJtoVVxpmkfW4RAwUjXHORlz2mlEdha9ZxwjyE8YEJSad6Mekt2pM8ilLtiLmAkLxyb+KtrlqaaMTXKY/OR/xrJdSQZrA66wi7UN4ejNw//oXuQ6QUqx7qJGJhz5OhYH0u/WSLHS6XlexixVumul+wRQ9M7iqfIbWXdexTubvKEJ5jFUtYru7ipndlCx64rYHFcU/ak8BPdI9yms/2kF2pI5NHOkWxb+Q8GoR+bqHVzM38w7BzWqDuUM3M/jWkzjUU97cLaubuI28a0uzT3HtR3vXc+jGutpOXS2AcYgXs/oPnMrsWZ5waO5nzK0sa8mV53BOYUD76JiP09q23bZr3bbNbQcm/hz8H9i5I8uvsrIG+z8yJVncBIZZgl+/NlFop99t/FBnqSBeZkpWma+m+4ln/VCYBmPtS7V1zJsar8WKjnSJHm0wYUroDGYdpbpDm3mLBLb14mdJbejUv6kPt3jVqdnSTzex9NMXYuJHRUp3mSUnkRilPC05ztZM3AjPX5c+7woz5zJX+VqM2I0zqb6c62xYNEYeXncgpWxl21dCsOtVdi7bvOvJB/2uXCL9pElP+uwF2djfFH3gS1xNszNfRa74TK5YvkrvVD6QfpzpPJDbW7RFUd8fWKynur9cv8szyUWK+6csQ4GiifM35zf53sxNxRYG97x3MRqfn/u+552dBb1x3+v3R+Pe4OL07OL8zPezk+/K9HX3aPk/kZkRRteEAAA= diff --git a/charts/kubezero-metrics/templates/rules/kube-prometheus.yaml b/charts/kubezero-metrics/templates/rules/kube-prometheus.yaml index 96747eb0..81b2ef1c 100644 --- a/charts/kubezero-metrics/templates/rules/kube-prometheus.yaml +++ b/charts/kubezero-metrics/templates/rules/kube-prometheus.yaml @@ -36,6 +36,26 @@ spec: expr: vector(1) labels: severity: none + - alert: InfoInhibitor + annotations: + description: 'This is an alert that is used to inhibit info alerts. + + By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with + + other alerts. + + This alert fires whenever there''s a severity="info" alert, and stops firing when another alert with a + + severity of ''warning'' or ''critical'' starts firing on the same namespace. + + This alert should be routed to a null receiver and configured to inhibit alerts with severity="info". + + ' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/infoinhibitor + summary: Info-level alert inhibition. + expr: ALERTS{severity = "info"} == 1 unless on(namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 + labels: + severity: none - name: node-network rules: - alert: NodeNetworkInterfaceFlapping @@ -61,7 +81,7 @@ spec: record: instance:node_cpu:ratio - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) record: cluster:node_cpu:sum_rate5m - - expr: cluster:node_cpu_seconds_total:rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) + - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) record: cluster:node_cpu:ratio - name: kube-prometheus-general.rules rules: diff --git a/charts/kubezero-metrics/templates/rules/kubernetes.yaml b/charts/kubezero-metrics/templates/rules/kubernetes.yaml index 69d21e2c..7a18034a 100644 --- a/charts/kubezero-metrics/templates/rules/kubernetes.yaml +++ b/charts/kubezero-metrics/templates/rules/kubernetes.yaml @@ -25,7 +25,7 @@ spec: description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready summary: Pod has been in a non-ready state for more than 15 minutes. - expr: "sum by (namespace, pod) (\n max by(namespace, pod) (\n kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Pending|Unknown\"}\n ) * on(namespace, pod) group_left(owner_kind) topk by(namespace, pod) (\n 1, max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!=\"Job\"})\n )\n) > 0\n" + expr: "sum by (namespace, pod, cluster) (\n max by(namespace, pod, cluster) (\n kube_pod_status_phase{job=\"kube-state-metrics\", phase=~\"Pending|Unknown\"}\n ) * on(namespace, pod, cluster) group_left(owner_kind) topk by(namespace, pod, cluster) (\n 1, max by(namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!=\"Job\"})\n )\n) > 0\n" for: 15m labels: severity: warning @@ -79,16 +79,16 @@ spec: description: DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} has not finished or progressed for at least 15 minutes. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetrolloutstuck summary: DaemonSet rollout is stuck. - expr: "(\n (\n kube_daemonset_status_current_number_scheduled{job=\"kube-state-metrics\"}\n !=\n kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n ) or (\n kube_daemonset_status_number_misscheduled{job=\"kube-state-metrics\"}\n !=\n 0\n ) or (\n kube_daemonset_updated_number_scheduled{job=\"kube-state-metrics\"}\n !=\n kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n ) or (\n kube_daemonset_status_number_available{job=\"kube-state-metrics\"}\n !=\n kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n )\n) and (\n changes(kube_daemonset_updated_number_scheduled{job=\"kube-state-metrics\"}[5m])\n ==\n 0\n)\n" + expr: "(\n (\n kube_daemonset_status_current_number_scheduled{job=\"kube-state-metrics\"}\n !=\n kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n ) or (\n kube_daemonset_status_number_misscheduled{job=\"kube-state-metrics\"}\n !=\n 0\n ) or (\n kube_daemonset_status_updated_number_scheduled{job=\"kube-state-metrics\"}\n !=\n kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n ) or (\n kube_daemonset_status_number_available{job=\"kube-state-metrics\"}\n !=\n kube_daemonset_status_desired_number_scheduled{job=\"kube-state-metrics\"}\n )\n) and (\n changes(kube_daemonset_status_updated_number_scheduled{job=\"kube-state-metrics\"}[5m])\n ==\n 0\n)\n" for: 15m labels: severity: warning - alert: KubeContainerWaiting annotations: - description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + description: pod/{{`{{`}} $labels.pod {{`}}`}} in namespace {{`{{`}} $labels.namespace {{`}}`}} on container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting summary: Pod container waiting longer than 1 hour - expr: 'sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0 + expr: 'sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0 ' for: 1h @@ -114,15 +114,12 @@ spec: for: 15m labels: severity: warning - - alert: KubeJobCompletion + - alert: KubeJobNotCompleted annotations: - description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than 12 hours to complete. - runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobcompletion + description: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than {{`{{`}} "43200" | humanizeDuration {{`}}`}} to complete. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobnotcompleted summary: Job did not complete in time - expr: 'kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"} > 0 - - ' - for: 12h + expr: "time() - max by(namespace, job_name, cluster) (kube_job_status_start_time{job=\"kube-state-metrics\"}\n and\nkube_job_status_active{job=\"kube-state-metrics\"} > 0) > 43200\n" labels: severity: warning - alert: KubeJobFailed @@ -173,7 +170,7 @@ spec: severity: warning - alert: KubeMemoryOvercommit annotations: - description: Cluster has overcommitted memory resource requests for Pods by {{`{{`}} $value {{`}}`}} bytes and cannot tolerate node failure. + description: Cluster has overcommitted memory resource requests for Pods by {{`{{`}} $value | humanize {{`}}`}} bytes and cannot tolerate node failure. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit summary: Cluster has overcommitted memory resource requests. expr: 'sum(namespace_memory:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0 @@ -191,7 +188,7 @@ spec: description: Cluster has overcommitted CPU resource requests for Namespaces. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit summary: Cluster has overcommitted CPU resource requests. - expr: "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"cpu\"})\n /\nsum(kube_node_status_allocatable{resource=\"cpu\"})\n > 1.5\n" + expr: "sum(min without(resource) (kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=~\"(cpu|requests.cpu)\"}))\n /\nsum(kube_node_status_allocatable{resource=\"cpu\", job=\"kube-state-metrics\"})\n > 1.5\n" for: 5m labels: severity: warning @@ -200,7 +197,7 @@ spec: description: Cluster has overcommitted memory resource requests for Namespaces. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit summary: Cluster has overcommitted memory resource requests. - expr: "sum(kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=\"memory\"})\n /\nsum(kube_node_status_allocatable{resource=\"memory\",job=\"kube-state-metrics\"})\n > 1.5\n" + expr: "sum(min without(resource) (kube_resourcequota{job=\"kube-state-metrics\", type=\"hard\", resource=~\"(memory|requests.memory)\"}))\n /\nsum(kube_node_status_allocatable{resource=\"memory\", job=\"kube-state-metrics\"})\n > 1.5\n" for: 5m labels: severity: warning @@ -247,7 +244,7 @@ spec: description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup summary: PersistentVolume is filling up. - expr: "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\n" + expr: "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n" for: 1m labels: severity: critical @@ -256,7 +253,25 @@ spec: description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup summary: PersistentVolume is filling up. - expr: "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\n" + expr: "(\n kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_used_bytes{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_available_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n" + for: 1h + labels: + severity: warning + - alert: KubePersistentVolumeInodesFillingUp + annotations: + description: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} only has {{`{{`}} $value | humanizePercentage {{`}}`}} free inodes. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: "(\n kubelet_volume_stats_inodes_free{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_inodes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.03\nand\nkubelet_volume_stats_inodes_used{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n" + for: 1m + labels: + severity: critical + - alert: KubePersistentVolumeInodesFillingUp + annotations: + description: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is expected to run out of inodes within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} of its inodes are free. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup + summary: PersistentVolumeInodes are filling up. + expr: "(\n kubelet_volume_stats_inodes_free{job=\"kubelet\", metrics_path=\"/metrics\"}\n /\n kubelet_volume_stats_inodes{job=\"kubelet\", metrics_path=\"/metrics\"}\n) < 0.15\nand\nkubelet_volume_stats_inodes_used{job=\"kubelet\", metrics_path=\"/metrics\"} > 0\nand\npredict_linear(kubelet_volume_stats_inodes_free{job=\"kubelet\", metrics_path=\"/metrics\"}[6h], 4 * 24 * 3600) < 0\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_access_mode{ access_mode=\"ReadOnlyMany\"} == 1\nunless on(namespace, persistentvolumeclaim)\nkube_persistentvolumeclaim_labels{label_excluded_from_alerts=\"true\"} == 1\n" for: 1h labels: severity: warning @@ -278,7 +293,7 @@ spec: description: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeversionmismatch summary: Different semantic versions of Kubernetes components running. - expr: 'count(count by (git_version) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1 + expr: 'count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1 ' for: 15m @@ -289,7 +304,7 @@ spec: description: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} $value | humanizePercentage {{`}}`}} errors.' runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclienterrors summary: Kubernetes API server client is experiencing errors. - expr: "(sum(rate(rest_client_requests_total{code=~\"5..\"}[5m])) by (instance, job, namespace)\n /\nsum(rate(rest_client_requests_total[5m])) by (instance, job, namespace))\n> 0.01\n" + expr: "(sum(rate(rest_client_requests_total{code=~\"5..\"}[5m])) by (cluster, instance, job, namespace)\n /\nsum(rate(rest_client_requests_total[5m])) by (cluster, instance, job, namespace))\n> 0.01\n" for: 15m labels: severity: warning @@ -367,7 +382,7 @@ spec: rules: - alert: KubeClientCertificateExpiration annotations: - description: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days. + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration summary: Client certificate is about to expire. expr: 'apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 @@ -377,7 +392,7 @@ spec: severity: warning - alert: KubeClientCertificateExpiration annotations: - description: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours. + description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration summary: Client certificate is about to expire. expr: 'apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 @@ -385,22 +400,22 @@ spec: ' labels: severity: critical - - alert: AggregatedAPIErrors + - alert: KubeAggregatedAPIErrors annotations: - description: An aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. - runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/aggregatedapierrors - summary: An aggregated API has reported errors. - expr: 'sum by(name, namespace)(increase(aggregator_unavailable_apiservice_total[10m])) > 4 + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors + summary: Kubernetes aggregated API has reported errors. + expr: 'sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total[10m])) > 4 ' labels: severity: warning - - alert: AggregatedAPIDown + - alert: KubeAggregatedAPIDown annotations: - description: An aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. - runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/aggregatedapidown - summary: An aggregated API is down. - expr: '(1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85 + description: Kubernetes aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown + summary: Kubernetes aggregated API is down. + expr: '(1 - max by(name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85 ' for: 5m @@ -419,9 +434,9 @@ spec: severity: critical - alert: KubeAPITerminatedRequests annotations: - description: The apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + description: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests - summary: The apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + summary: The kubernetes apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. expr: 'sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20 ' @@ -457,16 +472,16 @@ spec: description: Kubelet '{{`{{`}} $labels.node {{`}}`}}' is running at {{`{{`}} $value | humanizePercentage {{`}}`}} of its Pod capacity. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubelettoomanypods summary: Kubelet is running at capacity. - expr: "count by(node) (\n (kube_pod_status_phase{job=\"kube-state-metrics\",phase=\"Running\"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job=\"kube-state-metrics\"})\n)\n/\nmax by(node) (\n kube_node_status_capacity{job=\"kube-state-metrics\",resource=\"pods\"} != 1\n) > 0.95\n" + expr: "count by(cluster, node) (\n (kube_pod_status_phase{job=\"kube-state-metrics\",phase=\"Running\"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job=\"kube-state-metrics\"})\n)\n/\nmax by(cluster, node) (\n kube_node_status_capacity{job=\"kube-state-metrics\",resource=\"pods\"} != 1\n) > 0.95\n" for: 15m labels: - severity: warning + severity: info - alert: KubeNodeReadinessFlapping annotations: description: The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodereadinessflapping summary: Node readiness status is flapping. - expr: 'sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2 + expr: 'sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (cluster, node) > 2 ' for: 15m @@ -488,7 +503,7 @@ spec: description: Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletpodstartuplatencyhigh summary: Kubelet Pod startup latency is too high. - expr: 'histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60 + expr: 'histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60 ' for: 15m @@ -595,96 +610,85 @@ spec: severity: critical - name: kube-apiserver-burnrate.rules rules: - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[1d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[1d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[1d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[1d]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[1d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"\ + 5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n" labels: verb: read record: apiserver_request:burnrate1d - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[1h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[1h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[1h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[1h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[1h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[1h]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[1h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"\ + 5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n" labels: verb: read record: apiserver_request:burnrate1h - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[2h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[2h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[2h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[2h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[2h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[2h]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[2h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"\ + 5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n" labels: verb: read record: apiserver_request:burnrate2h - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[30m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[30m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[30m]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[30m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"\ + ,code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n" labels: verb: read record: apiserver_request:burnrate30m - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[3d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[3d]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[3d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[3d]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[3d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[3d]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[3d]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"\ + 5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n" labels: verb: read record: apiserver_request:burnrate3d - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[5m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[5m]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[5m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[5m]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[5m]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[5m]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[5m]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"\ + 5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n" labels: verb: read record: apiserver_request:burnrate5m - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[6h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[6h]))\n +\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[6h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[6h]))\n -\n (\n (\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[6h]))\n or\n vector(0)\n )\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[6h]))\n +\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[6h]))\n )\n )\n +\n # errors\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"\ + 5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n" labels: verb: read record: apiserver_request:burnrate6h - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1d]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[1d]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1d]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[1d]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[1d]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1d]))\n" labels: verb: write record: apiserver_request:burnrate1d - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1h]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[1h]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1h]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[1h]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[1h]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1h]))\n" labels: verb: write record: apiserver_request:burnrate1h - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[2h]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[2h]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[2h]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[2h]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[2h]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[2h]))\n" labels: verb: write record: apiserver_request:burnrate2h - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[30m]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[30m]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[30m]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[30m]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[30m]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[30m]))\n" labels: verb: write record: apiserver_request:burnrate30m - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[3d]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[3d]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[3d]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[3d]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[3d]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[3d]))\n" labels: verb: write record: apiserver_request:burnrate3d - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[5m]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[5m]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[5m]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[5m]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[5m]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[5m]))\n" labels: verb: write record: apiserver_request:burnrate5m - - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[6h]))\n -\n sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[6h]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[6h]))\n" + - expr: "(\n (\n # too slow\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[6h]))\n -\n sum by (cluster) (rate(apiserver_request_slo_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[6h]))\n )\n +\n sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[6h]))\n" labels: verb: write record: apiserver_request:burnrate6h - name: kube-apiserver-histogram.rules rules: - - expr: 'histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET"}[5m]))) > 0 + - expr: 'histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 ' labels: quantile: '0.99' verb: read - record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile - - expr: 'histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))) > 0 + record: cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile + - expr: 'histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_slo_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0 ' labels: quantile: '0.99' verb: write - record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile - - expr: 'histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod)) - - ' - labels: - quantile: '0.99' - record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile - - expr: 'histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod)) - - ' - labels: - quantile: '0.9' - record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile - - expr: 'histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod)) - - ' - labels: - quantile: '0.5' - record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile + record: cluster_quantile:apiserver_request_slo_duration_seconds:histogram_quantile - interval: 3m name: kube-apiserver-availability.rules rules: @@ -704,16 +708,32 @@ spec: labels: verb: write record: code:apiserver_request_total:increase30d - - expr: "1 - (\n (\n # write too slow\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~\"POST|PUT|PATCH|DELETE\"}[30d]))\n -\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[30d]))\n ) +\n (\n # read too slow\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~\"LIST|GET\"}[30d]))\n -\n (\n (\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30d]))\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30d]))\n )\n ) +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{code=~\"5..\"}\ - \ or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d)\n" + - expr: 'sum by (cluster, verb, scope) (increase(apiserver_request_slo_duration_seconds_count[1h])) + + ' + record: cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase1h + - expr: 'sum by (cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase1h[30d]) * 24 * 30) + + ' + record: cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d + - expr: 'sum by (cluster, verb, scope, le) (increase(apiserver_request_slo_duration_seconds_bucket[1h])) + + ' + record: cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase1h + - expr: 'sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase1h[30d]) * 24 * 30) + + ' + record: cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d + - expr: "1 - (\n (\n # write too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n -\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n ) +\n (\n # read too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"\ + cluster\",le=\"30\"})\n )\n ) +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d)\n" labels: verb: all record: apiserver_request:availability30d - - expr: "1 - (\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\"}[30d]))\n -\n (\n # too slow\n (\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"}[30d]))\n or\n vector(0)\n )\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"}[30d]))\n +\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",scope=\"cluster\",le=\"40\"}[30d]))\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n" + - expr: "1 - (\n sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n -\n (\n # too slow\n (\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n or\n vector(0)\n )\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n +\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"})\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n" labels: verb: read record: apiserver_request:availability30d - - expr: "1 - (\n (\n # too slow\n sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~\"POST|PUT|PATCH|DELETE\"}[30d]))\n -\n sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"}[30d]))\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\"})\n" + - expr: "1 - (\n (\n # too slow\n sum by (cluster) (cluster_verb_scope:apiserver_request_slo_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n -\n sum by (cluster) (cluster_verb_scope_le:apiserver_request_slo_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n )\n +\n # errors\n sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\"})\n" labels: verb: write record: apiserver_request:availability30d @@ -757,19 +777,19 @@ spec: record: node_namespace_pod_container:container_memory_cache - expr: "container_memory_swap{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", image!=\"\"}\n* on (namespace, pod) group_left(node) topk by(namespace, pod) (1,\n max by(namespace, pod, node) (kube_pod_info{node!=\"\"})\n)\n" record: node_namespace_pod_container:container_memory_swap - - expr: "kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" + - expr: "kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests - expr: "sum by (namespace, cluster) (\n sum by (namespace, pod, cluster) (\n max by (namespace, pod, container, cluster) (\n kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"}\n ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n" record: namespace_memory:kube_pod_container_resource_requests:sum - - expr: "kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" + - expr: "kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests - expr: "sum by (namespace, cluster) (\n sum by (namespace, pod, cluster) (\n max by (namespace, pod, container, cluster) (\n kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"}\n ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n" record: namespace_cpu:kube_pod_container_resource_requests:sum - - expr: "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" + - expr: "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n)\n" record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits - expr: "sum by (namespace, cluster) (\n sum by (namespace, pod, cluster) (\n max by (namespace, pod, container, cluster) (\n kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"}\n ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n" record: namespace_memory:kube_pod_container_resource_limits:sum - - expr: "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n )\n" + - expr: "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on (namespace, pod, cluster)\ngroup_left() max by (namespace, pod, cluster) (\n (kube_pod_status_phase{phase=~\"Pending|Running\"} == 1)\n )\n" record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits - expr: "sum by (namespace, cluster) (\n sum by (namespace, pod, cluster) (\n max by (namespace, pod, container, cluster) (\n kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"}\n ) * on(namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (\n kube_pod_status_phase{phase=~\"Pending|Running\"} == 1\n )\n )\n)\n" record: namespace_cpu:kube_pod_container_resource_limits:sum @@ -785,6 +805,10 @@ spec: labels: workload_type: statefulset record: namespace_workload_pod:kube_pod_owner:relabel + - expr: "max by (cluster, namespace, workload, pod) (\n label_replace(\n kube_pod_owner{job=\"kube-state-metrics\", owner_kind=\"Job\"},\n \"workload\", \"$1\", \"owner_name\", \"(.*)\"\n )\n)\n" + labels: + workload_type: job + record: namespace_workload_pod:kube_pod_owner:relabel - name: kube-scheduler.rules rules: - expr: 'histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) @@ -843,27 +867,33 @@ spec: record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile - name: node.rules rules: - - expr: "topk by(namespace, pod) (1,\n max by (node, namespace, pod) (\n label_replace(kube_pod_info{job=\"kube-state-metrics\",node!=\"\"}, \"pod\", \"$1\", \"pod\", \"(.*)\")\n))\n" + - expr: "topk by(cluster, namespace, pod) (1,\n max by (cluster, node, namespace, pod) (\n label_replace(kube_pod_info{job=\"kube-state-metrics\",node!=\"\"}, \"pod\", \"$1\", \"pod\", \"(.*)\")\n))\n" record: 'node_namespace_pod:kube_pod_info:' - expr: "count by (cluster, node) (sum by (node, cpu) (\n node_cpu_seconds_total{job=\"node-exporter\"}\n* on (namespace, pod) group_left(node)\n topk by(namespace, pod) (1, node_namespace_pod:kube_pod_info:)\n))\n" record: node:node_num_cpu:sum - expr: "sum(\n node_memory_MemAvailable_bytes{job=\"node-exporter\"} or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"} +\n node_memory_Cached_bytes{job=\"node-exporter\"} +\n node_memory_MemFree_bytes{job=\"node-exporter\"} +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n) by (cluster)\n" record: :node_memory_MemAvailable_bytes:sum + - expr: 'sum(rate(node_cpu_seconds_total{job="node-exporter",mode!="idle",mode!="iowait",mode!="steal"}[5m])) / + + count(sum(node_cpu_seconds_total{job="node-exporter"}) by (cluster, instance, cpu)) + + ' + record: cluster:node_cpu:ratio_rate5m - name: kubelet.rules rules: - - expr: 'histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + - expr: 'histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) ' labels: quantile: '0.99' record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile - - expr: 'histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + - expr: 'histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) ' labels: quantile: '0.9' record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile - - expr: 'histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + - expr: 'histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster, instance, le) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) ' labels: diff --git a/charts/kubezero-metrics/templates/rules/node-exporter.yaml b/charts/kubezero-metrics/templates/rules/node-exporter.yaml index f3588629..6c466324 100644 --- a/charts/kubezero-metrics/templates/rules/node-exporter.yaml +++ b/charts/kubezero-metrics/templates/rules/node-exporter.yaml @@ -14,7 +14,7 @@ spec: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup summary: Filesystem is predicted to run out of space within the next 24 hours. - expr: "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 40\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n" + expr: "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 15\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n" for: 1h labels: severity: warning @@ -23,7 +23,7 @@ spec: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup summary: Filesystem is predicted to run out of space within the next 4 hours. - expr: "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 15\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 4*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n" + expr: "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 10\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 4*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n" for: 1h labels: severity: critical @@ -188,13 +188,13 @@ spec: severity: critical - name: node-exporter.rules rules: - - expr: "count without (cpu) (\n count without (mode) (\n node_cpu_seconds_total{job=\"node-exporter\"}\n )\n)\n" + - expr: "count without (cpu, mode) (\n node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}\n)\n" record: instance:node_num_cpu:sum - - expr: "1 - avg without (cpu, mode) (\n rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\"}[5m])\n)\n" + - expr: "1 - avg without (cpu) (\n sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\"}[5m]))\n)\n" record: instance:node_cpu_utilisation:rate5m - expr: "(\n node_load1{job=\"node-exporter\"}\n/\n instance:node_num_cpu:sum{job=\"node-exporter\"}\n)\n" record: instance:node_load1_per_cpu:ratio - - expr: "1 - (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n" + - expr: "1 - (\n (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"}\n +\n node_memory_Cached_bytes{job=\"node-exporter\"}\n +\n node_memory_MemFree_bytes{job=\"node-exporter\"}\n +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n )\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n" record: instance:node_memory_utilisation:ratio - expr: 'rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) diff --git a/charts/kubezero-metrics/templates/rules/prometheus-operator.yaml b/charts/kubezero-metrics/templates/rules/prometheus-operator.yaml index f8d3f776..3079b4e5 100644 --- a/charts/kubezero-metrics/templates/rules/prometheus-operator.yaml +++ b/charts/kubezero-metrics/templates/rules/prometheus-operator.yaml @@ -25,7 +25,7 @@ spec: description: Errors while performing watch operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorwatcherrors summary: Errors while performing watch operations in controller. - expr: '(sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[10m])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[10m]))) > 0.4 + expr: '(sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="prometheus-operator",namespace="monitoring"}[5m])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="prometheus-operator",namespace="monitoring"}[5m]))) > 0.4 ' for: 15m @@ -69,7 +69,7 @@ spec: description: Prometheus operator in {{`{{`}} $labels.namespace {{`}}`}} namespace isn't ready to reconcile {{`{{`}} $labels.controller {{`}}`}} resources. runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatornotready summary: Prometheus operator not ready - expr: 'min by(namespace, controller) (max_over_time(prometheus_operator_ready{job="prometheus-operator",namespace="monitoring"}[5m]) == 0) + expr: 'min by (controller,namespace) (max_over_time(prometheus_operator_ready{job="prometheus-operator",namespace="monitoring"}[5m]) == 0) ' for: 5m @@ -86,4 +86,19 @@ spec: for: 5m labels: severity: warning + - name: config-reloaders + rules: + - alert: ConfigReloaderSidecarErrors + annotations: + description: 'Errors encountered while the {{`{{`}}$labels.pod{{`}}`}} config-reloader sidecar attempts to sync config in {{`{{`}}$labels.namespace{{`}}`}} namespace. + + As a result, configuration for service running in {{`{{`}}$labels.pod{{`}}`}} may be stale and cannot be updated anymore.' + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/configreloadersidecarerrors + summary: config-reloader sidecar has not had a successful reload for 10m + expr: 'max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0 + + ' + for: 10m + labels: + severity: warning diff --git a/charts/kubezero-metrics/templates/rules/prometheus.yaml b/charts/kubezero-metrics/templates/rules/prometheus.yaml index 3cbc4141..8672671c 100644 --- a/charts/kubezero-metrics/templates/rules/prometheus.yaml +++ b/charts/kubezero-metrics/templates/rules/prometheus.yaml @@ -181,6 +181,28 @@ spec: for: 15m labels: severity: warning + - alert: PrometheusScrapeBodySizeLimitHit + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured body_size_limit. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapebodysizelimithit + summary: Prometheus has dropped some targets that exceeded body size limit. + expr: 'increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0 + + ' + for: 15m + labels: + severity: warning + - alert: PrometheusScrapeSampleLimitHit + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed {{`{{`}} printf "%.0f" $value {{`}}`}} scrapes in the last 5m because some targets exceeded the configured sample_limit. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapesamplelimithit + summary: Prometheus has failed scrapes that have exceeded the configured sample limit. + expr: 'increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0 + + ' + for: 15m + labels: + severity: warning - alert: PrometheusTargetSyncFailure annotations: description: '{{`{{`}} printf "%.0f" $value {{`}}`}} targets in Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} have failed to sync because invalid configuration was supplied.' diff --git a/charts/kubezero-storage/Chart.yaml b/charts/kubezero-storage/Chart.yaml index cbfe3ead..f5e83336 100644 --- a/charts/kubezero-storage/Chart.yaml +++ b/charts/kubezero-storage/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-storage description: KubeZero umbrella chart for all things storage incl. AWS EBS/EFS, openEBS-lvm, gemini type: application -version: 0.7.0 +version: 0.7.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -28,7 +28,7 @@ dependencies: condition: gemini.enabled # repository: https://charts.fairwinds.com/stable - name: aws-ebs-csi-driver - version: 2.10.1 + version: 2.11.0 condition: aws-ebs-csi-driver.enabled # repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: aws-efs-csi-driver diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md index cbe7b1f4..8f3281d4 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md @@ -1,9 +1,15 @@ # Helm chart +# v2.11.0 +* Bump app/driver to version `v1.11.3` +* Add support for leader election tuning for `csi-provisioner` and `csi-attacher` ([#1371](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1371), [@moogzy](https://github.com/moogzy)) +* Change `fsGroupPolicy` to `File` ([#1377](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1377), [@ConnorJC3](https://github.com/ConnorJC3)) +* Allow all taint for `csi-node` by default ([#1381](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1381), [@gtxu](https://github.com/gtxu)) + ## v2.10.1 * Bump app/driver to version `v1.11.2` -## 2.10.0 +## v2.10.0 * Implement securityContext for containers * Add securityContext for node pod * Utilize more secure defaults for securityContext diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml index 3fc90afa..03c789cf 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml @@ -3,7 +3,7 @@ annotations: - kind: added description: Custom controller.updateStrategy to set controller deployment strategy. apiVersion: v2 -appVersion: 1.11.2 +appVersion: 1.11.3 description: A Helm chart for AWS EBS CSI Driver home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver keywords: @@ -19,4 +19,4 @@ maintainers: name: aws-ebs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver -version: 2.10.1 +version: 2.11.0 diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml index 08ff7656..6afb4a00 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -170,7 +170,18 @@ spec: {{- if .Values.controller.extraCreateMetadata }} - --extra-create-metadata {{- end}} - - --leader-election=true + - --leader-election={{ .Values.sidecars.provisioner.leaderElection.enabled | required "leader election state for csi-provisioner is required, must be set to true || false." }} + {{- if .Values.sidecars.provisioner.leaderElection.enabled }} + {{- if .Values.sidecars.provisioner.leaderElection.leaseDuration }} + - --leader-election-lease-duration={{ .Values.sidecars.provisioner.leaderElection.leaseDuration }} + {{- end }} + {{- if .Values.sidecars.provisioner.leaderElection.renewDeadline}} + - --leader-election-renew-deadline={{ .Values.sidecars.provisioner.leaderElection.renewDeadline }} + {{- end }} + {{- if .Values.sidecars.provisioner.leaderElection.retryPeriod }} + - --leader-election-retry-period={{ .Values.sidecars.provisioner.leaderElection.retryPeriod }} + {{- end }} + {{- end }} - --default-fstype={{ .Values.controller.defaultFsType }} env: - name: ADDRESS @@ -202,7 +213,18 @@ spec: args: - --csi-address=$(ADDRESS) - --v={{ .Values.sidecars.attacher.logLevel }} - - --leader-election=true + - --leader-election={{ .Values.sidecars.attacher.leaderElection.enabled | required "leader election state for csi-attacher is required, must be set to true || false." }} + {{- if .Values.sidecars.attacher.leaderElection.enabled }} + {{- if .Values.sidecars.attacher.leaderElection.leaseDuration }} + - --leader-election-lease-duration={{ .Values.sidecars.attacher.leaderElection.leaseDuration }} + {{- end }} + {{- if .Values.sidecars.attacher.leaderElection.renewDeadline}} + - --leader-election-renew-deadline={{ .Values.sidecars.attacher.leaderElection.renewDeadline }} + {{- end }} + {{- if .Values.sidecars.attacher.leaderElection.retryPeriod }} + - --leader-election-retry-period={{ .Values.sidecars.attacher.leaderElection.retryPeriod }} + {{- end }} + {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml index 6598355a..baf05409 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml @@ -7,3 +7,4 @@ metadata: spec: attachRequired: true podInfoOnMount: false + fsGroupPolicy: File diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml index 636dac0c..f57b949d 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml @@ -40,8 +40,6 @@ spec: {{- if .Values.node.tolerateAllTaints }} - operator: Exists {{- else }} - - key: CriticalAddonsOnly - operator: Exists - operator: Exists effect: NoExecute tolerationSeconds: 300 diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml index f8d85697..ac68406a 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml @@ -40,8 +40,6 @@ spec: {{- if .Values.node.tolerateAllTaints }} - operator: Exists {{- else }} - - key: CriticalAddonsOnly - operator: Exists - operator: Exists effect: NoExecute tolerationSeconds: 300 diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml index 2d7076c1..dc31ef10 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml @@ -22,6 +22,16 @@ sidecars: tag: "v3.1.0" logLevel: 2 resources: {} + # Tune leader lease election for csi-provisioner. + # Leader election is on by default. + leaderElection: + enabled: true + # Optional values to tune lease behavior. + # The arguments provided must be in an acceptable time.ParseDuration format. + # Ref: https://pkg.go.dev/flag#Duration + # leaseDuration: "15s" + # renewDeadline: "10s" + # retryPeriod: "5s" securityContext: readOnlyRootFilesystem: true allowPrivilegeEscalation: false @@ -31,6 +41,16 @@ sidecars: pullPolicy: IfNotPresent repository: k8s.gcr.io/sig-storage/csi-attacher tag: "v3.4.0" + # Tune leader lease election for csi-attacher. + # Leader election is on by default. + leaderElection: + enabled: true + # Optional values to tune lease behavior. + # The arguments provided must be in an acceptable time.ParseDuration format. + # Ref: https://pkg.go.dev/flag#Duration + # leaseDuration: "15s" + # renewDeadline: "10s" + # retryPeriod: "5s" logLevel: 2 resources: {} securityContext: @@ -183,7 +203,7 @@ node: nodeSelector: {} podAnnotations: {} podLabels: {} - tolerateAllTaints: false + tolerateAllTaints: true tolerations: [] resources: {} serviceAccount: diff --git a/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json b/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json index a38a3a4b..560a4e29 100644 --- a/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json +++ b/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json @@ -18,7 +18,7 @@ "subdir": "contrib/mixin" } }, - "version": "74aa38ec10bc22d34ffd204f46df6e460b78d855", + "version": "19002cfc689fba2b8f56605e5797bf79f8b61fdd", "sum": "W/Azptf1PoqjyMwJON96UY69MFugDA4IAYiKURscryc=" }, { @@ -38,7 +38,7 @@ "subdir": "grafana-builder" } }, - "version": "c132c4afcf17491718539db4c2d94c0ea4346120", + "version": "dbf6fc14105c28b6fd0253005f7ca2da37d3d4e1", "sum": "tDR6yT2GVfw0wTU12iZH+m01HrbIr6g/xN+/8nzNkU0=" }, { @@ -58,7 +58,7 @@ "subdir": "lib/promgrafonnet" } }, - "version": "eb98d4f74e8ac9c30b1f0e815b07bed31da76c8f", + "version": "5e44626d70c2bf2d35c37f3fee5a6261a5335cc6", "sum": "zv7hXGui6BfHzE9wPatHI/AGZa4A2WKo6pq7ZdqBsps=" }, { diff --git a/charts/kubezero-storage/values.yaml b/charts/kubezero-storage/values.yaml index 974db463..459a35fc 100644 --- a/charts/kubezero-storage/values.yaml +++ b/charts/kubezero-storage/values.yaml @@ -63,11 +63,6 @@ gemini: aws-ebs-csi-driver: enabled: false - # starting with 1.6 the ebs-plugin panics with "could not get number of attached ENIs" - # somewhere related to metadata / volumeattach limits and nitro instances ... AWS as usual - #image: - # tag: v1.5.3 - controller: replicaCount: 1 logLevel: 2 diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index f8a034d7..8e26a5d5 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero description: KubeZero - Root App of Apps chart type: application -version: 1.23.10-3 +version: 1.23.11 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero/README.md b/charts/kubezero/README.md index 831bedd9..6763f39e 100644 --- a/charts/kubezero/README.md +++ b/charts/kubezero/README.md @@ -68,4 +68,4 @@ Kubernetes: `>= 1.20.0` | storage.targetRevision | string | `"0.7.0"` | | ---------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.9.1](https://github.com/norwoodj/helm-docs/releases/v1.9.1) +Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0) diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 4a13dce2..be51130f 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -22,7 +22,7 @@ cert-manager: storage: enabled: false - targetRevision: 0.7.0 + targetRevision: 0.7.1 aws-ebs-csi-driver: enabled: false aws-efs-csi-driver: @@ -52,7 +52,7 @@ istio-private-ingress: metrics: enabled: false namespace: monitoring - targetRevision: 0.8.1 + targetRevision: 0.8.4 istio: grafana: {} prometheus: {} diff --git a/charts/manticore/README.md b/charts/manticore/README.md index 113f38c8..aa20decf 100644 --- a/charts/manticore/README.md +++ b/charts/manticore/README.md @@ -29,4 +29,4 @@ Kubernetes: `>= 1.20.0` | manticoresearch.worker.volume.size | string | `"4Gi"` | | ---------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.9.1](https://github.com/norwoodj/helm-docs/releases/v1.9.1) +Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)