From f00b6e163cf9398792afd6a8f55ee6350b094377 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 17 May 2021 12:13:03 +0200 Subject: [PATCH 01/45] chore: update helm-docs --- charts/kubezero-aws-ebs-csi-driver/README.md | 4 ++-- charts/kubezero-aws-efs-csi-driver/README.md | 4 ++-- charts/kubezero-kiam/README.md | 10 +++++----- charts/kubezero-logging/README.md | 6 +++--- charts/kubezero-metrics/README.md | 2 +- charts/kubezero-redis/README.md | 1 + charts/kubezero-timecapsule/README.md | 8 ++++---- charts/kubezero/README.md | 2 ++ 8 files changed, 20 insertions(+), 17 deletions(-) diff --git a/charts/kubezero-aws-ebs-csi-driver/README.md b/charts/kubezero-aws-ebs-csi-driver/README.md index a76dc2c..93b343c 100644 --- a/charts/kubezero-aws-ebs-csi-driver/README.md +++ b/charts/kubezero-aws-ebs-csi-driver/README.md @@ -1,6 +1,6 @@ # kubezero-aws-ebs-csi-driver -![Version: 0.4.4](https://img.shields.io/badge/Version-0.4.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.9.0](https://img.shields.io/badge/AppVersion-0.9.0-informational?style=flat-square) +![Version: 0.5.1](https://img.shields.io/badge/Version-0.5.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.10.0](https://img.shields.io/badge/AppVersion-0.10.0-informational?style=flat-square) KubeZero Umbrella Chart for aws-ebs-csi-driver @@ -23,7 +23,7 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| -| https://kubernetes-sigs.github.io/aws-ebs-csi-driver | aws-ebs-csi-driver | 0.9.14 | +| https://kubernetes-sigs.github.io/aws-ebs-csi-driver | aws-ebs-csi-driver | 0.10.0 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## IAM Role diff --git a/charts/kubezero-aws-efs-csi-driver/README.md b/charts/kubezero-aws-efs-csi-driver/README.md index 0f393ce..6a17d54 100644 --- a/charts/kubezero-aws-efs-csi-driver/README.md +++ b/charts/kubezero-aws-efs-csi-driver/README.md @@ -1,6 +1,6 @@ # kubezero-aws-efs-csi-driver -![Version: 0.3.2](https://img.shields.io/badge/Version-0.3.2-informational?style=flat-square) ![AppVersion: 1.2.0](https://img.shields.io/badge/AppVersion-1.2.0-informational?style=flat-square) +![Version: 0.3.5](https://img.shields.io/badge/Version-0.3.5-informational?style=flat-square) ![AppVersion: 1.2.0](https://img.shields.io/badge/AppVersion-1.2.0-informational?style=flat-square) KubeZero Umbrella Chart for aws-efs-csi-driver @@ -23,7 +23,7 @@ Kubernetes: `>=1.18.0-0` | Repository | Name | Version | |------------|------|---------| -| | aws-efs-csi-driver | 1.2.1 | +| | aws-efs-csi-driver | 1.2.2 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Storage Class diff --git a/charts/kubezero-kiam/README.md b/charts/kubezero-kiam/README.md index 61c6a05..643d6f6 100644 --- a/charts/kubezero-kiam/README.md +++ b/charts/kubezero-kiam/README.md @@ -1,6 +1,6 @@ # kubezero-kiam -![Version: 0.3.2](https://img.shields.io/badge/Version-0.3.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 4.0](https://img.shields.io/badge/AppVersion-4.0-informational?style=flat-square) +![Version: 0.3.4](https://img.shields.io/badge/Version-0.3.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 4.0](https://img.shields.io/badge/AppVersion-4.0-informational?style=flat-square) KubeZero Umbrella Chart for Kiam @@ -53,9 +53,9 @@ By default all access to the meta-data service is blocked, expect for: | kiam.agent.prometheus.servicemonitor.enabled | bool | `false` | | | kiam.agent.prometheus.servicemonitor.interval | string | `"30s"` | | | kiam.agent.prometheus.servicemonitor.labels.release | string | `"metrics"` | | -| kiam.agent.resources.limits.memory | string | `"20Mi"` | | +| kiam.agent.resources.limits.memory | string | `"64Mi"` | | | kiam.agent.resources.requests.cpu | string | `"50m"` | | -| kiam.agent.resources.requests.memory | string | `"20Mi"` | | +| kiam.agent.resources.requests.memory | string | `"16Mi"` | | | kiam.agent.sslCertHostPath | string | `"/etc/ssl/certs"` | | | kiam.agent.tlsCerts.caFileName | string | `"ca.crt"` | | | kiam.agent.tlsCerts.certFileName | string | `"tls.crt"` | | @@ -74,9 +74,9 @@ By default all access to the meta-data service is blocked, expect for: | kiam.server.prometheus.servicemonitor.enabled | bool | `false` | | | kiam.server.prometheus.servicemonitor.interval | string | `"30s"` | | | kiam.server.prometheus.servicemonitor.labels.release | string | `"metrics"` | | -| kiam.server.resources.limits.memory | string | `"100Mi"` | | +| kiam.server.resources.limits.memory | string | `"128Mi"` | | | kiam.server.resources.requests.cpu | string | `"50m"` | | -| kiam.server.resources.requests.memory | string | `"50Mi"` | | +| kiam.server.resources.requests.memory | string | `"64Mi"` | | | kiam.server.service.port | int | `6444` | | | kiam.server.service.targetPort | int | `6444` | | | kiam.server.sslCertHostPath | string | `"/etc/ssl/certs"` | | diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md index 171c670..47b0b7c 100644 --- a/charts/kubezero-logging/README.md +++ b/charts/kubezero-logging/README.md @@ -18,9 +18,9 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| +| | eck-operator | 1.5.0 | | | fluent-bit | 0.15.4 | | | fluentd | 0.2.2 | -| https://helm.elastic.co | eck-operator | 1.5.0 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Changes from upstream @@ -105,10 +105,10 @@ Kubernetes: `>= 1.18.0` | fluentd.env[1].name | string | `"OUTPUT_PASSWORD"` | | | fluentd.env[1].valueFrom.secretKeyRef.key | string | `"elastic"` | | | fluentd.env[1].valueFrom.secretKeyRef.name | string | `"logging-es-elastic-user"` | | -| fluentd.fileConfigs."00_system.conf" | string | `"\n workers 2\n"` | | +| fluentd.fileConfigs."00_system.conf" | string | `"\n root_dir /var/log/fluentd\n # log_level debug\n workers 2\n"` | | | fluentd.fileConfigs."01_sources.conf" | string | `"\n @type http\n @label @KUBERNETES\n port 9880\n bind 0.0.0.0\n keepalive_timeout 30\n\n\n\n @type forward\n @label @KUBERNETES\n port 24224\n bind 0.0.0.0\n # skip_invalid_event true\n send_keepalive_packet true\n \n self_hostname \"#{ENV['HOSTNAME']}\"\n shared_key {{ .Values.shared_key }}\n \n"` | | | fluentd.fileConfigs."02_filters.conf" | string | `""` | | -| fluentd.fileConfigs."04_outputs.conf" | string | `""` | | +| fluentd.fileConfigs."04_outputs.conf" | string | `""` | | | fluentd.image.repository | string | `"fluent/fluentd-kubernetes-daemonset"` | | | fluentd.image.tag | string | `"v1.12-debian-elasticsearch7-1"` | | | fluentd.istio.enabled | bool | `false` | | diff --git a/charts/kubezero-metrics/README.md b/charts/kubezero-metrics/README.md index 335ab29..16c8e81 100644 --- a/charts/kubezero-metrics/README.md +++ b/charts/kubezero-metrics/README.md @@ -49,7 +49,7 @@ Kubernetes: `>= 1.18.0` | kube-prometheus-stack.grafana."grafana.ini"."auth.anonymous".enabled | bool | `true` | | | kube-prometheus-stack.grafana."grafana.ini".alerting.enabled | bool | `false` | | | kube-prometheus-stack.grafana."grafana.ini".analytics.check_for_updates | bool | `false` | | -| kube-prometheus-stack.grafana."grafana.ini".dashboards.default_home_dashboard_path | string | `"/tmp/dashboards/zdt-home.json"` | | +| kube-prometheus-stack.grafana."grafana.ini".dashboards.default_home_dashboard_path | string | `"/tmp/dashboards/home.json"` | | | kube-prometheus-stack.grafana."grafana.ini".dashboards.min_refresh_interval | string | `"30s"` | | | kube-prometheus-stack.grafana."grafana.ini".date_formats.default_timezone | string | `"UTC"` | | | kube-prometheus-stack.grafana."grafana.ini".security.cookie_secure | bool | `true` | | diff --git a/charts/kubezero-redis/README.md b/charts/kubezero-redis/README.md index 223be04..2617a26 100644 --- a/charts/kubezero-redis/README.md +++ b/charts/kubezero-redis/README.md @@ -51,6 +51,7 @@ https://grafana.com/grafana/dashboards/11835 ## Redis # Resources +- https://ot-container-kit.github.io/redis-operator/ - https://github.com/helm/charts/tree/master/stable/redis - https://github.com/rustudorcalin/deploying-redis-cluster - diff --git a/charts/kubezero-timecapsule/README.md b/charts/kubezero-timecapsule/README.md index 83a0581..9940ea4 100644 --- a/charts/kubezero-timecapsule/README.md +++ b/charts/kubezero-timecapsule/README.md @@ -24,10 +24,10 @@ Kubernetes: `>= 1.18.0` | Key | Type | Default | Description | |-----|------|---------|-------------| -| gemini.resources.limits.cpu | string | `"200m"` | | -| gemini.resources.limits.memory | string | `"256Mi"` | | -| gemini.resources.requests.cpu | string | `"50m"` | | -| gemini.resources.requests.memory | string | `"64Mi"` | | +| gemini.resources.limits.cpu | string | `"400m"` | | +| gemini.resources.limits.memory | string | `"128Mi"` | | +| gemini.resources.requests.cpu | string | `"20m"` | | +| gemini.resources.requests.memory | string | `"32Mi"` | | ---------------------------------------------- Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0) diff --git a/charts/kubezero/README.md b/charts/kubezero/README.md index e6aa98b..d692d84 100644 --- a/charts/kubezero/README.md +++ b/charts/kubezero/README.md @@ -30,6 +30,7 @@ Kubernetes: `>= 1.18.0` | argocd.enabled | bool | `false` | | | argocd.istio.enabled | bool | `false` | | | argocd.namespace | string | `"argocd"` | | +| aws-ebs-csi-driver.crds | bool | `true` | | | aws-ebs-csi-driver.enabled | bool | `false` | | | aws-efs-csi-driver.enabled | bool | `false` | | | aws-node-termination-handler.enabled | bool | `false` | | @@ -59,6 +60,7 @@ Kubernetes: `>= 1.18.0` | metrics.istio.grafana | object | `{}` | | | metrics.istio.prometheus | object | `{}` | | | metrics.namespace | string | `"monitoring"` | | +| timecapsule.enabled | bool | `false` | | ---------------------------------------------- Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0) -- 2.40.1 From a729aedfc05ccccd36580da5178ff5abda8cbaf5 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 28 May 2021 17:16:36 +0200 Subject: [PATCH 02/45] feat: kubeadm for v1.20.1 --- charts/kubeadm/.helmignore | 2 +- charts/kubeadm/Chart.yaml | 2 +- charts/kubeadm/README.md.gotmpl | 4 ++ charts/kubeadm/scripts/minor.sh | 3 ++ charts/kubeadm/scripts/patch.sh | 3 ++ .../templates/ClusterConfiguration.yaml | 39 ++++++++++++++----- .../kubeadm/templates/InitConfiguration.yaml | 3 +- .../kubeadm/templates/JoinConfiguration.yaml | 5 ++- charts/kubeadm/templates/admin-kubectl.yaml | 2 +- .../kubeadm/templates/patches/coredns0.yaml | 2 +- charts/kubeadm/templates/patches/etcd0.yaml | 1 + .../02-aws-iam-authenticator-mappings.yaml | 2 +- charts/kubeadm/values.yaml | 28 ++++++++----- 13 files changed, 70 insertions(+), 26 deletions(-) create mode 100755 charts/kubeadm/scripts/minor.sh create mode 100755 charts/kubeadm/scripts/patch.sh diff --git a/charts/kubeadm/.helmignore b/charts/kubeadm/.helmignore index 0b1f83c..cc1438c 100644 --- a/charts/kubeadm/.helmignore +++ b/charts/kubeadm/.helmignore @@ -1,2 +1,2 @@ -*.sh *.md +*.md.gotmpl diff --git a/charts/kubeadm/Chart.yaml b/charts/kubeadm/Chart.yaml index b46b7d0..dd5eeb1 100644 --- a/charts/kubeadm/Chart.yaml +++ b/charts/kubeadm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubeadm description: KubeZero Kubeadm golden config type: application -version: 1.20.0 +version: 1.20.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubeadm/README.md.gotmpl b/charts/kubeadm/README.md.gotmpl index ab835c6..dc0e843 100644 --- a/charts/kubeadm/README.md.gotmpl +++ b/charts/kubeadm/README.md.gotmpl @@ -35,3 +35,7 @@ Installs the Istio control plane - https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration - https://github.com/awslabs/amazon-eks-ami + +### Etcd +- https://itnext.io/breaking-down-and-fixing-etcd-cluster-d81e35b9260d + diff --git a/charts/kubeadm/scripts/minor.sh b/charts/kubeadm/scripts/minor.sh new file mode 100755 index 0000000..30111fb --- /dev/null +++ b/charts/kubeadm/scripts/minor.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "Minor uppgrade from 1.19!" diff --git a/charts/kubeadm/scripts/patch.sh b/charts/kubeadm/scripts/patch.sh new file mode 100755 index 0000000..cd03a42 --- /dev/null +++ b/charts/kubeadm/scripts/patch.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +echo "Patch upgrade within 1.20" diff --git a/charts/kubeadm/templates/ClusterConfiguration.yaml b/charts/kubeadm/templates/ClusterConfiguration.yaml index e7da166..43820e9 100644 --- a/charts/kubeadm/templates/ClusterConfiguration.yaml +++ b/charts/kubeadm/templates/ClusterConfiguration.yaml @@ -2,40 +2,61 @@ apiVersion: kubeadm.k8s.io/v1beta2 kind: ClusterConfiguration metadata: name: kubezero-clusterconfiguration -kubernetesVersion: {{ .Values.clusterVersion }} +kubernetesVersion: {{ .Chart.Version }} clusterName: {{ .Values.clusterName }} -controlPlaneEndpoint: {{ .Values.apiEndpoint }} +controlPlaneEndpoint: {{ .Values.api.endpoint }} networking: podSubnet: 10.244.0.0/16 etcd: local: extraArgs: + #name: {{ .Values.etcd.nodeName }} + ### DNS discovery + #discovery-srv: {{ .Values.domain }} + #discovery-srv-name: {{ .Values.clusterName }} + #initial-cluster: + ### Regular + #{{- if .Values.etcd.initialCluster }} + #initial-cluster: {{ .Values.etcd.initialCluster }} + #{{- end }} + #initial-advertise-peer-urls: "https://{{ .Values.etcd.nodeName }}:2380" + #advertise-client-urls: "https://{{ .Values.etcd.nodeName }}:2379" + initial-cluster-token: etcd-{{ .Values.clusterName }} listen-metrics-urls: "http://{{ .Values.listenAddress }}:2381" logger: "zap" # log-level: "warn" - {{- with .Values.etcdExtraArgs }} + {{- with .Values.etcd.extraArgs }} {{- toYaml . | nindent 6 }} {{- end }} + # These will only be used to create the etcd certs but removed for Init/Join kudeadm calls allowing us to sneak in aliases for etcd nodes + serverCertSANs: + - "{{ .Values.etcd.nodeName }}" + - "{{ .Values.etcd.nodeName }}.{{ .Values.domain }}" + - "{{ .Values.domain }}" + peerCertSANs: + - "{{ .Values.etcd.nodeName }}" + - "{{ .Values.etcd.nodeName }}.{{ .Values.domain }}" + - "{{ .Values.domain }}" controllerManager: extraArgs: profiling: "false" bind-address: {{ .Values.listenAddress }} terminated-pod-gc-threshold: "300" - leader-elect: {{ .Values.clusterHighAvailable | quote }} + leader-elect: {{ .Values.highAvailable | quote }} logging-format: json feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" "platform" .Values.platform ) | trimSuffix "," | quote }} scheduler: extraArgs: profiling: "false" bind-address: {{ .Values.listenAddress }} - leader-elect: {{ .Values.clusterHighAvailable | quote }} + leader-elect: {{ .Values.highAvailable | quote }} logging-format: json feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" "platform" .Values.platform ) | trimSuffix "," | quote }} apiServer: certSANs: - - {{ regexSplit ":" .Values.apiEndpoint -1 | first }} + - {{ regexSplit ":" .Values.api.endpoint -1 | first }} extraArgs: - etcd-servers: {{ .Values.allEtcdEndpoints }} + etcd-servers: {{ ternary .Values.api.allEtcdEndpoints "https://127.0.0.1:2379" .Values.highAvailable }} profiling: "false" audit-log-path: "/var/log/kubernetes/audit.log" audit-policy-file: /etc/kubernetes/apiserver/audit-policy.yaml @@ -50,11 +71,11 @@ apiServer: {{- end }} feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" "platform" .Values.platform ) | trimSuffix "," | quote }} enable-admission-plugins: NodeRestriction,EventRateLimit - {{- if .Values.clusterHighAvailable }} + {{- if .Values.highAvailable }} goaway-chance: ".001" {{- end }} logging-format: json - {{- with .Values.apiExtraArgs }} + {{- with .Values.api.extraArgs }} {{- toYaml . | nindent 4 }} {{- end }} extraVolumes: diff --git a/charts/kubeadm/templates/InitConfiguration.yaml b/charts/kubeadm/templates/InitConfiguration.yaml index 1b9deb3..466ba00 100644 --- a/charts/kubeadm/templates/InitConfiguration.yaml +++ b/charts/kubeadm/templates/InitConfiguration.yaml @@ -3,10 +3,11 @@ kind: InitConfiguration metadata: name: kubezero-initconfiguration localAPIEndpoint: - bindPort: {{ regexSplit ":" .Values.apiEndpoint -1 | last }} + bindPort: {{ .Values.api.listenPort }} nodeRegistration: ignorePreflightErrors: - Swap - DirAvailable--var-lib-etcd + - KubeletVersion kubeletExtraArgs: node-labels: {{ .Values.nodeLabels | quote }} diff --git a/charts/kubeadm/templates/JoinConfiguration.yaml b/charts/kubeadm/templates/JoinConfiguration.yaml index 6d3d44a..017f497 100644 --- a/charts/kubeadm/templates/JoinConfiguration.yaml +++ b/charts/kubeadm/templates/JoinConfiguration.yaml @@ -5,12 +5,13 @@ discovery: kubeConfigPath: /root/.kube/config controlPlane: localAPIEndpoint: - advertiseAddress: IP_ADDRESS - bindPort: {{ regexSplit ":" .Values.apiEndpoint -1 | last }} + advertiseAddress: {{ .Values.serviceIp }} + bindPort: {{ .Values.api.listenPort }} nodeRegistration: ignorePreflightErrors: - DirAvailable--var-lib-etcd - FileAvailable--etc-kubernetes-pki-ca.crt - Swap + - KubeletVersion kubeletExtraArgs: node-labels: {{ .Values.nodeLabels | quote }} diff --git a/charts/kubeadm/templates/admin-kubectl.yaml b/charts/kubeadm/templates/admin-kubectl.yaml index bdbf724..62ad49f 100644 --- a/charts/kubeadm/templates/admin-kubectl.yaml +++ b/charts/kubeadm/templates/admin-kubectl.yaml @@ -3,7 +3,7 @@ apiVersion: v1 kind: Config clusters: - cluster: - server: https://{{ .Values.apiEndpoint }} + server: https://{{ .Values.api.endpoint }} name: {{ .Values.clusterName }} contexts: - context: diff --git a/charts/kubeadm/templates/patches/coredns0.yaml b/charts/kubeadm/templates/patches/coredns0.yaml index ef42b4e..208e743 100644 --- a/charts/kubeadm/templates/patches/coredns0.yaml +++ b/charts/kubeadm/templates/patches/coredns0.yaml @@ -1,5 +1,5 @@ spec: - replicas: {{ ternary 3 1 .Values.clusterHighAvailable }} + replicas: {{ ternary 3 1 .Values.highAvailable }} template: spec: containers: diff --git a/charts/kubeadm/templates/patches/etcd0.yaml b/charts/kubeadm/templates/patches/etcd0.yaml index 1ef4e10..b33390b 100644 --- a/charts/kubeadm/templates/patches/etcd0.yaml +++ b/charts/kubeadm/templates/patches/etcd0.yaml @@ -5,3 +5,4 @@ spec: requests: cpu: 200m memory: 192Mi + ephemeral-storage: 1Gi diff --git a/charts/kubeadm/templates/resources/02-aws-iam-authenticator-mappings.yaml b/charts/kubeadm/templates/resources/02-aws-iam-authenticator-mappings.yaml index 70ba334..7310ef0 100644 --- a/charts/kubeadm/templates/resources/02-aws-iam-authenticator-mappings.yaml +++ b/charts/kubeadm/templates/resources/02-aws-iam-authenticator-mappings.yaml @@ -5,7 +5,7 @@ kind: IAMIdentityMapping metadata: name: kubezero-worker-nodes spec: - arn: {{ .Values.WorkerNodeRole }} + arn: {{ .Values.workerNodeRole }} username: system:node:{{ "{{" }}EC2PrivateDNSName{{ "}}" }} groups: # For now use masters, define properly with 1.20 diff --git a/charts/kubeadm/values.yaml b/charts/kubeadm/values.yaml index 6260a55..a861fdd 100644 --- a/charts/kubeadm/values.yaml +++ b/charts/kubeadm/values.yaml @@ -1,16 +1,26 @@ -clusterVersion: 1.19.0 -listenAddress: 0.0.0.0 clusterName: pleasechangeme -apiEndpoint: kube-api.changeme.org:6443 -etcdExtraArgs: {} -apiExtraArgs: {} -clusterHighAvailable: false -allEtcdEndpoints: "" +domain: changeme.org + +serviceIp: set_via_cmdline + +api: + endpoint: kube-api.changeme.org:6443 + listenPort: 6443 + allEtcdEndpoints: "" + extraArgs: {} + +etcd: + nodeName: set_via_cmdline + extraArgs: {} + +highAvailable: false +listenAddress: 0.0.0.0 + # supported values aws,bare-metal platform: "aws" # Set to false for openrc, eg. on Gentoo or Alpine systemd: true protectKernelDefaults: true -WorkerNodeRole: "arn:aws:iam::000000000000:role/KubernetesNode" -KubeAdminRole: "arn:aws:iam::000000000000:role/KubernetesNode" +workerNodeRole: "arn:aws:iam::000000000000:role/KubernetesNode" +kubeAdminRole: "arn:aws:iam::000000000000:role/KubernetesNode" -- 2.40.1 From a67d1674e80ac16898cefa98e5bba789575b3254 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 28 May 2021 17:16:58 +0200 Subject: [PATCH 03/45] chore: more logging tuning --- charts/kubezero-logging/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 46515e3..21f83eb 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -185,15 +185,15 @@ fluentd: reload_on_failure true request_timeout 60s suppress_type_name true - slow_flush_log_threshold 40.0 + slow_flush_log_threshold 50.0 # bulk_message_request_threshold 2097152 @type file_single - chunk_limit_size 16MB - total_limit_size 4GB + chunk_limit_size 2MB + total_limit_size 2GB flush_mode interval - flush_thread_count 2 + flush_thread_count 1 flush_interval 10s flush_at_shutdown true retry_type exponential_backoff -- 2.40.1 From 1913391eadd6bef65bc3b1edff4bb3ba21d64409 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 1 Jun 2021 18:40:34 +0200 Subject: [PATCH 04/45] feat: kube-prometheus-stack version upgrade incl. latest state metrics and dashboards for the metrics systems themselves, fix for gathering all prometheus rules across all namespaces --- charts/kubezero-metrics/Chart.yaml | 4 +- charts/kubezero-metrics/README.md.gotmpl | 10 +- .../charts/kube-prometheus-stack/Chart.yaml | 15 +- .../charts/kube-prometheus-stack/README.md | 7 +- .../charts/grafana/Chart.yaml | 2 +- .../charts/grafana/templates/_pod.tpl | 4 +- .../charts/grafana/templates/deployment.yaml | 4 +- .../charts/grafana/templates/hpa.yaml | 20 + .../charts/grafana/templates/secret.yaml | 2 +- .../charts/grafana/templates/statefulset.yaml | 2 +- .../charts/grafana/values.yaml | 16 + .../charts/kube-state-metrics/Chart.yaml | 7 +- .../charts/kube-state-metrics/README.md | 32 +- .../templates/deployment.yaml | 63 +- .../kube-state-metrics/templates/role.yaml | 12 +- .../templates/rolebinding.yaml | 2 +- .../charts/kube-state-metrics/values.yaml | 8 +- .../prometheus-node-exporter/Chart.yaml | 2 +- .../templates/psp.yaml | 4 + .../prometheus-node-exporter/values.yaml | 1 + .../crds/crd-alertmanagerconfigs.yaml | 2 +- .../crds/crd-alertmanagers.yaml | 2 +- .../crds/crd-podmonitors.yaml | 2 +- .../crds/crd-probes.yaml | 2 +- .../crds/crd-prometheuses.yaml | 2 +- .../crds/crd-prometheusrules.yaml | 2 +- .../crds/crd-servicemonitors.yaml | 2 +- .../crds/crd-thanosrulers.yaml | 2 +- .../templates/_helpers.tpl | 6 + .../templates/alertmanager/ingress.yaml | 5 +- .../alertmanager/ingressperreplica.yaml | 5 +- .../exporters/kubelet/servicemonitor.yaml | 20 +- .../node-exporter/servicemonitor.yaml | 4 +- .../grafana/configmap-dashboards.yaml | 2 +- .../dashboards-1.14/cluster-total.yaml | 2 +- .../dashboards-1.14/controller-manager.yaml | 46 +- .../grafana/dashboards-1.14/etcd.yaml | 4 +- .../k8s-resources-cluster.yaml | 108 +- .../k8s-resources-namespace.yaml | 102 +- .../dashboards-1.14/k8s-resources-node.yaml | 20 +- .../dashboards-1.14/k8s-resources-pod.yaml | 114 +- .../k8s-resources-workload.yaml | 106 +- .../k8s-resources-workloads-namespace.yaml | 178 +- .../dashboards-1.14/namespace-by-pod.yaml | 4 +- .../namespace-by-workload.yaml | 6 +- .../node-cluster-rsrc-use.yaml | 34 +- .../dashboards-1.14/node-rsrc-use.yaml | 34 +- .../grafana/dashboards-1.14/nodes.yaml | 12 +- .../grafana/dashboards-1.14/pod-total.yaml | 6 +- .../grafana/dashboards-1.14/prometheus.yaml | 24 +- .../grafana/dashboards-1.14/proxy.yaml | 56 +- .../grafana/dashboards-1.14/scheduler.yaml | 64 +- .../dashboards-1.14/workload-total.yaml | 8 +- .../templates/grafana/dashboards/etcd.yaml | 1116 ------------- .../dashboards/k8s-cluster-rsrc-use.yaml | 959 ----------- .../grafana/dashboards/k8s-node-rsrc-use.yaml | 986 ----------- .../dashboards/k8s-resources-cluster.yaml | 1479 ----------------- .../dashboards/k8s-resources-namespace.yaml | 963 ----------- .../grafana/dashboards/k8s-resources-pod.yaml | 1006 ----------- .../dashboards/k8s-resources-workload.yaml | 936 ----------- .../k8s-resources-workloads-namespace.yaml | 972 ----------- .../templates/grafana/dashboards/nodes.yaml | 1383 --------------- .../dashboards/persistentvolumesusage.yaml | 573 ------- .../templates/grafana/dashboards/pods.yaml | 680 -------- .../grafana/dashboards/statefulset.yaml | 926 ----------- .../templates/prometheus/ingress.yaml | 5 +- .../prometheus/ingressThanosSidecar.yaml | 5 +- .../prometheus/ingressperreplica.yaml | 5 +- .../prometheus/podDisruptionBudget.yaml | 2 +- .../templates/prometheus/prometheus.yaml | 4 +- .../templates/prometheus/service.yaml | 2 +- .../prometheus/serviceThanosSidecar.yaml | 2 +- .../serviceThanosSidecarExternal.yaml | 2 +- .../prometheus/serviceperreplica.yaml | 2 +- .../charts/kube-prometheus-stack/values.yaml | 10 +- .../dashboards/metrics-dashboards.yaml | 10 + .../templates/grafana-dashboards-metrics.yaml | 15 + charts/kubezero-metrics/update.sh | 3 +- charts/kubezero-metrics/values.yaml | 1 + 79 files changed, 624 insertions(+), 12626 deletions(-) create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/hpa.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/etcd.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-cluster-rsrc-use.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-node-rsrc-use.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-cluster.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-namespace.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-pod.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-workload.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-workloads-namespace.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/nodes.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/persistentvolumesusage.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/pods.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/statefulset.yaml create mode 100644 charts/kubezero-metrics/dashboards/metrics-dashboards.yaml create mode 100644 charts/kubezero-metrics/templates/grafana-dashboards-metrics.yaml diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index 0dedf46..78c5595 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -16,11 +16,11 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: kube-prometheus-stack - version: 15.4.4 + version: 16.1.2 # Switch back to upstream once all alerts are fixed eg. etcd gpcr # repository: https://prometheus-community.github.io/helm-charts - name: prometheus-adapter - version: 2.12.3 + version: 2.13 repository: https://prometheus-community.github.io/helm-charts condition: prometheus-adapter.enabled kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-metrics/README.md.gotmpl b/charts/kubezero-metrics/README.md.gotmpl index a29542d..ce00b33 100644 --- a/charts/kubezero-metrics/README.md.gotmpl +++ b/charts/kubezero-metrics/README.md.gotmpl @@ -17,9 +17,7 @@ # Dashboards -## Etcs -- https://grafana.com/grafana/dashboards/3070 - -## ElasticSearch -- https://grafana.com/grafana/dashboards/266 - +## Alertmanager +- https://grafana.com/api/dashboards/9578/revisions/4/download +## Prometheus +- https://grafana.com/api/dashboards/3662/revisions/2/download diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml index 54a73d5..ba03554 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml @@ -6,21 +6,24 @@ annotations: url: https://github.com/prometheus-operator/kube-prometheus artifacthub.io/operator: "true" apiVersion: v2 -appVersion: 0.47.0 +appVersion: 0.48.0 dependencies: - condition: kubeStateMetrics.enabled name: kube-state-metrics repository: https://prometheus-community.github.io/helm-charts - version: 2.13.* + version: 3.1.* - condition: nodeExporter.enabled name: prometheus-node-exporter repository: https://prometheus-community.github.io/helm-charts - version: 1.17.* + version: 1.18.* - condition: grafana.enabled name: grafana repository: https://grafana.github.io/helm-charts - version: 6.8.* -description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. + version: 6.9.* +description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, + and Prometheus rules combined with documentation and scripts to provide easy to + operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus + Operator. home: https://github.com/prometheus-operator/kube-prometheus icon: https://raw.githubusercontent.com/prometheus/prometheus.github.io/master/assets/prometheus_logo-cb55bb5c346.png keywords: @@ -44,4 +47,4 @@ sources: - https://github.com/prometheus-community/helm-charts - https://github.com/prometheus-operator/kube-prometheus type: application -version: 15.4.4 +version: 16.1.2 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md index eeaad6b..747c2b9 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md @@ -35,7 +35,7 @@ _See [helm install](https://helm.sh/docs/helm/helm_install/) for command documen By default this chart installs additional, dependent charts: -- [kubernetes/kube-state-metrics](https://github.com/kubernetes/kube-state-metrics/tree/master/charts/kube-state-metrics) +- [prometheus-community/kube-state-metrics](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics) - [prometheus-community/prometheus-node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) - [grafana/grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) @@ -83,6 +83,9 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. +### From 15.x to 16.x +Version 16 upgrades kube-state-metrics to v2.0.0. This includes changed command-line arguments and removed metrics, see this [blog post](https://kubernetes.io/blog/2021/04/13/kube-state-metrics-v-2-0/). This version also removes Grafana dashboards that supported Kubernetes 1.14 or earlier. + ### From 14.x to 15.x Version 15 upgrades prometheus-operator from 0.46.x to 0.47.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRDs manually before updating: @@ -392,7 +395,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: labels: - app: prometheus + app.kubernetes.io/name: prometheus prometheus: prometheus-migration-prometheus name: prometheus-prometheus-migration-prometheus-db-prometheus-prometheus-migration-prometheus-0 namespace: monitoring diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml index 167090c..899ba61 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml @@ -19,4 +19,4 @@ name: grafana sources: - https://github.com/grafana/grafana type: application -version: 6.8.3 +version: 6.9.1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl index 98d5d4e..d74ee4d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl @@ -318,14 +318,14 @@ containers: containerPort: 3000 protocol: TCP env: - {{- if not .Values.env.GF_SECURITY_ADMIN_USER }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} - name: GF_SECURITY_ADMIN_USER valueFrom: secretKeyRef: name: {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} key: {{ .Values.admin.userKey | default "admin-user" }} {{- end }} - {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} - name: GF_SECURITY_ADMIN_PASSWORD valueFrom: secretKeyRef: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/deployment.yaml index 4d77794..1c9ae86 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/deployment.yaml @@ -14,7 +14,9 @@ metadata: {{ toYaml . | indent 4 }} {{- end }} spec: + {{- if not .Values.autoscaling.enabled }} replicas: {{ .Values.replicas }} + {{- end }} revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} selector: matchLabels: @@ -34,7 +36,7 @@ spec: checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} -{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +{{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} {{- end }} {{- if .Values.envRenderSecret }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/hpa.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/hpa.yaml new file mode 100644 index 0000000..9c186d7 --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/hpa.yaml @@ -0,0 +1,20 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "grafana.fullname" . }} + labels: + app.kubernetes.io/name: {{ template "grafana.name" . }} + helm.sh/chart: {{ template "grafana.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ template "grafana.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: +{{ toYaml .Values.autoscaling.metrics | indent 4 }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml index 57d2e5f..ac804e2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml @@ -1,4 +1,4 @@ -{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +{{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/statefulset.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/statefulset.yaml index b2b4616..ad3dd06 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/statefulset.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/statefulset.yaml @@ -27,7 +27,7 @@ spec: checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} - {{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} + {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }} checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} {{- end }} {{- with .Values.podAnnotations }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml index 5782ae0..a81db06 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml @@ -22,6 +22,22 @@ serviceAccount: replicas: 1 +## Create HorizontalPodAutoscaler object for deployment type +# +autoscaling: + enabled: false +# minReplicas: 1 +# maxReplicas: 10 +# metrics: +# - type: Resource +# resource: +# name: cpu +# targetAverageUtilization: 60 +# - type: Resource +# resource: +# name: memory +# targetAverageUtilization: 60 + ## See `kubectl explain poddisruptionbudget.spec` for more ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ podDisruptionBudget: {} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml index b7f6ddf..02394f6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml @@ -1,5 +1,5 @@ -apiVersion: v1 -appVersion: 1.9.8 +apiVersion: v2 +appVersion: 2.0.0 description: Install kube-state-metrics to generate and expose cluster-level metrics home: https://github.com/kubernetes/kube-state-metrics/ keywords: @@ -15,4 +15,5 @@ maintainers: name: kube-state-metrics sources: - https://github.com/kubernetes/kube-state-metrics/ -version: 2.13.2 +type: application +version: 3.1.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md index e93a3d2..531ac17 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md @@ -5,7 +5,7 @@ Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state ## Get Repo Info ```console -helm repo add kube-state-metrics https://kubernetes.github.io/kube-state-metrics +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update ``` @@ -14,11 +14,7 @@ _See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation ## Install Chart ```console -# Helm 3 -$ helm install [RELEASE_NAME] kube-state-metrics/kube-state-metrics [flags] - -# Helm 2 -$ helm install --name [RELEASE_NAME] kube-state-metrics/kube-state-metrics [flags] +helm install [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] ``` _See [configuration](#configuration) below._ @@ -28,11 +24,7 @@ _See [helm install](https://helm.sh/docs/helm/helm_install/) for command documen ## Uninstall Chart ```console -# Helm 3 -$ helm uninstall [RELEASE_NAME] - -# Helm 2 -# helm delete --purge [RELEASE_NAME] +helm uninstall [RELEASE_NAME] ``` This removes all the Kubernetes components associated with the chart and deletes the release. @@ -42,19 +34,29 @@ _See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command doc ## Upgrading Chart ```console -# Helm 3 or 2 -$ helm upgrade [RELEASE_NAME] kube-state-metrics/kube-state-metrics [flags] +helm upgrade [RELEASE_NAME] prometheus-community/kube-state-metrics [flags] ``` _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ -### From stable/kube-state-metrics +### Migrating from stable/kube-state-metrics and kubernetes/kube-state-metrics You can upgrade in-place: 1. [get repo info](#get-repo-info) 1. [upgrade](#upgrading-chart) your existing release name using the new chart repo + +## Upgrading to v3.0.0 + +v3.0.0 includes kube-state-metrics v2.0, see the [changelog](https://github.com/kubernetes/kube-state-metrics/blob/release-2.0/CHANGELOG.md) for major changes on the application-side. + +The upgraded chart now the following changes: +* Dropped support for helm v2 (helm v3 or later is required) +* collectors key was renamed to resources +* namespace key was renamed to namespaces + + ## Configuration See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: @@ -63,4 +65,4 @@ See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_h helm show values kube-state-metrics/kube-state-metrics ``` -You may also `helm show values` on this chart's [dependencies](#dependencies) for additional options. +You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options. diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml index 5f6b644..090d880 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml @@ -69,91 +69,91 @@ spec: {{- end }} {{ end }} {{ if .Values.collectors.certificatesigningrequests }} - - --collectors=certificatesigningrequests + - --resources=certificatesigningrequests {{ end }} {{ if .Values.collectors.configmaps }} - - --collectors=configmaps + - --resources=configmaps {{ end }} {{ if .Values.collectors.cronjobs }} - - --collectors=cronjobs + - --resources=cronjobs {{ end }} {{ if .Values.collectors.daemonsets }} - - --collectors=daemonsets + - --resources=daemonsets {{ end }} {{ if .Values.collectors.deployments }} - - --collectors=deployments + - --resources=deployments {{ end }} {{ if .Values.collectors.endpoints }} - - --collectors=endpoints + - --resources=endpoints {{ end }} {{ if .Values.collectors.horizontalpodautoscalers }} - - --collectors=horizontalpodautoscalers + - --resources=horizontalpodautoscalers {{ end }} {{ if .Values.collectors.ingresses }} - - --collectors=ingresses + - --resources=ingresses {{ end }} {{ if .Values.collectors.jobs }} - - --collectors=jobs + - --resources=jobs {{ end }} {{ if .Values.collectors.limitranges }} - - --collectors=limitranges + - --resources=limitranges {{ end }} {{ if .Values.collectors.mutatingwebhookconfigurations }} - - --collectors=mutatingwebhookconfigurations + - --resources=mutatingwebhookconfigurations {{ end }} {{ if .Values.collectors.namespaces }} - - --collectors=namespaces + - --resources=namespaces {{ end }} {{ if .Values.collectors.networkpolicies }} - - --collectors=networkpolicies + - --resources=networkpolicies {{ end }} {{ if .Values.collectors.nodes }} - - --collectors=nodes + - --resources=nodes {{ end }} {{ if .Values.collectors.persistentvolumeclaims }} - - --collectors=persistentvolumeclaims + - --resources=persistentvolumeclaims {{ end }} {{ if .Values.collectors.persistentvolumes }} - - --collectors=persistentvolumes + - --resources=persistentvolumes {{ end }} {{ if .Values.collectors.poddisruptionbudgets }} - - --collectors=poddisruptionbudgets + - --resources=poddisruptionbudgets {{ end }} {{ if .Values.collectors.pods }} - - --collectors=pods + - --resources=pods {{ end }} {{ if .Values.collectors.replicasets }} - - --collectors=replicasets + - --resources=replicasets {{ end }} {{ if .Values.collectors.replicationcontrollers }} - - --collectors=replicationcontrollers + - --resources=replicationcontrollers {{ end }} {{ if .Values.collectors.resourcequotas }} - - --collectors=resourcequotas + - --resources=resourcequotas {{ end }} {{ if .Values.collectors.secrets }} - - --collectors=secrets + - --resources=secrets {{ end }} {{ if .Values.collectors.services }} - - --collectors=services + - --resources=services {{ end }} {{ if .Values.collectors.statefulsets }} - - --collectors=statefulsets + - --resources=statefulsets {{ end }} {{ if .Values.collectors.storageclasses }} - - --collectors=storageclasses + - --resources=storageclasses {{ end }} {{ if .Values.collectors.validatingwebhookconfigurations }} - - --collectors=validatingwebhookconfigurations + - --resources=validatingwebhookconfigurations {{ end }} {{ if .Values.collectors.verticalpodautoscalers }} - - --collectors=verticalpodautoscalers + - --resources=verticalpodautoscalers {{ end }} {{ if .Values.collectors.volumeattachments }} - - --collectors=volumeattachments + - --resources=volumeattachments {{ end }} -{{ if .Values.namespace }} - - --namespace={{ .Values.namespace | join "," }} +{{ if .Values.namespaces }} + - --namespaces={{ tpl .Values.namespaces $ | join "," }} {{ end }} {{ if .Values.autosharding.enabled }} - --pod=$(POD_NAME) @@ -176,6 +176,9 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" ports: - containerPort: 8080 +{{- if .Values.selfMonitor.enabled }} + - containerPort: 8081 +{{- end }} livenessProbe: httpGet: path: /healthz diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml index c493f16..25c8bc8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml @@ -1,11 +1,8 @@ -{{- if and (eq $.Values.rbac.create true) (not .Values.rbac.useExistingRole) -}} -{{- if eq .Values.rbac.useClusterRole false }} -{{- range (split "," $.Values.namespace) }} -{{- end }} -{{- end -}} +{{- if and (eq .Values.rbac.create true) (not .Values.rbac.useExistingRole) -}} +{{- range (split "," .Values.namespaces) }} --- apiVersion: rbac.authorization.k8s.io/v1 -{{- if eq .Values.rbac.useClusterRole false }} +{{- if eq $.Values.rbac.useClusterRole false }} kind: Role {{- else }} kind: ClusterRole @@ -17,7 +14,7 @@ metadata: app.kubernetes.io/managed-by: {{ $.Release.Service }} app.kubernetes.io/instance: {{ $.Release.Name }} name: {{ template "kube-state-metrics.fullname" $ }} -{{- if eq .Values.rbac.useClusterRole false }} +{{- if eq $.Values.rbac.useClusterRole false }} namespace: {{ . }} {{- end }} rules: @@ -190,3 +187,4 @@ rules: verbs: ["list", "watch"] {{ end -}} {{- end -}} +{{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml index 732174a..72a1a2e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml @@ -1,5 +1,5 @@ {{- if and (eq .Values.rbac.create true) (eq .Values.rbac.useClusterRole false) -}} -{{- range (split "," $.Values.namespace) }} +{{- range (split "," $.Values.namespaces) }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml index aaf97bd..62e178d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml @@ -2,7 +2,7 @@ prometheusScrape: true image: repository: k8s.gcr.io/kube-state-metrics/kube-state-metrics - tag: v1.9.8 + tag: v2.0.0 pullPolicy: IfNotPresent imagePullSecrets: [] @@ -110,7 +110,7 @@ podAnnotations: {} podDisruptionBudget: {} # Available collectors for kube-state-metrics. By default all available -# collectors are enabled. +# resources are enabled. collectors: certificatesigningrequests: true configmaps: true @@ -147,8 +147,8 @@ kubeconfig: # base64 encoded kube-config file secret: -# Namespace to be enabled for collecting resources. By default all namespaces are collected. -# namespace: "" +# Comma-separated list of namespaces to be enabled for collecting resources. By default all namespaces are collected. +namespaces: "" ## Override the deployment namespace ## diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml index b045551..a9c26d7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml @@ -14,4 +14,4 @@ maintainers: name: prometheus-node-exporter sources: - https://github.com/prometheus/node_exporter/ -version: 1.17.0 +version: 1.18.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/psp.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/psp.yaml index f00506c..34afb14 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/psp.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/psp.yaml @@ -6,6 +6,10 @@ metadata: name: {{ template "prometheus-node-exporter.fullname" . }} namespace: {{ template "prometheus-node-exporter.namespace" . }} labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: + {{ toYaml .Values.rbac.pspAnnotations | indent 4 }} +{{- end}} spec: privileged: false # Required to prevent escalations to root. diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml index 5cb6981..400ce11 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml @@ -72,6 +72,7 @@ rbac: ## If true, create & use Pod Security Policy resources ## https://kubernetes.io/docs/concepts/policy/pod-security-policy/ pspEnabled: true + pspAnnotations: {} # for deployments that have node_exporter deployed outside of the cluster, list # their addresses here diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml index e8c2828..a6988a7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml index 41128a6..486ee7e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml index 4529496..c16e955 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml index 6e5af51..4b3f92c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml index be23c9f..cea3585 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml index cd8705a..1c33519 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml index 543546a..1c8a4a3 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml index aae4b9e..9112650 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.47.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl index 8336cb8..9ee060b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl @@ -116,3 +116,9 @@ Allow the release namespace to be overridden for multi-namespace deployments in {{- define "kube-prometheus-stack.ingress.isStable" -}} {{- eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1" -}} {{- end -}} + +{{/* Check Ingress supports pathType */}} +{{/* pathType was added to networking.k8s.io/v1beta1 in Kubernetes 1.18 */}} +{{- define "kube-prometheus-stack.ingress.supportsPathType" -}} + {{- or (eq (include "kube-prometheus-stack.ingress.isStable" .) "true") (and (eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18.x" (include "kube-prometheus-stack.ingress.kubeVersion" .))) -}} +{{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml index 8ade270..f337502 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml @@ -5,6 +5,7 @@ {{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} {{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} kind: Ingress metadata: @@ -34,7 +35,7 @@ spec: paths: {{- range $p := $paths }} - path: {{ tpl $p $ }} - {{- if $pathType }} + {{- if and $pathType $ingressSupportsPathType }} pathType: {{ $pathType }} {{- end }} backend: @@ -54,7 +55,7 @@ spec: paths: {{- range $p := $paths }} - path: {{ tpl $p $ }} - {{- if $pathType }} + {{- if and $pathType $ingressSupportsPathType }} pathType: {{ $pathType }} {{- end }} backend: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml index c55ec2a..f21bf96 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml @@ -4,6 +4,7 @@ {{- $servicePort := .Values.alertmanager.service.port -}} {{- $ingressValues := .Values.alertmanager.ingressPerReplica -}} {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} apiVersion: v1 kind: List metadata: @@ -12,7 +13,7 @@ metadata: items: {{ range $i, $e := until $count }} - kind: Ingress - apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} metadata: name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} namespace: {{ template "kube-prometheus-stack.namespace" $ }} @@ -38,7 +39,7 @@ items: paths: {{- range $p := $ingressValues.paths }} - path: {{ tpl $p $ }} - {{- if $pathType }} + {{- if and $pathType $ingressSupportsPathType }} pathType: {{ $pathType }} {{- end }} backend: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml index e802922..549b3ff 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml @@ -16,7 +16,7 @@ spec: interval: {{ .Values.kubelet.serviceMonitor.interval }} {{- end }} {{- if .Values.kubelet.serviceMonitor.proxyUrl }} - proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl}} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} {{- end }} tlsConfig: caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt @@ -38,6 +38,9 @@ spec: {{- if .Values.kubelet.serviceMonitor.interval }} interval: {{ .Values.kubelet.serviceMonitor.interval }} {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} {{- end }} @@ -62,6 +65,9 @@ spec: {{- if .Values.kubelet.serviceMonitor.interval }} interval: {{ .Values.kubelet.serviceMonitor.interval }} {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} honorLabels: true tlsConfig: caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt @@ -83,6 +89,9 @@ spec: {{- if .Values.kubelet.serviceMonitor.interval }} interval: {{ .Values.kubelet.serviceMonitor.interval }} {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} honorLabels: true tlsConfig: caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt @@ -102,6 +111,9 @@ spec: {{- if .Values.kubelet.serviceMonitor.interval }} interval: {{ .Values.kubelet.serviceMonitor.interval }} {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} honorLabels: true {{- if .Values.kubelet.serviceMonitor.metricRelabelings }} metricRelabelings: @@ -117,6 +129,9 @@ spec: {{- if .Values.kubelet.serviceMonitor.interval }} interval: {{ .Values.kubelet.serviceMonitor.interval }} {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} honorLabels: true {{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} metricRelabelings: @@ -132,6 +147,9 @@ spec: {{- if .Values.kubelet.serviceMonitor.interval }} interval: {{ .Values.kubelet.serviceMonitor.interval }} {{- end }} + {{- if .Values.kubelet.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubelet.serviceMonitor.proxyUrl }} + {{- end }} honorLabels: true {{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} metricRelabelings: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml index adcc419..2061b98 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml @@ -18,8 +18,8 @@ spec: {{- if .Values.nodeExporter.serviceMonitor.interval }} interval: {{ .Values.nodeExporter.serviceMonitor.interval }} {{- end }} - {{- if .Values.kubeApiServer.serviceMonitor.proxyUrl }} - proxyUrl: {{ .Values.kubeApiServer.serviceMonitor.proxyUrl}} + {{- if .Values.nodeExporter.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.nodeExporter.serviceMonitor.proxyUrl}} {{- end }} {{- if .Values.nodeExporter.serviceMonitor.scrapeTimeout }} scrapeTimeout: {{ .Values.nodeExporter.serviceMonitor.scrapeTimeout }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmap-dashboards.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmap-dashboards.yaml index fdd64ce..7ce2a7f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmap-dashboards.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmap-dashboards.yaml @@ -1,5 +1,5 @@ {{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} -{{- $files := .Files.Glob "dashboards/*.json" }} +{{- $files := .Files.Glob "dashboards-1.14/*.json" }} {{- if $files }} apiVersion: v1 kind: ConfigMapList diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml index 1d1c3e9..2fab7dc 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -1832,7 +1832,7 @@ data: ], "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/controller-manager.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/controller-manager.yaml index 76d561a..139b1a5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/controller-manager.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/controller-manager.yaml @@ -107,7 +107,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(up{job=\"kube-controller-manager\"})", + "expr": "sum(up{cluster=\"$cluster\", job=\"kube-controller-manager\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -176,7 +176,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_adds_total{job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", @@ -282,7 +282,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_depth{job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", @@ -388,7 +388,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name, le))", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", @@ -608,7 +608,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", @@ -714,7 +714,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", @@ -820,7 +820,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{job=\"kube-controller-manager\",instance=~\"$instance\"}", + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -913,7 +913,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{job=\"kube-controller-manager\",instance=~\"$instance\"}[5m])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -1006,7 +1006,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "go_goroutines{job=\"kube-controller-manager\",instance=~\"$instance\"}", + "expr": "go_goroutines{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -1090,6 +1090,32 @@ data: "allValue": null, "current": { + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + }, "datasource": "$datasource", "hide": 0, @@ -1100,7 +1126,7 @@ data: "options": [ ], - "query": "label_values(process_cpu_seconds_total{job=\"kube-controller-manager\"}, instance)", + "query": "label_values(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-controller-manager\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml index 66768d9..f8217d9 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'etcd' from https://raw.githubusercontent.com/etcd-io/website/master/content/docs/current/op-guide/grafana.json +Generated from 'etcd' from https://raw.githubusercontent.com/etcd-io/website/master/content/en/docs/v3.4/op-guide/grafana.json Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -28,6 +28,7 @@ data: "editable": true, "gnetId": null, "hideControls": false, + "id": 6, "links": [], "refresh": "10s", "rows": [ @@ -1110,7 +1111,6 @@ data: }, "timezone": "browser", "title": "etcd", - "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", "version": 215 } {{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml index b1aadb1..a3c678c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -163,7 +163,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"cpu\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -247,7 +247,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", + "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{resource=\"cpu\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -331,7 +331,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -415,7 +415,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"memory\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -499,7 +499,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", + "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"memory\"}) / sum(kube_node_status_allocatable{resource=\"memory\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -894,7 +894,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -903,7 +903,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", resource=\"cpu\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -912,7 +912,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"cpu\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -921,7 +921,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"cpu\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1321,7 +1321,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(kube_pod_container_resource_requests{cluster=\"$cluster\", resource=\"memory\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1330,7 +1330,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", resource=\"memory\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1339,7 +1339,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"memory\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1348,7 +1348,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"memory\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1705,7 +1705,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Current Network Usage", "titleSize": "h6" }, { @@ -1745,7 +1745,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1797,19 +1797,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1843,7 +1831,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1901,7 +1889,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Bandwidth", "titleSize": "h6" }, { @@ -1941,7 +1929,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1993,19 +1981,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -2039,7 +2015,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -2097,7 +2073,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Average Container Bandwidth by Namespace", "titleSize": "h6" }, { @@ -2137,7 +2113,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -2189,19 +2165,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -2235,7 +2199,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -2293,7 +2257,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Rate of Packets", "titleSize": "h6" }, { @@ -2333,7 +2297,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -2385,19 +2349,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -2431,7 +2383,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -2489,7 +2441,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Rate of Packets Dropped", "titleSize": "h6" } ], diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml index 4694b59..e20f344 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -78,7 +78,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"})", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -162,7 +162,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"})", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -246,7 +246,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"})", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -330,7 +330,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"})", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -422,8 +422,9 @@ data: "color": "#F2495C", "dashes": true, "fill": 0, + "hiddenSeries": true, "hideTooltip": true, - "legend": false, + "legend": true, "linewidth": 2, "stack": false }, @@ -432,8 +433,9 @@ data: "color": "#FF9830", "dashes": true, "fill": 0, + "hiddenSeries": true, "hideTooltip": true, - "legend": false, + "legend": true, "linewidth": 2, "stack": false } @@ -704,7 +706,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -713,7 +715,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -722,7 +724,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -731,7 +733,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -828,8 +830,9 @@ data: "color": "#F2495C", "dashes": true, "fill": 0, + "hiddenSeries": true, "hideTooltip": true, - "legend": false, + "legend": true, "linewidth": 2, "stack": false }, @@ -838,8 +841,9 @@ data: "color": "#FF9830", "dashes": true, "fill": 0, + "hiddenSeries": true, "hideTooltip": true, - "legend": false, + "legend": true, "linewidth": 2, "stack": false } @@ -1167,7 +1171,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1176,7 +1180,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1185,7 +1189,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1194,7 +1198,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1578,7 +1582,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Current Network Usage", "titleSize": "h6" }, { @@ -1618,7 +1622,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1670,19 +1674,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1716,7 +1708,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1774,7 +1766,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Bandwidth", "titleSize": "h6" }, { @@ -1814,7 +1806,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1866,19 +1858,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1912,7 +1892,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1970,7 +1950,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Rate of Packets", "titleSize": "h6" }, { @@ -2010,7 +1990,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -2062,19 +2042,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -2108,7 +2076,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -2166,7 +2134,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Rate of Packets Dropped", "titleSize": "h6" } ], @@ -2209,7 +2177,7 @@ data: ], "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", @@ -2236,7 +2204,7 @@ data: ], "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-node.yaml index b99e40f..73a3e88 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-node.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -321,7 +321,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -330,7 +330,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -339,7 +339,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -348,7 +348,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -749,7 +749,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -758,7 +758,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{node=~\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -767,7 +767,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -776,7 +776,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{node=~\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -901,7 +901,7 @@ data: ], "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", @@ -928,7 +928,7 @@ data: ], "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, node)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml index acee659..014b165 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -94,7 +94,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", cluster=\"$cluster\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}container{{`}}`}}", @@ -102,7 +102,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"})\n", + "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "requests", @@ -110,7 +110,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"})\n", + "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "limits", @@ -208,7 +208,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\", cluster=\"$cluster\"}[5m])) by (container) /sum(increase(container_cpu_cfs_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\", cluster=\"$cluster\"}[5m])) by (container)", + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[5m])) by (container) /sum(increase(container_cpu_cfs_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[5m])) by (container)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}container{{`}}`}}", @@ -450,7 +450,7 @@ data: ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -459,7 +459,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -468,7 +468,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -477,7 +477,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -486,7 +486,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -584,7 +584,7 @@ data: "dashes": true, "fill": 0, "hideTooltip": true, - "legend": false, + "legend": true, "linewidth": 2, "stack": false }, @@ -594,7 +594,7 @@ data: "dashes": true, "fill": 0, "hideTooltip": true, - "legend": false, + "legend": true, "linewidth": 2, "stack": false } @@ -605,7 +605,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\", image!=\"\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}container{{`}}`}}", @@ -613,7 +613,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"})\n", + "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "requests", @@ -621,7 +621,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"})\n", + "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "limits", @@ -913,7 +913,7 @@ data: ], "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\", image!=\"\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -922,7 +922,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -931,7 +931,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -940,7 +940,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container)", + "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -949,7 +949,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1072,12 +1072,12 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}pod{{`}}`}}", @@ -1124,19 +1124,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1171,12 +1159,12 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}pod{{`}}`}}", @@ -1229,7 +1217,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Bandwidth", "titleSize": "h6" }, { @@ -1270,12 +1258,12 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}pod{{`}}`}}", @@ -1322,19 +1310,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1369,12 +1345,12 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}pod{{`}}`}}", @@ -1427,7 +1403,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Rate of Packets", "titleSize": "h6" }, { @@ -1468,12 +1444,12 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}pod{{`}}`}}", @@ -1520,19 +1496,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1567,12 +1531,12 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ { - "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}pod{{`}}`}}", @@ -1625,7 +1589,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Rate of Packets Dropped", "titleSize": "h6" } ], @@ -1668,7 +1632,7 @@ data: ], "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", @@ -1695,7 +1659,7 @@ data: ], "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml index 0df1bea..38c2ead 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -321,7 +321,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -330,7 +330,7 @@ data: "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -339,7 +339,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -348,7 +348,7 @@ data: "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -692,7 +692,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -701,7 +701,7 @@ data: "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -710,7 +710,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -719,7 +719,7 @@ data: "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1076,7 +1076,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Current Network Usage", "titleSize": "h6" }, { @@ -1116,7 +1116,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1168,19 +1168,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1214,7 +1202,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1272,7 +1260,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Bandwidth", "titleSize": "h6" }, { @@ -1312,7 +1300,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1364,19 +1352,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1410,7 +1386,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1468,7 +1444,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Average Container Bandwidth by Pod", "titleSize": "h6" }, { @@ -1508,7 +1484,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1560,19 +1536,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1606,7 +1570,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1664,7 +1628,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Rate of Packets", "titleSize": "h6" }, { @@ -1704,7 +1668,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1756,19 +1720,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1802,7 +1754,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1860,7 +1812,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Rate of Packets Dropped", "titleSize": "h6" } ], @@ -1903,7 +1855,7 @@ data: ], "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", @@ -1930,7 +1882,7 @@ data: ], "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", @@ -1957,7 +1909,7 @@ data: ], "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", @@ -1984,7 +1936,7 @@ data: ], "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml index 9ff5394..0c53de7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -74,8 +74,9 @@ data: "color": "#F2495C", "dashes": true, "fill": 0, + "hiddenSeries": true, "hideTooltip": true, - "legend": false, + "legend": true, "linewidth": 2, "stack": false }, @@ -84,8 +85,9 @@ data: "color": "#FF9830", "dashes": true, "fill": 0, + "hiddenSeries": true, "hideTooltip": true, - "legend": false, + "legend": true, "linewidth": 2, "stack": false } @@ -403,7 +405,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -412,7 +414,7 @@ data: "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -421,7 +423,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -430,7 +432,7 @@ data: "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -527,8 +529,9 @@ data: "color": "#F2495C", "dashes": true, "fill": 0, + "hiddenSeries": true, "hideTooltip": true, - "legend": false, + "legend": true, "linewidth": 2, "stack": false }, @@ -537,8 +540,9 @@ data: "color": "#FF9830", "dashes": true, "fill": 0, + "hiddenSeries": true, "hideTooltip": true, - "legend": false, + "legend": true, "linewidth": 2, "stack": false } @@ -856,7 +860,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -865,7 +869,7 @@ data: "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -874,7 +878,7 @@ data: "step": 10 }, { - "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -883,7 +887,7 @@ data: "step": 10 }, { - "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1259,7 +1263,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Current Network Usage", "titleSize": "h6" }, { @@ -1299,7 +1303,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1351,19 +1355,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1397,7 +1389,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1455,7 +1447,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Bandwidth", "titleSize": "h6" }, { @@ -1495,7 +1487,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1547,19 +1539,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1593,7 +1573,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1651,7 +1631,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Average Container Bandwidth by Workload", "titleSize": "h6" }, { @@ -1691,7 +1671,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1743,19 +1723,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1789,7 +1757,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1847,7 +1815,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Rate of Packets", "titleSize": "h6" }, { @@ -1887,7 +1855,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -1939,19 +1907,7 @@ data: "show": false } ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ + }, { "aliasColors": { @@ -1985,7 +1941,7 @@ data: ], "spaceLength": 10, - "span": 12, + "span": 6, "stack": true, "steppedLine": false, "targets": [ @@ -2043,7 +1999,7 @@ data: "repeatIteration": null, "repeatRowId": null, "showTitle": true, - "title": "Network", + "title": "Rate of Packets Dropped", "titleSize": "h6" } ], @@ -2070,38 +2026,6 @@ data: "regex": "", "type": "datasource" }, - { - "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "deployment", - "value": "deployment" - }, - "datasource": "$datasource", - "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "type", - "options": [ - - ], - "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, { "allValue": null, "current": { @@ -2118,7 +2042,7 @@ data: ], "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", @@ -2129,6 +2053,38 @@ data: "type": "query", "useTags": false }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "allValue": null, "current": { @@ -2145,7 +2101,7 @@ data: ], "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml index c131e68..eb53ba0 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -1302,7 +1302,7 @@ data: ], "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", @@ -1333,7 +1333,7 @@ data: ], "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml index 097d7f5..6a01666 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -1542,7 +1542,7 @@ data: ], "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", @@ -1573,7 +1573,7 @@ data: ], "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, @@ -1605,7 +1605,7 @@ data: ], "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml index 62ab619..f486bcb 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -77,7 +77,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(\n instance:node_cpu_utilisation:rate1m{job=\"node-exporter\"}\n*\n instance:node_num_cpu:sum{job=\"node-exporter\"}\n)\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\"}))\n", + "expr": "(\n instance:node_cpu_utilisation:rate5m{job=\"node-exporter\"}\n*\n instance:node_num_cpu:sum{job=\"node-exporter\"}\n)\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\"}))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -92,7 +92,7 @@ data: "timeShift": null, "title": "CPU Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -178,7 +178,7 @@ data: "timeShift": null, "title": "CPU Saturation (load1 per CPU)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -276,7 +276,7 @@ data: "timeShift": null, "title": "Memory Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -347,7 +347,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance:node_vmstat_pgmajfault:rate1m{job=\"node-exporter\"}", + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -362,7 +362,7 @@ data: "timeShift": null, "title": "Memory Saturation (Major Page Faults)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -453,7 +453,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance:node_network_receive_bytes_excluding_lo:rate1m{job=\"node-exporter\"}", + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", @@ -461,7 +461,7 @@ data: "step": 10 }, { - "expr": "instance:node_network_transmit_bytes_excluding_lo:rate1m{job=\"node-exporter\"}", + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", @@ -476,7 +476,7 @@ data: "timeShift": null, "title": "Net Utilisation (Bytes Receive/Transmit)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -555,7 +555,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance:node_network_receive_drop_excluding_lo:rate1m{job=\"node-exporter\"}", + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", @@ -563,7 +563,7 @@ data: "step": 10 }, { - "expr": "instance:node_network_transmit_drop_excluding_lo:rate1m{job=\"node-exporter\"}", + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", @@ -578,7 +578,7 @@ data: "timeShift": null, "title": "Net Saturation (Drops Receive/Transmit)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -661,7 +661,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance_device:node_disk_io_time_seconds:rate1m{job=\"node-exporter\"}\n/ scalar(count(instance_device:node_disk_io_time_seconds:rate1m{job=\"node-exporter\"}))\n", + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\"}\n/ scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\"}))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", @@ -676,7 +676,7 @@ data: "timeShift": null, "title": "Disk IO Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -747,7 +747,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance_device:node_disk_io_time_weighted_seconds:rate1m{job=\"node-exporter\"}\n/ scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate1m{job=\"node-exporter\"}))\n", + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\"}\n/ scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\"}))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", @@ -762,7 +762,7 @@ data: "timeShift": null, "title": "Disk IO Saturation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -860,7 +860,7 @@ data: "timeShift": null, "title": "Disk Space Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml index cd21961..bfbe258 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -77,7 +77,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance:node_cpu_utilisation:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Utilisation", @@ -92,7 +92,7 @@ data: "timeShift": null, "title": "CPU Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -178,7 +178,7 @@ data: "timeShift": null, "title": "CPU Saturation (Load1 per CPU)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -276,7 +276,7 @@ data: "timeShift": null, "title": "Memory Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -347,7 +347,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance:node_vmstat_pgmajfault:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Major page faults", @@ -362,7 +362,7 @@ data: "timeShift": null, "title": "Memory Saturation (Major Page Faults)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -453,7 +453,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance:node_network_receive_bytes_excluding_lo:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Receive", @@ -461,7 +461,7 @@ data: "step": 10 }, { - "expr": "instance:node_network_transmit_bytes_excluding_lo:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Transmit", @@ -476,7 +476,7 @@ data: "timeShift": null, "title": "Net Utilisation (Bytes Receive/Transmit)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -555,7 +555,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance:node_network_receive_drop_excluding_lo:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Receive drops", @@ -563,7 +563,7 @@ data: "step": 10 }, { - "expr": "instance:node_network_transmit_drop_excluding_lo:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Transmit drops", @@ -578,7 +578,7 @@ data: "timeShift": null, "title": "Net Saturation (Drops Receive/Transmit)", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -661,7 +661,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance_device:node_disk_io_time_seconds:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}}", @@ -676,7 +676,7 @@ data: "timeShift": null, "title": "Disk IO Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -747,7 +747,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "instance_device:node_disk_io_time_weighted_seconds:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}}", @@ -762,7 +762,7 @@ data: "timeShift": null, "title": "Disk IO Saturation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -860,7 +860,7 @@ data: "timeShift": null, "title": "Disk Space Utilisation", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml index 2a29fc0..e3b3bd8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml @@ -94,7 +94,7 @@ data: { "expr": "(\n (1 - rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"}[$__interval]))\n/ ignoring(cpu) group_left\n count without (cpu)( node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", "format": "time_series", - "interval": "1m", + "interval": "$__rate_interval", "intervalFactor": 5, "legendFormat": "{{`{{`}}cpu{{`}}`}}", "refId": "A" @@ -530,7 +530,7 @@ data: { "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__interval])", "format": "time_series", - "interval": "1m", + "interval": "$__rate_interval", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}} read", "refId": "A" @@ -538,7 +538,7 @@ data: { "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__interval])", "format": "time_series", - "interval": "1m", + "interval": "$__rate_interval", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}} written", "refId": "B" @@ -546,7 +546,7 @@ data: { "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__interval])", "format": "time_series", - "interval": "1m", + "interval": "$__rate_interval", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}} io time", "refId": "C" @@ -760,7 +760,7 @@ data: { "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__interval])", "format": "time_series", - "interval": "1m", + "interval": "$__rate_interval", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}}", "refId": "A" @@ -854,7 +854,7 @@ data: { "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__interval])", "format": "time_series", - "interval": "1m", + "interval": "$__rate_interval", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}}", "refId": "A" diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml index 95abda4..7145287 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml @@ -1034,7 +1034,7 @@ data: ], "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", @@ -1065,7 +1065,7 @@ data: ], "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, @@ -1097,7 +1097,7 @@ data: ], "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)", - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus.yaml index 7095fb7..0cd3e7b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus.yaml @@ -220,7 +220,7 @@ data: "timeShift": null, "title": "Prometheus Stats", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -319,7 +319,7 @@ data: "timeShift": null, "title": "Target Sync", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -405,7 +405,7 @@ data: "timeShift": null, "title": "Targets", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -503,7 +503,7 @@ data: "timeShift": null, "title": "Average Scrape Interval Duration", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -613,7 +613,7 @@ data: "timeShift": null, "title": "Scrape failures", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -699,7 +699,7 @@ data: "timeShift": null, "title": "Appended Samples", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -797,7 +797,7 @@ data: "timeShift": null, "title": "Head Series", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -883,7 +883,7 @@ data: "timeShift": null, "title": "Head Chunks", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -981,7 +981,7 @@ data: "timeShift": null, "title": "Query Rate", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -1067,7 +1067,7 @@ data: "timeShift": null, "title": "Stage Duration", "tooltip": { - "shared": true, + "shared": false, "sort": 0, "value_type": "individual" }, @@ -1133,7 +1133,7 @@ data: "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { "selected": true, "text": "All", @@ -1161,7 +1161,7 @@ data: "useTags": false }, { - "allValue": null, + "allValue": ".+", "current": { "selected": true, "text": "All", diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/proxy.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/proxy.yaml index 8c6b77c..8f8559e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/proxy.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/proxy.yaml @@ -107,7 +107,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(up{job=\"kube-proxy\"})", + "expr": "sum(up{cluster=\"$cluster\", job=\"kube-proxy\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -176,7 +176,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "rate", @@ -269,7 +269,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -375,7 +375,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "rate", @@ -468,7 +468,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{job=\"kube-proxy\", instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -574,28 +574,28 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"2..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"3..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"4..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\",code=~\"5..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -688,7 +688,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-proxy\",instance=~\"$instance\",verb=\"POST\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\",verb=\"POST\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", @@ -794,7 +794,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-proxy\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", @@ -900,7 +900,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{job=\"kube-proxy\",instance=~\"$instance\"}", + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -993,7 +993,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{job=\"kube-proxy\",instance=~\"$instance\"}[5m])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -1086,7 +1086,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "go_goroutines{job=\"kube-proxy\",instance=~\"$instance\"}", + "expr": "go_goroutines{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -1170,6 +1170,32 @@ data: "allValue": null, "current": { + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + }, "datasource": "$datasource", "hide": 0, @@ -1180,7 +1206,7 @@ data: "options": [ ], - "query": "label_values(kubeproxy_network_programming_duration_seconds_bucket{job=\"kube-proxy\"}, instance)", + "query": "label_values(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/scheduler.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/scheduler.yaml index 860fdc8..3dbf4ab 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/scheduler.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/scheduler.yaml @@ -107,7 +107,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(up{job=\"kube-scheduler\"})", + "expr": "sum(up{cluster=\"$cluster\", job=\"kube-scheduler\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -176,28 +176,28 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} e2e", "refId": "A" }, { - "expr": "sum(rate(scheduler_binding_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} binding", "refId": "B" }, { - "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} scheduling algorithm", "refId": "C" }, { - "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} volume", @@ -290,28 +290,28 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} e2e", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} binding", "refId": "B" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} scheduling algorithm", "refId": "C" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}} volume", @@ -417,28 +417,28 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "2xx", "refId": "A" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "3xx", "refId": "B" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "4xx", "refId": "C" }, { - "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "5xx", @@ -531,7 +531,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", @@ -637,7 +637,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", @@ -743,7 +743,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{job=\"kube-scheduler\", instance=~\"$instance\"}", + "expr": "process_resident_memory_bytes{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -836,7 +836,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])", + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -929,7 +929,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "go_goroutines{job=\"kube-scheduler\",instance=~\"$instance\"}", + "expr": "go_goroutines{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -1013,6 +1013,32 @@ data: "allValue": null, "current": { + }, + "datasource": "$datasource", + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + }, "datasource": "$datasource", "hide": 0, @@ -1023,7 +1049,7 @@ data: "options": [ ], - "query": "label_values(process_cpu_seconds_total{job=\"kube-scheduler\"}, instance)", + "query": "label_values(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-scheduler\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml index 07f5353..75f95bd 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml @@ -1212,7 +1212,7 @@ data: ], "query": "label_values(kube_pod_info, cluster)", - "refresh": 1, + "refresh": 2, "regex": "", "sort": 0, "tagValuesQuery": "", @@ -1243,7 +1243,7 @@ data: ], "query": "label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)", - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, @@ -1275,7 +1275,7 @@ data: ], "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)", - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, @@ -1307,7 +1307,7 @@ data: ], "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", - "refresh": 1, + "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/etcd.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/etcd.yaml deleted file mode 100644 index 157843a..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/etcd.yaml +++ /dev/null @@ -1,1116 +0,0 @@ -{{- /* -Generated from 'etcd' from https://raw.githubusercontent.com/etcd-io/website/master/content/docs/current/op-guide/grafana.json -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.kubeEtcd.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ template "kube-prometheus-stack.namespace" . }} - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "etcd" | trunc 63 | trimSuffix "-" }} - annotations: -{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} - labels: - {{- if $.Values.grafana.sidecar.dashboards.label }} - {{ $.Values.grafana.sidecar.dashboards.label }}: "1" - {{- end }} - app: {{ template "kube-prometheus-stack.name" $ }}-grafana -{{ include "kube-prometheus-stack.labels" $ | indent 4 }} -data: - etcd.json: |- - { - "annotations": { - "list": [] - }, - "description": "etcd sample Grafana dashboard with Prometheus", - "editable": true, - "gnetId": null, - "hideControls": false, - "links": [], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "$datasource", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "id": 28, - "interval": null, - "isNew": true, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "targets": [ - { - "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", - "intervalFactor": 2, - "legendFormat": "", - "metric": "etcd_server_has_leader", - "refId": "A", - "step": 20 - } - ], - "thresholds": "", - "title": "Up", - "type": "singlestat", - "valueFontSize": "200%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 23, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 5, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "RPC Rate", - "metric": "grpc_server_started_total", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "RPC Failed Rate", - "metric": "grpc_server_handled_total", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "RPC Rate", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 41, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", - "intervalFactor": 2, - "legendFormat": "Watch Streams", - "metric": "grpc_server_handled_total", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", - "intervalFactor": 2, - "legendFormat": "Lease Streams", - "metric": "grpc_server_handled_total", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Active Streams", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "showTitle": false, - "title": "Row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "$datasource", - "decimals": null, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} DB Size", - "metric": "", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "DB Size", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 1, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} WAL fsync", - "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", - "refId": "A", - "step": 4 - }, - { - "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} DB fsync", - "metric": "etcd_disk_backend_commit_duration_seconds_bucket", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Disk Sync Duration", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 29, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_resident_memory_bytes{job=\"$cluster\"}", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} Resident Memory", - "metric": "process_resident_memory_bytes", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 5, - "id": 22, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[5m])", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic In", - "metric": "etcd_network_client_grpc_received_bytes_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Client Traffic In", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 5, - "id": 21, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[5m])", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic Out", - "metric": "etcd_network_client_grpc_sent_bytes_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Client Traffic Out", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 20, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[5m])) by (instance)", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic In", - "metric": "etcd_network_peer_received_bytes_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Peer Traffic In", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "$datasource", - "decimals": null, - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[5m])) by (instance)", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic Out", - "metric": "etcd_network_peer_sent_bytes_total", - "refId": "A", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Peer Traffic Out", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - }, - { - "collapse": false, - "editable": true, - "height": "250px", - "panels": [ - { - "aliasColors": {}, - "bars": false, - "datasource": "$datasource", - "editable": true, - "error": false, - "fill": 0, - "id": 40, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[5m]))", - "intervalFactor": 2, - "legendFormat": "Proposal Failure Rate", - "metric": "etcd_server_proposals_failed_total", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", - "intervalFactor": 2, - "legendFormat": "Proposal Pending Total", - "metric": "etcd_server_proposals_pending", - "refId": "B", - "step": 2 - }, - { - "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[5m]))", - "intervalFactor": 2, - "legendFormat": "Proposal Commit Rate", - "metric": "etcd_server_proposals_committed_total", - "refId": "C", - "step": 2 - }, - { - "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[5m]))", - "intervalFactor": 2, - "legendFormat": "Proposal Apply Rate", - "refId": "D", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Raft Proposals", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "datasource": "$datasource", - "decimals": 0, - "editable": true, - "error": false, - "fill": 0, - "id": 19, - "isNew": true, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} Total Leader Elections Per Day", - "metric": "etcd_server_leader_changes_seen_total", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Total Leader Elections Per Day", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "New row" - } - ], - "schemaVersion": 13, - "sharedCrosshair": false, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [], - "query": "label_values(etcd_server_has_leader, job)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "now": true, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "etcd", - "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", - "version": 215 - } -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-cluster-rsrc-use.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-cluster-rsrc-use.yaml deleted file mode 100644 index fbe5e31..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-cluster-rsrc-use.yaml +++ /dev/null @@ -1,959 +0,0 @@ -{{- /* -Generated from 'k8s-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ template "kube-prometheus-stack.namespace" . }} - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} - annotations: -{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} - labels: - {{- if $.Values.grafana.sidecar.dashboards.label }} - {{ $.Values.grafana.sidecar.dashboards.label }}: "1" - {{- end }} - app: {{ template "kube-prometheus-stack.name" $ }}-grafana -{{ include "kube-prometheus-stack.labels" $ | indent 4 }} -data: - k8s-cluster-rsrc-use.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "node:cluster_cpu_utilisation:ratio{cluster=\"$cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}node{{`}}`}}", - "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_cpu_saturation_load1:{cluster=\"$cluster\"} / scalar(sum(min(kube_pod_info{cluster=\"$cluster\"}) by (node)))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}node{{`}}`}}", - "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Saturation (Load1)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "node:cluster_memory_utilisation:ratio{cluster=\"$cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}node{{`}}`}}", - "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_memory_swap_io_bytes:sum_rate{cluster=\"$cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}node{{`}}`}}", - "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Saturation (Swap I/O)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(:kube_pod_info_node_count:{cluster=\"$cluster\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}node{{`}}`}}", - "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk IO Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_disk_saturation:avg_irate{cluster=\"$cluster\"} / scalar(:kube_pod_info_node_count:{cluster=\"$cluster\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}node{{`}}`}}", - "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk IO Saturation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Disk", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_net_utilisation:sum_irate{cluster=\"$cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}node{{`}}`}}", - "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Net Utilisation (Transmitted)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_net_saturation:sum_irate{cluster=\"$cluster\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}node{{`}}`}}", - "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Net Saturation (Dropped)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Network", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"} - node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"}) by (device,pod,namespace)) by (pod,namespace)\n/ scalar(sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"}) by (device,pod,namespace)))\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{cluster=\"$cluster\"}\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}node{{`}}`}}", - "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk Capacity", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Storage", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(:kube_pod_info_node_count:, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / USE Method / Cluster", - "uid": "a6e7d1362e1ddbb79db21d5bb40d7137", - "version": 0 - } -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-node-rsrc-use.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-node-rsrc-use.yaml deleted file mode 100644 index 6d01578..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-node-rsrc-use.yaml +++ /dev/null @@ -1,986 +0,0 @@ -{{- /* -Generated from 'k8s-node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ template "kube-prometheus-stack.namespace" . }} - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-node-rsrc-use" | trunc 63 | trimSuffix "-" }} - annotations: -{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} - labels: - {{- if $.Values.grafana.sidecar.dashboards.label }} - {{ $.Values.grafana.sidecar.dashboards.label }}: "1" - {{- end }} - app: {{ template "kube-prometheus-stack.name" $ }}-grafana -{{ include "kube-prometheus-stack.labels" $ | indent 4 }} -data: - k8s-node-rsrc-use.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_cpu_utilisation:avg1m{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Utilisation", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_cpu_saturation_load1:{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Saturation", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Saturation (Load1)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_memory_utilisation:{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Memory", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_memory_swap_io_bytes:sum_rate{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Swap IO", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Saturation (Swap I/O)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_disk_utilisation:avg_irate{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Utilisation", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk IO Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_disk_saturation:avg_irate{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Saturation", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk IO Saturation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Disk", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_net_utilisation:sum_irate{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Utilisation", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Net Utilisation (Transmitted)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_net_saturation:sum_irate{cluster=\"$cluster\", node=\"$node\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Saturation", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Net Saturation (Dropped)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Net", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "node:node_filesystem_usage:{cluster=\"$cluster\"}\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{cluster=\"$cluster\", node=\"$node\"}\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}device{{`}}`}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Disk", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(:kube_pod_info_node_count:, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "node", - "multi": false, - "name": "node", - "options": [ - - ], - "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / USE Method / Node", - "uid": "4ac4f123aae0ff6dbaf4f4f66120033b", - "version": 0 - } -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-cluster.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-cluster.yaml deleted file mode 100644 index ecfbb3c..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-cluster.yaml +++ /dev/null @@ -1,1479 +0,0 @@ -{{- /* -Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ template "kube-prometheus-stack.namespace" . }} - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-cluster" | trunc 63 | trimSuffix "-" }} - annotations: -{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} - labels: - {{- if $.Values.grafana.sidecar.dashboards.label }} - {{ $.Values.grafana.sidecar.dashboards.label }}: "1" - {{- end }} - app: {{ template "kube-prometheus-stack.name" $ }}-grafana -{{ include "kube-prometheus-stack.labels" $ | indent 4 }} -data: - k8s-resources-cluster.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "100px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[1m]))", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "CPU Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(node:node_num_cpu:sum{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "CPU Requests Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(node:node_num_cpu:sum{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "CPU Limits Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "1 - sum(:node_memory_MemFreeCachedBuffers_bytes:sum{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Memory Utilisation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Memory Requests Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "percentunit", - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Memory Limits Commitment", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Headlines", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}namespace{{`}}`}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Pods", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": true, - "linkTooltip": "Drill down to pods", - "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "Workloads", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": true, - "linkTooltip": "Drill down to workloads", - "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #G", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Namespace", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down to pods", - "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", - "pattern": "namespace", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "G", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}namespace{{`}}`}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage (w/o cache)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Pods", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": true, - "linkTooltip": "Drill down to pods", - "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "Workloads", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": true, - "linkTooltip": "Drill down to workloads", - "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #G", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Namespace", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down to pods", - "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", - "pattern": "namespace", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "G", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Requests by Namespace", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Requests", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(:kube_pod_info_node_count:, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Cluster", - "uid": "efa86fd1d0c121a26444b636a3f509a8", - "version": 0 - } -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-namespace.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-namespace.yaml deleted file mode 100644 index 64da49c..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-namespace.yaml +++ /dev/null @@ -1,963 +0,0 @@ -{{- /* -Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ template "kube-prometheus-stack.namespace" . }} - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-namespace" | trunc 63 | trimSuffix "-" }} - annotations: -{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} - labels: - {{- if $.Values.grafana.sidecar.dashboards.label }} - {{ $.Values.grafana.sidecar.dashboards.label }}: "1" - {{- end }} - app: {{ template "kube-prometheus-stack.name" $ }}-grafana -{{ include "kube-prometheus-stack.labels" $ | indent 4 }} -data: - k8s-resources-namespace.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}pod_name{{`}}`}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Pod", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", - "pattern": "pod", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"}) by (pod_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}pod_name{{`}}`}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage (w/o cache)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Usage (RSS)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Cache)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #G", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Swap", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #H", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Pod", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", - "pattern": "pod", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "G", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "H", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Quota", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(:kube_pod_info_node_count:, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Namespace (Pods)", - "uid": "85a562078cdf77779eaa1add43ccec1e", - "version": 0 - } -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-pod.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-pod.yaml deleted file mode 100644 index 78cd045..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-pod.yaml +++ /dev/null @@ -1,1006 +0,0 @@ -{{- /* -Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ template "kube-prometheus-stack.namespace" . }} - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-pod" | trunc 63 | trimSuffix "-" }} - annotations: -{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} - labels: - {{- if $.Values.grafana.sidecar.dashboards.label }} - {{ $.Values.grafana.sidecar.dashboards.label }}: "1" - {{- end }} - app: {{ template "kube-prometheus-stack.name" $ }}-grafana -{{ include "kube-prometheus-stack.labels" $ | indent 4 }} -data: - k8s-resources-pod.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", cluster=\"$cluster\"}) by (container_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}container_name{{`}}`}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Container", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "container", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}container_name{{`}}`}} (RSS)", - "legendLink": null, - "step": 10 - }, - { - "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}container_name{{`}}`}} (Cache)", - "legendLink": null, - "step": 10 - }, - { - "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}container_name{{`}}`}} (Swap)", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Usage (RSS)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Cache)", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #G", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Usage (Swap", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #H", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Container", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "container", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "G", - "step": 10 - }, - { - "expr": "sum(label_replace(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "H", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Quota", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(:kube_pod_info_node_count:, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "pod", - "multi": false, - "name": "pod", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Pod", - "uid": "6581e46e4e5c7ba40a07646395ef7b23", - "version": 0 - } -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-workload.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-workload.yaml deleted file mode 100644 index af1a153..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-workload.yaml +++ /dev/null @@ -1,936 +0,0 @@ -{{- /* -Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ template "kube-prometheus-stack.namespace" . }} - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workload" | trunc 63 | trimSuffix "-" }} - annotations: -{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} - labels: - {{- if $.Values.grafana.sidecar.dashboards.label }} - {{ $.Values.grafana.sidecar.dashboards.label }}: "1" - {{- end }} - app: {{ template "kube-prometheus-stack.name" $ }}-grafana -{{ include "kube-prometheus-stack.labels" $ | indent 4 }} -data: - k8s-resources-workload.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}pod{{`}}`}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Pod", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", - "pattern": "pod", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}pod{{`}}`}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Pod", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", - "pattern": "pod", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Quota", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(:kube_pod_info_node_count:, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "workload", - "multi": false, - "name": "workload", - "options": [ - - ], - "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}, workload)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "type", - "multi": false, - "name": "type", - "options": [ - - ], - "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Workload", - "uid": "a164a7f0339f99e89cea5cb47e9be617", - "version": 0 - } -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-workloads-namespace.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-workloads-namespace.yaml deleted file mode 100644 index c11f51b..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/k8s-resources-workloads-namespace.yaml +++ /dev/null @@ -1,972 +0,0 @@ -{{- /* -Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ template "kube-prometheus-stack.namespace" . }} - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workloads-namespace" | trunc 63 | trimSuffix "-" }} - annotations: -{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} - labels: - {{- if $.Values.grafana.sidecar.dashboards.label }} - {{ $.Values.grafana.sidecar.dashboards.label }}: "1" - {{- end }} - app: {{ template "kube-prometheus-stack.name" $ }}-grafana -{{ include "kube-prometheus-stack.labels" $ | indent 4 }} -data: - k8s-resources-workloads-namespace.json: |- - { - "annotations": { - "list": [ - - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "links": [ - - ], - "refresh": "10s", - "rows": [ - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Running Pods", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "CPU Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "CPU Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Workload", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", - "pattern": "workload", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "Workload Type", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "workload_type", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "count(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}) by (workload, workload_type)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "CPU Quota", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Usage", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "hidden" - }, - { - "alias": "Running Pods", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #A", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "Memory Usage", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #B", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #C", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Requests %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #D", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Memory Limits", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #E", - "thresholds": [ - - ], - "type": "number", - "unit": "bytes" - }, - { - "alias": "Memory Limits %", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "Value #F", - "thresholds": [ - - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Workload", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "Drill down", - "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", - "pattern": "workload", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "Workload Type", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTooltip": "Drill down", - "linkUrl": "", - "pattern": "workload_type", - "thresholds": [ - - ], - "type": "number", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "/.*/", - "thresholds": [ - - ], - "type": "string", - "unit": "short" - } - ], - "targets": [ - { - "expr": "count(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}) by (workload, workload_type)", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 10 - }, - { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "C", - "step": 10 - }, - { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 10 - }, - { - "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "E", - "step": 10 - }, - { - "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", - "format": "table", - "instant": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "F", - "step": 10 - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Quota", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Memory Quota", - "titleSize": "h6" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(:kube_pod_info_node_count:, cluster)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "prod", - "value": "prod" - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Compute Resources / Namespace (Workloads)", - "uid": "a87fb0d919ec0ea5f6543124e16c42a5", - "version": 0 - } -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/nodes.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/nodes.yaml deleted file mode 100644 index 82d332a..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/nodes.yaml +++ /dev/null @@ -1,1383 +0,0 @@ -{{- /* -Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ template "kube-prometheus-stack.namespace" . }} - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes" | trunc 63 | trimSuffix "-" }} - annotations: -{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} - labels: - {{- if $.Values.grafana.sidecar.dashboards.label }} - {{ $.Values.grafana.sidecar.dashboards.label }}: "1" - {{- end }} - app: {{ template "kube-prometheus-stack.name" $ }}-grafana -{{ include "kube-prometheus-stack.labels" $ | indent 4 }} -data: - nodes.json: |- - { - "__inputs": [ - - ], - "__requires": [ - - ], - "annotations": { - "list": [ - - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ - - ], - "refresh": "", - "rows": [ - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 2, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(node_load1{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "load 1m", - "refId": "A" - }, - { - "expr": "max(node_load5{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "load 5m", - "refId": "B" - }, - { - "expr": "max(node_load15{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "load 15m", - "refId": "C" - }, - { - "expr": "count(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", mode=\"user\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "logical cores", - "refId": "D" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "System load", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 3, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}cpu{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Usage Per Core", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 4, - "legend": { - "alignAsTable": "true", - "avg": "true", - "current": "true", - "max": "false", - "min": "false", - "rightSide": "true", - "show": "true", - "total": "false", - "values": "true" - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 9, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max (sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m])) ) * 100\n", - "format": "time_series", - "intervalFactor": 10, - "legendFormat": "{{`{{`}} cpu {{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Utilization", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": 100, - "min": 0, - "show": true - }, - { - "format": "percent", - "label": null, - "logBase": 1, - "max": 100, - "min": 0, - "show": true - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "$datasource", - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 5, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "avg(sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m]))) * 100\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "80, 90", - "title": "CPU Usage", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 6, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 9, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(\n node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory used", - "refId": "A" - }, - { - "expr": "max(node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory buffers", - "refId": "B" - }, - { - "expr": "max(node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory cached", - "refId": "C" - }, - { - "expr": "max(node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "memory free", - "refId": "D" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "$datasource", - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 7, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(\n (\n (\n node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "80, 90", - "title": "Memory Usage", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 8, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - { - "alias": "read", - "yaxis": 1 - }, - { - "alias": "io time", - "yaxis": 2 - } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(rate(node_disk_read_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "read", - "refId": "A" - }, - { - "expr": "max(rate(node_disk_written_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "written", - "refId": "B" - }, - { - "expr": "max(rate(node_disk_io_time_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "io time", - "refId": "C" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk I/O", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max by (namespace, pod, device) ((node_filesystem_size_bytes{cluster=\"$cluster\", fstype=~\"ext[234]|btrfs|xfs|zfs\", instance=\"$instance\", job=\"node-exporter\"} - node_filesystem_avail_bytes{cluster=\"$cluster\", fstype=~\"ext[234]|btrfs|xfs|zfs\", instance=\"$instance\", job=\"node-exporter\"}) / node_filesystem_size_bytes{cluster=\"$cluster\", fstype=~\"ext[234]|btrfs|xfs|zfs\", instance=\"$instance\", job=\"node-exporter\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "disk used", - "refId": "A" - }, - { - "expr": "max by (namespace, pod, device) (node_filesystem_avail_bytes{cluster=\"$cluster\", fstype=~\"ext[234]|btrfs|xfs|zfs\", instance=\"$instance\", job=\"node-exporter\"} / node_filesystem_size_bytes{cluster=\"$cluster\", fstype=~\"ext[234]|btrfs|xfs|zfs\", instance=\"$instance\", job=\"node-exporter\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "disk free", - "refId": "B" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Disk Space Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 10, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(rate(node_network_receive_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}device{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Network Received", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 11, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(rate(node_network_transmit_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}device{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Network Transmitted", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 12, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 9, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(\n node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "inodes used", - "refId": "A" - }, - { - "expr": "max(node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "inodes free", - "refId": "B" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Inodes Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "$datasource", - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 13, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(\n (\n (\n node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "80, 90", - "title": "Inodes Usage", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": null, - "multi": false, - "name": "instance", - "options": [ - - ], - "query": "label_values(node_boot_time_seconds{cluster=\"$cluster\", job=\"node-exporter\"}, instance)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Nodes", - "uid": "fa49a4706d07a042595b664c87fb33ea", - "version": 0 - } -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/persistentvolumesusage.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/persistentvolumesusage.yaml deleted file mode 100644 index 3c51e11..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/persistentvolumesusage.yaml +++ /dev/null @@ -1,573 +0,0 @@ -{{- /* -Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ template "kube-prometheus-stack.namespace" . }} - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "persistentvolumesusage" | trunc 63 | trimSuffix "-" }} - annotations: -{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} - labels: - {{- if $.Values.grafana.sidecar.dashboards.label }} - {{ $.Values.grafana.sidecar.dashboards.label }}: "1" - {{- end }} - app: {{ template "kube-prometheus-stack.name" $ }}-grafana -{{ include "kube-prometheus-stack.labels" $ | indent 4 }} -data: - persistentvolumesusage.json: |- - { - "__inputs": [ - - ], - "__requires": [ - - ], - "annotations": { - "list": [ - - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ - - ], - "refresh": "", - "rows": [ - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 9, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Used Space", - "refId": "A" - }, - { - "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Free Space", - "refId": "B" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Volume Space Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "$datasource", - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 3, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "(\n kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "80, 90", - "title": "Volume Space Usage", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 9, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Used inodes", - "refId": "A" - }, - { - "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": " Free inodes", - "refId": "B" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Volume inodes Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "$datasource", - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": true, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 5, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "80, 90", - "title": "Volume inodes Usage", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kubelet_volume_stats_capacity_bytes, cluster)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\"}, namespace)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "PersistentVolumeClaim", - "multi": false, - "name": "volume", - "options": [ - - ], - "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\"}, persistentvolumeclaim)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-7d", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Persistent Volumes", - "uid": "919b92a8e8041bd567af9edab12c840c", - "version": 0 - } -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/pods.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/pods.yaml deleted file mode 100644 index e2534d7..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/pods.yaml +++ /dev/null @@ -1,680 +0,0 @@ -{{- /* -Generated from 'pods' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ template "kube-prometheus-stack.namespace" . }} - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "pods" | trunc 63 | trimSuffix "-" }} - annotations: -{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} - labels: - {{- if $.Values.grafana.sidecar.dashboards.label }} - {{ $.Values.grafana.sidecar.dashboards.label }}: "1" - {{- end }} - app: {{ template "kube-prometheus-stack.name" $ }}-grafana -{{ include "kube-prometheus-stack.labels" $ | indent 4 }} -data: - pods.json: |- - { - "__inputs": [ - - ], - "__requires": [ - - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "$datasource", - "enable": true, - "expr": "time() == BOOL timestamp(rate(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[2m]) > 0)", - "hide": false, - "iconColor": "rgba(215, 44, 44, 1)", - "name": "Restarts", - "showIn": 0, - "tags": [ - "restart" - ], - "type": "rows" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ - - ], - "refresh": "", - "rows": [ - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(container_name) (container_memory_usage_bytes{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name=~\"$container\", container_name!=\"POD\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Current: {{`{{`}} container_name {{`}}`}}", - "refId": "A" - }, - { - "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Requested: {{`{{`}} container {{`}}`}}", - "refId": "B" - }, - { - "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Limit: {{`{{`}} container {{`}}`}}", - "refId": "C" - }, - { - "expr": "sum by(container_name) (container_memory_cache{job=\"kubelet\", namespace=\"$namespace\", pod_name=~\"$pod\", container_name=~\"$container\", container_name!=\"POD\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Cache: {{`{{`}} container_name {{`}}`}}", - "refId": "D" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 3, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (container_name) (rate(container_cpu_usage_seconds_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", image!=\"\", pod_name=\"$pod\", container_name=~\"$container\", container_name!=\"POD\"}[1m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Current: {{`{{`}} container_name {{`}}`}}", - "refId": "A" - }, - { - "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Requested: {{`{{`}} container {{`}}`}}", - "refId": "B" - }, - { - "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Limit: {{`{{`}} container {{`}}`}}", - "refId": "C" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sort_desc(sum by (pod_name) (rate(container_network_receive_bytes_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}[1m])))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "RX: {{`{{`}} pod_name {{`}}`}}", - "refId": "A" - }, - { - "expr": "sort_desc(sum by (pod_name) (rate(container_network_transmit_bytes_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}[1m])))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "TX: {{`{{`}} pod_name {{`}}`}}", - "refId": "B" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Network I/O", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max by (container) (kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Restarts: {{`{{`}} container {{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Total Restarts Per Container", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_pod_info, cluster)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Pod", - "multi": false, - "name": "pod", - "options": [ - - ], - "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=~\"$namespace\"}, pod)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": true, - "label": "Container", - "multi": false, - "name": "container", - "options": [ - - ], - "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}, container)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / Pods", - "uid": "ab4f13a9892a76a4d21ce8c2445bf4ea", - "version": 0 - } -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/statefulset.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/statefulset.yaml deleted file mode 100644 index 04b563b..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/statefulset.yaml +++ /dev/null @@ -1,926 +0,0 @@ -{{- /* -Generated from 'statefulset' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: {{ template "kube-prometheus-stack.namespace" . }} - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "statefulset" | trunc 63 | trimSuffix "-" }} - annotations: -{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} - labels: - {{- if $.Values.grafana.sidecar.dashboards.label }} - {{ $.Values.grafana.sidecar.dashboards.label }}: "1" - {{- end }} - app: {{ template "kube-prometheus-stack.name" $ }}-grafana -{{ include "kube-prometheus-stack.labels" $ | indent 4 }} -data: - statefulset.json: |- - { - "__inputs": [ - - ], - "__requires": [ - - ], - "annotations": { - "list": [ - - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "hideControls": false, - "id": null, - "links": [ - - ], - "refresh": "", - "rows": [ - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 2, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "cores", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}[3m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "CPU", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 3, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "GB", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(container_memory_usage_bytes{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}) / 1024^3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Memory", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 4, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "Bps", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 4, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(rate(container_network_transmit_bytes_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\",pod_name=~\"$statefulset.*\"}[3m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Network", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "height": "100px", - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 5, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Desired Replicas", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 6, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Replicas of current version", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 7, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(kube_statefulset_status_observed_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Observed Generation", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 8, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 3, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "max(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Metadata Generation", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "0", - "value": "null" - } - ], - "valueName": "current" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - - }, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "replicas specified", - "refId": "A" - }, - { - "expr": "max(kube_statefulset_status_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "replicas created", - "refId": "B" - }, - { - "expr": "min(kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "ready", - "refId": "C" - }, - { - "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "replicas of current version", - "refId": "D" - }, - { - "expr": "min(kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "updated", - "refId": "E" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Replicas", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - } - ], - "schemaVersion": 14, - "style": "dark", - "tags": [ - "kubernetes-mixin" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": null, - "name": "datasource", - "options": [ - - ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, - "includeAll": false, - "label": "cluster", - "multi": false, - "name": "cluster", - "options": [ - - ], - "query": "label_values(kube_statefulset_metadata_generation, cluster)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [ - - ], - "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\"}, namespace)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - - }, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Name", - "multi": false, - "name": "statefulset", - "options": [ - - ], - "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", namespace=\"$namespace\"}, statefulset)", - "refresh": 2, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [ - - ], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Kubernetes / StatefulSets", - "uid": "a31c1f46e6f727cb37c0d731a7245005", - "version": 0 - } -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml index 67f6ece..3992789 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml @@ -5,6 +5,7 @@ {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} + {{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} kind: Ingress metadata: @@ -34,7 +35,7 @@ spec: paths: {{- range $p := $paths }} - path: {{ tpl $p $ }} - {{- if $pathType }} + {{- if and $pathType $ingressSupportsPathType }} pathType: {{ $pathType }} {{- end }} backend: @@ -54,7 +55,7 @@ spec: paths: {{- range $p := $paths }} - path: {{ tpl $p $ }} - {{- if $pathType }} + {{- if and $pathType $ingressSupportsPathType }} pathType: {{ $pathType }} {{- end }} backend: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml index 5a4d6e1..ace4058 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml @@ -5,6 +5,7 @@ {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} {{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} kind: Ingress metadata: @@ -33,7 +34,7 @@ spec: paths: {{- range $p := $paths }} - path: {{ tpl $p $ }} - {{- if $pathType }} + {{- if and $pathType $ingressSupportsPathType }} pathType: {{ $pathType }} {{- end }} backend: @@ -53,7 +54,7 @@ spec: paths: {{- range $p := $paths }} - path: {{ tpl $p $ }} - {{- if $pathType }} + {{- if and $pathType $ingressSupportsPathType }} pathType: {{ $pathType }} {{- end }} backend: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml index a89c1a9..df63199 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml @@ -4,6 +4,7 @@ {{- $servicePort := .Values.prometheus.servicePerReplica.port -}} {{- $ingressValues := .Values.prometheus.ingressPerReplica -}} {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "kube-prometheus-stack.ingress.supportsPathType" .) "true" -}} apiVersion: v1 kind: List metadata: @@ -12,7 +13,7 @@ metadata: items: {{ range $i, $e := until $count }} - kind: Ingress - apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" $ }} metadata: name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} namespace: {{ template "kube-prometheus-stack.namespace" $ }} @@ -38,7 +39,7 @@ items: paths: {{- range $p := $ingressValues.paths }} - path: {{ tpl $p $ }} - {{- if $pathType }} + {{- if and $pathType $ingressSupportsPathType }} pathType: {{ $pathType }} {{- end }} backend: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/podDisruptionBudget.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/podDisruptionBudget.yaml index 573317a..cce4a85 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/podDisruptionBudget.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/podDisruptionBudget.yaml @@ -16,6 +16,6 @@ spec: {{- end }} selector: matchLabels: - app: prometheus + app.kubernetes.io/name: prometheus prometheus: {{ template "kube-prometheus-stack.fullname" . }}-prometheus {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml index 7b47e38..6f00a8f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml @@ -218,7 +218,7 @@ spec: - topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} labelSelector: matchExpressions: - - {key: app, operator: In, values: [prometheus]} + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-prometheus]} {{- else if eq .Values.prometheus.prometheusSpec.podAntiAffinity "soft" }} podAntiAffinity: @@ -228,7 +228,7 @@ spec: topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} labelSelector: matchExpressions: - - {key: app, operator: In, values: [prometheus]} + - {key: app.kubernetes.io/name, operator: In, values: [prometheus]} - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-prometheus]} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/service.yaml index 8676b81..c642006 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/service.yaml @@ -51,7 +51,7 @@ spec: {{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} {{- end }} selector: - app: prometheus + app.kubernetes.io/name: prometheus prometheus: {{ template "kube-prometheus-stack.fullname" . }}-prometheus {{- if .Values.prometheus.service.sessionAffinity }} sessionAffinity: {{ .Values.prometheus.service.sessionAffinity }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecar.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecar.yaml index 7c33379..c3d52ef 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecar.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecar.yaml @@ -25,6 +25,6 @@ spec: nodePort: {{ .Values.prometheus.thanosService.nodePort }} {{- end }} selector: - app: prometheus + app.kubernetes.io/name: prometheus prometheus: {{ template "kube-prometheus-stack.fullname" . }}-prometheus {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecarExternal.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecarExternal.yaml index f9a0331..99668f4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecarExternal.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecarExternal.yaml @@ -23,6 +23,6 @@ spec: nodePort: {{ .Values.prometheus.thanosServiceExternal.nodePort }} {{- end }} selector: - app: prometheus + app.kubernetes.io/name: prometheus prometheus: {{ template "kube-prometheus-stack.fullname" . }}-prometheus {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceperreplica.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceperreplica.yaml index 1a55433..470ce79 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceperreplica.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceperreplica.yaml @@ -38,7 +38,7 @@ items: port: {{ $serviceValues.port }} targetPort: {{ $serviceValues.targetPort }} selector: - app: prometheus + app.kubernetes.io/name: prometheus prometheus: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus statefulset.kubernetes.io/pod-name: prometheus-{{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} type: "{{ $serviceValues.type }}" diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml index 5178836..109820a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml @@ -396,7 +396,7 @@ alertmanager: ## image: repository: quay.io/prometheus/alertmanager - tag: v0.21.0 + tag: v0.22.0 sha: "" ## If true then the user will be responsible to provide a secret with alertmanager configuration @@ -1342,7 +1342,7 @@ prometheusOperator: enabled: true image: repository: jettech/kube-webhook-certgen - tag: v1.5.0 + tag: v1.5.2 sha: "" pullPolicy: IfNotPresent resources: {} @@ -1540,7 +1540,7 @@ prometheusOperator: ## image: repository: quay.io/prometheus-operator/prometheus-operator - tag: v0.47.0 + tag: v0.48.0 sha: "" pullPolicy: IfNotPresent @@ -1556,7 +1556,7 @@ prometheusOperator: ## prometheusConfigReloaderImage: repository: quay.io/prometheus-operator/prometheus-config-reloader - tag: v0.47.0 + tag: v0.48.0 sha: "" ## Set the prometheus config reloader side-car CPU limit @@ -1899,7 +1899,7 @@ prometheus: ## image: repository: quay.io/prometheus/prometheus - tag: v2.26.0 + tag: v2.27.1 sha: "" ## Tolerations for use with node taints diff --git a/charts/kubezero-metrics/dashboards/metrics-dashboards.yaml b/charts/kubezero-metrics/dashboards/metrics-dashboards.yaml new file mode 100644 index 0000000..7310eec --- /dev/null +++ b/charts/kubezero-metrics/dashboards/metrics-dashboards.yaml @@ -0,0 +1,10 @@ +configmap: grafana-dashboards-metrics +gzip: true +folder: Metrics +dashboards: +- name: AlertManager + url: https://grafana.com/api/dashboards/9578/revisions/4/download + tags: [] +- name: Prometheus + url: https://grafana.com/api/dashboards/3662/revisions/2/download + tags: [] diff --git a/charts/kubezero-metrics/templates/grafana-dashboards-metrics.yaml b/charts/kubezero-metrics/templates/grafana-dashboards-metrics.yaml new file mode 100644 index 0000000..070f453 --- /dev/null +++ b/charts/kubezero-metrics/templates/grafana-dashboards-metrics.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" (include "kubezero-lib.fullname" $) "grafana-dashboards-metrics" | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} + labels: + grafana_dashboard: "1" +{{ include "kubezero-lib.labels" . | indent 4 }} + annotations: + k8s-sidecar-target-directory: Metrics +binaryData: + AlertManager.json.gz: + H4sIAAAAAAAC/+2deXPbRpqH/9en6IWzO3JWlkjwTlWmypYniXdzTexJ1WzspUGwSWEMAjQO24zK+9m3uwEQVwMgKYgmoN9WbcYCiKPv52m8DdyeEaJMp4a19j1X+Yb88eYi2OLQ977hULGNbSHkVvyX7fM2a8q2KktHW2iWplxEO4y5dLOlrcTvv8/u+EAd17Atvm942b3sKGLH5wv55daaRU3JxdY30kslNycuVH6NueZpru07Os1eaO3YK+rdUN/NX+1Xyb7EJVnKDkqba1hLk7qe5uUv+VKyb+dUSq/maTOT5i/0Kr05fw3236DKaJZls/the3mdCS6qmIbrbWtQfCtsz8w3TO8FP1P3It6aKAB5xrLfUEvc0zfEc3ya2H5jzCVbDd22rm3TdvgJneVMO+9cELXbZf8ZDC5I93Hy1FGqn8ZpIf9BnprU8VK3ENcX92Zma85cCfd9vsin9C5pop/W4sb1G81aUvecVUOduu6UlbzjTT1jRacuZSmcu7fEsNhWS6ff/t9r5avoj9fK5z/U1ZvH5K+ko+Qza6GZbmFuPZoturNOR5ZBv1FxB6k0uDf2R1GgneRGj65FI1il8k9b/jfd8HqiRDea2m14Jv3Odlaal7jYNo/F/745C3NbmVNXd4y1F9bL51GZEH5DrJ2QOMuDolyxXmhJHcI2OobukoXtEHvmUucD/zVvVOyX9oKwQ4hu+iwJDtGsOVnbrmuwYiJzOpv5S/bjS3HXCp0bXqb8lKVFvRe8ZU0Go3GwhXdKr2zb9Ix1VO2Dxmf5phn8xS6lhQnpDgaDXq8/UkfjTl/sNQ3rnaQ/5mUmKqOtxw2V5XDcnYdbnCX1npma9S5Tz4L8Fnln64k+zNveq/LLYmHohmYSdhF/Ra2gcfBMSuZo4tCwgfBbjrf6jsk33nje2v3m6iruUy8N+4qd2b3S+NlYxgb/CE97Je/OonQb1sK+a8K/N7wf/Jk06akqE7RjVjsskjtklyQvDe/Gn13q9iqR+lRi7z+trz4anpcqrDix4T6i6brtWx5hf96Q+E6JQ03No3OSuY9dku4FpxZpj1vki1/KE/zeZ42fN4g7JvonzTB56xYDkizliU7CZ12BS1byI3YqZcf21+7l0rbZIC0SzLoYf3X16N+C/40z9Im41nGy4MVv19KU/5dtWITtZOnm6f3NsPdNr8MOuTRWV9p6ffXoyrFZgh/FafyGdeOO8enSdpZZZhAkIunSdNs0tbVL59khinWixvxXOyaMYDDLUMRH9rfaT2z4lBmVNvzv7JAd9MW94fbv+O7eSHoMarG+2ixoCY79UV6ouqbf0Fds3LZ9L9HzR6m2nWea/o7XHyufdrH/d830qXyXmyAttu2ROpno/WFyYBX8o/ZGjHvUyQXpMxTqXI4nKQZ6NO/3tZ4WjbZx0is5ho3EurHSRJZtc1tZbMdxNh47iaq11PwlTRfkSvsUpa/bSRbYyrCiHZ0Mckgoxrth4nJjm/MftVlQgoW/+Elz3lGRc7zR5KpEYXUbZKpbr6K2deW1rR//abHO8YNmZivFdtCPC2LF2hlrqa+CutaVbU9XhfjeEwj3gecn8Wzi0U9eogKQcFfiphO3XXAyh+Np+cnU+GS5qsXK/TmrXb+ynkhYaKLwFZ4bYsdPtkBWVtcti+psFFJSv3nFr5zJPEZs3sL4JHQlu/E72/JeGn+KUw46/57Y79D8MWJb4SEi+T+xHqs42xesvfDj+P2lM8gLblz5+eppZoe9PaAk79w1q8KslmSa0sIwzbT3qP0+N58ea/wDlTf8rppq+At+oXxT4aeOlWDRnw8mYyXfBqXNR1AxO9hfWen8DMarkuzaig/HkPMkIE25Oc6nvN+9LVKex+mMjLug0Jgcg7rpn0TN7ztN90RSu6ndrPgFzCtPy4pi26u4mdRGg8bP/mrG6Iphc3Szbn7skHm/aETJ6jdOVj+xt7z62WKo/3bHmhe12crKJ374c9gJaB+WBYOeqAGZXmyX4SQldmIqgrDW72mGFZFZVkHijN3qn+H9xSXhxMUF8deEV4ILYmrsaMaztjYXW4TiGaxkGPVq7Ghf56bNGsWlkhjL4hKYJEtg53FikhkneruNE+qwbEBYs5SHt5Xq/lj22aJNp4CQ58sPVJtTJ7eHj8+pJLCS4zeVvEteKlE3kbtZ19uYZYOPZhqaaBwcgdJ1jtWHhPP/k/3fk59+evL8Ofnhh29Wq2/cTINda9wmLPm5oqZ0Y8zn1FIqh7Htbb3Iz0ZEdBUOP8kMLkKvBGv1Bxdk0A/+v3M5SXdMuxFZ/LsB29sdsb6bn5WdbfRYSfzqzV2yM4FuampHAWxky8CQZlyqQ/zjjbSMXCYH1jJ9nG8ZCWLcvfh+D6cnUXr7ld4HWb4du/D+EfTMD7nwOocUnhAj8ujp/qVnCRwpKL09Su7HeChF6R1Wes9OovQ49QTQYxYXpKJT6e4H34uGRXldVpTpFHWySewWJ2anYt+zy8VAuWcRX11+fbV/Sw0xdKciy0vlrqrMB8/zx+QJ2VuXydf8gcZ5tOUieur1mIiZ7KlJFx4pefS4v4anniqHAs4P83KPQA9084sd8yqVVSw1C2M55WY4DTrEaaiBIr3sBlfrykQfN83Pdk/zjkllnf+JpO26tFlsH2uEtxcpvitE3s/ApOI5muXyO5YEO0TtNNiMeXvZ7Es8dbWm/PGYYYnn46m5l7DbSE2YRFXEsi36QOb+u+qOkzqdEab/Mf1/pOn/7oTHPLEuRHQenfH+c/90oOkdery5f1Yt0igT9i/TFeV9kdu46f/rMJjI5ZXoIcz7t3vQTI2P177jUMsj1nacZFXJ+EADwnQf/Jg42HFMxIiIEbExI2I+QPbII2LQudzyQYN++1oJupzXygVp7pPxVLeJUbLto6Trr1lf4rp0jpEyGCnHu42UPYyUGCmPFjrGugZVHfL4MdFBdA8YLBcTqs7V0xks456n2QNmrgfFoPlA1NI1TMpD3R76kKl2dxsyu/EPMWhi0MSgucOgGfUxbXTMKG0PJAb73pdUDQu63Th+uHxN1c+2ZywMPVxEvvuiKhHPch0N3bfx5WeakxuQxKLtH6m19EQaO6ntVPbzPYfrBJmJUTuZKN5lzw3XMyzdI7zWL51wh+vrN0Rzya+81T33vc0FeWkyhhFPs12b2NYl+cUiFl1qwfzIJ8MlmkP56damtmH0t9BYfZ6nL5hEA0P0N939x9thRbmPisp9HA+rdEkFiiUuxYptaT11X8mXvvOanNuoBygk6Tb5gva/rdbeRv5igP+hjp3fw3o7yakY/0i2OsbyhjV92TsGEl1ycmsQTc+GFU8zlcyO50EUfeaQ4Ke5zaKFF2ITHzDcbCg/3/jRmHupdiyDm9xwnxowlTVlFd/ytGWeldf8KEebG76brDPB9nw7YuU2p45YdKAszOQaU4euaTBA5KLIw33PDYcRSNi+Ei/2CMaSXz5Qh1XlndYdXH0nGkkujEvfjriTSb/f0TKddDJ4JGqCT/5ZgQk6lXUzLIX6u1zu8PckrOn8x4Ar0vt2Hrxdf3XOehaHai5Nj+KpPmEqqlnh09M/vuLvgwkG5DePH5PZhoeGbbuq2kf3oGuIg/hub5M94+dM0NEhkU+75kvQgR47e+Sv4kgrinKnHA3qPElnLCnM2d0pKzWIr+h3AXOnlIBv/40uw9fCZA54eWMsvPwRMhwIBlNO9eSrfDcRL2e/TXbLbHycF3bMnWwPO41Aw7Dmxgdj7mumkkfs+O1HyVcMfeLjcfryM19/FzTbdDCtsgp72sxqh0juMr+WDy7bIeGPN7lb3GifyvrCogXZQdXhEi+L/zXt5TPWfHJ1LRhCcz8PxtDc5ry8FLfek7vPXFvY5AtdMI2EHsT2H+mH7U2nXmZz2kD73He2r3tJdpi8Sc5FJGDncjIRoMr+Qd77muWxDscla7Eec9vj5Fm0U82iwz1ZtKsWwegAMBokZC8UzV75qCwqmXpqHZDytpPD0VSW7b6eg51sQEpPpp4gst4wIbZZH7GaRl3HOcuUC9ZplNDalL+IyNI3UTz+NBhud8W242MtS9L7nRFsT7iV52BzsvDOhMtT+540mnDJ3I/mgjLjJl+cA+ati3nBu6fHu8eYnO6OC8BwvNvk9NOCZ+uNm5UOggR45y0e6Wxnnt8GT0XeXpC3cUjBW0I9PQ/u/fonkbsTefkMy7FdgHm29hWT+T6QXfTAu4qM5a9GuYdpWl5qf3KXaC0dB1WyaKo2DL/dnY/jel10SnUxGA3onSZ206/UqQuSZZFMhQ9gBZmJ1n2E2VpxnZNHrafJsGI+ZwGiwiwiZhG/BICwwYGythiFKwaT+on374Zkwv71dmHw9wu95Qa0tl0j8eTbmpO3rPOwzQ8cUuzMk/GjPPZW+3JiGYNYHt6D5aguFoHFiM7U4Xzfx8pVQBO0j4pH2Y15Sh30B9OofzjgQWzQb3zBiajb26jvOnki+i3TDcfdLv8gAvAIePRwJp1SQ1x9c05qwZxGry+Zcyrp6iUQVwhyZTBXAnS7QZ0M7F6yYnS3H0pxdduhDPG4Z7Pxnb+AN9p1QzXTu7kkvB8jP7C7MNhv1/xTK7KPi3xDXlt/Ja9uGOwFH2ohwXVnlNEfsVjz/ouYKdfpOno4bbBezP5ohRciM42viWG72A2Sj9Q0+bt9xemouC/+/hrXXvCX32nmE/EO4PCrV/wmtp+AWTu2Z7PKcpm6GRGCHrJrh4d1ji/IxxvGIewv3rXpTIpd8jpgL3MT3tPmtXL52nptsQGDrHhG8fdyEZdSEn1L4ePHj5c66819M/4oChtivCuPmpRfe3N5463MR2GmPgnOmxnwMshbWKu3NbufeQ9bBn9lFTyu5J3EtnT9FJV9MJKMmPnbkHJxORsX83ExI5dwchkrF/NyPtUybi5m5wJ+3oGhKzi6jKVLeLqcqau4upqtgwLQbcaRvwfdV6YibM+Z3SO43AziAaQFlAinT3Llk0F/NlnMF5PZ4sny/Se3r+QO2wbblx+XOuxzYQUos4dsB5/m+ViSw2Bu8bKt/P3qxa8gSPcBo8JbTte0IhEoloEqISiUAmkOyF9SF77vJ+jjpmJo2e3ddDty/w6EL6X869SIFox5+RPLiL+8FEpfJik3gFILKDWBtA2ESUqmqMADCl2gzAcKnGAnL8g3MLkfFDhCqSeUuUKxLxQ7Q4E35JOQ9wdpuyjj81JGL+P0ElYv4fVCZs8lrsnpKGybG3nFKnCRUh9JX6h0kudo5A+mf6hMPzyA6fudMaC+pVCfY4RqnA9+8SL39VJ13O1NBmPJb3/l0x4vJHp4XDMYLQb6ZDQaDYdP3n10VHdXM8gdBzOAGcAMYAYwA5gBzABm0Aoz6KoHqcEEagA1aLwauA59MphNxoPxbDHoP3HG7rtPu+qB9FgoAhQBigBFgCJAEaAIUIR2KML4EEXIVC0oAhShqYow7g8G+nCw6I+erFYfzff7KELuWCgCFAGKAEWAIkARWqYIh68sWNqua6y30hB+/JHotm954jO0nNP5elA+pHGM336gVmdlKt54YW34n6zfIGJgu/yi4fL9cjjuIbIG4fIIl28BzuYq9HFoturruMcE2biAMt33HjR7cYe8CN/oLKrFiWVIeY2VvQnw1Nk+MzRL31oHuAfcA+4fwvx/g7F9eAC297tdcDvmtOuY084YICLiAf+Af8A/4B/wD/gH/AP+v0DQeyX9q6B/0H/j6R9B77AAWAAsABYAC4AFwAIergWMD7KAHiwAFtAKC0BcOywAFgALgAXAAmABsIDdLOCQKP9fst92coU8WNsOMrAC70bzyL8Yx7G+kq/CjcYR7g/Zzz/JDzbpwkseemeZGN5lGcC43CVGA6hEnSqR+1bml1cKLAeo2QIi/BANvdIAxpPOoqPmf1H5Va4ajOAIK13ln9iKeFjkzpRn1F5f2foSi2DDPr+Kjw+0hV2yKbiDk8+osmrfSI0Ixm6erKuwEkAloBJQiQf4QKH9kjA8QBL63T4soeWWcOQHDxnvxOIDqAZUA6oB1YBqQDWgGlCNFqiGfIlDpWvgiQRcoz2u0bylDnAOOAecA84B54BzwDngHM1yjvFBzjGEc8A5WuUczVpYAeeAc8A54BxwDjgHnAOrM1ySMgvN8+hq7bn8e2eM12zLYlSU8gvyS9ZLjEXiBOHit+g8X3RJRq9iefcQX2Zou4xgSUbN/vBd0L63fYNgGBiEHI1TuXSybPzS16nrLnyzqlTv1yPSmRWu7z7VPNutGTTSKFJpgk3AJmATeILRGk8YHuAJ/e4IooCnFnU+tRjikxCwDdgGbAO2AduAbcA2YBtttA3pwoxq3cBzCehGe3SjeQszoB3QDmgHtAPaAe2AdkA7Gqcd44O0YwLtgHa0SjuatTYD2gHtgHZAO6Ad0A5oR6u0o/6VGi6DyMS3l1yXdfgumW0Ib73krb9m16RvCbEd8paNP+aU9TN8g3RpOXcalzVFwioS76Qo2zPfnvWLLuLodyq0BQ9L2m4tWMRRs2hcRU388uurQ/XhYtdrXX4ddz/S68VCM1QndNA9/EJBp1dxkUV/PpiMmyZDrr86L2f8qLee8pFhL7h/zIeN85W7FJR4VNYXo9jtbXTtz5/rN6M9ci5qFk3Jve1IvWMONuuTgyHS4JuDcCW4Eh7RPBgLGh5iQWoHGoSHN3U+vOljiQpcCi4Fl4JLwaXgUnApuBRcqmkuJV2AUy1TXcgUZKo1MtW8BTiQKkgVpApSBamCVEGqIFWQqpOSqvFBUqVCqiBVrZKqZi0vglRBqiBVkCpIFaQKUgWpOi2pqmuFlGv8SR/U+qhhuXcNetCulmsX1kfBlGBK+5kSHyagS3fSpSZl4YNwpgB9oEwno0yzjServ1AmKNNxnkO1WYaGB8hQX4UN4SFUrQ+hMn6NZVJQKigVlApKBaWCUkGpoFRQqoavlqp0qj6cCk7VGqfCaim4FdwKbgW3glvBreBWcCu41b0tmqp0qwHcCm7VKrfCoim4FdwKbgW3glvBreBWcKsa3Kr+70u996nPOsmchUkdKz5s7fjWqSyNGqjldjXEGynaLldYGlWzD/2aat/z1n58thx8g16ugniPCbWVxXJXKyjPj2CwOImc+HswbkW31jawFzkNssd7EUD2eGrScGYfHsDsfXUIaMcTkTqfiGQ0ECt4QP4gf5A/yB/kD/IH+YP8j7TQpBr9R0B/oH9r0L95C02gAFAAKAAUAAoABYACQAFqXw9RrQBjKAAUoFUK0Kz1EFAAKAAUAAoABYACtF0BzjKXkzVLjtFxJ7NtAA5LxVki/6Ob44vrTG3t5lqiFKqVDNwJmE6+bikb+i4guhd9iXFb9gE8q+Ptkcqa80+6mp/66oXjrh3oVnzOstfFW6/abiJYPFCzPFzNHZtjeumiY13Xep3Ova+1XmiGWXEns0V3dvidnLziyBf52qxc+erR+XRpu66xjiB//w9MHkz9N8acFjaBA5Tg9vYd3dzHUuk9czCs/cfJxAPyKbw/Up1fz46SX7yJ+g492fwKupAdsuu6CaYZfHBzQbbFQYLiiN0T1nlC1gnjxEOne7Gx4673qPKsfr8Lz8ITnzqf+GTVHQs+YGuwNdgabA22BluDrcHWYGuwNfkanWpdwzu1oGst0rUGfg0G2gZtg7ZB26Bt0DZoG7QN2vagtG18kLb1oG3QtnZpW8M+NANtg7ZB26Bt0DZoG7QN2gZtIw39zs6RV6pVfEO018Fr89rudlipVrON/bwgrDflq+2dDWGH245bZkPjSWfRUVtrQzeG69mMIFbT975meYzszrsXjuYVwGqO7f1g/oDxpW5bc3caIMaun4pssCudIsw+D0sDPAueBc/iMcQR1/pUfe2+j3cq4ClEvU8hMvKDtT7gXfAueBe8C94F74J3yf2ulqgE3gGAF8DbHuBt3moJgC/AF+AL8AX4AnwBvrXFm1eCL77iDfBtGfg2K94c4AvwBfgCfAG+AF9E7B41YldVqyJ2sRoTEbuI2AXPHsyz8tVkLuNaVj3cDL7y79nZa42l9VgL8Orm1ihhx1rGaLGWvfziuXhARlmLKbvzhn8U7+ftx2tJnOXge3wjD4yPye17j2Kuovd+H+vtMLdd89x2D1HMcAA4ABwADgAHgAPAAeAAXzSyu1oCxpAASEB7JACR3ZAByABkADIAGYAMQAYQ7b6PDEwgA5CBdskAot0hA5AByABkADIAGYAMSGXgLHO5uJV+n26PSvyLsAE4LBVnifyPbo6P0qa2dnMtUQr4SgY0BdiridcMZoP0BdD3umeZsg9Afrw9UFlzFEvXcixySOZht1suRRO867HtToQ1DrCYo1tMAN88i1gmnyxpp0pS55+XOa6ciAtPgyrUjEwqqu7NcpM4RSyTgoInLAPhInARuAgeTNT8vvUKBO8POmBwPJeo87nEpI+FCgB5gDxAHiAPkAfIA+QB8nW8SL6S5LsgeZB8a0geqw1A9CB6ED2IHkQPogfRt/AN+ZVEr4LoQfStInosGQDRg+hB9CB6ED2IHkT/EKPiR+XY3+3gw1htx36ExYPU75HUd/yUVRJNv9Tnq+6KqdF9N/x7VdI0gVTxgSpQKuadaw4JH1VNO+PTU5h2rnXaOas0iAkHxYJiQbGgWFAsKBYUe1A8dCXG4u0iwNgWYSwCooGzwFngLHAWOAucbV0wcCXODoCzwNl24SyigYGzwFngLHAWOAucfXCRsGqv4v3QIyAvAmERCAtIPRRSJSjqWtqatWJv6hp/0uls41G3EEC/AHFGt0f47bUDOFNJAm+eEm+K6g/mBHO2IK61iib7A6yrwgxqva9TGCGsFUwKJgWTgknBpGBSRKnuD6WY4wSUtgdKEaQKOAWcAk4Bp4BTwOmJx5xWw+kYcAo4bRWcIuQUcAo4BZwCTgGngNPGR5BOqt6lio+iIYQUIaSAzoOhs2hJ05Y8c6uaXH+145ImckXu8/QHA++NMaeFtRs0LEsS1lOdIhGDhkHDrYhtnVTN1E7AuZiprfftAF0Et4KWQcugZdAyaBm0DFpuTtRtFS4PO8Bl4HJ7cBlht8BmYDOwGdgMbAY2A5sPjAeuxGZEUwCbW4bNCAgGNgObgc3AZmAzsBmRyvcUqdwbVLzrVgVaI1AZgcr7wPC16TOkdIhJFx5ZU4rFcWlOXeo5Qj2JxXFRuf2LNSk6Lyq5JiLn9zb5/prwQ8CaYE2wJqZo640DrqLI/hAYiRnaet/YoCIMGCwKFgWLgkXBomBRRNnuDqM9wChgtDUw2rwgW0ApoBRQCigFlAJKH0gMazWU9gGlgNJWQWmzQlgBpYBSQCmgFFD6MKH0LHO5ZHNkKVfi7WGVd9i9nyVyPbolPkqY2trNtT0pXioZzBFYqSZgMBu6GeCkepYp8fDFsup2blNZcxJIV2zEviYzsatWvKVXxdvLEPyK4FesBKuHo13DpKxW3AM217346vaW3ZNHP39uOEi/DHOc6LZveWS2ISJdxLZA0ydD06dNoiBqTPPuHAdbRZT94QBEiVneWt9UkJEUBMKCS8Gl4FJwKbgUXIqY2F3BdAgwBZi2B0zx5lkAKgAVgApABaACUE8/PrYaUEcAVABquwAV73gFoAJQAagAVAAqALUNEaXjiojSHlZ5IaIUEaV7cScf6nUx1gfoudAM8yGTp2HpDmVjy7kUQac8k1hOT0Xjbdpb/7NlLehuD1S9qC0LN9NAcY6TjzVkVUGz2GbVsyZRfaL0gfRAeiA95pzrD9YdV005jwHrmHKudco5438I1gXyA/mB/EB+ID+QH8gP5L/vOOhK5scrH8D8LWL+5sVBg/3B/mB/sD/YH+wP9gf7H8T+40PYf9QB+4P928X+zQoxB/uD/cH+YH+wP9gf7I/o/eNG76v9quh9BAS1XRAQvV8z0mPVaJJGbwzXsxlFrBh4apbHGOi8e8HEshxPs99dmQZ0sSOpni7yR+lqy1rUdKpAs/hmC0gWs9gHM+rwAEbtj7pgVExi1zuJPUbQOkgXpAvSBemCdEG6IN2jxGpXo64K1AXqtgd18c5qIC+QF8gL5AXyAnkfYohyNfL2gLxA3nYhL96CDeQF8gJ5gbxAXiAvInMlXNzrVETm9vHaDkTmIjIXJHswycp51bW0NWvIHtvyJ53ONl4TvvESY2l0+4TffluoNJUoQOkJQaloHwBTgGkLAm2rkLM/wqdcMBVb71Rsf4JAW4ArwBXgCnAFuAJcAa6HxM1Wk+sA5ApybQ+5Im4WBAuCBcGCYEGwINgWhMFWE+wQBAuCbRfBIgwWBAuCBcGCYEGwINgHEdU6rIpqxWovRLUiqhVgel9gml2IVYSnFyRa3vXta6VzOZmcOLG2bSFWLmEgV6zFArVi3rXekNdh1bTrCDyKadd6p117CHkF1YJqQbWgWlAtqBZUW388bCXW4rNewNoWYS3iYYG3wFvgLfAWeAu8bX2wbCXe4t1YwNuW4S2CZYG3wFvgLfAWeAu8ffCRtH21IpK2h6/iIpIWkbSg1pqpdanvzKunCKffX5PWcSlLk0gSgBRACiDFfGvNQbJVqNkfd4CamG6t9xNdXQTJAlgBrABWACuAFcAKYN0n/rWaWDE5CmJtEbEi/hXkCnIFuYJcQa4g1waHtlaTqwpyBbm2i1wR2gpyBbmCXEGuINd2kOtZ5nL59qjEu8Ja77DbP0tkfHRXfCwxtbWba35SDFUyOCTwU018uDUbURqsqOqdZQo9jCIdbZe9KGtODOm6feohuXOqGytN3LR6R6of3SFWt9uriNXF58nqg3oGG1bAoQjUbW2g7tWPxsrwyOXXV2Xofd0dXHdH+V9s+5X8IB/3DZ2GgThDO3q+dmw2trhTfe1H3D0VzbWQvv/4ajqNWPnN44NZfOnY/vrZJne7slsORxNHW7nS34v9idtSJL94cyE7aTSUCuLL/eDzRT23xruc/e9JVKv8PZ1VnGMrMvwcSh2ac7utC58l7mU7rEN7tnllCD5Wnr68zv9mbZuGzsuaDbALzTe9UkeS7HPZMd/tVrGCnkxaGLLiuZUXWUWRbns26V5puaZKlppz2aGfLw65w6rLrahmSa+W2/amsnKx8TAnNNXkH3U5jCvO3/kzOl3b8ynrbDyN9WbOlJWvILOpyTvpoDPSbbb1lv0u2/88JrMNOWc7Du97DmgDwfBxy29I1gi2tfdZAwz/+td/EN9lA/GVS1iO3Yfaq1D7Q9T+tLUYev8AFqV+AQMeHmDA/TG++9JqAz72M63MjArWD8Cj4dHwaHg0PBoeDY+GR8Oj4dHw6FP2aOnKpmqR7kOkIdItEenmLWuCUEOoIdQQagg1hBpCDaGGUEOoIdQnItTjg4QasdkQ6jYJdbNWW0KoIdQQagg1hBpCDaGGUEOoIdQQ6lo+P3SXJc1qp2pJ8wja3GZtxpJmiO59i27kuAy4WGZa3nRFV7azmc42Hv1S7xOqUc5IdNOklEz/ptwnsqey9OSonbALl+dOE97B9JPI4wDdAe6nBO6i2gPcAe6tBPfhAeDeHw8B7njeVePzrhFWYgL/gf/Af+A/8B/4D/wH/n/BBWTV/I+Je/B/W/gfC8jgAfAAeAA8AB4AD4AHPEwPGB/kAWN4ADygRR6AdS/wAHgAPAAeAA+AB+CDbGFr/C3sAUu/yHYWHso7Gt5v8L29TnAIQ5kbutJ+p44bQtE42OxtgivMNedd8MvtIqzgL7pam4ykrOU2yYwo3eQSuMR33WKcvk2PoktZRxRCkGzVXow50V6ZLkXDX/JLcYalm/6cPjUlpL2tXAnBIQnzSfxyxa5pSM4QdgOK/Ch77Uk6XUV8hjWkg7xXJcqrm9q6pJ8yQ7fivjPW/3DMlxtLl9xcVCcSN5f9ZF6isDTT/D3K5eQ1EoWYzOsdDJEXlmEZEZmLZE+D9ZXnqW+wznzDnE8Na2Enh+jK4kx3sXFpvsgPTdsCzBwTlZ9kNKssvb3TExesKivYq8uvt0D4Wjn/439fK2/+8/FrJc3TFWUejqXJmsOamyhY9+/RnYucmgbDUdGdX2zZLpUGSdsV2/Y8N893wejpk4cVNsjixA7fpa+CCyd9OOgh+Vj6OeicgmXBYbe0CGhIseyPT7oRDTBACbcpqcPWBuMAJz44LKrtKu/kkKwMEv1ut5P4o5f8o7uK/z1I/Lub/KPXSe6JmUVRE//uhut330Rp4KiaqJ2VV0meeJg8cfIqaj/5R6z9ymievN/oXlLZ96dtxf1GPE49TRR/sMsXEzUKpdqTydQ1wjHmw3YcUkdnn/8feEUcaIKyBAA= + Prometheus.json.gz: + H4sIAAAAAAAC/+1dbXfTOrb+zq/Q8b13TcuENklfoKw7dy1oKXSGAqcpZ2YucL0cW0186tg+ltwXOp3ffrckO7Fj2XFC4ipFLFqIpejl0dbej7a2pbsnCBmm6fphTInxEn2Bzwjd8d+Q4lsjDE+No555/u7N6UnPaKVJntXHHksLo2CE6RDHZJLoYGJHbkjdwGdZXqFJJkRwdIUjdO3SoeQxfIxcO1MUvQ15ExyLWiSIIxtP0kIvHrj+iVPWDJH+IenEp0kOnuEefn9rCQAi/EfsRlgCQVr/ILIuLN+aFO460scpZG+nE6B3JMFjd2tvq/2sj6nVSZrSklcaWj6AXKwyHEorzD7OVFddhwxYtxRSvwimrMrOVnurvUDfiOsPPEyoRYtV9iRptXsprY1afQ8XKzrPPy7WMRYcy/cDaA+kMskRlRqeS+hYjiZNQTmo5RBCHuzz2l8iGsU4+/wmjDhC8WjDHlr+AJONyQCZduBfuAPTswgFYfYCyzFJbNuYEJO6I4baKDQJhmwOuXN9+Ozb+C///mr8Z/rhq3H/pdMefdvcRP1btJE+3sw2bug6rGkXlkeybXOh2MPAC3gDo0F/o91CB/st1DnIfdtzRy5DptNuZ56mqItW57Agw+D6hAGfzU8oDln+vVE2azrOlocjaiTP71urGQM7iH26wUYiDidg5rGcQhH9N+osAqW10d3bE2hyROcA1Amu/aWiyf/99iTBdVrJv8UUQWcjih3oFZSb6PhEEaJDL4gdBFj7CFvE9W5RP3Y9/oRg5Fhk2A+syCFbaEhpSF5ubye6dcsORtsg6k5sU7Jts2K2v/ofYVpeufgaBRep0UAXMJpoMqSoC1oIoa/+Z4IvYg9dBBGKmYbJ2h0aoFHguxTSbkEsMklbX/0zfOWyuU/QNfagGfgXjpKBHZdOiYgx8DHlpmhnf78rnjCNfB4EHnXDFHY+7oeBT6PAI9nxFyrJjz2Pf/Jc/5JbI6FoInwB1mnIcN5pC1k1ouBaYq/swPOskBRkyxhidzBk0tLda08sJFOLJKOtsnOFlWbZQ3wOCiSIaaZ5k3Qmqq8t+3IQwZxwigKd5vnN8uLijEpT8y3gz4Xs74Ls7+2Kn/ZWTp1kcu08h5nRPWih3TbL9qIkH/QbdZ53IdseL+35ppHJ9S3XrBpqgk8B2x1ZHMCdqYTc3PiEoSCfWgPMxDUOmUpGThwxWYRiQQIJRRFmeRAoEZg7V5aHQhy5gQMSfMj1Pc8pnomZxT7/Kc39J+REQcjmPLL6wRXeyrcUZH9ksSE0/MDH+bSBFQ/w2H6NH4+sm3TUcgpGJLp+mjidxPSMTBCYXhkyKQ485z1jj2RWrlMrusTROFsm132uA3zqdPOPElhkQpufW5n+hiEMyLnQfZ2ytKKs5nHLaOArhhDTMBTf0CmRREkyqyqXcN+qV3gkZGJW4d184aUCD6N9BDL/KQDkSHHIDYYiTzwNuOGCiev72AZtbxTynbMWSYAPQcwv3Bv27f8yZCnHoBh77ndePjRgOhNoQfFt2fPsd/emv8rBOgW9OHvwmBHh8wRaX4SViq4ZH7ZfSRKD8Rdrok5AAReUBzyMLkFIJXPywvW8PD3Y6YBS67yAXy8OmFbrvCgovwvWoKLqFTMB54gbL63L9OTBTqGc7MQunYvcLkKZ8cgvjhTQgwGmNQYhZVrW1WADfkxQaBEnsjnGlaevrd+DPn8E/zIyO1ak3zbRUybPm8URm6hFVjgQ5MjFpJgtLenYsikHq1vI4uEB9p3jcXnFQkBKxVJVIjkJDdtt15SbsYpkWBoHjG8fTEHtUs5PjM/C2mTgmMqYcD3J0muiSKonJs9Sb3YFnG3+Zc6JlSqzueYW/1K6+LfjKALrOvluRmrvpvgIiK7EPNRlBDnD3+NeDaCPQ9ceIivC6Ojj3z+gs5O3787Rh49//wXY6fHJPxDzrvzyy7TJrsKc8UhGzBgPjsRaosCthF3cq2P8QuAmSWUFtQ09CqQahGuEd9hycCRNDSJaVGEAsIQ0MNySMkpVi1xZ0luvjk22PNfik4WBVpQwGF08mbz/hD/PTk+fHR2hd+9ejkYviUQphBaFCeWXl5nOLBgqB/vGXFZ+3NxisZwxJ1Z4eqyqKHV9Wj0Pta5Lrwvzc1HYM5y7WzUm26bJqJJp/gvswb84Wd2WjFFWkX75NmsMC+mxz1fgbCZEdMEhZh8qR9mwsYyLzB5n5jpoi7/LGuV8mQ8xxkyDyVcOeQFImcGsQS92VNZ56bOZfR8bV8rWevOJT7nxn5dF1eZMzEtVSZJyntF56FEN8lN3pcLozHFC0jukhPUcCnvv3aKjvBuMZYLVAGGdkndpPGgiaTZjeFgfiUTt/oiTpGgbfthHkmNE5wG1PABp1McRc4Zwcip4N3J9FJaUsub+i0pew2laR/sv1sl/MdN9sae9F0p6Lypn4nKdF2yDKLNFR4nTN4eY7cxxdXeXs72tsk25lXotxmb59bJ9EjD/2J9WV/xbYqaFNegVO6DdEismGSXJ67oT8/j5wa7mB5ofaH7wM/ADexjDDH7c/KCEEJziURDdokOOgGYEme+NvwODEmJrWgElT09gSK1krS1JPguuT5zpJCb558kA5GNGxsNiUWShgZdzp4nUFOzhvjwAr0Y8ys7+usSjrJuvZcgMaN7fEsUeRgydEGEmeCJ6EY1cQrCDHGHPiRdcy3Nu/Wy0a1/TLk271pJ2dZugXbKJpnxUCSNe8JOJKsnQsETXBZHppraUmEI9mlyZ1uRl2fiTZjja0uNKOq1OmePmVNiLMd0gOsDkoTw5msIIASSXbhhOOAyIckBBWp0kPBwRmEDWAP90DOaFZjCawWgG85MzmEQ7agozpjC9xF5oDqM5TKMhL5BogbkEhmJRNHQpf5GHWKMQaA1/pQ9ZvoOucYRRhH/npuSnIy0HmrRo0qJJy89DWkRpZqIbTXxjY+wAXxF60eR6UZOXSeCMFTm3qJdYEs1bNG9ZPm/5kAvSJZymsIL6FsEAjMt8K2CxxMENqO8F9iVPF+/hOy65/Oloy46mLZq2aNryE9EWFrSTHF1jMp0Yg9rQPGXMU86EcThOkNFMRTOVFTCVXjxiNMXyvPEuULKQ+MkISLetCYgmIGtJQHY1AVHPb4L+jL7689SfVOvEoefaFsWZ0wcbbwSYTjO4MPvMQJIHqz6IHPaVhWt/tPtu2nmlcMD3H7FrXyZ7VmT5Id9zHEHIj8k4TMnqXV7R9q1Iyp8Mdp7ke+wP6JBb8UIaLvvaAuzzFdBO4JPPvuMoQOkakO/h4SgKomkO6nJL1CmSt52890icKFW0f9bVoMSSpaIkTwVuU5bi+iUpmZdipukpU2gl3+LyPfvYPmZ4iWxpwhKuXUeMXR3SWqBlBeJihOMjGKUDH7JvR5bjxqz4vWKaXF4AbtDu/Pgj48ILphSU0MzpCU2SdgPLsXGZmCa8aOpgAFCDtvT4E66wQa++F6SpmL4Q73B9O8IWwRs+GDhGdgEk+9J0XHY0LH9gCk9wYuFKbdoeP1n4f1C7yUPYjoWT+lCQdH5M8xJOJmktAN9cAcZqgVgI9TUWeU2sAQzzIU5qgVgMNqpA8bAhFFkgrTkJoZ32X6qF4JtxOyuQO2oIOeKY1neAykwOZlYbulesqegsOUO6HL03zaEHloPEnhmFttrQHfJ2orNPhxW4HTeHmwN6zguCyzhUG7ejDz30nrezAre3zeGG7e56zNU3h90aM/Vdc8gN7DXRcm8P39RA7qQ55EYWmPoh2NW1gO80aW0NDP/aHIZBiH2+5FkPED+mza2B4t+aQ5FGLl0XOTznba2B3/uG8JvDw67YmkOElL/nt8SU43j6MDjO3ClQjNCk7UTnaTsrMP3woJjKNj4Um+QphuhjTNnW/Wve1gpEP6qAaHYvR8m5nqD5kbWzAsxPTYEZYcwjWczvsBbAOAT0lDY+/5s2swK8X5sCj4WW2cEotITrUmVH63nv6DU6nLS1Ar6zJuHLHLdp+gE1L5iaUW3qisOPPwQUHfNYrnLsek1iVxLVqBZ4Ir6wArLzHz9vvPyKgOxZ5IU4H5bWG7oXVJ6YbLQeZ/cL30j2C+n4Arm76b05K8KOfHcuuXqjLduYM9N9ddd33CvXia3cxvW9dA8+f+koT7qxblxSbFY/ti/FNlPxcgpjlOzP0eKFGWlIl+Rb5buQ453GL99Ku3Br3dQJV5vIqDiMvyh66Y23klESGYLBa4sUwuYym6/S6zrE7qshmTbyIwlnT+/aXZG254e6IU0q7Ud2zqkV/zC1c7+swAeju9cOb4x1Dn4oj1/o6PAFHb4wHb6wL41eKN4Vtezghdp3nvwwW5Hf0lTgMp2ZXOZu3OL7e2OlN6islNF8DmFeELTBxxo7m5rOPASdyVyh1K6Q8MJ7EHmq8znUNGdemjNtNUp5TusR2Pw9bfO1za9n8xuIWPzx09Sb9F38sL3fVcLe98S5iqjq6HZt9BXzYUhGSRv3R+bDyMRe6Rc3Zr17oXmM5jFq85i5bo3TPGZ+HiP24Q6D2KeavqhNX3ol0qvpy3Loy2NwTex0tU1/7Da9ehJyceWnOsDS9ApLwgRyV4H7eAAs/Ao/++ccR1eoSyJYcAeqFRnDstB6kUVos0lmkTRsGaSitWzQEplSD7RSYa9zmZ9STEyMPtpGZ5IuaVamKCuTMOimSY32La3atzSlrxTwL+Xlv9Pe2nuxtbPVab980WZt+o/Xr3Z3Xh2UT2gljhI5Yy8YsAN3p64MsIAaADVykmsCanPgXU2BtVtrmpE+3IkioInKSJWQazMV9PkOzOoAyVorj5cakTqvUq0i3h4hCEQa9bAd+JpqqU61khHblo2XdoU9MrZVav71vp6MRRUuXLr17eS6JX6rEr7BdsxWloGPrOSkYBSCjtuqy6v2Na/SvEopT1/ZK6wmk2uTzYC5KNUm6t+ijaQIyLo4vRq6DpZL36Lca9KqSvalvJdL6J0eDA3iGktzLsU5FzMjmmjpPcfp819tCnMJpbqMnUbP9K241nEBctHVL1JpdqEWuyg4bhKKwWiFx9sN9p+tQ4kJ2eu6bbr8jfEpooGeslsX2s26c2pSCtVDmM75sHBKocmE2mTiFNS/m8wZzSke9VZZdjZqh80s8nOgyY8mP+qQH0EMnFhohpTl3Om462X7QY4SiDVxUT3IZxWcRcf3PAJHyPS+S4R/53fjlVzGWB653dYcQHOAtdheWcaNe42GrqTNTbaSkTfjdOBVBFiv/PpAdXarxj1BtMaxwa8fAuzFr0lUB+ZAnHvbn3WK8OEDAjz/RZA/cPnjigAOZhwsfKT4WuMsZSQ9GSPRiw2lzqaauRKRjKH2nj427+nUIGsHqnTlNcQo9VHxlVcM1G4QBXGIJve71T/IU6+/9PpLnfUX2w5GT9H0HvTkDshp76xkDxpVv6C5PU/pNnu/ba7yFfQKv1Gcqb0CYWWnip0xTTa5+1G7ifX+tmZoa+gqrmAbOtxNsw0Fw92GlIZmhP+AMaUTEjBy7SioYgKVLymKaLeh5Tsejhq/LOXuLql5rfeL352ff0JnYlw0HVgbOjCZNnrrWPOBKj6gzxTXfEDZ3V9oJRRtgvWJbus4Hma8YOe5Nm7YPcDrXGsGMNEp6A0fDvQrG44xGUCyGCVNCn6SUDLtHlj/SDLm9XtW2L9AX330TJyNxP83cgnBDnJizN+584Jr+cYHz0wuXWYY0tygoQIK+sRB0J7ItRER1xjUjlHTXgvNUhT0Wki3MNx0b1Ye31O9edEoPSm+CL/6CKlauAldozp8p7yVDYfz1cIvUb+qA9gTzVxGCNlDhThlNsyCCI3DMjQbVpwNu/KR0oT4kUU0pU4DHdNU/6VQfZS/JtsFsq3OOaZ+QN0LFuYvqA6gKXiOqsE/qh9M+iGLJ+phfSGP6uQlN2Kavzxm/uJLh1pzmFkcRl8xqDmMOg7DDWZuNjbRM5ThMXbgX7gD07MINSPsBexg9ti2MSGZFzHTIzDq7nNuo/22PhZjbg506voxxcB+XOgJ6olhuIg9dMgHCZ3x8dG8SPW4Jz6KmhHpLc4KaqCvadHUQB1qUIsQgCXSVxGv7Egsbe0VsvbLel1dDOoiVMDoGJoJrINvROhL7RSpDO56e4hc/yoQHiRU3A6bxZZeaLak2ZI6m0HjEJhBYA5seUx4CTdq5RhU9kD0NI/ePZqfPYF+4fdobqPuSDMmpf0jD80kNCNaNSMaWFGfnePASA62c69qzs2OskL7eE/ycQmCv7AI71t97xZdR4E/+AWhTx67kh4NsRfWv0lG31P36KlStQbjAs+vkPQ8xrhloa05rlTTgi/uwCqMzUoIGXCxER5BnZSYvOtm/5biOcPff/h8y6KUL0bMCj1YiJ112guHeJfA2RiQSzvHsmIa1HjHYEkIMlpnDsFwmOSWNIXkAmCBDQLb4voDJIgoYm1GFMwSXihAfknwwSJLbdxgAQKm3mL2f6GzUpeE0xBbYbOzdQGsWCOhJc9ighc6rWyZWLmOh5WHyqmcfMfNAOXDcKkvVKhaqN42glWEOXF3VIcrbWcFYO8aAUxt3c6xgiZSPKpA6mTVSI1syx5i5SeiaObsqfjXhvBSW7YStKCRFVD9beVQseWb+pLFWjlbsN43g5bicsWxqhar01UD5eMbyqi74qqdNRMN7AqkPqwaqYAOcaS4SPE2zhCpj6sGiruUlNdUvJWzNdWnZtBSW6wEVtVi9evKgVIbokpwzmqD81CbswE6xaMgukWfCdsX2jg++cfpm029T/uw+7Rc3pvc39SBa48qhF3f2qxjsgobgDsKRLAnV7Klhtb0eJNn3ePcQvnIrGajsP6ILZ+6Hr6/RywkS5SwaEiWGmZfX/KM9Mmc2uY/Kpv/XNt8bfOVsfmykwirjf9813fsZWKzRXGNx2Y/BiJwzockvX8VbaM9HaOtOhnQ96w+/jjt11Fwif0Wcgd+EOFZMdpPktYaxB7ikfUbjohoUWdXPKa3yWGHVnTJCwOFPpgoFIPiUehBN/zBeEaBtSQ0I7eTUWZRe78xyS/gneEQWbJUh9qkoYUZnQEawvZiB7/ijCc//UpEyBjFoK2KuZPJbYBRydYZhOJVt7xiNfhtEiy7uFYC9C+UmjNkxOmz6AgnuVOcmBZYTzDmyYG6WVPEjAHT4HmzAk8HmL9Lur31lJm8r8bGl//7anz78+Z29suJJs3MBDZyHH3ya9pMI59a6A97Js+cqFfR4UxCTPC5KChLwsZD+ihkIWUTCwtEHObJyv2cwz7mNpOx33oqG/32eo9+ntR6/MZ3GXGn+IYbjc6Upb9KamAJsiXbTHGZorOz5GU6+0RgBOMrEZgnZRaxqtPl3S7r+JT5La9KtrZK69opr2tn2XXtl9e1v+y6Ot0KELtLr80pr8xZcl3d8rq6y67reXldz5dd1067vDKWttzaDipqO1h6bZ0XFdXxxCcyMiq1PJ1hC+3Azz78gCTDL6eFuvDzHH4AqBY6YL+mqhzreDsmFNaJT7J1Mcp7L+ifyzVcQvwuxIrS8IPrZ2OlBAvB5JmR+1rownormnw5MXvjRX5WNRp7mQMqO+3Mh53sh85kCWpklqNGJ/thp51Nmcx6I6MBxrPwW9oH5gUoquzyWrIF72cLztbS3c1+mMCfnUcTWc6j/j3gDhcjIeWSK7W6W23EnEBXLr4Wua7G9L7beXL//12AtdY0TgEA diff --git a/charts/kubezero-metrics/update.sh b/charts/kubezero-metrics/update.sh index e3ea8f3..9f3d1cc 100755 --- a/charts/kubezero-metrics/update.sh +++ b/charts/kubezero-metrics/update.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION=15.4.4 +VERSION=16.1.2 rm -rf charts/kube-prometheus-stack curl -L -s -o - https://github.com/prometheus-community/helm-charts/releases/download/kube-prometheus-stack-${VERSION}/kube-prometheus-stack-${VERSION}.tgz | tar xfz - -C charts @@ -11,6 +11,7 @@ patch -p0 -i metrics-zdt.patch --no-backup-if-mismatch # Create ZDT dashboard configmap cd dashboards +../sync_grafana_dashboards.py metrics-dashboards.yaml ../templates/grafana-dashboards-metrics.yaml ../sync_grafana_dashboards.py k8s-dashboards.yaml ../templates/grafana-dashboards-k8s.yaml ../sync_grafana_dashboards.py zdt-dashboards.yaml ../templates/grafana-dashboards-zdt.yaml cd - diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index dc0ba52..cb46f63 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -104,6 +104,7 @@ kube-prometheus-stack: # gather all monitors podMonitorSelectorNilUsesHelmValues: false serviceMonitorSelectorNilUsesHelmValues: false + ruleSelectorNilUsesHelmValues: false resources: requests: -- 2.40.1 From 3a5ca22499ebff8dd1e452c52df38bd8f3e59def Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 15 Jun 2021 13:03:52 +0200 Subject: [PATCH 05/45] feat: Add Redis Grafana Dashboard --- charts/kubezero-redis/Chart.yaml | 2 +- charts/kubezero-redis/dashboards.yaml | 9 +++++++++ .../templates/grafana-dashboards.yaml | 13 +++++++++++++ charts/kubezero-redis/update.sh | 4 ++++ 4 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 charts/kubezero-redis/dashboards.yaml create mode 100644 charts/kubezero-redis/templates/grafana-dashboards.yaml create mode 100755 charts/kubezero-redis/update.sh diff --git a/charts/kubezero-redis/Chart.yaml b/charts/kubezero-redis/Chart.yaml index 5749769..d3fae86 100644 --- a/charts/kubezero-redis/Chart.yaml +++ b/charts/kubezero-redis/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-redis description: KubeZero Umbrella Chart for Redis HA type: application -version: 0.2.1 +version: 0.2.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-redis/dashboards.yaml b/charts/kubezero-redis/dashboards.yaml new file mode 100644 index 0000000..1793bd9 --- /dev/null +++ b/charts/kubezero-redis/dashboards.yaml @@ -0,0 +1,9 @@ +configmap: grafana-dashboards +gzip: true +# folder: Logging +condition: 'or .Values.redis.metrics.enabled ( index .Values "redis-cluster" "metrics" "enabled")' +dashboards: +- name: redis + url: https://grafana.com/api/dashboards/11835/revisions/1/download + tags: + - Redis diff --git a/charts/kubezero-redis/templates/grafana-dashboards.yaml b/charts/kubezero-redis/templates/grafana-dashboards.yaml new file mode 100644 index 0000000..ac38010 --- /dev/null +++ b/charts/kubezero-redis/templates/grafana-dashboards.yaml @@ -0,0 +1,13 @@ +{{- if or .Values.redis.metrics.enabled ( index .Values "redis-cluster" "metrics" "enabled") }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" (include "kubezero-lib.fullname" $) "grafana-dashboards" | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} + labels: + grafana_dashboard: "1" +{{ include "kubezero-lib.labels" . | indent 4 }} +binaryData: + redis.json.gz: + H4sIAAAAAAAC/+1daVPcOBr+Pr9C5Zndgq0OsQ0dmqmaDySBmdQAyQYyW7UD1aW2RbcWX2PJQIdifvvq8CHZ6jMNNMEfOKxX1vGejw5Ldz8AYPX7OEoySqyfwZ/sGYA78ZtRIhgilmq9P+1/+vzx+ODst4Mvp1anIAdwgAJO/5TGIaIjlJGK6CPipTihOI54lopAx4ko1IcUkjhLPVTRkiAb4uiDz+mJoVBJP8mbpVQrMtyz3xcd2aUU/ZXhFBk6VdQ/TOEljGBVOPaNyQUTfq0TrlFK8t692epuuXkjOubqEhgxZjUrS0bGqtRkpaLpdZhYiicyM2qy0VSls2Vv2Uv0jeBoGCBCIW1WeWqgNXtZihNGUczyMiqXp6zeCjChpXSrRjHKIMMB/cBLcjpVqsIcc6dZHhTBQcDpNM2Qkj7CviEVe3H0Lg7ilBeYDgdww+4A13HYr263A5xNteii6/tVX8A/wX6AUqo1oZIlGQ1imPpWTrsXfy9+yMVQN7DPyMcEvC/eApdxCqpOAkk+uE3ilKIUOFu3HYApuInTKwJuMB2BEQpCwETCGPA65blfjSBA+Qtb5+l5xH8+XIJxnIEQEy5fIDKCEIVxOgYZxQH+KvrWAUmAIEEgjH18OQbnVghvZbZzC1zDIEMAR/IfsjWGoVQfixVHayKwhhGiwiU4Tm+7K5O4gZzFcUBxwgi2SBRqF2VBIJ9Yq2HOHKfb23Z2uju2280LCHB0JXyDVDChwAZf4UFvhM5wiOKMKoVLGpf8W+hdDdM4i3jdlzAgSKf/wTtoJhFFeUGuP+4O05vujvyxt/Y0DZI5tneZZrl7HbBj8yw9Q54uozi7LsvSFaXsbhY6dFG551nWwPTLwyEUXLHLRIN8ZHqaCjPQ+8l0MIRUOIOq3CHMhqg0Y5HEdKNglGPbSndCHBUENZmM4pt6Zdx0Rszhj+LAP+KBiUzLcQzTKyREwLtRWFjVxhT7n2Kit3LEHneVwngTXOX5ttbKMX9uFC20dK96jJieMjOoaxcmJ+imzmVda3PWJQkzxDPpNRxTuq5oVYcUpyQNksaAoluq6BPISbzoMvG+M72wFEbDGYW5VWENzWTK8J4p56eYsYboGmFxFgnCcSwcMrOkKEIeRb6l5TnjNdc4GidlAKmkkcSEXuJbHaHkiYdxRE/xV1FP1/6HQk9R8x2RNvEVwZNjmEyRxSWzQf4eb7TONSp7Y5283q8R4vKFKQwlCVN2pjo1o7vEQaBHr20WuZiPZb96e9xxOD3Nt1zyepo2xUtWy5HFuNwH7W1rBeRma7Q44VVYMVlYh4swHSI6hW8sRImqmd5ssJ9+zIBEnzKfvSGCUz9L+APDuH2CmLr45A5HLMxFHvrl73Prp+Lh3Lr/8ycOhaU9Xmxu6ryufJkojqAUI6JnUWzZMlMOoUcFo1yNHKAhivzDsgb9ZeaZU+w105nGSaRc0wpCEQ+KTs+2p2hF6QpJjd+YCvdufRF8a+J2E64Tdq0q/66q/II6Xflj3mLrlzn1vnAjM1VfZCyGC16WpiiiZijbRvoVRfoojtALCfbuzGDPxw1cqT4yn5Q2xxASDDhuiwZaNNCigZWiARn7S4XoewFmvn9y7F862D9mSHeXiuchOsyVyQn19NMRvmxYRgkA3kmWLYkAei0CmI4ADLF7GQjQBBJPjQASxGqK6EOAgNoE3BNggJ0aBthZEQZwWgzQYoAWA6wUAzA9Af8C+TSAnHzuZ4SBgcGYoslQAIDXQHuHzyrMeGXzO8MPPbuz110FiDiWSwNfCByil4IkYIAheVfEfcW3DGDaCFJioecIRUM6Eq5NS0em7LOC+YJxGwsbdrSEX1Poc/Cn4QEeI/X+zB01e7Wo2ZsnavooOpWGU2+zaIc7IzIKA9KbBq+HBndVyLJJYYZvSsWRIXXSNEVMoclHypWvgtDoPnehxBDs0Q32haK40yDAPEFRCX76GulRWV6jVTmyg8MmzE14dVxrMv5uV09vSpAx3EcpEm76MogV85e+sgBLtY6xwOUhk7EwF+JdNWrhbi9B/pGMdDpt/pEkpMVUsheHIYx80k/S2EOEhxIh38kzyk54sYbzyPuLRY0dW3elgnPVOv0c8UQRoRJKahEjRJ/RMNfJ2gszBqq5WMDBLfIypuUsgBPkKeGmXClWhyDkMyIMXuRrxE17hkzozTEHiVNaG5wIW+4XMc3LwiyAFF8jq4lpqr0n6gaPW3iLa5Y4yLwrqZ5qn3mzc5PWJucrrF3LbR43lc7HYONjeDttfKAs6Y44J3T9yzcE6Y3ghHj4FhKk78goHWwju/SwjeQmWpw89Fi7djYMZNwUOgMOQ5MuivQjdF02WtsSsmbwQ5lLcJ8TJnHerACUOC0oeVJQojXt+8UkWAElV2gsmtAfYUpm4ZHuMngk3/1W05ZVAhXe9NWMcOfBKp0lmMs3vKHnyl7Z+AUZ/PZ5gsHfmCox/HcsugyYcwCnTwYFceTja+xnMHixULCmXUsBLLtFgd+MAuv4wfrx7aHz1rabivliJ6mcWZs5d5cAhLsPDwh5/DgIEzqeQPsvSuPV4Mj68ttawEieACABX3k32+mt5TZKLLA68tArHbwNa4MFF10HWj34mxfiMWteEMetB147404ETFgmauftVg/WhAY/JBJq4drzm7R7DnDMcVeAxxRmrdMM3Voiq3bVcCWrhhGi8gtrCR4eZA5pQbRwdwdEi8D9/aKQan7opHMgzujjsOCbZ8sYc2RjJ3NnzTHVCaL8w17w4fXHxcBUHTO1WOqJsVS7Bvpy4NTu085uOTtLwKnuEmiKC3KfnJlPWpiFtWrZF4FaKR6O6KnxJIclUZhx32oLwpYDYRq/VoLBSBYWu3/9gVjKmwg5NsFgDDb8wdPCDn/AIAf4vjZnyfmdDxSFcjXu/dt2Me5BMYn2AW4LSVpI8nwgSWOGZylM4my3UzwtulgvdPEKNLP3WUk4xdFwyntPB0eimIKigY++7PbcmGVm1PJbrNYDuh3kvQLXBJzE9FX5/DsTSIvi2t31LYxTikfX2BMqZf3Y27MPbVfVnxCF4nwBv4/5SKgvMjc/Hzq3nF13y9ndsrecn/ecrn1udf4XD1h6dYAnP6zCXEmKvADiULZh+/BNb3u73d41/wSY6ywBNnst1nwZWNPsCIU3kBuFctvTfFXhhdzpH+M/+uokw1cbyvqcgC/MGwmgNcfanJwtK1LXccd73qO12UdWZ3ju/h+L4avjq2z4M8az4uvQg1o32p1mLYZtpyLXGhz2nngqcil06Oy8ZHjotFORjzMVSePkaqPbAdh0VsV0eAHEERVPu+zphX5jq1WF1bTu9D0YBHmflt+5tgaHVYB3vCPLHFSxqj1a7Uxai0JeMgpZbNJpRVNMvWVAxJsWL0z47u+RoIL73X3vN+Vg5FWfi+xMCtPrEY/lxUglP4DXOPXYFJPbqNtG3WcSdctb1Jjpcavivd62pYJbxBuhEP5RXr3mOjKZjoP8FrL0SuZkbrISvLQaqyyaojDhM1rRcJ772arYdGc6otN4NZt6Umf93r+apIvpZdUIceQFmY/2zefVTrnHkMsxY/ZveG367YhaTKo8Ekv+K0PpeMKlfJqYHC11iG5rgy2LXOHkSxqcjiPPdMpT81bAuj1r2hUUB2/XvIQirvsF79Pz0SWOcHFFneBzX7qX8nKeDuB8FJFQOxp4aSGeFMUtIsPI9NIs+S3Un0qs7reJ1RBimG0K2ZF/F02zdGqjDzzNnDlXGtlJhZARdCYLMqK7NdCku5LxHMOUDwzEdEAS+6vRrk+xD8QhwAsoF6u8H9XeWUq3FunhqvXNafVtXmkIYfC0QvBCQKbVu6X1sA77ZmhgUffDaeACff5ONbN2PatA4CUYKo/yj29eOQW2L07rZ3hQey3BDN2n1cs5w8q76VQAbnWVoYpjKw/b6oN6aHtX+d9RH7ZtlaKMQFzlfye/kPai6AMfCCq6NLMWteA3asFqLe6O+qAsCez6anuLtmjs+xqL8bE1SOMbNkLN4as+3pv3ntwN0624UputTN5zfPv+6OTz71/+81WmVtcY7/xw/38XppizYHsAAA== +{{- end }} diff --git a/charts/kubezero-redis/update.sh b/charts/kubezero-redis/update.sh new file mode 100755 index 0000000..596912b --- /dev/null +++ b/charts/kubezero-redis/update.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# Fetch dashboards from Grafana.com and update ZDT CM +../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/grafana-dashboards.yaml -- 2.40.1 From 27ac03d9a0f0572af140e65f06f81c8cd0fa758e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 15 Jun 2021 16:14:29 +0200 Subject: [PATCH 06/45] Switch to file buffer --- charts/kubezero-logging/values.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 21f83eb..2269328 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -186,19 +186,22 @@ fluentd: request_timeout 60s suppress_type_name true slow_flush_log_threshold 50.0 - # bulk_message_request_threshold 2097152 + + # Retry failed bulk requests + # https://github.com/uken/fluent-plugin-elasticsearch#unrecoverable-error-types + unrecoverable_error_types ["out_of_memory_error"] + bulk_message_request_threshold 2097152 - @type file_single - chunk_limit_size 2MB + @type file + chunk_limit_size 4MB total_limit_size 2GB flush_mode interval flush_thread_count 1 - flush_interval 10s + flush_interval 30s flush_at_shutdown true retry_type exponential_backoff retry_timeout 2h - flush_thread_interval 30s overflow_action drop_oldest_chunk disable_chunk_backup true -- 2.40.1 From a459525847f31d3270c5f603670075618f5a26e5 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 15 Jun 2021 16:14:29 +0200 Subject: [PATCH 07/45] fix: Switch fluentd ES to file buffer --- charts/kubezero-logging/values.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 21f83eb..2269328 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -186,19 +186,22 @@ fluentd: request_timeout 60s suppress_type_name true slow_flush_log_threshold 50.0 - # bulk_message_request_threshold 2097152 + + # Retry failed bulk requests + # https://github.com/uken/fluent-plugin-elasticsearch#unrecoverable-error-types + unrecoverable_error_types ["out_of_memory_error"] + bulk_message_request_threshold 2097152 - @type file_single - chunk_limit_size 2MB + @type file + chunk_limit_size 4MB total_limit_size 2GB flush_mode interval flush_thread_count 1 - flush_interval 10s + flush_interval 30s flush_at_shutdown true retry_type exponential_backoff retry_timeout 2h - flush_thread_interval 30s overflow_action drop_oldest_chunk disable_chunk_backup true -- 2.40.1 From 23912192ade0b4c6a98f9ba082fde2cd91f2d540 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 25 Jun 2021 18:00:38 +0200 Subject: [PATCH 08/45] chore: update docs and scripts --- docs/SA_OIDC.yaml | 15 +++ docs/Upgrade-2.18.md | 56 --------- docs/Upgrade-2.20.md | 21 ++-- docs/aws_sidecar.md | 22 ++++ docs/etcd_hackery.md | 4 +- ...ion.sh => deprecated_setClusterVersion.sh} | 0 scripts/upgrade_120.sh | 117 ++++++++++++++++++ 7 files changed, 170 insertions(+), 65 deletions(-) create mode 100644 docs/SA_OIDC.yaml delete mode 100644 docs/Upgrade-2.18.md create mode 100644 docs/aws_sidecar.md rename scripts/{setClusterVersion.sh => deprecated_setClusterVersion.sh} (100%) mode change 100755 => 100644 create mode 100755 scripts/upgrade_120.sh diff --git a/docs/SA_OIDC.yaml b/docs/SA_OIDC.yaml new file mode 100644 index 0000000..ba0c857 --- /dev/null +++ b/docs/SA_OIDC.yaml @@ -0,0 +1,15 @@ +# Service Account Tokens + +## Federation with AWS IAM + +### Discovery +- public S3 location for openid and jwks config files +- synchronized from the api-server to S3 during version upgrades +service-account-issuer: `arn:aws:s3:::${ConfigBucketName}/k8s/${ClusterName}` +api-audiences: `sts.amazonaws.com` + +## Projection + +## Resources +- https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#service-account-token-volume-projection +- https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#service-account-issuer-discovery diff --git a/docs/Upgrade-2.18.md b/docs/Upgrade-2.18.md deleted file mode 100644 index bc9fd1d..0000000 --- a/docs/Upgrade-2.18.md +++ /dev/null @@ -1,56 +0,0 @@ -# Upgrade to KubeZero V2.18.0 (Argoless) - -## (optional) Upgrade control plane nodes / worker nodes -- Set kube version in the controller config to eg. `1.18` -- Update kube-controller and worker stacks with latest CFN code - -- terminate controller node(s) -- once all controller nodes successfully upgraded replace worker nodes in a rolling fashion via. drain / terminate / rinse-repeat - -## ArgoCD -- disable all auto-sync and "prune" features to prevent that eg. namespaces from previous apps get removed - - either remove auto-sync from old values.yaml and run deploy one last time, trigger kubezero sync ! - - or disable manual via Argo UI starting with Kubezero app itself - -- uninstall argo helm chart: -`helm uninstall kubezero -n argocd` - -- remove all "argocd.argoproj.io/instance" labels from namespaces to prevent namespace removal later on: - `./scripts/remove_argo_ns.sh` - -## KubeZero - Part 1 -- migrate values.yaml to new structure, adapt as needed - & update new central kubezero location in git and merge cluster configs - -- upgrade all CRDs: - `./bootstrap.sh crds all clusters/$CLUSTER ../../../kubezero/charts` - -- upgrade first components: - `./bootstrap.sh deploy calico,cert-manager,kiam,aws-ebs-csi-driver,aws-efs-csi-driver clusters/$CLUSTER ../../../kubezero/charts` - -## Istio -Due to changes of the ingress namespace resource the upgrade will cause a brief (~3-5 min) ingress service interruption ! - - - delete istio operators, to remove all pieces, remove operator itself - `./scripts/delete_istio_17.sh` - - deploy istio and istio-ingress via bootstrap.sh - `./bootstrap.sh deploy istio,istio-ingress clusters/$CLUSTER ../../../kubezero/charts` - - patch all VirtualServices via script to new namespace - `./scripts/patch_vs.sh` - -Ingress service interruption ends. - -## KubeZero - Part 2 - -- push kubezero & cluster config to git - -- if logging incl. ElasticSearch is installed remove the old ECK operator first: - `./scripts/remove_old_eck.sh` - -- upgrade all remaining components and install new ArgoCD: - `./bootstrap.sh deploy all clusters/$CLUSTER ../../../kubezero/charts` - -## Verification / Tests -- verify argocd incl. kubezero app -- verify all argo apps status -- verify all the things diff --git a/docs/Upgrade-2.20.md b/docs/Upgrade-2.20.md index b610787..14fd510 100644 --- a/docs/Upgrade-2.20.md +++ b/docs/Upgrade-2.20.md @@ -2,16 +2,23 @@ # CloudBender ## Changes -- controller node names are now strictly tight to the AZ they are in: AZ1 -> controller00, AZ2 -> controller01 etc. to prevent controller03 from happening in case AWS launches new instances before the old ones are actually terminated +### Single node control plane +- Control ## Upgrade -- Set Kubernetes version in the controller config to eg. `1.20` -- Update controller and worker stacks with latest CFN code +- Set the specific wanted Kubernetes version in the controller config to eg. `v1.20.2` +- configure your AWS CLI profile as well as your kubectl context to cluster you want to upgrade. +- verify your config ... -- Upgrade requires careful replacement in case existing control planes are shuffled otherwise: ( this might reduce the number of online controllers temporarily to 1 ! ) - - manually set controller ASG to Min/Maz 0 for the ASG currently hosting controller00 - - terminate controller node in AZ1 which will return as controller00 - - replace controller01 and 02 in similar fashion +- run ./scripts/upgrade_120.sh +- update the CFN stack for kube-control-plane + +### Single node control plane +- will automatically be upgraded and the controller node replaced as part of the CFN update + +### Clustered control plane +- replace controller instances one by one in no particular order +- once confirmed that the upgraded 1.20 control plane is working as expected update the clustered control plane CFN stack once more with `LBType: none` to remove the AWS NLB fronting the Kubernetes API which is not required anymore. - replace worker nodes in a rolling fashion via. drain / terminate / rinse-repeat diff --git a/docs/aws_sidecar.md b/docs/aws_sidecar.md new file mode 100644 index 0000000..7f75e1a --- /dev/null +++ b/docs/aws_sidecar.md @@ -0,0 +1,22 @@ +``` + - args: + - -c + - sleep 3600 + command: + - /bin/sh + env: + - name: AWS_ROLE_ARN + value: + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/sts.amazonaws.com/serviceaccount/token + image: amazon/aws-cli + imagePullPolicy: Always + name: busybox + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/run/secrets/sts.amazonaws.com/serviceaccount/ + name: aws-token + readOnly: true +``` diff --git a/docs/etcd_hackery.md b/docs/etcd_hackery.md index 0551229..773bcad 100644 --- a/docs/etcd_hackery.md +++ b/docs/etcd_hackery.md @@ -4,8 +4,8 @@ https://github.com/jpbetz/auger ## Change a PV object -etcdctl get /registry/persistentvolumes/services-dsny-cache | auger decode > pv.yaml +etcdctl get /registry/persistentvolumes/my_pv | auger decode > pv.yaml vi pv.yaml -cat pv.yaml | auger encode | etcdctl put /registry/persistentvolumes/services-dsny-cache +cat pv.yaml | auger encode | etcdctl put /registry/persistentvolumes/my_pv diff --git a/scripts/setClusterVersion.sh b/scripts/deprecated_setClusterVersion.sh old mode 100755 new mode 100644 similarity index 100% rename from scripts/setClusterVersion.sh rename to scripts/deprecated_setClusterVersion.sh diff --git a/scripts/upgrade_120.sh b/scripts/upgrade_120.sh new file mode 100755 index 0000000..eef12c9 --- /dev/null +++ b/scripts/upgrade_120.sh @@ -0,0 +1,117 @@ +#!/bin/bash +set -e + +# Get all kube-control-plane ASGs in the current account and region +asgs=$(aws autoscaling describe-auto-scaling-groups --output json | jq .AutoScalingGroups[].AutoScalingGroupName -r | grep kube-control-plane) + +for asg in $asgs; do + hooks=$(aws autoscaling describe-lifecycle-hooks --auto-scaling-group-name $asg --output json | jq '.LifecycleHooks[] | select (.LifecycleTransition=="autoscaling:EC2_INSTANCE_TERMINATING") | .LifecycleHookName' -r) + + for hook in $hooks; do + echo "Delete Lifecycle hook $hook of ASG $asg ? to abort" + read + aws autoscaling delete-lifecycle-hook --lifecycle-hook-name $hook --auto-scaling-group-name $asg + done +done + +# unset any AWS_DEFAULT_PROFILE as it will break aws-iam-auth +unset AWS_DEFAULT_PROFILE + +nodes=$(kubectl get nodes -l node-role.kubernetes.io/master -o json | jq .items[].metadata.name -r) + +for node in $nodes; do + echo "Deploying upgrade job on $node..." + + cat <<'EOF' > _job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: kubezero-upgrade + namespace: kube-system +spec: + template: + spec: + hostNetwork: true + hostIPC: true + hostPID: true + containers: + - name: busybox + image: busybox + command: + - /bin/sh + - -c + - | + cat <<'EOF' > /host/tmp/upgrade.sh + #!/bin/bash -ex + for l in $(cat /etc/environment); do + export $l + done + my_ip=$(ec2metadata --local-ipv4) + my_id=$(ec2metadata --instance-id) + clusterName=$(yq r /etc/kubezero/kubezero.yaml clusterName) + my_asg=$(aws ec2 describe-tags --filters "Name=resource-id,Values=${my_id}" --output json | jq '.Tags[] | select(.Key=="aws:cloudformation:logical-id") | .Value' -r) + + [ $my_asg == "KubeControlAsgAZ1" ] && nodename="etcd0-$clusterName" + [ $my_asg == "KubeControlAsgAZ2" ] && nodename="etcd1-$clusterName" + [ $my_asg == "KubeControlAsgAZ3" ] && nodename="etcd2-$clusterName" + zone_name=$(hostname -d) + + zone_id=$(aws route53 list-hosted-zones --query 'HostedZones[?Name==`'"$zone_name"'.`].Id' --output text | cut --delimiter="/" --fields=3) + cat < route53.json + { "Changes": [ { "Action": "UPSERT", "ResourceRecordSet": { "Name": "${nodename}.${zone_name}", "Type": "A", "TTL": 30, "ResourceRecords": [ { "Value": "$my_ip" } ] } } ] } + EOF2 + + echo "Updating DNS entry for $nodename to $my_ip" + aws route53 change-resource-record-sets --hosted-zone-id $zone_id --change-batch file://route53.json + + echo "Adding additional control shutdown commands" + if [ ! -f /usr/local/sbin/drain_delete_node.sh ]; then + cat < /usr/local/sbin/drain_delete_node.sh + #!/bin/bash -ex + export LC_TYPE=en_US.UTF-8 + export KUBECONFIG=/root/.kube/config + kubeadm reset phase update-cluster-status + kubeadm reset phase remove-etcd-member + EOF3 + chmod +x /usr/local/sbin/drain_delete_node.sh + sed -e 's,/usr/local/sbin/backup_control_plane.sh&,/usr/local/sbin/drain_delete_node.sh,' -i /usr/local/sbin/cloudbender_shutdown.sh + fi + + echo "Patching ClusterConfig to re-create new etcd server certificates" + yq w /etc/kubezero/kubeadm/templates/ClusterConfiguration.yaml etcd.local.serverCertSANs[+] $nodename > /etc/kubernetes/kubeadm-recert.yaml + rm -f /etc/kubernetes/pki/etcd/server.* + kubeadm init phase certs etcd-server --config=/etc/kubernetes/kubeadm-recert.yaml 2>/dev/null + kill -s HUP $(ps -e | grep etcd | awk '{print $1}') + echo "Waiting for etcd to accept connections again...might take 30s or more" + while true; do + etcdctl member list -w simple 1>/dev/null 2>&1 && break || true + sleep 3 + done + EOF + chmod +x /host/tmp/upgrade.sh + chroot /host bash -c /tmp/upgrade.sh + volumeMounts: + - name: host + mountPath: /host + securityContext: + privileged: true + volumes: + - name: host + hostPath: + path: / + type: Directory + nodeSelector: + kubernetes.io/hostname: __node__ + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + restartPolicy: Never +EOF + + # set controller node + sed -i -e "s/__node__/$node/" _job.yaml + + kubectl apply -f _job.yaml + kubectl wait -n kube-system --timeout 300s --for=condition=complete job/kubezero-upgrade + kubectl delete -f _job.yaml +done -- 2.40.1 From 61add69904e61e5cb24dac44ed5a504a645f90c9 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 29 Jun 2021 17:39:44 +0200 Subject: [PATCH 09/45] fix: JWT aud for istio, cleanup --- charts/kubeadm/Chart.yaml | 2 +- .../templates/ClusterConfiguration.yaml | 24 +++++++++---------- .../kubeadm/templates/InitConfiguration.yaml | 3 +++ .../kubeadm/templates/JoinConfiguration.yaml | 3 +++ .../templates/resources/10-runtimeClass.yaml | 2 ++ .../resources/20-oicd-public-rbac.yaml | 13 ++++++++++ 6 files changed, 33 insertions(+), 14 deletions(-) create mode 100644 charts/kubeadm/templates/resources/20-oicd-public-rbac.yaml diff --git a/charts/kubeadm/Chart.yaml b/charts/kubeadm/Chart.yaml index dd5eeb1..1918978 100644 --- a/charts/kubeadm/Chart.yaml +++ b/charts/kubeadm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubeadm description: KubeZero Kubeadm golden config type: application -version: 1.20.1 +version: 1.20.8 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubeadm/templates/ClusterConfiguration.yaml b/charts/kubeadm/templates/ClusterConfiguration.yaml index 43820e9..9d424bc 100644 --- a/charts/kubeadm/templates/ClusterConfiguration.yaml +++ b/charts/kubeadm/templates/ClusterConfiguration.yaml @@ -10,17 +10,10 @@ networking: etcd: local: extraArgs: - #name: {{ .Values.etcd.nodeName }} ### DNS discovery #discovery-srv: {{ .Values.domain }} #discovery-srv-name: {{ .Values.clusterName }} #initial-cluster: - ### Regular - #{{- if .Values.etcd.initialCluster }} - #initial-cluster: {{ .Values.etcd.initialCluster }} - #{{- end }} - #initial-advertise-peer-urls: "https://{{ .Values.etcd.nodeName }}:2380" - #advertise-client-urls: "https://{{ .Values.etcd.nodeName }}:2379" initial-cluster-token: etcd-{{ .Values.clusterName }} listen-metrics-urls: "http://{{ .Values.listenAddress }}:2381" logger: "zap" @@ -42,21 +35,21 @@ controllerManager: profiling: "false" bind-address: {{ .Values.listenAddress }} terminated-pod-gc-threshold: "300" - leader-elect: {{ .Values.highAvailable | quote }} + # leader-elect: {{ .Values.highAvailable | quote }} logging-format: json feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" "platform" .Values.platform ) | trimSuffix "," | quote }} scheduler: extraArgs: profiling: "false" bind-address: {{ .Values.listenAddress }} - leader-elect: {{ .Values.highAvailable | quote }} + # leader-elect: {{ .Values.highAvailable | quote }} logging-format: json feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" "platform" .Values.platform ) | trimSuffix "," | quote }} apiServer: certSANs: - {{ regexSplit ":" .Values.api.endpoint -1 | first }} extraArgs: - etcd-servers: {{ ternary .Values.api.allEtcdEndpoints "https://127.0.0.1:2379" .Values.highAvailable }} + etcd-servers: {{ .Values.api.allEtcdEndpoints }} profiling: "false" audit-log-path: "/var/log/kubernetes/audit.log" audit-policy-file: /etc/kubernetes/apiserver/audit-policy.yaml @@ -67,13 +60,18 @@ apiServer: tls-cipher-suites: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" admission-control-config-file: /etc/kubernetes/apiserver/admission-configuration.yaml {{- if eq .Values.platform "aws" }} + service-account-issuer: "{{ .Values.serviceAccountIssuer }}" + service-account-jwks-uri: "{{ .Values.serviceAccountIssuer }}/openid/v1/jwks" + api-audiences: "istio-ca,sts.amazonaws.com" authentication-token-webhook-config-file: /etc/kubernetes/apiserver/aws-iam-authenticator.yaml + {{- else }} + api-audiences: "istio-ca" {{- end }} feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" "platform" .Values.platform ) | trimSuffix "," | quote }} enable-admission-plugins: NodeRestriction,EventRateLimit - {{- if .Values.highAvailable }} - goaway-chance: ".001" - {{- end }} + # {{- if .Values.highAvailable }} + # goaway-chance: ".001" + # {{- end }} logging-format: json {{- with .Values.api.extraArgs }} {{- toYaml . | nindent 4 }} diff --git a/charts/kubeadm/templates/InitConfiguration.yaml b/charts/kubeadm/templates/InitConfiguration.yaml index 466ba00..969e9bf 100644 --- a/charts/kubeadm/templates/InitConfiguration.yaml +++ b/charts/kubeadm/templates/InitConfiguration.yaml @@ -11,3 +11,6 @@ nodeRegistration: - KubeletVersion kubeletExtraArgs: node-labels: {{ .Values.nodeLabels | quote }} + {{- with .Values.providerID }} + provider-id: {{ . }} + {{- end }} diff --git a/charts/kubeadm/templates/JoinConfiguration.yaml b/charts/kubeadm/templates/JoinConfiguration.yaml index 017f497..72953b8 100644 --- a/charts/kubeadm/templates/JoinConfiguration.yaml +++ b/charts/kubeadm/templates/JoinConfiguration.yaml @@ -15,3 +15,6 @@ nodeRegistration: - KubeletVersion kubeletExtraArgs: node-labels: {{ .Values.nodeLabels | quote }} + {{- with .Values.providerID }} + provider-id: {{ . }} + {{- end }} diff --git a/charts/kubeadm/templates/resources/10-runtimeClass.yaml b/charts/kubeadm/templates/resources/10-runtimeClass.yaml index ed979d2..54610a4 100644 --- a/charts/kubeadm/templates/resources/10-runtimeClass.yaml +++ b/charts/kubeadm/templates/resources/10-runtimeClass.yaml @@ -1,3 +1,4 @@ +{{- if .Values.disabledfor120 }} apiVersion: node.k8s.io/v1 kind: RuntimeClass metadata: @@ -6,3 +7,4 @@ handler: runc overhead: podFixed: memory: 16Mi +{{- end }} diff --git a/charts/kubeadm/templates/resources/20-oicd-public-rbac.yaml b/charts/kubeadm/templates/resources/20-oicd-public-rbac.yaml new file mode 100644 index 0000000..927881b --- /dev/null +++ b/charts/kubeadm/templates/resources/20-oicd-public-rbac.yaml @@ -0,0 +1,13 @@ +{{- if eq .Values.platform "aws" }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: oidc-public +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:service-account-issuer-discovery +subjects: + - kind: Group + name: system:unauthenticated +{{- end }} -- 2.40.1 From 08d3a940ecd583abcbf09d90fa2f46a40a59effa Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 30 Jun 2021 12:32:37 +0200 Subject: [PATCH 10/45] feat: Version bump of all things kubezero-logging to tackle logging hickups --- charts/kubezero-logging/Chart.yaml | 10 +- .../charts/eck-operator/Chart.yaml | 4 +- .../charts/eck-operator/crds/all-crds.yaml | 609 +++++++++++++++++- .../eck-operator/templates/_helpers.tpl | 14 + .../eck-operator/templates/cluster-roles.yaml | 12 + .../templates/operator-namespace.yaml | 1 - .../eck-operator/templates/podMonitor.yaml | 36 ++ .../eck-operator/templates/statefulset.yaml | 13 +- .../charts/eck-operator/values.yaml | 34 + .../charts/fluent-bit/Chart.yaml | 11 +- .../charts/fluent-bit/templates/_pod.tpl | 12 +- .../templates/configmap-dashboards.yaml | 2 +- .../charts/fluent-bit/values.yaml | 12 + .../charts/fluentd/Chart.yaml | 2 +- .../charts/fluentd/templates/_pod.tpl | 8 +- .../templates/configmap-dashboards.yaml | 1 + .../charts/fluentd/templates/hpa.yaml | 39 ++ .../fluentd/templates/prometheusrules.yaml | 5 +- .../fluentd/templates/servicemonitor.yaml | 9 + .../charts/fluentd/values.yaml | 89 +++ charts/kubezero-logging/fluent-bit.patch | 12 - charts/kubezero-logging/update.sh | 6 +- charts/kubezero-logging/values.yaml | 13 +- 23 files changed, 895 insertions(+), 59 deletions(-) create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/podMonitor.yaml create mode 100644 charts/kubezero-logging/charts/fluentd/templates/hpa.yaml diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index 38c97ef..ced132b 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application -version: 0.6.5 -appVersion: 1.5.0 +version: 0.7.0 +appVersion: 1.6.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -19,13 +19,13 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: eck-operator - version: 1.5.0 + version: 1.6.0 # repository: https://helm.elastic.co condition: eck-operator.enabled - name: fluentd - version: 0.2.2 + version: 0.2.6 condition: fluentd.enabled - name: fluent-bit - version: 0.15.4 + version: 0.15.14 condition: fluent-bit.enabled kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-logging/charts/eck-operator/Chart.yaml b/charts/kubezero-logging/charts/eck-operator/Chart.yaml index 5cd7a0d..36b05d3 100644 --- a/charts/kubezero-logging/charts/eck-operator/Chart.yaml +++ b/charts/kubezero-logging/charts/eck-operator/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.5.0 +appVersion: 1.6.0 description: 'A Helm chart for deploying the Elastic Cloud on Kubernetes (ECK) operator: the official Kubernetes operator for orchestrating Elasticsearch, Kibana, APM Server, Enterprise Search, and Beats.' home: https://github.com/elastic/cloud-on-k8s icon: https://helm.elastic.co/icons/eck.png @@ -17,4 +17,4 @@ maintainers: name: Elastic name: eck-operator type: application -version: 1.5.0 +version: 1.6.0 diff --git a/charts/kubezero-logging/charts/eck-operator/crds/all-crds.yaml b/charts/kubezero-logging/charts/eck-operator/crds/all-crds.yaml index 5e41c56..0dbf2a9 100644 --- a/charts/kubezero-logging/charts/eck-operator/crds/all-crds.yaml +++ b/charts/kubezero-logging/charts/eck-operator/crds/all-crds.yaml @@ -10,8 +10,8 @@ metadata: app.kubernetes.io/instance: 'logging' app.kubernetes.io/managed-by: 'Helm' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '1.5.0' - helm.sh/chart: 'eck-operator-crds-1.5.0' + app.kubernetes.io/version: '1.6.0' + helm.sh/chart: 'eck-operator-crds-1.6.0' name: agents.agent.k8s.elastic.co spec: additionalPrinterColumns: @@ -191,6 +191,13 @@ spec: type: string outputName: type: string + serviceName: + description: ServiceName is the name of an existing Kubernetes + service which will be used to make requests to the referenced + object. It has to be in the same namespace as the referenced + resource. If left empty the default HTTP service of the referenced + resource will be used. + type: string required: - name type: object @@ -239,7 +246,7 @@ spec: type: array serviceAccountName: description: ServiceAccountName is used to check access from the current - resource to a Elasticsearch resource in a different namespace. Can + resource to an Elasticsearch resource in a different namespace. Can only be used if ECK is enforcing RBAC on references. type: string version: @@ -297,8 +304,8 @@ metadata: app.kubernetes.io/instance: 'logging' app.kubernetes.io/managed-by: 'Helm' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '1.5.0' - helm.sh/chart: 'eck-operator-crds-1.5.0' + app.kubernetes.io/version: '1.6.0' + helm.sh/chart: 'eck-operator-crds-1.6.0' name: apmservers.apm.k8s.elastic.co spec: additionalPrinterColumns: @@ -366,6 +373,13 @@ spec: description: Namespace of the Kubernetes object. If empty, defaults to the current namespace. type: string + serviceName: + description: ServiceName is the name of an existing Kubernetes service + which will be used to make requests to the referenced object. + It has to be in the same namespace as the referenced resource. + If left empty the default HTTP service of the referenced resource + will be used. + type: string required: - name type: object @@ -757,6 +771,13 @@ spec: description: Namespace of the Kubernetes object. If empty, defaults to the current namespace. type: string + serviceName: + description: ServiceName is the name of an existing Kubernetes service + which will be used to make requests to the referenced object. + It has to be in the same namespace as the referenced resource. + If left empty the default HTTP service of the referenced resource + will be used. + type: string required: - name type: object @@ -873,8 +894,8 @@ metadata: app.kubernetes.io/instance: 'logging' app.kubernetes.io/managed-by: 'Helm' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '1.5.0' - helm.sh/chart: 'eck-operator-crds-1.5.0' + app.kubernetes.io/version: '1.6.0' + helm.sh/chart: 'eck-operator-crds-1.6.0' name: beats.beat.k8s.elastic.co spec: additionalPrinterColumns: @@ -1055,6 +1076,13 @@ spec: description: Namespace of the Kubernetes object. If empty, defaults to the current namespace. type: string + serviceName: + description: ServiceName is the name of an existing Kubernetes service + which will be used to make requests to the referenced object. + It has to be in the same namespace as the referenced resource. + If left empty the default HTTP service of the referenced resource + will be used. + type: string required: - name type: object @@ -1074,6 +1102,13 @@ spec: description: Namespace of the Kubernetes object. If empty, defaults to the current namespace. type: string + serviceName: + description: ServiceName is the name of an existing Kubernetes service + which will be used to make requests to the referenced object. + It has to be in the same namespace as the referenced resource. + If left empty the default HTTP service of the referenced resource + will be used. + type: string required: - name type: object @@ -1182,8 +1217,535 @@ metadata: app.kubernetes.io/instance: 'logging' app.kubernetes.io/managed-by: 'Helm' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '1.5.0' - helm.sh/chart: 'eck-operator-crds-1.5.0' + app.kubernetes.io/version: '1.6.0' + helm.sh/chart: 'eck-operator-crds-1.6.0' + name: elasticmapsservers.maps.k8s.elastic.co +spec: + additionalPrinterColumns: + - JSONPath: .status.health + name: health + type: string + - JSONPath: .status.availableNodes + description: Available nodes + name: nodes + type: integer + - JSONPath: .status.version + description: ElasticMapsServer version + name: version + type: string + - JSONPath: .metadata.creationTimestamp + name: age + type: date + group: maps.k8s.elastic.co + names: + categories: + - elastic + kind: ElasticMapsServer + listKind: ElasticMapsServerList + plural: elasticmapsservers + shortNames: + - ems + singular: elasticmapsserver + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + description: ElasticMapsServer represents an Elastic Map Server resource in + a Kubernetes cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: MapsSpec holds the specification of an Elastic Maps Server + instance. + properties: + config: + description: 'Config holds the ElasticMapsServer configuration. See: + https://www.elastic.co/guide/en/kibana/current/maps-connect-to-ems.html#elastic-maps-server-configuration' + type: object + configRef: + description: ConfigRef contains a reference to an existing Kubernetes + Secret holding the Elastic Maps Server configuration. Configuration + settings are merged and have precedence over settings specified in + `config`. + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + count: + description: Count of Elastic Maps Server instances to deploy. + format: int32 + type: integer + elasticsearchRef: + description: ElasticsearchRef is a reference to an Elasticsearch cluster + running in the same Kubernetes cluster. + properties: + name: + description: Name of the Kubernetes object. + type: string + namespace: + description: Namespace of the Kubernetes object. If empty, defaults + to the current namespace. + type: string + serviceName: + description: ServiceName is the name of an existing Kubernetes service + which will be used to make requests to the referenced object. + It has to be in the same namespace as the referenced resource. + If left empty the default HTTP service of the referenced resource + will be used. + type: string + required: + - name + type: object + http: + description: HTTP holds the HTTP layer configuration for Elastic Maps + Server. + properties: + service: + description: Service defines the template for the associated Kubernetes + Service object. + properties: + metadata: + description: ObjectMeta is the metadata of the service. The + name and namespace provided here are managed by ECK and will + be ignored. + type: object + spec: + description: Spec is the specification of the service. + properties: + allocateLoadBalancerNodePorts: + description: allocateLoadBalancerNodePorts defines if NodePorts + will be automatically allocated for services with type + LoadBalancer. Default is "true". It may be set to "false" + if the cluster load-balancer does not rely on NodePorts. + allocateLoadBalancerNodePorts may only be set for services + with type LoadBalancer and will be cleared if the type + is changed to any other type. This field is alpha-level + and is only honored by servers that enable the ServiceLBNodePortControl + feature. + type: boolean + clusterIP: + description: 'clusterIP is the IP address of the service + and is usually assigned randomly. If an address is specified + manually, is in-range (as per system configuration), and + is not in use, it will be allocated to the service; otherwise + creation of the service will fail. This field may not + be changed through updates unless the type field is also + being changed to ExternalName (which requires this field + to be blank) or the type field is being changed from ExternalName + (in which case this field may optionally be specified, + as describe above). Valid values are "None", empty string + (""), or a valid IP address. Setting this to "None" makes + a "headless service" (no virtual IP), which is useful + when direct endpoint connections are preferred and proxying + is not required. Only applies to types ClusterIP, NodePort, + and LoadBalancer. If this field is specified when creating + a Service of type ExternalName, creation will fail. This + field will be wiped when updating a Service to type ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + clusterIPs: + description: "ClusterIPs is a list of IP addresses assigned + to this service, and are usually assigned randomly. If + an address is specified manually, is in-range (as per + system configuration), and is not in use, it will be allocated + to the service; otherwise creation of the service will + fail. This field may not be changed through updates unless + the type field is also being changed to ExternalName (which + requires this field to be empty) or the type field is + being changed from ExternalName (in which case this field + may optionally be specified, as describe above). Valid + values are \"None\", empty string (\"\"), or a valid IP + address. Setting this to \"None\" makes a \"headless + service\" (no virtual IP), which is useful when direct + endpoint connections are preferred and proxying is not + required. Only applies to types ClusterIP, NodePort, + and LoadBalancer. If this field is specified when creating + a Service of type ExternalName, creation will fail. This + field will be wiped when updating a Service to type ExternalName. + \ If this field is not specified, it will be initialized + from the clusterIP field. If this field is specified, + clients must ensure that clusterIPs[0] and clusterIP have + the same value. \n Unless the \"IPv6DualStack\" feature + gate is enabled, this field is limited to one value, which + must be the same as the clusterIP field. If the feature + gate is enabled, this field may hold a maximum of two + entries (dual-stack IPs, in either order). These IPs + must correspond to the values of the ipFamilies field. + Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy + field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" + items: + type: string + type: array + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this + service. These IPs are not managed by Kubernetes. The + user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external + load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that + discovery mechanisms will return as an alias for this + service (e.g. a DNS CNAME record). No proxying will be + involved. Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires Type to be + type: string + externalTrafficPolicy: + description: externalTrafficPolicy denotes if this Service + desires to route external traffic to node-local or cluster-wide + endpoints. "Local" preserves the client source IP and + avoids a second hop for LoadBalancer and Nodeport type + services, but risks potentially imbalanced traffic spreading. + "Cluster" obscures the client source IP and may cause + a second hop to another node, but should have good overall + load-spreading. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. This only applies when type + is set to LoadBalancer and externalTrafficPolicy is set + to Local. If a value is specified, is in-range, and is + not in use, it will be used. If not specified, a value + will be automatically allocated. External systems (e.g. + load-balancers) can use this port to determine if a given + node holds endpoints for this service or not. If this + field is specified when creating a Service which does + not need it, creation will fail. This field will be wiped + when updating a Service to no longer need it (e.g. changing + type). + format: int32 + type: integer + ipFamilies: + description: "IPFamilies is a list of IP families (e.g. + IPv4, IPv6) assigned to this service, and is gated by + the \"IPv6DualStack\" feature gate. This field is usually + assigned automatically based on cluster configuration + and the ipFamilyPolicy field. If this field is specified + manually, the requested family is available in the cluster, + and ipFamilyPolicy allows it, it will be used; otherwise + creation of the service will fail. This field is conditionally + mutable: it allows for adding or removing a secondary + IP family, but it does not allow changing the primary + IP family of the Service. Valid values are \"IPv4\" and + \"IPv6\". This field only applies to Services of types + ClusterIP, NodePort, and LoadBalancer, and does apply + to \"headless\" services. This field will be wiped when + updating a Service to type ExternalName. \n This field + may hold a maximum of two entries (dual-stack families, + in either order). These families must correspond to the + values of the clusterIPs field, if specified. Both clusterIPs + and ipFamilies are governed by the ipFamilyPolicy field." + items: + description: IPFamily represents the IP Family (IPv4 or + IPv6). This type is used to express the family of an + IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + ipFamilyPolicy: + description: IPFamilyPolicy represents the dual-stack-ness + requested or required by this Service, and is gated by + the "IPv6DualStack" feature gate. If there is no value + provided, then this field will be set to SingleStack. + Services can be "SingleStack" (a single IP family), "PreferDualStack" + (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise + fail). The ipFamilies and clusterIPs fields depend on + the value of this field. This field will be wiped when + updating a service to type ExternalName. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer + LoadBalancer will get created with the IP specified in + this field. This feature depends on whether the underlying + cloud-provider supports specifying the loadBalancerIP + when a load balancer is created. This field will be ignored + if the cloud-provider does not support the feature.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client + IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this + service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + appProtocol: + description: The application protocol for this port. + This field follows standard Kubernetes label syntax. + Un-prefixed names are reserved for IANA standard + service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). + Non-standard protocols should use prefixed names + such as mycompany.com/my-custom-protocol. This is + a beta field that is guarded by the ServiceAppProtocol + feature gate and enabled by default. + type: string + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field + in the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this + service is exposed when type is NodePort or LoadBalancer. Usually + assigned by the system. If a value is specified, + in-range, and not in use it will be used, otherwise + the operation will fail. If not specified, a port + will be allocated if this Service requires one. If + this field is specified when creating a Service + which does not need it, creation will fail. This + field will be wiped when updating a Service to no + longer need it (e.g. changing type from NodePort + to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this + service. + format: int32 + type: integer + protocol: + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access + on the pods targeted by the service. Number must + be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored + for services with clusterIP=None, and should be + omitted or set equal to the ''port'' field. More + info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + required: + - port + type: object + type: array + publishNotReadyAddresses: + description: publishNotReadyAddresses indicates that any + agent which deals with endpoints for this Service should + disregard any indications of ready/not-ready. The primary + use case for setting this field is for a StatefulSet's + Headless Service to propagate SRV DNS records for its + Pods for the purpose of peer discovery. The Kubernetes + controllers that generate Endpoints and EndpointSlice + resources for Services interpret this to mean that all + endpoints are considered "ready" even if the Pods themselves + are not. Agents which consume only Kubernetes generated + endpoints through the Endpoints or EndpointSlice resources + can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only + applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: + https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of + Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + topologyKeys: + description: topologyKeys is a preference-order list of + topology keys which implementations of services should + use to preferentially sort endpoints when accessing this + Service, it can not be used at the same time as externalTrafficPolicy=Local. + Topology keys must be valid label keys and at most 16 + keys may be specified. Endpoints are chosen based on the + first topology key with available backends. If this field + is specified and all entries have no backends that match + the topology of the client, the service has no backends + for that client and connections should fail. The special + value "*" may be used to mean "any topology". This catch-all + value, if used, only makes sense as the last value in + the list. If this is not specified or empty, no topology + constraints will be applied. This field is alpha-level + and is only honored by servers that enable the ServiceTopology + feature. + items: + type: string + type: array + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ClusterIP" allocates + a cluster-internal IP address for load-balancing to endpoints. + Endpoints are determined by the selector or if that is + not specified, by manual construction of an Endpoints + object or EndpointSlice objects. If clusterIP is "None", + no virtual IP is allocated and the endpoints are published + as a set of endpoints rather than a virtual IP. "NodePort" + builds on ClusterIP and allocates a port on every node + which routes to the same endpoints as the clusterIP. "LoadBalancer" + builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the + same endpoints as the clusterIP. "ExternalName" aliases + this service to the specified externalName. Several other + fields do not apply to ExternalName services. More info: + https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + type: object + tls: + description: TLS defines options for configuring TLS for HTTP. + properties: + certificate: + description: "Certificate is a reference to a Kubernetes secret + that contains the certificate and private key for enabling + TLS. The referenced secret should contain the following: \n + - `ca.crt`: The certificate authority (optional). - `tls.crt`: + The certificate (or a chain). - `tls.key`: The private key + to the first certificate in the certificate chain." + properties: + secretName: + description: SecretName is the name of the secret. + type: string + type: object + selfSignedCertificate: + description: SelfSignedCertificate allows configuring the self-signed + certificate generated by the operator. + properties: + disabled: + description: Disabled indicates that the provisioning of + the self-signed certifcate should be disabled. + type: boolean + subjectAltNames: + description: SubjectAlternativeNames is a list of SANs to + include in the generated HTTP TLS certificate. + items: + description: SubjectAlternativeName represents a SAN entry + in a x509 certificate. + properties: + dns: + description: DNS is the DNS name of the subject. + type: string + ip: + description: IP is the IP address of the subject. + type: string + type: object + type: array + type: object + type: object + type: object + image: + description: Image is the Elastic Maps Server Docker image to deploy. + type: string + podTemplate: + description: PodTemplate provides customisation options (labels, annotations, + affinity rules, resource requests, and so on) for the Elastic Maps + Server pods + type: object + serviceAccountName: + description: ServiceAccountName is used to check access from the current + resource to a resource (eg. Elasticsearch) in a different namespace. + Can only be used if ECK is enforcing RBAC on references. + type: string + version: + description: Version of Elastic Maps Server. + type: string + required: + - version + type: object + status: + description: MapsStatus defines the observed state of Elastic Maps Server + properties: + associationStatus: + description: AssociationStatus is the status of an association resource. + type: string + availableNodes: + description: AvailableNodes is the number of available replicas in the + deployment. + format: int32 + type: integer + health: + description: Health of the deployment. + type: string + version: + description: 'Version of the stack resource currently running. During + version upgrades, multiple versions may run in parallel: this value + specifies the lowest version currently running.' + type: string + type: object + version: v1alpha1 + versions: + - name: v1alpha1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: eck-operator-crds/templates/all-crds.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.5.0 + creationTimestamp: null + labels: + app.kubernetes.io/instance: 'logging' + app.kubernetes.io/managed-by: 'Helm' + app.kubernetes.io/name: 'eck-operator-crds' + app.kubernetes.io/version: '1.6.0' + helm.sh/chart: 'eck-operator-crds-1.6.0' name: elasticsearches.elasticsearch.k8s.elastic.co spec: additionalPrinterColumns: @@ -1985,6 +2547,13 @@ spec: description: Namespace of the Kubernetes object. If empty, defaults to the current namespace. type: string + serviceName: + description: ServiceName is the name of an existing Kubernetes + service which will be used to make requests to the referenced + object. It has to be in the same namespace as the referenced + resource. If left empty the default HTTP service of the + referenced resource will be used. + type: string required: - name type: object @@ -2478,8 +3047,8 @@ metadata: app.kubernetes.io/instance: 'logging' app.kubernetes.io/managed-by: 'Helm' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '1.5.0' - helm.sh/chart: 'eck-operator-crds-1.5.0' + app.kubernetes.io/version: '1.6.0' + helm.sh/chart: 'eck-operator-crds-1.6.0' name: enterprisesearches.enterprisesearch.k8s.elastic.co spec: additionalPrinterColumns: @@ -2558,6 +3127,13 @@ spec: description: Namespace of the Kubernetes object. If empty, defaults to the current namespace. type: string + serviceName: + description: ServiceName is the name of an existing Kubernetes service + which will be used to make requests to the referenced object. + It has to be in the same namespace as the referenced resource. + If left empty the default HTTP service of the referenced resource + will be used. + type: string required: - name type: object @@ -3002,8 +3578,8 @@ metadata: app.kubernetes.io/instance: 'logging' app.kubernetes.io/managed-by: 'Helm' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '1.5.0' - helm.sh/chart: 'eck-operator-crds-1.5.0' + app.kubernetes.io/version: '1.6.0' + helm.sh/chart: 'eck-operator-crds-1.6.0' name: kibanas.kibana.k8s.elastic.co spec: additionalPrinterColumns: @@ -3071,6 +3647,13 @@ spec: description: Namespace of the Kubernetes object. If empty, defaults to the current namespace. type: string + serviceName: + description: ServiceName is the name of an existing Kubernetes service + which will be used to make requests to the referenced object. + It has to be in the same namespace as the referenced resource. + If left empty the default HTTP service of the referenced resource + will be used. + type: string required: - name type: object diff --git a/charts/kubezero-logging/charts/eck-operator/templates/_helpers.tpl b/charts/kubezero-logging/charts/eck-operator/templates/_helpers.tpl index 3ba51a6..25b20ed 100644 --- a/charts/kubezero-logging/charts/eck-operator/templates/_helpers.tpl +++ b/charts/kubezero-logging/charts/eck-operator/templates/_helpers.tpl @@ -303,6 +303,20 @@ RBAC permissions - update - patch - delete +- apiGroups: + - maps.k8s.elastic.co + resources: + - elasticmapsservers + - elasticmapsservers/status + - elasticmapsservers/finalizers # needed for ownerReferences with blockOwnerDeletion on OCP + verbs: + - get + - list + - watch + - create + - update + - patch + - delete {{- end -}} {{/* diff --git a/charts/kubezero-logging/charts/eck-operator/templates/cluster-roles.yaml b/charts/kubezero-logging/charts/eck-operator/templates/cluster-roles.yaml index 2fd7b49..342792f 100644 --- a/charts/kubezero-logging/charts/eck-operator/templates/cluster-roles.yaml +++ b/charts/kubezero-logging/charts/eck-operator/templates/cluster-roles.yaml @@ -35,6 +35,12 @@ rules: - apiGroups: ["beat.k8s.elastic.co"] resources: ["beats"] verbs: ["get", "list", "watch"] + - apiGroups: ["agent.k8s.elastic.co"] + resources: ["agents"] + verbs: ["get", "list", "watch"] + - apiGroups: ["maps.k8s.elastic.co"] + resources: ["elasticmapsservers"] + verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -60,4 +66,10 @@ rules: - apiGroups: ["beat.k8s.elastic.co"] resources: ["beats"] verbs: ["create", "delete", "deletecollection", "patch", "update"] + - apiGroups: ["agent.k8s.elastic.co"] + resources: ["agents"] + verbs: ["create", "delete", "deletecollection", "patch", "update"] + - apiGroups: ["maps.k8s.elastic.co"] + resources: ["elasticmapsservers"] + verbs: ["create", "delete", "deletecollection", "patch", "update"] {{- end -}} diff --git a/charts/kubezero-logging/charts/eck-operator/templates/operator-namespace.yaml b/charts/kubezero-logging/charts/eck-operator/templates/operator-namespace.yaml index 1061cdf..c3841f8 100644 --- a/charts/kubezero-logging/charts/eck-operator/templates/operator-namespace.yaml +++ b/charts/kubezero-logging/charts/eck-operator/templates/operator-namespace.yaml @@ -6,5 +6,4 @@ metadata: name: {{ .Release.Namespace }} labels: name: {{ .Release.Namespace }} - {{- include "eck-operator.labels" $ | nindent 4 }} {{- end -}} diff --git a/charts/kubezero-logging/charts/eck-operator/templates/podMonitor.yaml b/charts/kubezero-logging/charts/eck-operator/templates/podMonitor.yaml new file mode 100644 index 0000000..54a5966 --- /dev/null +++ b/charts/kubezero-logging/charts/eck-operator/templates/podMonitor.yaml @@ -0,0 +1,36 @@ +{{- $metricsPort := int .Values.config.metricsPort -}} +{{- if and .Values.podMonitor.enabled (gt $metricsPort 0) }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ include "eck-operator.fullname" . }} + namespace: {{ ternary .Values.podMonitor.namespace .Release.Namespace (not (empty .Values.podMonitor.namespace)) }} + labels: {{- include "eck-operator.labels" . | nindent 4 }} + {{- if .Values.podMonitor.labels }} + {{- toYaml .Values.podMonitor.labels | nindent 4 }} + {{- end }} + {{- with .Values.podMonitor.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.podMonitor.podTargetLabels }} + podTargetLabels: {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - port: metrics + path: /metrics + {{- if .Values.podMonitor.interval }} + interval: {{ .Values.podMonitor.interval }} + {{- end }} + {{- if .Values.podMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.podMonitor.scrapeTimeout }} + {{- end }} + {{- if .Values.podMonitor.podMetricsEndpointConfig }} + {{- toYaml .Values.podMonitor.podMetricsEndpointConfig | nindent 6 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: {{- include "eck-operator.selectorLabels" . | nindent 6 }} +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-logging/charts/eck-operator/templates/statefulset.yaml b/charts/kubezero-logging/charts/eck-operator/templates/statefulset.yaml index adb9889..6f60830 100644 --- a/charts/kubezero-logging/charts/eck-operator/templates/statefulset.yaml +++ b/charts/kubezero-logging/charts/eck-operator/templates/statefulset.yaml @@ -1,3 +1,4 @@ +{{- $metricsPort := int .Values.config.metricsPort -}} --- apiVersion: apps/v1 kind: StatefulSet @@ -24,6 +25,9 @@ spec: {{- end }} labels: {{- include "eck-operator.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} spec: terminationGracePeriodSeconds: 10 serviceAccountName: {{ include "eck-operator.serviceAccountName" . }} @@ -73,11 +77,18 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - {{- if .Values.webhook.enabled }} + {{- if or (gt $metricsPort 0) .Values.webhook.enabled }} ports: + {{- if (gt $metricsPort 0) }} + - containerPort: {{ .Values.config.metricsPort }} + name: metrics + protocol: TCP + {{- end }} + {{- if .Values.webhook.enabled }} - containerPort: 9443 name: https-webhook protocol: TCP + {{- end }} {{- end }} volumeMounts: - mountPath: "/conf" diff --git a/charts/kubezero-logging/charts/eck-operator/values.yaml b/charts/kubezero-logging/charts/eck-operator/values.yaml index 5d43ff5..8091d6d 100644 --- a/charts/kubezero-logging/charts/eck-operator/values.yaml +++ b/charts/kubezero-logging/charts/eck-operator/values.yaml @@ -40,6 +40,9 @@ resources: # podAnnotations define the annotations that should be added to the operator pod. podAnnotations: {} +## podLabels define additional labels that should be added to the operator pod. +podLabels: {} + # podSecurityContext defines the pod security context for the operator pod. podSecurityContext: runAsNonRoot: true @@ -165,6 +168,37 @@ config: # Can be disabled if cluster-wide storage class RBAC access is not available. validateStorageClass: true +# Prometheus PodMonitor configuration +# Reference: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#podmonitor +podMonitor: + + # enabled determines whether a podMonitor should deployed to scrape the eck metrics. + # This requires the prometheus operator and the config.metricsPort not to be 0 + enabled: false + + # labels adds additional labels to the podMonitor + labels: {} + + # annotations adds additional annotations to the podMonitor + annotations: {} + + # namespace determines in which namespace the podMonitor will be deployed. + # If not set the podMonitor will be created in the namespace to release is installed into + # namespace: monitoring + + # interval specifies the interval at which metrics should be scraped + interval: 5m + + # scrapeTimeout specifies the timeout after which the scrape is ended + scrapeTimeout: 30s + + # podTargetLabels transfers labels on the Kubernetes Pod onto the target. + podTargetLabels: [] + + # podMetricsEndpointConfig allows to add an extended configuration to the podMonitor + podMetricsEndpointConfig: {} + # honorTimestamps: true + # Internal use only internal: # manifestGen specifies whether the chart is running under manifest generator. diff --git a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml index 546461f..e3e351d 100644 --- a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml +++ b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml @@ -1,9 +1,10 @@ annotations: artifacthub.io/changes: | - - Upgrade fluent-bit image to v1.7.3 -apiVersion: v2 -appVersion: 1.7.3 -description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems. + - Upgrade fluent-bit image to v1.7.8 +apiVersion: v1 +appVersion: 1.7.8 +description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD + family operating systems. home: https://fluentbit.io/ icon: https://fluentbit.io/assets/img/logo1-default.png keywords: @@ -20,4 +21,4 @@ maintainers: name: fluent-bit sources: - https://github.com/fluent/fluent-bit/ -version: 0.15.4 +version: 0.15.14 diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl b/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl index 8239767..22cd951 100644 --- a/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl +++ b/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl @@ -13,6 +13,14 @@ securityContext: dnsConfig: {{- toYaml . | nindent 2 }} {{- end }} +{{- with .Values.hostAliases }} +hostAliases: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- if .Values.initContainers }} +initContainers: + {{- toYaml .Values.initContainers | nindent 2 }} +{{- end }} #hostNetwork: true #dnsPolicy: ClusterFirstWithHostNet containers: @@ -23,11 +31,11 @@ containers: imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if .Values.env }} env: - {{- toYaml .Values.env | nindent 4 }} + {{- toYaml .Values.env | nindent 6 }} {{- end }} {{- if .Values.envFrom }} envFrom: - {{- toYaml .Values.envFrom | nindent 4 }} + {{- toYaml .Values.envFrom | nindent 6 }} {{- end }} {{- if .Values.args }} args: diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/configmap-dashboards.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/configmap-dashboards.yaml index 52cac5a..d2b212a 100644 --- a/charts/kubezero-logging/charts/fluent-bit/templates/configmap-dashboards.yaml +++ b/charts/kubezero-logging/charts/fluent-bit/templates/configmap-dashboards.yaml @@ -4,7 +4,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: dashboard-{{ trimSuffix ".json" (base $path) }} + name: {{ include "fluent-bit.fullname" $ }}-dashboard-{{ trimSuffix ".json" (base $path) }} {{- with $.Values.dashboards.annotations }} annotations: {{- toYaml . | nindent 4 -}} diff --git a/charts/kubezero-logging/charts/fluent-bit/values.yaml b/charts/kubezero-logging/charts/fluent-bit/values.yaml index 38b0033..5725ccf 100644 --- a/charts/kubezero-logging/charts/fluent-bit/values.yaml +++ b/charts/kubezero-logging/charts/fluent-bit/values.yaml @@ -47,6 +47,13 @@ dnsConfig: {} # - name: ndots # value: "2" # - name: edns0 + +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "foo.local" + # - "bar.local" + securityContext: {} # capabilities: @@ -261,3 +268,8 @@ daemonSetVolumeMounts: args: [] command: [] + +initContainers: [] + # - name: do-something + # image: busybox + # command: ['do', 'something'] diff --git a/charts/kubezero-logging/charts/fluentd/Chart.yaml b/charts/kubezero-logging/charts/fluentd/Chart.yaml index 3fcba93..7a33e77 100644 --- a/charts/kubezero-logging/charts/fluentd/Chart.yaml +++ b/charts/kubezero-logging/charts/fluentd/Chart.yaml @@ -10,4 +10,4 @@ name: fluentd sources: - https://github.com/fluent/fluentd/ - https://github.com/fluent/fluentd-kubernetes-daemonset -version: 0.2.2 +version: 0.2.6 diff --git a/charts/kubezero-logging/charts/fluentd/templates/_pod.tpl b/charts/kubezero-logging/charts/fluentd/templates/_pod.tpl index 3961bb6..0cb10b0 100644 --- a/charts/kubezero-logging/charts/fluentd/templates/_pod.tpl +++ b/charts/kubezero-logging/charts/fluentd/templates/_pod.tpl @@ -44,13 +44,9 @@ containers: protocol: {{ $port.protocol }} {{- end }} livenessProbe: - httpGet: - path: /metrics - port: metrics + {{- toYaml .Values.livenessProbe | nindent 6 }} readinessProbe: - httpGet: - path: /metrics - port: metrics + {{- toYaml .Values.readinessProbe | nindent 6 }} resources: {{- toYaml .Values.resources | nindent 8 }} volumeMounts: diff --git a/charts/kubezero-logging/charts/fluentd/templates/configmap-dashboards.yaml b/charts/kubezero-logging/charts/fluentd/templates/configmap-dashboards.yaml index 862114e..ce047b0 100644 --- a/charts/kubezero-logging/charts/fluentd/templates/configmap-dashboards.yaml +++ b/charts/kubezero-logging/charts/fluentd/templates/configmap-dashboards.yaml @@ -4,6 +4,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: dashboard-{{ trimSuffix ".json" (base $path) }} + namespace: {{ $.Values.dashboards.namespace | default $.Release.Namespace }} labels: {{- include "fluentd.labels" $ | nindent 4 }} {{- range $key, $val := $.Values.dashboards.labels }} diff --git a/charts/kubezero-logging/charts/fluentd/templates/hpa.yaml b/charts/kubezero-logging/charts/fluentd/templates/hpa.yaml new file mode 100644 index 0000000..b2e8282 --- /dev/null +++ b/charts/kubezero-logging/charts/fluentd/templates/hpa.yaml @@ -0,0 +1,39 @@ +{{- if and ( eq .Values.kind "Deployment" ) .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "fluentd.fullname" . }} + labels: + {{- include "fluentd.labels" . | nindent 4 }} +spec: + {{- if .Values.autoscaling.behavior }} + behavior: + {{- toYaml .Values.autoscaling.behavior | nindent 4 }} + {{- end }} + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "fluentd.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + type: Utilization + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + type: Utilization + {{- end }} + {{- if .Values.autoscaling.customRules -}} + {{- toYaml .Values.autoscaling.customRules | nindent 4}} + {{- end -}} +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-logging/charts/fluentd/templates/prometheusrules.yaml b/charts/kubezero-logging/charts/fluentd/templates/prometheusrules.yaml index 961e6e7..26361b7 100644 --- a/charts/kubezero-logging/charts/fluentd/templates/prometheusrules.yaml +++ b/charts/kubezero-logging/charts/fluentd/templates/prometheusrules.yaml @@ -14,7 +14,8 @@ metadata: spec: {{- with .Values.metrics.prometheusRule.rules }} groups: - - name: {{ template "fluentd.fullname" . }} - rules: {{- toYaml . | nindent 4 }} + - name: {{ template "fluentd.fullname" $ }} + rules: + {{- toYaml . | nindent 4 }} {{- end }} {{- end }} diff --git a/charts/kubezero-logging/charts/fluentd/templates/servicemonitor.yaml b/charts/kubezero-logging/charts/fluentd/templates/servicemonitor.yaml index 4b542bd..521b6c5 100644 --- a/charts/kubezero-logging/charts/fluentd/templates/servicemonitor.yaml +++ b/charts/kubezero-logging/charts/fluentd/templates/servicemonitor.yaml @@ -12,6 +12,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: + jobLabel: {{ .Values.metrics.serviceMonitor.jobLabel | default .Release.Name }} endpoints: - port: metrics path: /metrics @@ -21,6 +22,14 @@ spec: {{- with .Values.metrics.serviceMonitor.scrapeTimeout }} scrapeTimeout: {{ . }} {{- end }} +{{- if .Values.metrics.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.metrics.serviceMonitor.metricRelabelings | indent 6) . }} +{{- end }} +{{- if .Values.metrics.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.metrics.serviceMonitor.relabelings | indent 6 }} +{{- end }} {{- if .Values.metrics.serviceMonitor.namespaceSelector }} namespaceSelector: {{ toYaml .Values.metrics.serviceMonitor.namespaceSelector | indent 4 -}} diff --git a/charts/kubezero-logging/charts/fluentd/values.yaml b/charts/kubezero-logging/charts/fluentd/values.yaml index a2e7b5e..06c1a27 100644 --- a/charts/kubezero-logging/charts/fluentd/values.yaml +++ b/charts/kubezero-logging/charts/fluentd/values.yaml @@ -49,6 +49,30 @@ securityContext: {} # runAsNonRoot: true # runAsUser: 1000 +# Congigure the livessProbe +# Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +livenessProbe: + httpGet: + path: /metrics + port: metrics + # initialDelaySeconds: 0 + # periodSeconds: 10 + # timeoutSeconds: 1 + # successThreshold: 1 + # failureThreshold: 3 + +# Congigure the readinessProbe +# Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +readinessProbe: + httpGet: + path: /metrics + port: metrics + # initialDelaySeconds: 0 + # periodSeconds: 10 + # timeoutSeconds: 1 + # successThreshold: 1 + # failureThreshold: 3 + resources: {} # requests: # cpu: 10m @@ -56,6 +80,33 @@ resources: {} # limits: # memory: 128Mi +## only available if kind is Deployment +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + ## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics + customRules: [] + # - type: Pods + # pods: + # metric: + # name: packets-per-second + # target: + # type: AverageValue + # averageValue: 1k + ## see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-configurable-scaling-behavior + # behavior: + # scaleDown: + # policies: + # - type: Pods + # value: 4 + # periodSeconds: 60 + # - type: Percent + # value: 10 + # periodSeconds: 60 + # priorityClassName: "system-node-critical" nodeSelector: {} @@ -138,6 +189,26 @@ metrics: release: prometheus-operator namespace: "" namespaceSelector: {} + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - sourceLabels: [__name__] + # separator: ; + # regex: ^fluentd_output_status_buffer_(oldest|newest)_.+ + # replacement: $1 + # action: drop + ## relabel configs to apply to samples after ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + ## Additional serviceMonitor config + ## + # jobLabel: fluentd # scrapeInterval: 30s # scrapeTimeout: 5s # honorLabels: true @@ -147,6 +218,24 @@ metrics: additionalLabels: {} namespace: "" rules: [] + # - alert: FluentdDown + # expr: up{job="fluentd"} == 0 + # for: 5m + # labels: + # context: fluentd + # severity: warning + # annotations: + # summary: "Fluentd Down" + # description: "{{ $labels.pod }} on {{ $labels.nodename }} is down" + # - alert: FluentdScrapeMissing + # expr: absent(up{job="fluentd"} == 1) + # for: 15m + # labels: + # context: fluentd + # severity: warning + # annotations: + # summary: "Fluentd Scrape Missing" + # description: "Fluentd instance has disappeared from Prometheus target discovery" ## Grafana Monitoring Dashboard ## diff --git a/charts/kubezero-logging/fluent-bit.patch b/charts/kubezero-logging/fluent-bit.patch index c34ce1b..40e1844 100644 --- a/charts/kubezero-logging/fluent-bit.patch +++ b/charts/kubezero-logging/fluent-bit.patch @@ -1,15 +1,3 @@ -diff -tubrN charts/fluent-bit/Chart.yaml charts/fluent-bit.zdt/Chart.yaml ---- charts/fluent-bit/Chart.yaml 2021-04-06 13:55:48.000000000 +0200 -+++ charts/fluent-bit.zdt/Chart.yaml 2021-04-07 11:49:25.955685229 +0200 -@@ -1,7 +1,7 @@ - annotations: - artifacthub.io/changes: | - - Upgrade fluent-bit image to v1.7.3 --apiVersion: v1 -+apiVersion: v2 - appVersion: 1.7.3 - description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems. - home: https://fluentbit.io/ diff -tubrN charts/fluent-bit/templates/_pod.tpl charts/fluent-bit.zdt/templates/_pod.tpl --- charts/fluent-bit/templates/_pod.tpl 2021-04-06 13:55:48.000000000 +0200 +++ charts/fluent-bit.zdt/templates/_pod.tpl 2021-04-07 11:53:45.609030378 +0200 diff --git a/charts/kubezero-logging/update.sh b/charts/kubezero-logging/update.sh index 9089e39..9c0dd23 100755 --- a/charts/kubezero-logging/update.sh +++ b/charts/kubezero-logging/update.sh @@ -1,8 +1,8 @@ #!/bin/bash -ECK_VERSION=1.5.0 -FLUENT_BIT_VERSION=0.15.4 -FLUENTD_VERSION=0.2.2 +ECK_VERSION=1.6.0 +FLUENT_BIT_VERSION=0.15.14 +FLUENTD_VERSION=0.2.6 # fix ECK crds handling to adhere to proper helm v3 support which also fixes ArgoCD applyong updates on upgrades helm repo list | grep elastic -qc || { helm repo add elastic https://helm.elastic.co; helm repo update; } diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 2269328..df80173 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -11,7 +11,7 @@ eck-operator: node-role.kubernetes.io/master: "" # Version for ElasticSearch and Kibana have to match so we define it at top-level -version: 7.11.1 +version: 7.13.2 elastic_password: "" # super_secret_elastic_password @@ -46,7 +46,7 @@ fluentd: enabled: false image: repository: fluent/fluentd-kubernetes-daemonset - tag: v1.12-debian-elasticsearch7-1 + tag: v1-debian-elasticsearch istio: enabled: false @@ -192,13 +192,16 @@ fluentd: unrecoverable_error_types ["out_of_memory_error"] bulk_message_request_threshold 2097152 - + @type file - chunk_limit_size 4MB - total_limit_size 2GB + flush_mode interval flush_thread_count 1 flush_interval 30s + + chunk_limit_size 4MB + total_limit_size 2GB + flush_at_shutdown true retry_type exponential_backoff retry_timeout 2h -- 2.40.1 From e3e39e1be7b08b53378547512c3daa23d9f99016 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 30 Jun 2021 12:34:02 +0200 Subject: [PATCH 11/45] feat: convert cert-manager to use service account tokens rather than kiam, version bump of cert-manager --- charts/kubezero-cert-manager/Chart.yaml | 8 ++++---- charts/kubezero-cert-manager/README.md | 11 ++--------- charts/kubezero-cert-manager/values.yaml | 24 ++++++++++++++++++++++-- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/charts/kubezero-cert-manager/Chart.yaml b/charts/kubezero-cert-manager/Chart.yaml index bb8ab08..8e47ceb 100644 --- a/charts/kubezero-cert-manager/Chart.yaml +++ b/charts/kubezero-cert-manager/Chart.yaml @@ -2,20 +2,20 @@ apiVersion: v2 name: kubezero-cert-manager description: KubeZero Umbrella Chart for cert-manager type: application -version: 0.5.0 +version: 0.6.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: - kubezero - cert-manager maintainers: - - name: Quarky9 + dependencies: - name: kubezero-lib version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: cert-manager - version: 1.2.0 - repository: https://charts.jetstack.io + version: 1.4.0 condition: cert-manager.enabled + repository: https://charts.jetstack.io kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-cert-manager/README.md b/charts/kubezero-cert-manager/README.md index bfab896..c985dd6 100644 --- a/charts/kubezero-cert-manager/README.md +++ b/charts/kubezero-cert-manager/README.md @@ -1,24 +1,18 @@ # kubezero-cert-manager -![Version: 0.5.0](https://img.shields.io/badge/Version-0.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.6.1](https://img.shields.io/badge/Version-0.6.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero Umbrella Chart for cert-manager **Homepage:** -## Maintainers - -| Name | Email | Url | -| ---- | ------ | --- | -| Quarky9 | | | - ## Requirements Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| -| https://charts.jetstack.io | cert-manager | 1.2.0 | +| https://charts.jetstack.io | cert-manager | 1.4.0 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## AWS - IAM Role @@ -44,7 +38,6 @@ If your resolvers need additional sercrets like CloudFlare API tokens etc. make | cert-manager.ingressShim.defaultIssuerKind | string | `"ClusterIssuer"` | | | cert-manager.ingressShim.defaultIssuerName | string | `"letsencrypt-dns-prod"` | | | cert-manager.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | -| cert-manager.podAnnotations | object | `{}` | | | cert-manager.prometheus.servicemonitor.enabled | bool | `false` | | | cert-manager.tolerations[0].effect | string | `"NoSchedule"` | | | cert-manager.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | diff --git a/charts/kubezero-cert-manager/values.yaml b/charts/kubezero-cert-manager/values.yaml index 9a7bade..3fb9601 100644 --- a/charts/kubezero-cert-manager/values.yaml +++ b/charts/kubezero-cert-manager/values.yaml @@ -23,8 +23,28 @@ cert-manager: leaderElection: namespace: "cert-manager" - podAnnotations: {} - # iam.amazonaws.com/role: "" + # On AWS enable Projected Service Accounts to assume IAM role + #extraEnv: + #- name: AWS_ROLE_ARN + # value: "" + #- name: AWS_WEB_IDENTITY_TOKEN_FILE + # value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + #- name: AWS_STS_REGIONAL_ENDPOINTS + # value: regional + + #volumes: + #- name: aws-token + # projected: + # sources: + # - serviceAccountToken: + # path: token + # expirationSeconds: 86400 + # audience: "sts.amazonaws.com" + + #volumeMounts: + #- name: aws-token + # mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" + # readOnly: true tolerations: - key: node-role.kubernetes.io/master -- 2.40.1 From f2a82fd7f9debb180f51350cf8918c2f73d5ec50 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 30 Jun 2021 12:37:05 +0200 Subject: [PATCH 12/45] feat: convert all kubezero modules to use service account tokens --- charts/kubezero/bootstrap.sh | 4 +-- .../templates/aws-ebs-csi-driver.yaml | 16 ++++++++---- .../templates/aws-efs-csi-driver.yaml | 18 ++++++------- charts/kubezero/templates/cert-manager.yaml | 23 ++++++++++++++-- charts/kubezero/templates/istio-ingress.yaml | 26 +++++++++++++++++++ 5 files changed, 69 insertions(+), 18 deletions(-) diff --git a/charts/kubezero/bootstrap.sh b/charts/kubezero/bootstrap.sh index ea45a2e..869f28f 100755 --- a/charts/kubezero/bootstrap.sh +++ b/charts/kubezero/bootstrap.sh @@ -72,8 +72,8 @@ function delete_ns() { # Extract crds via helm calls and apply delta=crds only function _crds() { - helm template $(chart_location $chart) --namespace $namespace --name-template $release --skip-crds -f $TMPDIR/values.yaml > $TMPDIR/helm-no-crds.yaml - helm template $(chart_location $chart) --namespace $namespace --name-template $release --include-crds -f $TMPDIR/values.yaml > $TMPDIR/helm-crds.yaml + helm template $(chart_location $chart) --namespace $namespace --name-template $release --skip-crds --set ${release}.installCRDs=false > $TMPDIR/helm-no-crds.yaml + helm template $(chart_location $chart) --namespace $namespace --name-template $release --include-crds --set ${release}.installCRDs=true > $TMPDIR/helm-crds.yaml diff -e $TMPDIR/helm-no-crds.yaml $TMPDIR/helm-crds.yaml | head -n-1 | tail -n+2 > $TMPDIR/crds.yaml [ -s $TMPDIR/crds.yaml ] && kubectl apply -f $TMPDIR/crds.yaml } diff --git a/charts/kubezero/templates/aws-ebs-csi-driver.yaml b/charts/kubezero/templates/aws-ebs-csi-driver.yaml index 3f404db..a797f6f 100644 --- a/charts/kubezero/templates/aws-ebs-csi-driver.yaml +++ b/charts/kubezero/templates/aws-ebs-csi-driver.yaml @@ -1,10 +1,16 @@ {{- define "aws-ebs-csi-driver-values" }} aws-ebs-csi-driver: - replicaCount: {{ ternary 2 1 .Values.HighAvailableControlplane }} - podAnnotations: - iam.amazonaws.com/role: {{ index .Values "aws-ebs-csi-driver" "IamArn" | quote }} - extraVolumeTags: - Name: {{ .Values.ClusterName }} + controller: + replicaCount: {{ ternary 2 1 .Values.HighAvailableControlplane }} + k8sTagClusterId: {{ .Values.ClusterName }} + env: + ebsPlugin: + - name: AWS_ROLE_ARN + value: {{ index .Values "aws-ebs-csi-driver" "IamArn" | quote }} + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + - name: AWS_STS_REGIONAL_ENDPOINTS + value: regional {{- end }} diff --git a/charts/kubezero/templates/aws-efs-csi-driver.yaml b/charts/kubezero/templates/aws-efs-csi-driver.yaml index a3de0f9..1f87e93 100644 --- a/charts/kubezero/templates/aws-efs-csi-driver.yaml +++ b/charts/kubezero/templates/aws-efs-csi-driver.yaml @@ -1,19 +1,19 @@ {{- define "aws-efs-csi-driver-values" }} -{{- with index .Values "aws-efs-csi-driver" "nodeSelector" }} +{{ with index .Values "aws-efs-csi-driver" "IamArn" }} aws-efs-csi-driver: - nodeSelector: - {{- toYaml . | nindent 4 }} + controller: + extraEnv: + - name: AWS_ROLE_ARN + value: "{{ . }}" + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + - name: AWS_STS_REGIONAL_ENDPOINTS + value: regional {{- end }} {{- with index .Values "aws-efs-csi-driver" "PersistentVolumes" }} PersistentVolumes: {{- toYaml . | nindent 2 }} {{- end }} -{{- if index .Values "aws-efs-csi-driver" "EfsId" }} -PersistentVolume: - create: true - EfsId: {{ index .Values "aws-efs-csi-driver" "EfsId" }} - Name: {{ default "kubezero-efs-pv" ( index .Values "aws-efs-csi-driver" "PVName" ) }} -{{- end }} {{- end }} diff --git a/charts/kubezero/templates/cert-manager.yaml b/charts/kubezero/templates/cert-manager.yaml index 40f06be..642b870 100644 --- a/charts/kubezero/templates/cert-manager.yaml +++ b/charts/kubezero/templates/cert-manager.yaml @@ -4,8 +4,27 @@ localCA: enabled: true {{ with index .Values "cert-manager" "IamArn" }} cert-manager: - podAnnotations: - iam.amazonaws.com/role: "{{ . }}" + extraEnv: + - name: AWS_ROLE_ARN + value: "{{ . }}" + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + - name: AWS_STS_REGIONAL_ENDPOINTS + value: regional + + volumes: + - name: aws-token + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 86400 + audience: "sts.amazonaws.com" + + volumeMounts: + - name: aws-token + mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" + readOnly: true {{- end }} {{- with index .Values "cert-manager" "clusterIssuer" }} diff --git a/charts/kubezero/templates/istio-ingress.yaml b/charts/kubezero/templates/istio-ingress.yaml index 0520737..ccf1621 100644 --- a/charts/kubezero/templates/istio-ingress.yaml +++ b/charts/kubezero/templates/istio-ingress.yaml @@ -16,9 +16,22 @@ istio-ingress: {{- toYaml . | nindent 6 }} {{- end }} {{- with index .Values "istio-ingress" "public" "dnsNames" }} + # Legacy dnsNames: {{- toYaml . | nindent 2 }} + + certificates: + - name: ingress-cert + dnsNames: + {{- toYaml . | nindent 4 }} {{- end }} + # New multi cert gateway + {{- range $cert := (index .Values "istio-ingress" "public" "certificates") }} + - name: {{ $cert.name }} + dnsNames: + {{- toYaml $cert.dnsNames | nindent 4 }} + {{- end }} + proxyProtocol: {{ default false (index .Values "istio-ingress" "public" "proxyProtocol") }} {{- end }} {{- if index .Values "istio-ingress" "private" }} @@ -32,9 +45,22 @@ istio-private-ingress: {{- toYaml . | nindent 6 }} {{- end }} {{- with index .Values "istio-ingress" "private" "dnsNames" }} + # Legacy dnsNames: {{- toYaml . | nindent 2 }} + + certificates: + - name: private-ingress-cert + dnsNames: + {{- toYaml . | nindent 4 }} {{- end }} + # New multi cert gateway + {{- range $cert := (index .Values "istio-ingress" "private" "certificates") }} + - name: {{ $cert.name }} + dnsNames: + {{- toYaml $cert.dnsNames | nindent 4 }} + {{- end }} + proxyProtocol: {{ default false (index .Values "istio-ingress" "private" "proxyProtocol") }} {{- end }} {{- end }} -- 2.40.1 From ff722835aba1f532c43943efe18d0bd514140d8f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 30 Jun 2021 12:47:50 +0200 Subject: [PATCH 13/45] chore: minor fix for dashboard utility script --- charts/kubezero-metrics/sync_grafana_dashboards.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-metrics/sync_grafana_dashboards.py b/charts/kubezero-metrics/sync_grafana_dashboards.py index 232c053..22fc818 100755 --- a/charts/kubezero-metrics/sync_grafana_dashboards.py +++ b/charts/kubezero-metrics/sync_grafana_dashboards.py @@ -37,7 +37,7 @@ with open(config_file, 'r') as yaml_contents: configmap = '' if 'condition' in config: # use index function to make go template happy if '-' in names - if '-' in config['condition']: + if '-' in config['condition'] and "index" not in config['condition']: tokens = config['condition'].split('.') configmap = '''{{- if index .Values %(condition)s }} ''' % {'condition': ' '.join(f'"{w}"' for w in tokens[2:])} -- 2.40.1 From dd867d8e580b48e605f81eaab3e5560d8de161d9 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 30 Jun 2021 18:26:06 +0200 Subject: [PATCH 14/45] feat: metrics version bump --- charts/kubezero-metrics/Chart.yaml | 4 +- .../charts/kube-prometheus-stack/Chart.yaml | 8 +- .../charts/grafana/Chart.yaml | 4 +- .../charts/grafana/README.md | 9 +- .../charts/grafana/templates/_helpers.tpl | 64 +- .../charts/grafana/templates/_pod.tpl | 9 + .../charts/grafana/templates/ingress.yaml | 28 +- .../charts/grafana/templates/role.yaml | 2 +- .../charts/grafana/templates/rolebinding.yaml | 2 +- .../charts/grafana/templates/secret.yaml | 2 +- .../charts/grafana/values.yaml | 26 +- .../charts/kube-state-metrics/Chart.yaml | 2 +- .../charts/kube-state-metrics/README.md | 2 +- .../templates/clusterrolebinding.yaml | 2 +- .../templates/deployment.yaml | 183 +- .../templates/psp-clusterrolebinding.yaml | 2 +- .../kube-state-metrics/templates/role.yaml | 56 +- .../templates/rolebinding.yaml | 2 +- .../kube-state-metrics/templates/service.yaml | 6 +- .../templates/serviceaccount.yaml | 2 +- .../templates/stsdiscovery-rolebinding.yaml | 2 +- .../charts/kube-state-metrics/values.yaml | 84 +- .../prometheus-node-exporter/Chart.yaml | 2 +- .../templates/monitor.yaml | 3 + .../templates/psp.yaml | 2 +- .../prometheus-node-exporter/values.yaml | 4 + .../crds/crd-alertmanagerconfigs.yaml | 2 +- .../crds/crd-alertmanagers.yaml | 2 +- .../crds/crd-podmonitors.yaml | 2 +- .../crds/crd-probes.yaml | 2 +- .../crds/crd-prometheuses.yaml | 2 +- .../crds/crd-prometheusrules.yaml | 2 +- .../crds/crd-servicemonitors.yaml | 2 +- .../crds/crd-thanosrulers.yaml | 2 +- .../templates/_helpers.tpl | 7 +- .../kube-api-server/servicemonitor.yaml | 4 +- .../kube-state-metrics/serviceMonitor.yaml | 18 + .../exporters/kubelet/servicemonitor.yaml | 2 +- .../node-exporter/servicemonitor.yaml | 5 + .../grafana/configmap-dashboards.yaml | 2 +- .../grafana/configmaps-datasources.yaml | 6 +- .../alertmanager-overview.yaml | 610 +++ .../grafana/dashboards-1.14/apiserver.yaml | 4 +- .../dashboards-1.14/cluster-total.yaml | 4 +- .../dashboards-1.14/controller-manager.yaml | 16 +- .../grafana/dashboards-1.14/etcd.yaml | 2 +- .../grafana/dashboards-1.14/k8s-coredns.yaml | 2 +- .../k8s-resources-cluster.yaml | 516 +- .../k8s-resources-namespace.yaml | 508 +- .../dashboards-1.14/k8s-resources-node.yaml | 12 +- .../dashboards-1.14/k8s-resources-pod.yaml | 707 ++- .../k8s-resources-workload.yaml | 12 +- .../k8s-resources-workloads-namespace.yaml | 12 +- .../grafana/dashboards-1.14/kubelet.yaml | 4469 ++++++++--------- .../dashboards-1.14/namespace-by-pod.yaml | 4 +- .../namespace-by-workload.yaml | 4 +- .../node-cluster-rsrc-use.yaml | 4 +- .../dashboards-1.14/node-rsrc-use.yaml | 4 +- .../grafana/dashboards-1.14/nodes.yaml | 22 +- .../persistentvolumesusage.yaml | 16 +- .../grafana/dashboards-1.14/pod-total.yaml | 4 +- .../prometheus-remote-write.yaml | 4 +- .../grafana/dashboards-1.14/prometheus.yaml | 4 +- .../grafana/dashboards-1.14/proxy.yaml | 4 +- .../grafana/dashboards-1.14/scheduler.yaml | 36 +- .../grafana/dashboards-1.14/statefulset.yaml | 4 +- .../dashboards-1.14/workload-total.yaml | 4 +- .../job-patch/job-createSecret.yaml | 6 +- .../job-patch/job-patchWebhook.yaml | 6 +- .../mutatingWebhookConfiguration.yaml | 4 +- .../validatingWebhookConfiguration.yaml | 4 +- .../prometheus-operator/deployment.yaml | 5 + .../rules-1.14/alertmanager.rules.yaml | 147 +- .../prometheus/rules-1.14/general.rules.yaml | 10 +- .../prometheus/rules-1.14/k8s.rules.yaml | 28 +- .../kube-apiserver-availability.rules.yaml | 90 +- .../rules-1.14/kube-apiserver-slos.yaml | 2 +- .../rules-1.14/kube-apiserver.rules.yaml | 150 +- .../kube-prometheus-general.rules.yaml | 2 +- .../kube-prometheus-node-recording.rules.yaml | 2 +- .../rules-1.14/kube-scheduler.rules.yaml | 2 +- .../rules-1.14/kube-state-metrics.yaml | 30 +- .../prometheus/rules-1.14/kubelet.rules.yaml | 2 +- .../rules-1.14/kubernetes-apps.yaml | 25 +- .../rules-1.14/kubernetes-resources.yaml | 2 +- .../rules-1.14/kubernetes-storage.yaml | 2 +- .../kubernetes-system-apiserver.yaml | 2 +- .../kubernetes-system-controller-manager.yaml | 2 +- .../rules-1.14/kubernetes-system-kubelet.yaml | 2 +- .../kubernetes-system-scheduler.yaml | 2 +- .../rules-1.14/kubernetes-system.yaml | 2 +- .../rules-1.14/node-exporter.rules.yaml | 34 +- .../prometheus/rules-1.14/node-exporter.yaml | 24 +- .../prometheus/rules-1.14/node-network.yaml | 5 +- .../prometheus/rules-1.14/node.rules.yaml | 2 +- .../rules-1.14/prometheus-operator.yaml | 2 +- .../prometheus/rules-1.14/prometheus.yaml | 85 +- .../prometheus/rules/alertmanager.rules.yaml | 63 - .../templates/prometheus/rules/etcd.yaml | 179 - .../prometheus/rules/general.rules.yaml | 56 - .../templates/prometheus/rules/k8s.rules.yaml | 83 - .../rules/kube-apiserver.rules.yaml | 39 - .../kube-prometheus-node-alerting.rules.yaml | 47 - .../kube-prometheus-node-recording.rules.yaml | 41 - .../rules/kube-scheduler.rules.yaml | 63 - .../prometheus/rules/kubernetes-absent.yaml | 159 - .../prometheus/rules/kubernetes-apps.yaml | 200 - .../rules/kubernetes-resources.yaml | 121 - .../prometheus/rules/kubernetes-storage.yaml | 72 - .../prometheus/rules/kubernetes-system.yaml | 184 - .../prometheus/rules/node-network.yaml | 57 - .../templates/prometheus/rules/node-time.yaml | 37 - .../prometheus/rules/node.rules.yaml | 202 - .../prometheus/rules/prometheus-operator.yaml | 49 - .../prometheus/rules/prometheus.rules.yaml | 139 - .../charts/kube-prometheus-stack/values.yaml | 61 +- charts/kubezero-metrics/update.sh | 2 +- 117 files changed, 5304 insertions(+), 4791 deletions(-) create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/alertmanager-overview.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/alertmanager.rules.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/etcd.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/general.rules.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/k8s.rules.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-apiserver.rules.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-prometheus-node-alerting.rules.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-prometheus-node-recording.rules.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-scheduler.rules.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-absent.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-apps.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-resources.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-storage.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-system.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/node-network.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/node-time.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/node.rules.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/prometheus-operator.yaml delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/prometheus.rules.yaml diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index 78c5595..0c2a44c 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -16,11 +16,11 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: kube-prometheus-stack - version: 16.1.2 + version: 16.12.0 # Switch back to upstream once all alerts are fixed eg. etcd gpcr # repository: https://prometheus-community.github.io/helm-charts - name: prometheus-adapter - version: 2.13 + version: 2.14.2 repository: https://prometheus-community.github.io/helm-charts condition: prometheus-adapter.enabled kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml index ba03554..fdad40a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml @@ -6,12 +6,12 @@ annotations: url: https://github.com/prometheus-operator/kube-prometheus artifacthub.io/operator: "true" apiVersion: v2 -appVersion: 0.48.0 +appVersion: 0.48.1 dependencies: - condition: kubeStateMetrics.enabled name: kube-state-metrics repository: https://prometheus-community.github.io/helm-charts - version: 3.1.* + version: 3.3.* - condition: nodeExporter.enabled name: prometheus-node-exporter repository: https://prometheus-community.github.io/helm-charts @@ -19,7 +19,7 @@ dependencies: - condition: grafana.enabled name: grafana repository: https://grafana.github.io/helm-charts - version: 6.9.* + version: 6.13.* description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus @@ -47,4 +47,4 @@ sources: - https://github.com/prometheus-community/helm-charts - https://github.com/prometheus-operator/kube-prometheus type: application -version: 16.1.2 +version: 16.12.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml index 899ba61..5f0c749 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 7.5.5 +appVersion: 8.0.3 description: The leading tool for querying and visualizing time series and metrics. home: https://grafana.net icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png @@ -19,4 +19,4 @@ name: grafana sources: - https://github.com/grafana/grafana type: application -version: 6.9.1 +version: 6.13.5 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md index 8719100..7bbf61e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md @@ -59,8 +59,8 @@ This version requires Helm >= 3.1.0. | `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | | `priorityClassName` | Name of Priority Class to assign pods | `nil` | | `image.repository` | Image repository | `grafana/grafana` | -| `image.tag` | Image tag (`Must be >= 5.0.0`) | `7.4.5` | -| `image.sha` | Image sha (optional) | `2b56f6106ddc376bb46d974230d530754bf65a640dfbc5245191d72d3b49efc6` | +| `image.tag` | Image tag (`Must be >= 5.0.0`) | `8.0.3` | +| `image.sha` | Image sha (optional) | `80c6d6ac633ba5ab3f722976fb1d9a138f87ca6a9934fcd26a5fc28cbde7dbfa` | | `image.pullPolicy` | Image pull policy | `IfNotPresent` | | `image.pullSecrets` | Image pull secrets | `{}` | | `service.enabled` | Enable grafana service | `true` | @@ -83,7 +83,7 @@ This version requires Helm >= 3.1.0. | `ingress.path` | Ingress accepted path | `/` | | `ingress.pathType` | Ingress type of path | `Prefix` | | `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` | -| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-alb-ingress-controller/guide/ingress/annotation/#actions). | `[]` | +| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-alb-ingress-controller/guide/ingress/annotation/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]` | | `ingress.tls` | Ingress TLS configuration | `[]` | | `resources` | CPU/Memory resource requests/limits | `{}` | | `nodeSelector` | Node labels for pod assignment | `{}` | @@ -116,6 +116,7 @@ This version requires Helm >= 3.1.0. | `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. | `{}` | | `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | | `envRenderSecret` | Sensible environment variables passed to pods and stored as secret | `{}` | +| `enableServiceLinks` | Inject Kubernetes services as environment variables. | `true` | | `extraSecretMounts` | Additional grafana server secret mounts | `[]` | | `extraVolumeMounts` | Additional grafana server volume mounts | `[]` | | `extraConfigmapMounts` | Additional grafana server configMap volume mounts | `[]` | @@ -136,7 +137,7 @@ This version requires Helm >= 3.1.0. | `podLabels` | Pod labels | `{}` | | `podPortName` | Name of the grafana port on the pod | `grafana` | | `sidecar.image.repository` | Sidecar image repository | `quay.io/kiwigrid/k8s-sidecar` | -| `sidecar.image.tag` | Sidecar image tag | `1.10.7` | +| `sidecar.image.tag` | Sidecar image tag | `1.12.2` | | `sidecar.image.sha` | Sidecar image sha (optional) | `""` | | `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | | `sidecar.resources` | Sidecar resources | `{}` | diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl index 70e0558..868aa07 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl @@ -104,13 +104,67 @@ app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} +{{/* +Looks if there's an existing secret and reuse its password. If not it generates +new password and use it. +*/}} +{{- define "grafana.password" -}} +{{- $secret := (lookup "v1" "Secret" (include "grafana.namespace" .) (include "grafana.fullname" .) ) -}} + {{- if $secret -}} + {{- index $secret "data" "admin-password" -}} + {{- else -}} + {{- (randAlphaNum 40) | b64enc | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Get KubeVersion removing pre-release information. +*/}} +{{- define "grafana.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version (regexFind "v[0-9]+\\.[0-9]+\\.[0-9]+" .Capabilities.KubeVersion.Version) -}} +{{- end -}} + {{/* Return the appropriate apiVersion for rbac. */}} -{{- define "rbac.apiVersion" -}} -{{- if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} -{{- print "rbac.authorization.k8s.io/v1" -}} -{{- else -}} -{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- define "grafana.rbac.apiVersion" -}} + {{- if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} + {{- print "rbac.authorization.k8s.io/v1" -}} + {{- else -}} + {{- print "rbac.authorization.k8s.io/v1beta1" -}} + {{- end -}} {{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "grafana.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19.x" (include "grafana.kubeVersion" .)) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return if ingress is stable. +*/}} +{{- define "grafana.ingress.isStable" -}} + {{- eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "grafana.ingress.supportsIngressClassName" -}} + {{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18.x" (include "grafana.kubeVersion" .))) -}} +{{- end -}} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "grafana.ingress.supportsPathType" -}} + {{- or (eq (include "grafana.ingress.isStable" .) "true") (and (eq (include "grafana.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18.x" (include "grafana.kubeVersion" .))) -}} {{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl index d74ee4d..3a74e9e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl @@ -159,6 +159,7 @@ imagePullSecrets: - name: {{ . }} {{- end}} {{- end }} +enableServiceLinks: {{ .Values.enableServiceLinks }} containers: {{- if .Values.sidecar.dashboards.enabled }} - name: {{ template "grafana.name" . }}-sc-dashboard @@ -357,6 +358,14 @@ containers: - name: GF_RENDERING_CALLBACK_URL value: http://{{ template "grafana.fullname" . }}.{{ template "grafana.namespace" . }}:{{ .Values.service.port }}/{{ .Values.imageRenderer.grafanaSubPath }} {{ end }} + - name: GF_PATHS_DATA + value: {{ (get .Values "grafana.ini").paths.data }} + - name: GF_PATHS_LOGS + value: {{ (get .Values "grafana.ini").paths.logs }} + - name: GF_PATHS_PLUGINS + value: {{ (get .Values "grafana.ini").paths.plugins }} + - name: GF_PATHS_PROVISIONING + value: {{ (get .Values "grafana.ini").paths.provisioning }} {{- range $key, $value := .Values.envValueFrom }} - name: {{ $key | quote }} valueFrom: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/ingress.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/ingress.yaml index 44ebfc9..7699cec 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/ingress.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/ingress.yaml @@ -1,17 +1,13 @@ {{- if .Values.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "grafana.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "grafana.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "grafana.ingress.supportsPathType" .) "true" -}} {{- $fullName := include "grafana.fullname" . -}} {{- $servicePort := .Values.service.port -}} {{- $ingressPath := .Values.ingress.path -}} {{- $ingressPathType := .Values.ingress.pathType -}} {{- $extraPaths := .Values.ingress.extraPaths -}} -{{- $newAPI := .Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" -}} -{{- if $newAPI -}} -apiVersion: networking.k8s.io/v1 -{{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1/Ingress" }} -apiVersion: networking.k8s.io/v1beta1 -{{- else }} -apiVersion: extensions/v1beta1 -{{- end }} +apiVersion: {{ include "grafana.ingress.apiVersion" . }} kind: Ingress metadata: name: {{ $fullName }} @@ -28,7 +24,7 @@ metadata: {{- end }} {{- end }} spec: - {{- if .Values.ingress.ingressClassName }} + {{- if and $ingressSupportsIngressClassName .Values.ingress.ingressClassName }} ingressClassName: {{ .Values.ingress.ingressClassName }} {{- end -}} {{- if .Values.ingress.tls }} @@ -41,15 +37,15 @@ spec: - host: {{ tpl . $}} http: paths: -{{ if $extraPaths }} +{{- if $extraPaths }} {{ toYaml $extraPaths | indent 10 }} {{- end }} - path: {{ $ingressPath }} - {{- if $newAPI }} + {{- if $ingressSupportsPathType }} pathType: {{ $ingressPathType }} {{- end }} backend: - {{- if $newAPI }} + {{- if $ingressApiIsStable }} service: name: {{ $fullName }} port: @@ -63,18 +59,20 @@ spec: - http: paths: - backend: - {{- if $newAPI }} + {{- if $ingressApiIsStable }} service: name: {{ $fullName }} port: number: {{ $servicePort }} - pathType: {{ $ingressPathType }} {{- else }} serviceName: {{ $fullName }} servicePort: {{ $servicePort }} {{- end }} - {{- if $ingressPath }} + {{- if $ingressPath }} path: {{ $ingressPath }} {{- end }} + {{- if $ingressSupportsPathType }} + pathType: {{ $ingressPathType }} + {{- end }} {{- end -}} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/role.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/role.yaml index 54c3fb0..6a1890f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/role.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/role.yaml @@ -1,5 +1,5 @@ {{- if and .Values.rbac.create (not .Values.rbac.useExistingRole) -}} -apiVersion: {{ template "rbac.apiVersion" . }} +apiVersion: {{ template "grafana.rbac.apiVersion" . }} kind: Role metadata: name: {{ template "grafana.fullname" . }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/rolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/rolebinding.yaml index 34f1ad6..e010725 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/rolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/rolebinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.rbac.create -}} -apiVersion: {{ template "rbac.apiVersion" . }} +apiVersion: {{ template "grafana.rbac.apiVersion" . }} kind: RoleBinding metadata: name: {{ template "grafana.fullname" . }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml index ac804e2..6d06cf5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml @@ -17,7 +17,7 @@ data: {{- if .Values.adminPassword }} admin-password: {{ .Values.adminPassword | b64enc | quote }} {{- else }} - admin-password: {{ randAlphaNum 40 | b64enc | quote }} + admin-password: {{ template "grafana.password" . }} {{- end }} {{- end }} {{- if not .Values.ldap.existingSecret }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml index a81db06..f4bc7c4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml @@ -69,7 +69,7 @@ livenessProbe: image: repository: grafana/grafana - tag: 7.5.5 + tag: 8.0.3 sha: "" pullPolicy: IfNotPresent @@ -188,7 +188,7 @@ ingress: labels: {} path: / - # pathType is only for k8s > 1.19 + # pathType is only for k8s >= 1.1= pathType: Prefix hosts: @@ -382,6 +382,10 @@ envFromSecret: "" ## This can be useful for auth tokens, etc envRenderSecret: {} +# Inject Kubernetes services as environment variables. +# See https://kubernetes.io/docs/concepts/services-networking/connect-applications-service/#environment-variables +enableServiceLinks: true + ## Additional grafana server secret mounts # Defines additional mounts with secrets. Secrets must be manually created in the namespace. extraSecretMounts: [] @@ -446,13 +450,13 @@ datasources: {} # access: proxy # isDefault: true # - name: CloudWatch -# type: cloudwatch -# access: proxy -# uid: cloudwatch -# editable: false -# jsonData: -# authType: credentials -# defaultRegion: us-east-1 +# type: cloudwatch +# access: proxy +# uid: cloudwatch +# editable: false +# jsonData: +# authType: default +# defaultRegion: us-east-1 ## Configure notifiers ## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels @@ -532,7 +536,7 @@ dashboardsConfigMaps: {} ## grafana.ini: paths: - data: /var/lib/grafana/data + data: /var/lib/grafana/ logs: /var/log/grafana plugins: /var/lib/grafana/plugins provisioning: /etc/grafana/provisioning @@ -603,7 +607,7 @@ smtp: sidecar: image: repository: quay.io/kiwigrid/k8s-sidecar - tag: 1.10.7 + tag: 1.12.2 sha: "" imagePullPolicy: IfNotPresent resources: {} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml index 02394f6..759f5af 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml @@ -16,4 +16,4 @@ name: kube-state-metrics sources: - https://github.com/kubernetes/kube-state-metrics/ type: application -version: 3.1.0 +version: 3.3.1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md index 531ac17..7c2e169 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md @@ -62,7 +62,7 @@ The upgraded chart now the following changes: See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: ```console -helm show values kube-state-metrics/kube-state-metrics +helm show values prometheus-community/kube-state-metrics ``` You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options. diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrolebinding.yaml index af158c5..223c285 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrolebinding.yaml @@ -18,6 +18,6 @@ roleRef: {{- end }} subjects: - kind: ServiceAccount - name: {{ template "kube-state-metrics.fullname" . }} + name: {{ template "kube-state-metrics.serviceAccountName" . }} namespace: {{ template "kube-state-metrics.namespace" . }} {{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml index 090d880..a4c4c0b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml @@ -13,30 +13,30 @@ metadata: app.kubernetes.io/instance: "{{ .Release.Name }}" app.kubernetes.io/managed-by: "{{ .Release.Service }}" app.kubernetes.io/version: "{{ .Chart.AppVersion }}" -{{- if .Values.customLabels }} + {{- if .Values.customLabels }} {{ toYaml .Values.customLabels | indent 4 }} -{{- end }} + {{- end }} spec: selector: matchLabels: app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} replicas: {{ .Values.replicas }} -{{- if .Values.autosharding.enabled }} + {{- if .Values.autosharding.enabled }} serviceName: {{ template "kube-state-metrics.fullname" . }} volumeClaimTemplates: [] -{{- end }} + {{- end }} template: metadata: labels: app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} app.kubernetes.io/instance: "{{ .Release.Name }}" -{{- if .Values.customLabels }} + {{- if .Values.customLabels }} {{ toYaml .Values.customLabels | indent 8 }} -{{- end }} -{{- if .Values.podAnnotations }} + {{- end }} + {{- if .Values.podAnnotations }} annotations: {{ toYaml .Values.podAnnotations | indent 8 }} -{{- end }} + {{- end }} spec: hostNetwork: {{ .Values.hostNetwork }} serviceAccountName: {{ template "kube-state-metrics.serviceAccountName" . }} @@ -51,7 +51,7 @@ spec: {{- end }} containers: - name: {{ .Chart.Name }} -{{- if .Values.autosharding.enabled }} + {{- if .Values.autosharding.enabled }} env: - name: POD_NAME valueFrom: @@ -61,159 +61,94 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace -{{- end }} + {{- end }} args: -{{ if .Values.extraArgs }} + {{- if .Values.extraArgs }} {{- range .Values.extraArgs }} - {{ . }} {{- end }} -{{ end }} -{{ if .Values.collectors.certificatesigningrequests }} - - --resources=certificatesigningrequests -{{ end }} -{{ if .Values.collectors.configmaps }} - - --resources=configmaps -{{ end }} -{{ if .Values.collectors.cronjobs }} - - --resources=cronjobs -{{ end }} -{{ if .Values.collectors.daemonsets }} - - --resources=daemonsets -{{ end }} -{{ if .Values.collectors.deployments }} - - --resources=deployments -{{ end }} -{{ if .Values.collectors.endpoints }} - - --resources=endpoints -{{ end }} -{{ if .Values.collectors.horizontalpodautoscalers }} - - --resources=horizontalpodautoscalers -{{ end }} -{{ if .Values.collectors.ingresses }} - - --resources=ingresses -{{ end }} -{{ if .Values.collectors.jobs }} - - --resources=jobs -{{ end }} -{{ if .Values.collectors.limitranges }} - - --resources=limitranges -{{ end }} -{{ if .Values.collectors.mutatingwebhookconfigurations }} - - --resources=mutatingwebhookconfigurations -{{ end }} -{{ if .Values.collectors.namespaces }} - - --resources=namespaces -{{ end }} -{{ if .Values.collectors.networkpolicies }} - - --resources=networkpolicies -{{ end }} -{{ if .Values.collectors.nodes }} - - --resources=nodes -{{ end }} -{{ if .Values.collectors.persistentvolumeclaims }} - - --resources=persistentvolumeclaims -{{ end }} -{{ if .Values.collectors.persistentvolumes }} - - --resources=persistentvolumes -{{ end }} -{{ if .Values.collectors.poddisruptionbudgets }} - - --resources=poddisruptionbudgets -{{ end }} -{{ if .Values.collectors.pods }} - - --resources=pods -{{ end }} -{{ if .Values.collectors.replicasets }} - - --resources=replicasets -{{ end }} -{{ if .Values.collectors.replicationcontrollers }} - - --resources=replicationcontrollers -{{ end }} -{{ if .Values.collectors.resourcequotas }} - - --resources=resourcequotas -{{ end }} -{{ if .Values.collectors.secrets }} - - --resources=secrets -{{ end }} -{{ if .Values.collectors.services }} - - --resources=services -{{ end }} -{{ if .Values.collectors.statefulsets }} - - --resources=statefulsets -{{ end }} -{{ if .Values.collectors.storageclasses }} - - --resources=storageclasses -{{ end }} -{{ if .Values.collectors.validatingwebhookconfigurations }} - - --resources=validatingwebhookconfigurations -{{ end }} -{{ if .Values.collectors.verticalpodautoscalers }} - - --resources=verticalpodautoscalers -{{ end }} -{{ if .Values.collectors.volumeattachments }} - - --resources=volumeattachments -{{ end }} -{{ if .Values.namespaces }} - - --namespaces={{ tpl .Values.namespaces $ | join "," }} -{{ end }} -{{ if .Values.autosharding.enabled }} + {{- end }} + {{- if .Values.service.port }} + - --port={{ .Values.service.port | default 8080}} + {{- end }} + {{- if .Values.collectors }} + - --resources={{ .Values.collectors | join "," }} + {{- end }} + {{- if .Values.metricLabelsAllowlist }} + - --metric-labels-allowlist={{ .Values.metricLabelsAllowlist | join "," }} + {{- end }} + {{- if .Values.metricAllowlist }} + - --metric-allowlist={{ .Values.metricAllowlist | join "," }} + {{- end }} + {{- if .Values.metricDenylist }} + - --metric-denylist={{ .Values.metricDenylist | join "," }} + {{- end }} + {{- if .Values.namespaces }} + - --namespaces={{ tpl (.Values.namespaces | join ",") $ }} + {{- end }} + {{- if .Values.autosharding.enabled }} - --pod=$(POD_NAME) - --pod-namespace=$(POD_NAMESPACE) -{{ end }} -{{ if .Values.kubeconfig.enabled }} + {{- end }} + {{- if .Values.kubeconfig.enabled }} - --kubeconfig=/opt/k8s/.kube/config -{{ end }} -{{ if .Values.selfMonitor.telemetryHost }} + {{- end }} + {{- if .Values.selfMonitor.telemetryHost }} - --telemetry-host={{ .Values.selfMonitor.telemetryHost }} -{{ end }} - - --telemetry-port=8081 -{{- if .Values.kubeconfig.enabled }} + {{- end }} + - --telemetry-port={{ .Values.selfMonitor.telemetryPort | default 8081 }} + {{- if .Values.kubeconfig.enabled }} volumeMounts: - name: kubeconfig mountPath: /opt/k8s/.kube/ readOnly: true -{{- end }} + {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" ports: - - containerPort: 8080 -{{- if .Values.selfMonitor.enabled }} - - containerPort: 8081 -{{- end }} + - containerPort: {{ .Values.service.port | default 8080}} + {{- if .Values.selfMonitor.enabled }} + - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + {{- end }} livenessProbe: httpGet: path: /healthz - port: 8080 + port: {{ .Values.service.port | default 8080}} initialDelaySeconds: 5 timeoutSeconds: 5 readinessProbe: httpGet: path: / - port: 8080 + port: {{ .Values.service.port | default 8080}} initialDelaySeconds: 5 timeoutSeconds: 5 -{{- if .Values.resources }} + {{- if .Values.resources }} resources: {{ toYaml .Values.resources | indent 10 }} {{- end }} +{{- if .Values.containerSecurityContext }} + securityContext: +{{ toYaml .Values.containerSecurityContext | indent 10 }} +{{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.imagePullSecrets | indent 8 }} -{{- end }} -{{- if .Values.affinity }} + {{- end }} + {{- if .Values.affinity }} affinity: {{ toYaml .Values.affinity | indent 8 }} -{{- end }} -{{- if .Values.nodeSelector }} + {{- end }} + {{- if .Values.nodeSelector }} nodeSelector: {{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} -{{- if .Values.tolerations }} + {{- end }} + {{- if .Values.tolerations }} tolerations: {{ toYaml .Values.tolerations | indent 8 }} -{{- end }} -{{- if .Values.kubeconfig.enabled}} + {{- end }} + {{- if .Values.kubeconfig.enabled}} volumes: - name: kubeconfig secret: secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig -{{- end }} + {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml index feb97f2..b96503f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml @@ -14,6 +14,6 @@ roleRef: name: psp-{{ template "kube-state-metrics.fullname" . }} subjects: - kind: ServiceAccount - name: {{ template "kube-state-metrics.fullname" . }} + name: {{ template "kube-state-metrics.serviceAccountName" . }} namespace: {{ template "kube-state-metrics.namespace" . }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml index 25c8bc8..8643652 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml @@ -18,169 +18,169 @@ metadata: namespace: {{ . }} {{- end }} rules: -{{ if $.Values.collectors.certificatesigningrequests }} +{{ if has "certificatesigningrequests" $.Values.collectors }} - apiGroups: ["certificates.k8s.io"] resources: - certificatesigningrequests verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.configmaps }} +{{ if has "configmaps" $.Values.collectors }} - apiGroups: [""] resources: - configmaps verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.cronjobs }} +{{ if has "cronjobs" $.Values.collectors }} - apiGroups: ["batch"] resources: - cronjobs verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.daemonsets }} +{{ if has "daemonsets" $.Values.collectors }} - apiGroups: ["extensions", "apps"] resources: - daemonsets verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.deployments }} +{{ if has "deployments" $.Values.collectors }} - apiGroups: ["extensions", "apps"] resources: - deployments verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.endpoints }} +{{ if has "endpoints" $.Values.collectors }} - apiGroups: [""] resources: - endpoints verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.horizontalpodautoscalers }} +{{ if has "horizontalpodautoscalers" $.Values.collectors }} - apiGroups: ["autoscaling"] resources: - horizontalpodautoscalers verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.ingresses }} +{{ if has "ingresses" $.Values.collectors }} - apiGroups: ["extensions", "networking.k8s.io"] resources: - ingresses verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.jobs }} +{{ if has "jobs" $.Values.collectors }} - apiGroups: ["batch"] resources: - jobs verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.limitranges }} +{{ if has "limitranges" $.Values.collectors }} - apiGroups: [""] resources: - limitranges verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.mutatingwebhookconfigurations }} +{{ if has "mutatingwebhookconfigurations" $.Values.collectors }} - apiGroups: ["admissionregistration.k8s.io"] resources: - mutatingwebhookconfigurations verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.namespaces }} +{{ if has "namespaces" $.Values.collectors }} - apiGroups: [""] resources: - namespaces verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.networkpolicies }} +{{ if has "networkpolicies" $.Values.collectors }} - apiGroups: ["networking.k8s.io"] resources: - networkpolicies verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.nodes }} +{{ if has "nodes" $.Values.collectors }} - apiGroups: [""] resources: - nodes verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.persistentvolumeclaims }} +{{ if has "persistentvolumeclaims" $.Values.collectors }} - apiGroups: [""] resources: - persistentvolumeclaims verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.persistentvolumes }} +{{ if has "persistentvolumes" $.Values.collectors }} - apiGroups: [""] resources: - persistentvolumes verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.poddisruptionbudgets }} +{{ if has "poddisruptionbudgets" $.Values.collectors }} - apiGroups: ["policy"] resources: - poddisruptionbudgets verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.pods }} +{{ if has "pods" $.Values.collectors }} - apiGroups: [""] resources: - pods verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.replicasets }} +{{ if has "replicasets" $.Values.collectors }} - apiGroups: ["extensions", "apps"] resources: - replicasets verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.replicationcontrollers }} +{{ if has "replicationcontrollers" $.Values.collectors }} - apiGroups: [""] resources: - replicationcontrollers verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.resourcequotas }} +{{ if has "resourcequotas" $.Values.collectors }} - apiGroups: [""] resources: - resourcequotas verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.secrets }} +{{ if has "secrets" $.Values.collectors }} - apiGroups: [""] resources: - secrets verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.services }} +{{ if has "services" $.Values.collectors }} - apiGroups: [""] resources: - services verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.statefulsets }} +{{ if has "statefulsets" $.Values.collectors }} - apiGroups: ["apps"] resources: - statefulsets verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.storageclasses }} +{{ if has "storageclasses" $.Values.collectors }} - apiGroups: ["storage.k8s.io"] resources: - storageclasses verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.validatingwebhookconfigurations }} +{{ if has "validatingwebhookconfigurations" $.Values.collectors }} - apiGroups: ["admissionregistration.k8s.io"] resources: - validatingwebhookconfigurations verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.volumeattachments }} +{{ if has "volumeattachments" $.Values.collectors }} - apiGroups: ["storage.k8s.io"] resources: - volumeattachments verbs: ["list", "watch"] {{ end -}} -{{ if $.Values.collectors.verticalpodautoscalers }} +{{ if has "verticalpodautoscalers" $.Values.collectors }} - apiGroups: ["autoscaling.k8s.io"] resources: - verticalpodautoscalers diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml index 72a1a2e..6809e2b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml @@ -21,7 +21,7 @@ roleRef: {{- end }} subjects: - kind: ServiceAccount - name: {{ template "kube-state-metrics.fullname" $ }} + name: {{ template "kube-state-metrics.serviceAccountName" $ }} namespace: {{ template "kube-state-metrics.namespace" $ }} {{- end -}} {{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/service.yaml index 4f8e4a4..f03e6d6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/service.yaml @@ -23,16 +23,16 @@ spec: ports: - name: "http" protocol: TCP - port: {{ .Values.service.port }} + port: {{ .Values.service.port | default 8080}} {{- if .Values.service.nodePort }} nodePort: {{ .Values.service.nodePort }} {{- end }} - targetPort: 8080 + targetPort: {{ .Values.service.port | default 8080}} {{ if .Values.selfMonitor.enabled }} - name: "metrics" protocol: TCP port: {{ .Values.selfMonitor.telemetryPort | default 8081 }} - targetPort: 8081 + targetPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }} {{ end }} {{- if .Values.service.loadBalancerIP }} loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/serviceaccount.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/serviceaccount.yaml index 2e8a1ee..365ccec 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/serviceaccount.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/serviceaccount.yaml @@ -7,7 +7,7 @@ metadata: helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} - name: {{ template "kube-state-metrics.fullname" . }} + name: {{ template "kube-state-metrics.serviceAccountName" . }} namespace: {{ template "kube-state-metrics.namespace" . }} {{- if .Values.serviceAccount.annotations }} annotations: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml index 6a2e5bf..2e43277 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml @@ -15,6 +15,6 @@ roleRef: name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} subjects: - kind: ServiceAccount - name: {{ template "kube-state-metrics.fullname" . }} + name: {{ template "kube-state-metrics.serviceAccountName" . }} namespace: {{ template "kube-state-metrics.namespace" . }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml index 62e178d..a9853ba 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml @@ -88,6 +88,11 @@ securityContext: runAsUser: 65534 fsGroup: 65534 +## Specify security settings for a Container +## Allows overrides and additional options compared to (Pod) securityContext +## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +containerSecurityContext: {} + ## Node labels for pod assignment ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ nodeSelector: {} @@ -109,37 +114,56 @@ podAnnotations: {} # Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ podDisruptionBudget: {} -# Available collectors for kube-state-metrics. By default all available -# resources are enabled. +# Comma-separated list of metrics to be exposed. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricAllowlist: [] + +# Comma-separated list of metrics not to be enabled. +# This list comprises of exact metric names and/or regex patterns. +# The allowlist and denylist are mutually exclusive. +metricDenylist: [] + +# Comma-separated list of additional Kubernetes label keys that will be used in the resource's +# labels metric. By default the metric contains only name and namespace labels. +# To include additional labels, provide a list of resource names in their plural form and Kubernetes +# label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. +# A single '*' can be provided per resource instead to allow any labels, but that has +# severe performance implications (Example: '=pods=[*]'). +metricLabelsAllowlist: [] + # - namespaces=[k8s-label-1,k8s-label-n] + +# Available collectors for kube-state-metrics. +# By default, all available resources are enabled, comment out to disable. collectors: - certificatesigningrequests: true - configmaps: true - cronjobs: true - daemonsets: true - deployments: true - endpoints: true - horizontalpodautoscalers: true - ingresses: true - jobs: true - limitranges: true - mutatingwebhookconfigurations: true - namespaces: true - networkpolicies: true - nodes: true - persistentvolumeclaims: true - persistentvolumes: true - poddisruptionbudgets: true - pods: true - replicasets: true - replicationcontrollers: true - resourcequotas: true - secrets: true - services: true - statefulsets: true - storageclasses: true - validatingwebhookconfigurations: true - verticalpodautoscalers: false - volumeattachments: true + - certificatesigningrequests + - configmaps + - cronjobs + - daemonsets + - deployments + - endpoints + - horizontalpodautoscalers + - ingresses + - jobs + - limitranges + - mutatingwebhookconfigurations + - namespaces + - networkpolicies + - nodes + - persistentvolumeclaims + - persistentvolumes + - poddisruptionbudgets + - pods + - replicasets + - replicationcontrollers + - resourcequotas + - secrets + - services + - statefulsets + - storageclasses + - validatingwebhookconfigurations + - verticalpodautoscalers + - volumeattachments # Enabling kubeconfig will pass the --kubeconfig argument to the container kubeconfig: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml index a9c26d7..11152a7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml @@ -14,4 +14,4 @@ maintainers: name: prometheus-node-exporter sources: - https://github.com/prometheus/node_exporter/ -version: 1.18.0 +version: 1.18.2 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml index 2f7b6ae..88f9015 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml @@ -22,6 +22,9 @@ spec: {{- if $.Values.prometheus.monitor.tlsConfig }} tlsConfig: {{ toYaml $.Values.prometheus.monitor.tlsConfig | nindent 8 }} {{- end }} + {{- if .Values.prometheus.monitor.proxyUrl }} + proxyUrl: {{ .Values.prometheus.monitor.proxyUrl}} + {{- end }} {{- if .Values.prometheus.monitor.scrapeTimeout }} scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/psp.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/psp.yaml index 34afb14..ec1259e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/psp.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/psp.yaml @@ -8,7 +8,7 @@ metadata: labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} {{- if .Values.rbac.pspAnnotations }} annotations: - {{ toYaml .Values.rbac.pspAnnotations | indent 4 }} +{{ toYaml .Values.rbac.pspAnnotations | indent 4 }} {{- end}} spec: privileged: false diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml index 400ce11..f4e927f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml @@ -24,6 +24,10 @@ prometheus: bearerTokenFile: tlsConfig: {} + ## proxyUrl: URL of a proxy that should be used for scraping. + ## + proxyUrl: "" + relabelings: [] scrapeTimeout: 10s diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml index a6988a7..e84eb6b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml index 486ee7e..76f90f7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml index c16e955..745b866 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml index 4b3f92c..ca2ef5b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.1/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml index cea3585..ce033a6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml index 1c33519..8e625d6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml index 1c8a4a3..3bf6572 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml index 9112650..faf36d2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.48.1/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl index 9ee060b..61989b5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl @@ -98,7 +98,12 @@ Allow the release namespace to be overridden for multi-namespace deployments in {{/* Allow KubeVersion to be overridden. */}} {{- define "kube-prometheus-stack.ingress.kubeVersion" -}} - {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} + {{- $kubeVersion := default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} + {{/* Special use case for Amazon EKS, Google GKE */}} + {{- if and (regexMatch "\\d+\\.\\d+\\.\\d+-(?:eks|gke).+" $kubeVersion) (not .Values.kubeVersionOverride) -}} + {{- $kubeVersion = regexFind "\\d+\\.\\d+\\.\\d+" $kubeVersion -}} + {{- end -}} + {{- $kubeVersion -}} {{- end -}} {{/* Get Ingress API Version */}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-api-server/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-api-server/servicemonitor.yaml index f34c187..33a57b8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-api-server/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-api-server/servicemonitor.yaml @@ -22,9 +22,9 @@ spec: metricRelabelings: {{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.metricRelabelings | indent 6) . }} {{- end }} -{{- if .Values.kubeApiServer.relabelings }} +{{- if .Values.kubeApiServer.serviceMonitor.relabelings }} relabelings: -{{ toYaml .Values.kubeApiServer.relabelings | indent 6 }} +{{ toYaml .Values.kubeApiServer.serviceMonitor.relabelings | indent 6 }} {{- end }} tlsConfig: caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml index caeaa1e..8dead5f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml @@ -26,6 +26,24 @@ spec: relabelings: {{ toYaml .Values.kubeStateMetrics.serviceMonitor.relabelings | indent 4 }} {{- end }} +{{- if .Values.kubeStateMetrics.serviceMonitor.selfMonitor.enabled }} + - port: metrics + {{- if .Values.kubeStateMetrics.serviceMonitor.interval }} + interval: {{ .Values.kubeStateMetrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubeStateMetrics.serviceMonitor.proxyUrl }} + proxyUrl: {{ .Values.kubeStateMetrics.serviceMonitor.proxyUrl}} + {{- end }} + honorLabels: true +{{- if .Values.kubeStateMetrics.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeStateMetrics.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeStateMetrics.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.kubeStateMetrics.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} {{- if .Values.kubeStateMetrics.serviceMonitor.namespaceOverride }} namespaceSelector: matchNames: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml index 549b3ff..9933ecb 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml @@ -168,6 +168,6 @@ spec: - {{ .Values.kubelet.namespace }} selector: matchLabels: - app.kubernetes.io/managed-by: prometheus-operator + app.kubernetes.io/name: kubelet k8s-app: kubelet {{- end}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml index 2061b98..5615ba7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/node-exporter/servicemonitor.yaml @@ -13,6 +13,11 @@ spec: matchLabels: app: prometheus-node-exporter release: {{ $.Release.Name }} + {{- if (index .Values "prometheus-node-exporter" "namespaceOverride") }} + namespaceSelector: + matchNames: + - {{ index .Values "prometheus-node-exporter" "namespaceOverride" }} + {{- end }} endpoints: - port: metrics {{- if .Values.nodeExporter.serviceMonitor.interval }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmap-dashboards.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmap-dashboards.yaml index 7ce2a7f..ce5dd06 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmap-dashboards.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmap-dashboards.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if or (and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled) .Values.grafana.forceDeployDashboards }} {{- $files := .Files.Glob "dashboards-1.14/*.json" }} {{- if $files }} apiVersion: v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml index db62d53..c7c82ef 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled }} +{{- if or (and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled) .Values.grafana.forceDeployDatasources }} apiVersion: v1 kind: ConfigMap metadata: @@ -20,7 +20,11 @@ data: {{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} - name: Prometheus type: prometheus + {{- if .Values.grafana.sidecar.datasources.url }} + url: {{ .Values.grafana.sidecar.datasources.url }} + {{- else }} url: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} + {{- end }} access: proxy isDefault: true jsonData: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/alertmanager-overview.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/alertmanager-overview.yaml new file mode 100644 index 0000000..2135644 --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/alertmanager-overview.yaml @@ -0,0 +1,610 @@ +{{- /* +Generated from 'alertmanager-overview' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "alertmanager-overview" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + alertmanager-overview.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "30s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(alertmanager_alerts{namespace=\"$namespace\",service=\"$service\"}) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_alerts_received_total{namespace=\"$namespace\",service=\"$service\"}[5m])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Received", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_alerts_invalid_total{namespace=\"$namespace\",service=\"$service\"}[5m])) by (namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Invalid", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Alerts receive rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Alerts", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(alertmanager_notifications_total{namespace=\"$namespace\",service=\"$service\", integration=\"$integration\"}[5m])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total", + "refId": "A" + }, + { + "expr": "sum(rate(alertmanager_notifications_failed_total{namespace=\"$namespace\",service=\"$service\", integration=\"$integration\"}[5m])) by (integration,namespace,service,instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Failed", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notifications Send Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "integration", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=\"$namespace\",service=\"$service\", integration=\"$integration\"}[5m])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} 99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50,\n sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=\"$namespace\",service=\"$service\", integration=\"$integration\"}[5m])) by (le,namespace,service,instance)\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Median", + "refId": "B" + }, + { + "expr": "sum(rate(alertmanager_notification_latency_seconds_sum{namespace=\"$namespace\",service=\"$service\", integration=\"$integration\"}[5m])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=\"$namespace\",service=\"$service\", integration=\"$integration\"}[5m])) by (namespace,service,instance)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Average", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "$integration: Notification Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Notifications", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "alertmanager-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "service", + "options": [ + + ], + "query": "label_values(alertmanager_alerts, service)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "all", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": true, + "label": null, + "multi": false, + "name": "integration", + "options": [ + + ], + "query": "label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Alertmanager / Overview", + "uid": "alertmanager-overview", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/apiserver.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/apiserver.yaml index 1fa3a19..5ad5529 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/apiserver.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/apiserver.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'apiserver' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'apiserver' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.kubeApiServer.enabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeApiServer.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml index 2fab7dc..a10f752 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'cluster-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'cluster-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/controller-manager.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/controller-manager.yaml index 139b1a5..3717c8e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/controller-manager.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/controller-manager.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'controller-manager' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'controller-manager' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.kubeControllerManager.enabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeControllerManager.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -176,10 +176,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", "refId": "A" } ], @@ -282,10 +282,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", "refId": "A" } ], @@ -388,10 +388,10 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name, le))", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", "refId": "A" } ], diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml index f8217d9..2185b7d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml @@ -4,7 +4,7 @@ Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.kubeEtcd.enabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeEtcd.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-coredns.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-coredns.yaml index 3665b57..a8cfd81 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-coredns.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-coredns.yaml @@ -1,6 +1,6 @@ {{- /* Added manually, can be changed in-place. */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.coreDns.enabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.coreDns.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml index a3c678c..ef43b4d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -247,7 +247,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{resource=\"cpu\",cluster=\"$cluster\"})", + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"cpu\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -499,7 +499,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"memory\"}) / sum(kube_node_status_allocatable{resource=\"memory\",cluster=\"$cluster\"})", + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"memory\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -594,7 +594,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}namespace{{`}}`}}", @@ -885,7 +885,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -903,7 +903,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", resource=\"cpu\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -912,7 +912,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"cpu\"}) by (namespace)", + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -921,7 +921,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"cpu\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1321,7 +1321,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests{cluster=\"$cluster\", resource=\"memory\"}) by (namespace)", + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1330,7 +1330,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", resource=\"memory\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1339,7 +1339,7 @@ data: "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"memory\"}) by (namespace)", + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1348,7 +1348,7 @@ data: "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"memory\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2443,6 +2443,496 @@ data: "showTitle": true, "title": "Rate of Packets Dropped", "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(namespace) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\"}[5m])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(namespace) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" } ], "schemaVersion": 14, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml index e20f344..f14b36d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -78,7 +78,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -162,7 +162,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -256,7 +256,7 @@ data: "thresholds": "70,80", "timeFrom": null, "timeShift": null, - "title": "Memory Utilization (from requests)", + "title": "Memory Utilisation (from requests)", "tooltip": { "shared": false, "sort": 0, @@ -446,7 +446,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}pod{{`}}`}}", @@ -697,7 +697,7 @@ data: ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -715,7 +715,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -733,7 +733,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -2136,6 +2136,496 @@ data: "showTitle": true, "title": "Rate of Packets Dropped", "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" } ], "schemaVersion": 14, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-node.yaml index 73a3e88..cfdf0ac 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-node.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'k8s-resources-node' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'k8s-resources-node' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -77,7 +77,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}pod{{`}}`}}", @@ -312,7 +312,7 @@ data: ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -330,7 +330,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -348,7 +348,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml index 014b165..b813ee2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -94,7 +94,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}container{{`}}`}}", @@ -450,7 +450,7 @@ data: ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -468,7 +468,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -486,7 +486,7 @@ data: "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -634,7 +634,7 @@ data: ], "timeFrom": null, "timeShift": null, - "title": "Memory Usage", + "title": "Memory Usage (WSS)", "tooltip": { "shared": false, "sort": 0, @@ -725,7 +725,7 @@ data: "type": "hidden" }, { - "alias": "Memory Usage", + "alias": "Memory Usage (WSS)", "colorMode": null, "colors": [ @@ -1591,6 +1591,697 @@ data: "showTitle": true, "title": "Rate of Packets Dropped", "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[5m])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "ceil(sum by(pod) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[5m])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reads", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by(pod) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writes", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Pod - Read & Writes)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": -1, + "fill": 10, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceil(sum by(container) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "IOPS(Reads+Writes)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ThroughPut(Read+Write)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution(Containers)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "sort": { + "col": 4, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IOPS(Reads)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "IOPS(Reads + Writes)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": -1, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Throughput(Read)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Throughput(Read + Write)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum by(container) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Storage IO", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage IO - Distribution", + "titleSize": "h6" } ], "schemaVersion": 14, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml index 38c2ead..57b4088 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -77,7 +77,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}pod{{`}}`}}", @@ -312,7 +312,7 @@ data: ], "targets": [ { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -330,7 +330,7 @@ data: "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -348,7 +348,7 @@ data: "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml index 0c53de7..a485147 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -98,7 +98,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", @@ -396,7 +396,7 @@ data: "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -414,7 +414,7 @@ data: "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -432,7 +432,7 @@ data: "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/kubelet.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/kubelet.yaml index 7238299..613a6c2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/kubelet.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/kubelet.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'kubelet' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'kubelet' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.kubelet.enabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubelet.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -39,2385 +39,2100 @@ data: "id": null, "links": [ + ], + "panels": [ + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_node_name{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Running Kubelets", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 3, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Pods", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 4, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Running Container", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 5, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Actual Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 6, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Desired Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ + + ], + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 7, + "links": [ + + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "title": "Config Error Count", + "transparent": false, + "type": "stat" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 77 + }, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 77 + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 77 + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } ], "refresh": "10s", "rows": [ - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - }, - "id": 2, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(up{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "Up", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 3, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Running Pods", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 4, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Running Container", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 5, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Actual Volume Count", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 6, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Desired Volume Count", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - - }, - "id": 7, - "interval": null, - "links": [ - - ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Config Error Count", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (operation_type, instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Operation Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 9, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, operation_type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Operation Error Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 10, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Operation duration 99th quantile", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 11, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} pod", - "refId": "A" - }, - { - "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} worker", - "refId": "B" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Pod Start Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 12, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} pod", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} worker", - "refId": "B" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Pod Start Duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 13, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Storage Operation Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Storage Operation Error Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 15, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Storage Operation Duration 99th quantile", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 16, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, operation_type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}operation_type{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Cgroup manager operation rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 17, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Cgroup manager 99th quantile", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Pod lifecycle event generator", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 18, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "PLEG relist rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 19, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "PLEG relist interval", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 20, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "PLEG relist duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 21, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "2xx", - "refId": "A" - }, - { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "3xx", - "refId": "B" - }, - { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "4xx", - "refId": "C" - }, - { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "5xx", - "refId": "D" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "RPC Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 22, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, verb, url, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Request duration 99th quantile", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 23, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 24, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "CPU usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { - - }, - "id": 25, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ - - ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ - - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - } ], "schemaVersion": 14, "style": "dark", diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml index eb53ba0..288aeed 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'namespace-by-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'namespace-by-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml index 6a01666..f533068 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'namespace-by-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'namespace-by-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml index f486bcb..22f5aea 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'node-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'node-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.nodeExporter.enabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.nodeExporter.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml index bfbe258..52d24bf 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.nodeExporter.enabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.nodeExporter.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml index e3b3bd8..9da7444 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -92,9 +92,8 @@ data: "steppedLine": false, "targets": [ { - "expr": "(\n (1 - rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"}[$__interval]))\n/ ignoring(cpu) group_left\n count without (cpu)( node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "expr": "(\n (1 - rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"}[$__rate_interval]))\n/ ignoring(cpu) group_left\n count without (cpu)( node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", "format": "time_series", - "interval": "$__rate_interval", "intervalFactor": 5, "legendFormat": "{{`{{`}}cpu{{`}}`}}", "refId": "A" @@ -528,25 +527,22 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__interval])", + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__rate_interval])", "format": "time_series", - "interval": "$__rate_interval", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}} read", "refId": "A" }, { - "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__interval])", + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__rate_interval])", "format": "time_series", - "interval": "$__rate_interval", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}} written", "refId": "B" }, { - "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__interval])", + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__rate_interval])", "format": "time_series", - "interval": "$__rate_interval", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}} io time", "refId": "C" @@ -758,9 +754,8 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__interval])", + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval])", "format": "time_series", - "interval": "$__rate_interval", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}}", "refId": "A" @@ -852,9 +847,8 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__interval])", + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval])", "format": "time_series", - "interval": "$__rate_interval", "intervalFactor": 2, "legendFormat": "{{`{{`}}device{{`}}`}}", "refId": "A" diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml index 06fb315..93065ff 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -92,14 +92,14 @@ data: "steppedLine": false, "targets": [ { - "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "Used Space", "refId": "A" }, { - "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "Free Space", @@ -207,7 +207,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "max without(instance,node) (\n(\n kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100)\n", + "expr": "max without(instance,node) (\n(\n topk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n topk(1, kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -289,14 +289,14 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "Used inodes", "refId": "A" }, { - "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", "format": "time_series", "intervalFactor": 1, "legendFormat": " Free inodes", @@ -404,7 +404,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "max without(instance,node) (\nkubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100)\n", + "expr": "max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml index 7145287..d2842c3 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'pod-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'pod-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml index b33b738..a98581d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'prometheus-remote-write' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'prometheus-remote-write' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.prometheus.prometheusSpec.remoteWriteDashboards }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.prometheus.prometheusSpec.remoteWriteDashboards }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus.yaml index 0cd3e7b..71d3a62 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'prometheus' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'prometheus' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/proxy.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/proxy.yaml index 8f8559e..e6b9b98 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/proxy.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/proxy.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'proxy' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'proxy' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.kubeProxy.enabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeProxy.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/scheduler.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/scheduler.yaml index 3dbf4ab..82edd36 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/scheduler.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/scheduler.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'scheduler' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'scheduler' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.kubeScheduler.enabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled .Values.kubeScheduler.enabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -176,31 +176,31 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} e2e", + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", "refId": "A" }, { - "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} binding", + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", "refId": "B" }, { - "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} scheduling algorithm", + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", "refId": "C" }, { - "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} volume", + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", "refId": "D" } ], @@ -290,31 +290,31 @@ data: "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} e2e", + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} e2e", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} binding", + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} binding", "refId": "B" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} scheduling algorithm", + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} scheduling algorithm", "refId": "C" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{`{{`}}instance{{`}}`}} volume", + "legendFormat": "{{`{{`}}cluster{{`}}`}} {{`{{`}}instance{{`}}`}} volume", "refId": "D" } ], diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/statefulset.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/statefulset.yaml index 9b3ce7d..f1ffea5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/statefulset.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/statefulset.yaml @@ -4,7 +4,7 @@ Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -925,4 +925,4 @@ data: "uid": "a31c1f46e6f727cb37c0d731a7245005", "version": 0 } -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml index 75f95bd..dc1e4ed 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml @@ -1,10 +1,10 @@ {{- /* -Generated from 'workload-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Generated from 'workload-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml index f8afcb8..de81974 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -58,8 +58,8 @@ spec: tolerations: {{ toYaml . | indent 8 }} {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} securityContext: - runAsGroup: 2000 - runAsNonRoot: true - runAsUser: 2000 +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml index b2d8912..23dc9f4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -59,8 +59,8 @@ spec: tolerations: {{ toYaml . | indent 8 }} {{- end }} +{{- if .Values.prometheusOperator.admissionWebhooks.patch.securityContext }} securityContext: - runAsGroup: 2000 - runAsNonRoot: true - runAsUser: 2000 +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.securityContext | indent 8 }} +{{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml index b67df54..f42e33e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml @@ -5,8 +5,8 @@ metadata: name: {{ template "kube-prometheus-stack.fullname" . }}-admission {{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} annotations: - certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} - cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} {{- end }} labels: app: {{ template "kube-prometheus-stack.name" $ }}-admission diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml index 249488e..1439ed5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml @@ -5,8 +5,8 @@ metadata: name: {{ template "kube-prometheus-stack.fullname" . }}-admission {{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} annotations: - certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} - cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-admission" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-admission" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} {{- end }} labels: app: {{ template "kube-prometheus-stack.name" $ }}-admission diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml index 15b3684..00144ab 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml @@ -78,6 +78,11 @@ spec: {{- if .Values.prometheusOperator.prometheusInstanceNamespaces }} - --prometheus-instance-namespaces={{ .Values.prometheusOperator.prometheusInstanceNamespaces | join "," }} {{- end }} + {{- if .Values.prometheusOperator.thanosImage.sha }} + - --thanos-default-base-image={{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }}@sha256:{{ .Values.prometheusOperator.thanosImage.sha }} + {{- else }} + - --thanos-default-base-image={{ .Values.prometheusOperator.thanosImage.repository }}:{{ .Values.prometheusOperator.thanosImage.tag }} + {{- end }} {{- if .Values.prometheusOperator.thanosRulerInstanceNamespaces }} - --thanos-ruler-instance-namespaces={{ .Values.prometheusOperator.thanosRulerInstanceNamespaces | join "," }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/alertmanager.rules.yaml index 387a677..49e78b5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/alertmanager.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/alertmanager-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -26,41 +26,146 @@ spec: groups: - name: alertmanager.rules rules: - - alert: AlertmanagerConfigInconsistent + - alert: AlertmanagerFailedReload annotations: - message: 'The configuration of the instances of the Alertmanager cluster `{{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.service {{`}}`}}` are out of sync. - - {{`{{`}} range printf "alertmanager_config_hash{namespace=\"%s\",service=\"%s\"}" $labels.namespace $labels.service | query {{`}}`}} - - Configuration hash for pod {{`{{`}} .Labels.pod {{`}}`}} is "{{`{{`}} printf "%.f" .Value {{`}}`}}" - - {{`{{`}} end {{`}}`}} - - ' - expr: count by(namespace,service) (count_values by(namespace,service) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"})) != 1 + description: Configuration has failed to load for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-alertmanagerfailedreload + summary: Reloading an Alertmanager configuration has failed. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: 10m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} + - alert: AlertmanagerMembersInconsistent + annotations: + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} has only found {{`{{`}} $value {{`}}`}} members of the {{`{{`}}$labels.job{{`}}`}} cluster. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-alertmanagermembersinconsistent + summary: A member of an Alertmanager cluster has not found all other cluster members. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + < on (namespace,service) group_left + count by (namespace,service) (max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m])) + for: 10m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} + - alert: AlertmanagerFailedToSendAlerts + annotations: + description: Alertmanager {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} failed to send {{`{{`}} $value | humanizePercentage {{`}}`}} of notifications to {{`{{`}} $labels.integration {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-alertmanagerfailedtosendalerts + summary: An Alertmanager instance failed to send notifications. + expr: |- + ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 0.01 + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} + - alert: AlertmanagerClusterFailedToSendAlerts + annotations: + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration. + expr: |- + min by (namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + / + rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration=~`.*`}[5m]) + ) + > 0.01 for: 5m labels: severity: critical {{- if .Values.defaultRules.additionalRuleLabels }} {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} {{- end }} - - alert: AlertmanagerFailedReload + - alert: AlertmanagerClusterFailedToSendAlerts annotations: - message: Reloading Alertmanager's configuration has failed for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. - expr: alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} == 0 - for: 10m + description: The minimum notification failure rate to {{`{{`}} $labels.integration {{`}}`}} sent from any instance in the {{`{{`}}$labels.job{{`}}`}} cluster is {{`{{`}} $value | humanizePercentage {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-alertmanagerclusterfailedtosendalerts + summary: All Alertmanager instances in a cluster failed to send notifications to a non-critical integration. + expr: |- + min by (namespace,service, integration) ( + rate(alertmanager_notifications_failed_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + / + rate(alertmanager_notifications_total{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}", integration!~`.*`}[5m]) + ) + > 0.01 + for: 5m labels: severity: warning {{- if .Values.defaultRules.additionalRuleLabels }} {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} {{- end }} - - alert: AlertmanagerMembersInconsistent + - alert: AlertmanagerConfigInconsistent annotations: - message: Alertmanager has not found all other members of the cluster. + description: Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have different configurations. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-alertmanagerconfiginconsistent + summary: Alertmanager instances within the same cluster have different configurations. expr: |- - alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} - != on (service) GROUP_LEFT() - count by (service) (alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + count by (namespace,service) ( + count_values by (namespace,service) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + ) + != 1 + for: 20m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} + - alert: AlertmanagerClusterDown + annotations: + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have been up for less than half of the last 5m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-alertmanagerclusterdown + summary: Half or more of the Alertmanager instances within the same cluster are down. + expr: |- + ( + count by (namespace,service) ( + avg_over_time(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) < 0.5 + ) + / + count by (namespace,service) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 + for: 5m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} + - alert: AlertmanagerClusterCrashlooping + annotations: + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of Alertmanager instances within the {{`{{`}}$labels.job{{`}}`}} cluster have restarted at least 5 times in the last 10m.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-alertmanagerclustercrashlooping + summary: Half or more of the Alertmanager instances within the same cluster are crashlooping. + expr: |- + ( + count by (namespace,service) ( + changes(process_start_time_seconds{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[10m]) > 4 + ) + / + count by (namespace,service) ( + up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + ) + ) + >= 0.5 for: 5m labels: severity: critical diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/general.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/general.rules.yaml index 80771f4..e41175f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/general.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/general.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kube-prometheus-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -26,7 +26,9 @@ spec: rules: - alert: TargetDown annotations: - message: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' + description: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-targetdown + summary: One or more targets are unreachable. expr: 100 * (count(up == 0) BY (job, namespace, service) / count(up) BY (job, namespace, service)) > 10 for: 10m labels: @@ -36,7 +38,7 @@ spec: {{- end }} - alert: Watchdog annotations: - message: 'This is an alert meant to ensure that the entire alerting pipeline is functional. + description: 'This is an alert meant to ensure that the entire alerting pipeline is functional. This alert is always firing, therefore it should always be firing in Alertmanager @@ -47,6 +49,8 @@ spec: "DeadMansSnitch" integration in PagerDuty. ' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-watchdog + summary: An alert that should always be firing to certify that Alertmanager is working properly. expr: vector(1) labels: severity: none diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.yaml index 011a4a7..8f3314b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -26,11 +26,11 @@ spec: rules: - expr: |- sum by (cluster, namespace, pod, container) ( - rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) + irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) ) - record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate - expr: |- container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, @@ -77,6 +77,28 @@ spec: ) ) record: namespace_cpu:kube_pod_container_resource_requests:sum + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_memory:kube_pod_container_resource_limits:sum + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_limits:sum - expr: |- max by (cluster, namespace, workload, pod) ( label_replace( diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml index 7b00b54..8dda6b7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -29,56 +29,56 @@ spec: 1 - ( ( # write too slow - sum(increase(apiserver_request_duration_seconds_count{verb=~"POST|PUT|PATCH|DELETE"}[30d])) + sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~"POST|PUT|PATCH|DELETE"}[30d])) - - sum(increase(apiserver_request_duration_seconds_bucket{verb=~"POST|PUT|PATCH|DELETE",le="1"}[30d])) + sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~"POST|PUT|PATCH|DELETE",le="1"}[30d])) ) + ( # read too slow - sum(increase(apiserver_request_duration_seconds_count{verb=~"LIST|GET"}[30d])) + sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~"LIST|GET"}[30d])) - ( ( - sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30d])) + sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30d])) or vector(0) ) + - sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope="namespace",le="0.5"}[30d])) + sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope="namespace",le="0.5"}[30d])) + - sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope="cluster",le="5"}[30d])) + sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope="cluster",le="5"}[30d])) ) ) + # errors - sum(code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + sum by (cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) ) / - sum(code:apiserver_request_total:increase30d) + sum by (cluster) (code:apiserver_request_total:increase30d) labels: verb: all record: apiserver_request:availability30d - expr: |- 1 - ( - sum(increase(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[30d])) + sum by (cluster) (increase(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[30d])) - ( # too slow ( - sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30d])) + sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30d])) or vector(0) ) + - sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[30d])) + sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[30d])) + - sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[30d])) + sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[30d])) ) + # errors - sum(code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + sum by (cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) ) / - sum(code:apiserver_request_total:increase30d{verb="read"}) + sum by (cluster) (code:apiserver_request_total:increase30d{verb="read"}) labels: verb: read record: apiserver_request:availability30d @@ -86,74 +86,74 @@ spec: 1 - ( ( # too slow - sum(increase(apiserver_request_duration_seconds_count{verb=~"POST|PUT|PATCH|DELETE"}[30d])) + sum by (cluster) (increase(apiserver_request_duration_seconds_count{verb=~"POST|PUT|PATCH|DELETE"}[30d])) - - sum(increase(apiserver_request_duration_seconds_bucket{verb=~"POST|PUT|PATCH|DELETE",le="1"}[30d])) + sum by (cluster) (increase(apiserver_request_duration_seconds_bucket{verb=~"POST|PUT|PATCH|DELETE",le="1"}[30d])) ) + # errors - sum(code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + sum by (cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) ) / - sum(code:apiserver_request_total:increase30d{verb="write"}) + sum by (cluster) (code:apiserver_request_total:increase30d{verb="write"}) labels: verb: write record: apiserver_request:availability30d - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30 record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"2.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"2.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"2.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"2.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"2.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"2.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"2.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"2.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"2.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"2.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"2.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"2.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"3.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"3.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"3.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"3.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"3.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"3.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"3.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"3.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"3.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"3.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"3.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"3.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"4.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"4.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"4.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"4.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"4.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"4.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"4.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"4.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"4.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"4.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"4.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"4.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"5.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"5.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"5.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"5.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"5.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"5.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"5.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"5.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"5.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"5.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"5.."}[1h])) + - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"5.."}[1h])) record: code_verb:apiserver_request_total:increase1h - - expr: sum by (code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + - expr: sum by (cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) labels: verb: read record: code:apiserver_request_total:increase30d - - expr: sum by (code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + - expr: sum by (cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) labels: verb: write record: code:apiserver_request_total:increase30d diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml index 0f44ccc..22ede05 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml index eddc1e4..4bb373d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kube-apiserver.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'kube-apiserver.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -28,26 +28,26 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[1d])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[1d])) - ( ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[1d])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[1d])) or vector(0) ) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[1d])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[1d])) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[1d])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[1d])) ) ) + # errors - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d])) labels: verb: read record: apiserver_request:burnrate1d @@ -55,26 +55,26 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[1h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[1h])) - ( ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[1h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[1h])) or vector(0) ) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[1h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[1h])) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[1h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[1h])) ) ) + # errors - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h])) labels: verb: read record: apiserver_request:burnrate1h @@ -82,26 +82,26 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[2h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[2h])) - ( ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[2h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[2h])) or vector(0) ) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[2h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[2h])) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[2h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[2h])) ) ) + # errors - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h])) labels: verb: read record: apiserver_request:burnrate2h @@ -109,26 +109,26 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[30m])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[30m])) - ( ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30m])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30m])) or vector(0) ) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[30m])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[30m])) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[30m])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[30m])) ) ) + # errors - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m])) labels: verb: read record: apiserver_request:burnrate30m @@ -136,26 +136,26 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[3d])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[3d])) - ( ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[3d])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[3d])) or vector(0) ) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[3d])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[3d])) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[3d])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[3d])) ) ) + # errors - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d])) labels: verb: read record: apiserver_request:burnrate3d @@ -163,26 +163,26 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[5m])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[5m])) - ( ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[5m])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[5m])) or vector(0) ) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[5m])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[5m])) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[5m])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[5m])) ) ) + # errors - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) labels: verb: read record: apiserver_request:burnrate5m @@ -190,26 +190,26 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[6h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[6h])) - ( ( - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[6h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[6h])) or vector(0) ) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[6h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[6h])) + - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[6h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[6h])) ) ) + # errors - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h])) labels: verb: read record: apiserver_request:burnrate6h @@ -217,15 +217,15 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) - - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[1d])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[1d])) ) + - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) labels: verb: write record: apiserver_request:burnrate1d @@ -233,15 +233,15 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) - - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[1h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[1h])) ) + - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) labels: verb: write record: apiserver_request:burnrate1h @@ -249,15 +249,15 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) - - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[2h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[2h])) ) + - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) labels: verb: write record: apiserver_request:burnrate2h @@ -265,15 +265,15 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) - - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[30m])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[30m])) ) + - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) labels: verb: write record: apiserver_request:burnrate30m @@ -281,15 +281,15 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) - - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[3d])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[3d])) ) + - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) labels: verb: write record: apiserver_request:burnrate3d @@ -297,15 +297,15 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) - - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[5m])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[5m])) ) + - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) labels: verb: write record: apiserver_request:burnrate5m @@ -313,32 +313,32 @@ spec: ( ( # too slow - sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) - - sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[6h])) + sum by (cluster) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[6h])) ) + - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) ) / - sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) labels: verb: write record: apiserver_request:burnrate6h - - expr: sum by (code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + - expr: sum by (cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) labels: verb: read record: code_resource:apiserver_request_total:rate5m - - expr: sum by (code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + - expr: sum by (cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) labels: verb: write record: code_resource:apiserver_request_total:rate5m - - expr: histogram_quantile(0.99, sum by (le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET"}[5m]))) > 0 + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET"}[5m]))) > 0 labels: quantile: '0.99' verb: read record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile - - expr: histogram_quantile(0.99, sum by (le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))) > 0 + - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))) > 0 labels: quantile: '0.99' verb: write diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml index e54bee5..1b95d21 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kube-prometheus-general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kube-prometheus-general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kube-prometheus-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml index 27271f1..2d38ef8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kube-prometheus-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml index 24df268..4e44196 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-state-metrics.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-state-metrics.yaml index 0fa5032..f7e0c43 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-state-metrics.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-state-metrics.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kube-state-metrics' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kube-state-metrics' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kube-state-metrics-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -55,5 +55,33 @@ spec: severity: critical {{- if .Values.defaultRules.additionalRuleLabels }} {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} + - alert: KubeStateMetricsShardingMismatch + annotations: + description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatemetricsshardingmismatch + summary: kube-state-metrics sharding is misconfigured. + expr: stdvar (kube_state_metrics_total_shards{job="kube-state-metrics"}) != 0 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} + - alert: KubeStateMetricsShardsMissing + annotations: + description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatemetricsshardsmissing + summary: kube-state-metrics shards are missing. + expr: |- + 2^max(kube_state_metrics_total_shards{job="kube-state-metrics"}) - 1 + - + sum( 2 ^ max by (shard_ordinal) (kube_state_metrics_shard_ordinal{job="kube-state-metrics"}) ) + != 0 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubelet.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubelet.rules.yaml index 8712b9f..a073214 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubelet.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubelet.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubelet.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'kubelet.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml index 77bb40a..94c16cd 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -30,7 +30,10 @@ spec: description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is restarting {{`{{`}} printf "%.2f" $value {{`}}`}} times / 10 minutes. runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodcrashlooping summary: Pod is crash looping. - expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[10m]) * 60 * 5 > 0 + expr: |- + increase(kube_pod_container_status_restarts_total{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[10m]) > 0 + and + sum without (phase) (kube_pod_status_phase{phase!="Running",job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} == 1) for: 15m labels: severity: warning @@ -261,19 +264,19 @@ spec: runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubehpareplicasmismatch summary: HPA has not matched descired number of replicas. expr: |- - (kube_hpa_status_desired_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + (kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} != - kube_hpa_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) + kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) and - (kube_hpa_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > - kube_hpa_spec_min_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) + kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) and - (kube_hpa_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} < - kube_hpa_spec_max_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) + kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) and - changes(kube_hpa_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[15m]) == 0 + changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[15m]) == 0 for: 15m labels: severity: warning @@ -286,9 +289,9 @@ spec: runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubehpamaxedout summary: HPA is running at max replicas expr: |- - kube_hpa_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} == - kube_hpa_spec_max_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} for: 15m labels: severity: warning diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-resources.yaml index 27babbd..7a2ecbc 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-resources.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml index 527e6e3..3e06766 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml index c3110cf..96945b8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-system-apiserver' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'kubernetes-system-apiserver' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml index 3d1ace1..07ff37a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-system-controller-manager' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'kubernetes-system-controller-manager' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml index 5671b1c..1ea531f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml index 098f6fb..47376e3 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-system-scheduler' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'kubernetes-system-scheduler' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system.yaml index ea2f258..657e5c2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.rules.yaml index ddb7376..f734e8d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'node-exporter.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/node-exporter-prometheusRule.yaml +Generated from 'node-exporter.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/node-exporter-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -33,9 +33,9 @@ spec: record: instance:node_num_cpu:sum - expr: |- 1 - avg without (cpu, mode) ( - rate(node_cpu_seconds_total{job="node-exporter", mode="idle"}[1m]) + rate(node_cpu_seconds_total{job="node-exporter", mode="idle"}[5m]) ) - record: instance:node_cpu_utilisation:rate1m + record: instance:node_cpu_utilisation:rate5m - expr: |- ( node_load1{job="node-exporter"} @@ -50,30 +50,30 @@ spec: node_memory_MemTotal_bytes{job="node-exporter"} ) record: instance:node_memory_utilisation:ratio - - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[1m]) - record: instance:node_vmstat_pgmajfault:rate1m - - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[1m]) - record: instance_device:node_disk_io_time_seconds:rate1m - - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[1m]) - record: instance_device:node_disk_io_time_weighted_seconds:rate1m + - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) + record: instance:node_vmstat_pgmajfault:rate5m + - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[5m]) + record: instance_device:node_disk_io_time_seconds:rate5m + - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[5m]) + record: instance_device:node_disk_io_time_weighted_seconds:rate5m - expr: |- sum without (device) ( - rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[1m]) + rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m]) ) - record: instance:node_network_receive_bytes_excluding_lo:rate1m + record: instance:node_network_receive_bytes_excluding_lo:rate5m - expr: |- sum without (device) ( - rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[1m]) + rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m]) ) - record: instance:node_network_transmit_bytes_excluding_lo:rate1m + record: instance:node_network_transmit_bytes_excluding_lo:rate5m - expr: |- sum without (device) ( - rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[1m]) + rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m]) ) - record: instance:node_network_receive_drop_excluding_lo:rate1m + record: instance:node_network_receive_drop_excluding_lo:rate5m - expr: |- sum without (device) ( - rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[1m]) + rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m]) ) - record: instance:node_network_transmit_drop_excluding_lo:rate1m + record: instance:node_network_transmit_drop_excluding_lo:rate5m {{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml index 763c4a2..468e1db 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/node-exporter-prometheusRule.yaml +Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/node-exporter-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -27,6 +27,7 @@ spec: - alert: NodeFilesystemSpaceFillingUp annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemspacefillingup summary: Filesystem is predicted to run out of space within the next 24 hours. expr: |- ( @@ -45,6 +46,7 @@ spec: - alert: NodeFilesystemSpaceFillingUp annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemspacefillingup summary: Filesystem is predicted to run out of space within the next 4 hours. expr: |- ( @@ -63,6 +65,7 @@ spec: - alert: NodeFilesystemAlmostOutOfSpace annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutofspace summary: Filesystem has less than 5% space left. expr: |- ( @@ -79,6 +82,7 @@ spec: - alert: NodeFilesystemAlmostOutOfSpace annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutofspace summary: Filesystem has less than 3% space left. expr: |- ( @@ -95,6 +99,7 @@ spec: - alert: NodeFilesystemFilesFillingUp annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemfilesfillingup summary: Filesystem is predicted to run out of inodes within the next 24 hours. expr: |- ( @@ -113,6 +118,7 @@ spec: - alert: NodeFilesystemFilesFillingUp annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemfilesfillingup summary: Filesystem is predicted to run out of inodes within the next 4 hours. expr: |- ( @@ -131,6 +137,7 @@ spec: - alert: NodeFilesystemAlmostOutOfFiles annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutoffiles summary: Filesystem has less than 5% inodes left. expr: |- ( @@ -147,6 +154,7 @@ spec: - alert: NodeFilesystemAlmostOutOfFiles annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutoffiles summary: Filesystem has less than 3% inodes left. expr: |- ( @@ -163,6 +171,7 @@ spec: - alert: NodeNetworkReceiveErrs annotations: description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodenetworkreceiveerrs summary: Network interface is reporting many receive errors. expr: rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01 for: 1h @@ -174,6 +183,7 @@ spec: - alert: NodeNetworkTransmitErrs annotations: description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodenetworktransmiterrs summary: Network interface is reporting many transmit errors. expr: rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01 for: 1h @@ -185,6 +195,7 @@ spec: - alert: NodeHighNumberConntrackEntriesUsed annotations: description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodehighnumberconntrackentriesused summary: Number of conntrack are getting close to the limit. expr: (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75 labels: @@ -195,6 +206,7 @@ spec: - alert: NodeTextFileCollectorScrapeError annotations: description: Node Exporter text file collector failed to scrape. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodetextfilecollectorscrapeerror summary: Node Exporter text file collector failed to scrape. expr: node_textfile_scrape_error{job="node-exporter"} == 1 labels: @@ -204,7 +216,8 @@ spec: {{- end }} - alert: NodeClockSkewDetected annotations: - message: Clock on {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 300s. Ensure NTP is configured correctly on this host. + description: Clock on {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 300s. Ensure NTP is configured correctly on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodeclockskewdetected summary: Clock skew detected. expr: |- ( @@ -226,7 +239,8 @@ spec: {{- end }} - alert: NodeClockNotSynchronising annotations: - message: Clock on {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. + description: Clock on {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodeclocknotsynchronising summary: Clock not synchronising. expr: |- min_over_time(node_timex_sync_status[5m]) == 0 @@ -241,6 +255,7 @@ spec: - alert: NodeRAIDDegraded annotations: description: RAID array '{{`{{`}} $labels.device {{`}}`}}' on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-noderaiddegraded summary: RAID Array is degraded expr: node_md_disks_required - ignoring (state) (node_md_disks{state="active"}) > 0 for: 15m @@ -252,8 +267,9 @@ spec: - alert: NodeRAIDDiskFailure annotations: description: At least one device in RAID array on {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-noderaiddiskfailure summary: Failed device in RAID array - expr: node_md_disks{state="fail"} > 0 + expr: node_md_disks{state="failed"} > 0 labels: severity: warning {{- if .Values.defaultRules.additionalRuleLabels }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-network.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-network.yaml index 9a6955a..8964334 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-network.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-network.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'node-network' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'node-network' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kube-prometheus-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -26,7 +26,8 @@ spec: rules: - alert: NodeNetworkInterfaceFlapping annotations: - message: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing it's up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}}" + message: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing it's up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodenetworkinterfaceflapping expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 for: 2m labels: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node.rules.yaml index f24c555..c6dd9e1 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml +Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/prometheus-operator.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/prometheus-operator.yaml index d1c1f65..d3010bc 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/prometheus-operator.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/prometheus-operator.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'prometheus-operator' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'prometheus-operator' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/prometheus-operator-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/prometheus.yaml index c9c805e..52cc0d8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/prometheus.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'prometheus' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'prometheus' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/prometheus-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -29,6 +29,7 @@ spec: - alert: PrometheusBadConfig annotations: description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to reload its configuration. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusbadconfig summary: Failed Prometheus configuration reload. expr: |- # Without max_over_time, failed scrapes could create false negatives, see @@ -43,6 +44,7 @@ spec: - alert: PrometheusNotificationQueueRunningFull annotations: description: Alert notification queue of Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is running full. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusnotificationqueuerunningfull summary: Prometheus alert notification queue predicted to run full in less than 30m. expr: |- # Without min_over_time, failed scrapes could create false negatives, see @@ -61,6 +63,7 @@ spec: - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers annotations: description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.alertmanager{{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheuserrorsendingalertstosomealertmanagers summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager. expr: |- ( @@ -75,28 +78,11 @@ spec: severity: warning {{- if .Values.defaultRules.additionalRuleLabels }} {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: PrometheusErrorSendingAlertsToAnyAlertmanager - annotations: - description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% minimum errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to any Alertmanager.' - summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. - expr: |- - min without(alertmanager) ( - rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) - / - rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) - ) - * 100 - > 3 - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} {{- end }} - alert: PrometheusNotConnectedToAlertmanagers annotations: description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not connected to any Alertmanagers. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusnotconnectedtoalertmanagers summary: Prometheus is not connected to any Alertmanagers. expr: |- # Without max_over_time, failed scrapes could create false negatives, see @@ -111,6 +97,7 @@ spec: - alert: PrometheusTSDBReloadsFailing annotations: description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} reload failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheustsdbreloadsfailing summary: Prometheus has issues reloading blocks from disk. expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 for: 4h @@ -122,6 +109,7 @@ spec: - alert: PrometheusTSDBCompactionsFailing annotations: description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last 3h. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheustsdbcompactionsfailing summary: Prometheus has issues compacting blocks. expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 for: 4h @@ -133,8 +121,18 @@ spec: - alert: PrometheusNotIngestingSamples annotations: description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not ingesting samples. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusnotingestingsamples summary: Prometheus is not ingesting samples. - expr: rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) <= 0 + expr: |- + ( + rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) <= 0 + and + ( + sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + or + sum without(rule_group) (prometheus_rule_group_rules{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}) > 0 + ) + ) for: 10m labels: severity: warning @@ -144,6 +142,7 @@ spec: - alert: PrometheusDuplicateTimestamps annotations: description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with different values but duplicated timestamp. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusduplicatetimestamps summary: Prometheus is dropping samples with duplicate timestamps. expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 for: 10m @@ -155,6 +154,7 @@ spec: - alert: PrometheusOutOfOrderTimestamps annotations: description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with timestamps arriving out of order. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusoutofordertimestamps summary: Prometheus drops samples with out-of-order timestamps. expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 for: 10m @@ -166,15 +166,16 @@ spec: - alert: PrometheusRemoteStorageFailures annotations: description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}} + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusremotestoragefailures summary: Prometheus fails to send samples to remote storage. expr: |- ( - rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) / ( - rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + (rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) + - rate(prometheus_remote_storage_succeeded_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + (rate(prometheus_remote_storage_succeeded_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) or rate(prometheus_remote_storage_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m])) ) ) * 100 @@ -188,13 +189,14 @@ spec: - alert: PrometheusRemoteWriteBehind annotations: description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusremotewritebehind summary: Prometheus remote write is behind. expr: |- # Without max_over_time, failed scrapes could create false negatives, see # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. ( max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) - - on(job, instance) group_right + - ignoring(remote_name, url) group_right max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) ) > 120 @@ -207,6 +209,7 @@ spec: - alert: PrometheusRemoteWriteDesiredShards annotations: description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards for queue {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusremotewritedesiredshards summary: Prometheus remote write desired shards calculation wants to run more than configured max shards. expr: |- # Without max_over_time, failed scrapes could create false negatives, see @@ -225,6 +228,7 @@ spec: - alert: PrometheusRuleFailures annotations: description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to evaluate {{`{{`}} printf "%.0f" $value {{`}}`}} rules in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusrulefailures summary: Prometheus is failing rule evaluations. expr: increase(prometheus_rule_evaluation_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 for: 15m @@ -236,6 +240,7 @@ spec: - alert: PrometheusMissingRuleEvaluations annotations: description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has missed {{`{{`}} printf "%.0f" $value {{`}}`}} rule group evaluations in the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusmissingruleevaluations summary: Prometheus is missing rule evaluations due to slow rule group evaluation. expr: increase(prometheus_rule_group_iterations_missed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 for: 15m @@ -247,6 +252,7 @@ spec: - alert: PrometheusTargetLimitHit annotations: description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because the number of targets exceeded the configured target_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheustargetlimithit summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit. expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 for: 15m @@ -254,5 +260,36 @@ spec: severity: warning {{- if .Values.defaultRules.additionalRuleLabels }} {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} + - alert: PrometheusLabelLimitHit + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has dropped {{`{{`}} printf "%.0f" $value {{`}}`}} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheuslabellimithit + summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit. + expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} + - alert: PrometheusErrorSendingAlertsToAnyAlertmanager + annotations: + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% minimum errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to any Alertmanager.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheuserrorsendingalertstoanyalertmanager + summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. + expr: |- + min without (alertmanager) ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}",alertmanager!~``}[5m]) + ) + * 100 + > 3 + for: 15m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/alertmanager.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/alertmanager.rules.yaml deleted file mode 100644 index 7115984..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/alertmanager.rules.yaml +++ /dev/null @@ -1,63 +0,0 @@ -{{- /* -Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.alertmanager }} -{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} -{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} -{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager.rules" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: alertmanager.rules - rules: - - alert: AlertmanagerConfigInconsistent - annotations: - message: The configuration of the instances of the Alertmanager cluster `{{`{{`}}$labels.service{{`}}`}}` are out of sync. - expr: count_values("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) BY (service) / ON(service) GROUP_LEFT() label_replace(max(prometheus_operator_spec_replicas{job="{{ $operatorJob }}",namespace="{{ $namespace }}",controller="alertmanager"}) by (name, job, namespace, controller), "service", "$1", "name", "(.*)") != 1 - for: 5m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: AlertmanagerFailedReload - annotations: - message: Reloading Alertmanager's configuration has failed for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. - expr: alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} == 0 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: AlertmanagerMembersInconsistent - annotations: - message: Alertmanager has not found all other members of the cluster. - expr: |- - alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} - != on (service) GROUP_LEFT() - count by (service) (alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) - for: 5m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/etcd.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/etcd.yaml deleted file mode 100644 index ce4e87b..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/etcd.yaml +++ /dev/null @@ -1,179 +0,0 @@ -{{- /* -Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/website/master/content/en/docs/v3.4/op-guide/etcd3_alert.rules.yml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeEtcd.enabled .Values.defaultRules.rules.etcd }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: etcd - rules: - - alert: etcdInsufficientMembers - annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' - expr: sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2) - for: 3m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: etcdNoLeader - annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.' - expr: etcd_server_has_leader{job=~".*etcd.*"} == 0 - for: 1m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: etcdHighNumberOfLeaderChanges - annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": instance {{`{{`}} $labels.instance {{`}}`}} has seen {{`{{`}} $value {{`}}`}} leader changes within the last hour.' - expr: rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3 - for: 15m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: etcdHighNumberOfFailedGRPCRequests - annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' - expr: |- - 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method) - / - sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method) - > 1 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: etcdHighNumberOfFailedGRPCRequests - annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' - expr: |- - 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method) - / - sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method) - > 5 - for: 5m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: etcdGRPCRequestsSlow - annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": gRPC requests to {{`{{`}} $labels.grpc_method {{`}}`}} are taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' - expr: |- - histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le)) - > 0.15 - for: 10m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: etcdMemberCommunicationSlow - annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' - expr: |- - histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) - > 0.15 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: etcdHighNumberOfFailedProposals - annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last hour on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' - expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 - for: 15m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: etcdHighFsyncDurations - annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' - expr: |- - histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) - > 0.5 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: etcdHighCommitDurations - annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' - expr: |- - histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) - > 0.25 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: etcdHighNumberOfFailedHTTPRequests - annotations: - message: '{{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}' - expr: |- - sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) - BY (method) > 0.01 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: etcdHighNumberOfFailedHTTPRequests - annotations: - message: '{{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' - expr: |- - sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) - BY (method) > 0.05 - for: 10m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: etcdHTTPRequestsSlow - annotations: - message: etcd instance {{`{{`}} $labels.instance {{`}}`}} HTTP requests to {{`{{`}} $labels.method {{`}}`}} are slow. - expr: |- - histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) - > 0.15 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/general.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/general.rules.yaml deleted file mode 100644 index cde6feb..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/general.rules.yaml +++ /dev/null @@ -1,56 +0,0 @@ -{{- /* -Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.general }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "general.rules" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: general.rules - rules: - - alert: TargetDown - annotations: - message: '{{`{{`}} $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}} targets are down.' - expr: 100 * (count(up == 0) BY (job) / count(up) BY (job)) > 10 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: Watchdog - annotations: - message: 'This is an alert meant to ensure that the entire alerting pipeline is functional. - - This alert is always firing, therefore it should always be firing in Alertmanager - - and always fire against a receiver. There are integrations with various notification - - mechanisms that send a notification when this alert is not firing. For example the - - "DeadMansSnitch" integration in PagerDuty. - - ' - expr: vector(1) - labels: - severity: none -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/k8s.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/k8s.rules.yaml deleted file mode 100644 index 08aa7fe..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/k8s.rules.yaml +++ /dev/null @@ -1,83 +0,0 @@ -{{- /* -Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8s }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: k8s.rules - rules: - - expr: sum(rate(container_cpu_usage_seconds_total{job="kubelet", image!="", container_name!=""}[5m])) by (namespace) - record: namespace:container_cpu_usage_seconds_total:sum_rate - - expr: sum(container_memory_usage_bytes{job="kubelet", image!="", container_name!=""}) by (namespace) - record: namespace:container_memory_usage_bytes:sum - - expr: |- - sum by (namespace, pod_name, container_name) ( - rate(container_cpu_usage_seconds_total{job="kubelet", image!="", container_name!=""}[5m]) - ) - record: namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate - - expr: |- - sum by(namespace) ( - kube_pod_container_resource_requests_memory_bytes{job="kube-state-metrics"} - * on (endpoint, instance, job, namespace, pod, service) - group_left(phase) (kube_pod_status_phase{phase=~"^(Pending|Running)$"} == 1) - ) - record: namespace_name:kube_pod_container_resource_requests_memory_bytes:sum - - expr: |- - sum by (namespace) ( - kube_pod_container_resource_requests_cpu_cores{job="kube-state-metrics"} - * on (endpoint, instance, job, namespace, pod, service) - group_left(phase) (kube_pod_status_phase{phase=~"^(Pending|Running)$"} == 1) - ) - record: namespace_name:kube_pod_container_resource_requests_cpu_cores:sum - - expr: |- - sum( - label_replace( - label_replace( - kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, - "replicaset", "$1", "owner_name", "(.*)" - ) * on(replicaset, namespace) group_left(owner_name) kube_replicaset_owner{job="kube-state-metrics"}, - "workload", "$1", "owner_name", "(.*)" - ) - ) by (namespace, workload, pod) - labels: - workload_type: deployment - record: mixin_pod_workload - - expr: |- - sum( - label_replace( - kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, - "workload", "$1", "owner_name", "(.*)" - ) - ) by (namespace, workload, pod) - labels: - workload_type: daemonset - record: mixin_pod_workload - - expr: |- - sum( - label_replace( - kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, - "workload", "$1", "owner_name", "(.*)" - ) - ) by (namespace, workload, pod) - labels: - workload_type: statefulset - record: mixin_pod_workload -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-apiserver.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-apiserver.rules.yaml deleted file mode 100644 index e3a9296..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-apiserver.rules.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{- /* -Generated from 'kube-apiserver.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserver }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver.rules" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: kube-apiserver.rules - rules: - - expr: histogram_quantile(0.99, sum(rate(apiserver_request_latencies_bucket{job="apiserver"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: '0.99' - record: cluster_quantile:apiserver_request_latencies:histogram_quantile - - expr: histogram_quantile(0.9, sum(rate(apiserver_request_latencies_bucket{job="apiserver"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: '0.9' - record: cluster_quantile:apiserver_request_latencies:histogram_quantile - - expr: histogram_quantile(0.5, sum(rate(apiserver_request_latencies_bucket{job="apiserver"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: '0.5' - record: cluster_quantile:apiserver_request_latencies:histogram_quantile -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-prometheus-node-alerting.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-prometheus-node-alerting.rules.yaml deleted file mode 100644 index a8d5400..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-prometheus-node-alerting.rules.yaml +++ /dev/null @@ -1,47 +0,0 @@ -{{- /* -Generated from 'kube-prometheus-node-alerting.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeAlerting }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-alerting.rules" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: kube-prometheus-node-alerting.rules - rules: - - alert: NodeDiskRunningFull - annotations: - message: Device {{`{{`}} $labels.device {{`}}`}} of node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} will be full within the next 24 hours. - expr: '(node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[6h], 3600 * 24) < 0)' - for: 30m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: NodeDiskRunningFull - annotations: - message: Device {{`{{`}} $labels.device {{`}}`}} of node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} will be full within the next 2 hours. - expr: '(node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[30m], 3600 * 2) < 0)' - for: 10m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-prometheus-node-recording.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-prometheus-node-recording.rules.yaml deleted file mode 100644 index 87f072f..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-prometheus-node-recording.rules.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- /* -Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeRecording }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-recording.rules" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: kube-prometheus-node-recording.rules - rules: - - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[3m])) BY (instance) - record: instance:node_cpu:rate:sum - - expr: sum((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"})) BY (instance) - record: instance:node_filesystem_usage:sum - - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) - record: instance:node_network_receive_bytes:rate:sum - - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) - record: instance:node_network_transmit_bytes:rate:sum - - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance) - record: instance:node_cpu:ratio - - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) - record: cluster:node_cpu:sum_rate5m - - expr: cluster:node_cpu_seconds_total:rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) - record: cluster:node_cpu:ratio -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-scheduler.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-scheduler.rules.yaml deleted file mode 100644 index 46c8d1d..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kube-scheduler.rules.yaml +++ /dev/null @@ -1,63 +0,0 @@ -{{- /* -Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeScheduler }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-scheduler.rules" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: kube-scheduler.rules - rules: - - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: '0.99' - record: cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile - - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: '0.99' - record: cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile - - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: '0.99' - record: cluster_quantile:scheduler_binding_latency:histogram_quantile - - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: '0.9' - record: cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile - - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: '0.9' - record: cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile - - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: '0.9' - record: cluster_quantile:scheduler_binding_latency:histogram_quantile - - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: '0.5' - record: cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile - - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: '0.5' - record: cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile - - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_latency_microseconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod)) / 1e+06 - labels: - quantile: '0.5' - record: cluster_quantile:scheduler_binding_latency:histogram_quantile -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-absent.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-absent.yaml deleted file mode 100644 index 5c1ebce..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-absent.yaml +++ /dev/null @@ -1,159 +0,0 @@ -{{- /* -Generated from 'kubernetes-absent' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesAbsent }} -{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} -{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} -{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} -{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-absent" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: kubernetes-absent - rules: -{{- if .Values.alertmanager.enabled }} - - alert: AlertmanagerDown - annotations: - message: Alertmanager has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-alertmanagerdown - expr: absent(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} == 1) - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} -{{- if .Values.kubeDns.enabled }} - - alert: CoreDNSDown - annotations: - message: CoreDNS has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-corednsdown - expr: absent(up{job="kube-dns"} == 1) - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} -{{- if .Values.kubeApiServer.enabled }} - - alert: KubeAPIDown - annotations: - message: KubeAPI has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapidown - expr: absent(up{job="apiserver"} == 1) - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} -{{- if .Values.kubeControllerManager.enabled }} - - alert: KubeControllerManagerDown - annotations: - message: KubeControllerManager has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecontrollermanagerdown - expr: absent(up{job="kube-controller-manager"} == 1) - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} -{{- if .Values.kubeScheduler.enabled }} - - alert: KubeSchedulerDown - annotations: - message: KubeScheduler has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeschedulerdown - expr: absent(up{job="kube-scheduler"} == 1) - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} -{{- if .Values.kubeStateMetrics.enabled }} - - alert: KubeStateMetricsDown - annotations: - message: KubeStateMetrics has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatemetricsdown - expr: absent(up{job="kube-state-metrics"} == 1) - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} -{{- if .Values.prometheusOperator.kubeletService.enabled }} - - alert: KubeletDown - annotations: - message: Kubelet has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletdown - expr: absent(up{job="kubelet"} == 1) - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} -{{- if .Values.nodeExporter.enabled }} - - alert: NodeExporterDown - annotations: - message: NodeExporter has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodeexporterdown - expr: absent(up{job="node-exporter"} == 1) - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} - - alert: PrometheusDown - annotations: - message: Prometheus has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusdown - expr: absent(up{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} == 1) - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- if .Values.prometheusOperator.enabled }} - - alert: PrometheusOperatorDown - annotations: - message: PrometheusOperator has disappeared from Prometheus target discovery. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusoperatordown - expr: absent(up{job="{{ $operatorJob }}",namespace="{{ $namespace }}"} == 1) - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-apps.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-apps.yaml deleted file mode 100644 index e7a41ca..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-apps.yaml +++ /dev/null @@ -1,200 +0,0 @@ -{{- /* -Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesApps }} -{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: kubernetes-apps - rules: - - alert: KubePodCrashLooping - annotations: - message: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is restarting {{`{{`}} printf "%.2f" $value {{`}}`}} times / 5 minutes. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodcrashlooping - expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[15m]) * 60 * 5 > 0 - for: 1h - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubePodNotReady - annotations: - message: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than an hour. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodnotready - expr: sum by (namespace, pod) (kube_pod_status_phase{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown"}) > 0 - for: 1h - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeDeploymentGenerationMismatch - annotations: - message: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentgenerationmismatch - expr: |- - kube_deployment_status_observed_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - != - kube_deployment_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeDeploymentReplicasMismatch - annotations: - message: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than an hour. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentreplicasmismatch - expr: |- - kube_deployment_spec_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - != - kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - for: 1h - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeStatefulSetReplicasMismatch - annotations: - message: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetreplicasmismatch - expr: |- - kube_statefulset_status_replicas_ready{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - != - kube_statefulset_status_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeStatefulSetGenerationMismatch - annotations: - message: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetgenerationmismatch - expr: |- - kube_statefulset_status_observed_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - != - kube_statefulset_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeStatefulSetUpdateNotRolledOut - annotations: - message: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetupdatenotrolledout - expr: |- - max without (revision) ( - kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - unless - kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - ) - * - ( - kube_statefulset_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - != - kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - ) - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeDaemonSetRolloutStuck - annotations: - message: Only {{`{{`}} $value {{`}}`}}% of the desired Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are scheduled and ready. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetrolloutstuck - expr: |- - kube_daemonset_status_number_ready{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - / - kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} * 100 < 100 - for: 15m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeDaemonSetNotScheduled - annotations: - message: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.' - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetnotscheduled - expr: |- - kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - - - kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeDaemonSetMisScheduled - annotations: - message: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.' - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetmisscheduled - expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeCronJobRunning - annotations: - message: CronJob {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.cronjob {{`}}`}} is taking more than 1h to complete. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecronjobrunning - expr: time() - kube_cronjob_next_schedule_time{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 3600 - for: 1h - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeJobCompletion - annotations: - message: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than one hour to complete. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobcompletion - expr: kube_job_spec_completions{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - kube_job_status_succeeded{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 - for: 1h - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeJobFailed - annotations: - message: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobfailed - expr: kube_job_status_failed{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 - for: 1h - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-resources.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-resources.yaml deleted file mode 100644 index b34b442..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-resources.yaml +++ /dev/null @@ -1,121 +0,0 @@ -{{- /* -Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesResources }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-resources" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: kubernetes-resources - rules: - - alert: KubeCPUOvercommit - annotations: - message: Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit - expr: |- - sum(namespace_name:kube_pod_container_resource_requests_cpu_cores:sum) - / - sum(node:node_num_cpu:sum) - > - (count(node:node_num_cpu:sum)-1) / count(node:node_num_cpu:sum) - for: 5m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeMemOvercommit - annotations: - message: Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememovercommit - expr: |- - sum(namespace_name:kube_pod_container_resource_requests_memory_bytes:sum) - / - sum(node_memory_MemTotal_bytes) - > - (count(node:node_num_cpu:sum)-1) - / - count(node:node_num_cpu:sum) - for: 5m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeCPUOvercommit - annotations: - message: Cluster has overcommitted CPU resource requests for Namespaces. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit - expr: |- - sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"}) - / - sum(node:node_num_cpu:sum) - > 1.5 - for: 5m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeMemOvercommit - annotations: - message: Cluster has overcommitted memory resource requests for Namespaces. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememovercommit - expr: |- - sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"}) - / - sum(node_memory_MemTotal_bytes{job="node-exporter"}) - > 1.5 - for: 5m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeQuotaExceeded - annotations: - message: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} printf "%0.0f" $value {{`}}`}}% of its {{`{{`}} $labels.resource {{`}}`}} quota. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubequotaexceeded - expr: |- - 100 * kube_resourcequota{job="kube-state-metrics", type="used"} - / ignoring(instance, job, type) - (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) - > 90 - for: 15m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: CPUThrottlingHigh - annotations: - message: '{{`{{`}} printf "%0.0f" $value {{`}}`}}% throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container_name {{`}}`}} in pod {{`{{`}} $labels.pod_name {{`}}`}}.' - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-cputhrottlinghigh - expr: |- - 100 * sum(increase(container_cpu_cfs_throttled_periods_total{container_name!="", }[5m])) by (container_name, pod_name, namespace) - / - sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container_name, pod_name, namespace) - > 25 - for: 15m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-storage.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-storage.yaml deleted file mode 100644 index 6469fff..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-storage.yaml +++ /dev/null @@ -1,72 +0,0 @@ -{{- /* -Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesStorage }} -{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-storage" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: kubernetes-storage - rules: - - alert: KubePersistentVolumeUsageCritical - annotations: - message: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is only {{`{{`}} printf "%0.2f" $value {{`}}`}}% free. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumeusagecritical - expr: |- - 100 * kubelet_volume_stats_available_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}"} - / - kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}"} - < 3 - for: 1m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubePersistentVolumeFullInFourDays - annotations: - message: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} printf "%0.2f" $value {{`}}`}}% is available. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumefullinfourdays - expr: |- - 100 * ( - kubelet_volume_stats_available_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}"} - / - kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}"} - ) < 15 - and - predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}"}[6h], 4 * 24 * 3600) < 0 - for: 5m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubePersistentVolumeErrors - annotations: - message: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumeerrors - expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0 - for: 5m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-system.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-system.yaml deleted file mode 100644 index da23205..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/kubernetes-system.yaml +++ /dev/null @@ -1,184 +0,0 @@ -{{- /* -Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: kubernetes-system - rules: - - alert: KubeNodeNotReady - annotations: - message: '{{`{{`}} $labels.node {{`}}`}} has been unready for more than an hour.' - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodenotready - expr: kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0 - for: 1h - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeVersionMismatch - annotations: - message: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeversionmismatch - expr: count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"gitVersion","$1","gitVersion","(v[0-9]*.[0-9]*.[0-9]*).*"))) > 1 - for: 1h - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeClientErrors - annotations: - message: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} printf "%0.0f" $value {{`}}`}}% errors.' - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclienterrors - expr: |- - (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job) - / - sum(rate(rest_client_requests_total[5m])) by (instance, job)) - * 100 > 1 - for: 15m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeClientErrors - annotations: - message: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} printf "%0.0f" $value {{`}}`}} errors / second. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclienterrors - expr: sum(rate(ksm_scrape_error_total{job="kube-state-metrics"}[5m])) by (instance, job) > 0.1 - for: 15m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeletTooManyPods - annotations: - message: Kubelet {{`{{`}} $labels.instance {{`}}`}} is running {{`{{`}} $value {{`}}`}} Pods, close to the limit of 110. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubelettoomanypods - expr: kubelet_running_pod_count{job="kubelet"} > 110 * 0.9 - for: 15m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeAPILatencyHigh - annotations: - message: The API server has a 99th percentile latency of {{`{{`}} $value {{`}}`}} seconds for {{`{{`}} $labels.verb {{`}}`}} {{`{{`}} $labels.resource {{`}}`}}. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh - expr: cluster_quantile:apiserver_request_latencies:histogram_quantile{job="apiserver",quantile="0.99",subresource!="log",verb!~"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$"} > 1 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeAPILatencyHigh - annotations: - message: The API server has a 99th percentile latency of {{`{{`}} $value {{`}}`}} seconds for {{`{{`}} $labels.verb {{`}}`}} {{`{{`}} $labels.resource {{`}}`}}. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh - expr: cluster_quantile:apiserver_request_latencies:histogram_quantile{job="apiserver",quantile="0.99",subresource!="log",verb!~"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$"} > 4 - for: 10m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeAPIErrorsHigh - annotations: - message: API server is returning errors for {{`{{`}} $value {{`}}`}}% of requests. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh - expr: |- - sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) - / - sum(rate(apiserver_request_count{job="apiserver"}[5m])) * 100 > 3 - for: 10m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeAPIErrorsHigh - annotations: - message: API server is returning errors for {{`{{`}} $value {{`}}`}}% of requests. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh - expr: |- - sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) - / - sum(rate(apiserver_request_count{job="apiserver"}[5m])) * 100 > 1 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeAPIErrorsHigh - annotations: - message: API server is returning errors for {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.verb {{`}}`}} {{`{{`}} $labels.resource {{`}}`}} {{`{{`}} $labels.subresource {{`}}`}}. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh - expr: |- - sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) by (resource,subresource,verb) - / - sum(rate(apiserver_request_count{job="apiserver"}[5m])) by (resource,subresource,verb) * 100 > 10 - for: 10m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeAPIErrorsHigh - annotations: - message: API server is returning errors for {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.verb {{`}}`}} {{`{{`}} $labels.resource {{`}}`}} {{`{{`}} $labels.subresource {{`}}`}}. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh - expr: |- - sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) by (resource,subresource,verb) - / - sum(rate(apiserver_request_count{job="apiserver"}[5m])) by (resource,subresource,verb) * 100 > 5 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeClientCertificateExpiration - annotations: - message: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration - expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: KubeClientCertificateExpiration - annotations: - message: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration - expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/node-network.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/node-network.yaml deleted file mode 100644 index c75f1ae..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/node-network.yaml +++ /dev/null @@ -1,57 +0,0 @@ -{{- /* -Generated from 'node-network' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.network }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-network" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: node-network - rules: - - alert: NetworkReceiveErrors - annotations: - message: Network interface "{{`{{`}} $labels.device {{`}}`}}" showing receive errors on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}}" - expr: rate(node_network_receive_errs_total{job="node-exporter",device!~"veth.+"}[2m]) > 0 - for: 2m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: NetworkTransmitErrors - annotations: - message: Network interface "{{`{{`}} $labels.device {{`}}`}}" showing transmit errors on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}}" - expr: rate(node_network_transmit_errs_total{job="node-exporter",device!~"veth.+"}[2m]) > 0 - for: 2m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: NodeNetworkInterfaceFlapping - annotations: - message: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing it's up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}}" - expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 - for: 2m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/node-time.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/node-time.yaml deleted file mode 100644 index b7a2fc9..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/node-time.yaml +++ /dev/null @@ -1,37 +0,0 @@ -{{- /* -Generated from 'node-time' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.time }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-time" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: node-time - rules: - - alert: ClockSkewDetected - annotations: - message: Clock skew detected on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}}. Ensure NTP is configured correctly on this host. - expr: abs(node_timex_offset_seconds{job="node-exporter"}) > 0.03 - for: 2m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/node.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/node.rules.yaml deleted file mode 100644 index 2bc7af3..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/node.rules.yaml +++ /dev/null @@ -1,202 +0,0 @@ -{{- /* -Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.node }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node.rules" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: node.rules - rules: - - expr: sum(min(kube_pod_info) by (node)) - record: ':kube_pod_info_node_count:' - - expr: max(label_replace(kube_pod_info{job="kube-state-metrics"}, "pod", "$1", "pod", "(.*)")) by (node, namespace, pod) - record: 'node_namespace_pod:kube_pod_info:' - - expr: |- - count by (node) (sum by (node, cpu) ( - node_cpu_seconds_total{job="node-exporter"} - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - )) - record: node:node_num_cpu:sum - - expr: 1 - avg(rate(node_cpu_seconds_total{job="node-exporter",mode="idle"}[1m])) - record: :node_cpu_utilisation:avg1m - - expr: |- - 1 - avg by (node) ( - rate(node_cpu_seconds_total{job="node-exporter",mode="idle"}[1m]) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info:) - record: node:node_cpu_utilisation:avg1m - - expr: |- - node:node_cpu_utilisation:avg1m - * - node:node_num_cpu:sum - / - scalar(sum(node:node_num_cpu:sum)) - record: node:cluster_cpu_utilisation:ratio - - expr: |- - sum(node_load1{job="node-exporter"}) - / - sum(node:node_num_cpu:sum) - record: ':node_cpu_saturation_load1:' - - expr: |- - sum by (node) ( - node_load1{job="node-exporter"} - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - / - node:node_num_cpu:sum - record: 'node:node_cpu_saturation_load1:' - - expr: |- - 1 - - sum(node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"}) - / - sum(node_memory_MemTotal_bytes{job="node-exporter"}) - record: ':node_memory_utilisation:' - - expr: sum(node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"}) - record: :node_memory_MemFreeCachedBuffers_bytes:sum - - expr: sum(node_memory_MemTotal_bytes{job="node-exporter"}) - record: :node_memory_MemTotal_bytes:sum - - expr: |- - sum by (node) ( - (node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"}) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_memory_bytes_available:sum - - expr: |- - sum by (node) ( - node_memory_MemTotal_bytes{job="node-exporter"} - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_memory_bytes_total:sum - - expr: |- - (node:node_memory_bytes_total:sum - node:node_memory_bytes_available:sum) - / - node:node_memory_bytes_total:sum - record: node:node_memory_utilisation:ratio - - expr: |- - (node:node_memory_bytes_total:sum - node:node_memory_bytes_available:sum) - / - scalar(sum(node:node_memory_bytes_total:sum)) - record: node:cluster_memory_utilisation:ratio - - expr: |- - 1e3 * sum( - (rate(node_vmstat_pgpgin{job="node-exporter"}[1m]) - + rate(node_vmstat_pgpgout{job="node-exporter"}[1m])) - ) - record: :node_memory_swap_io_bytes:sum_rate - - expr: |- - 1 - - sum by (node) ( - (node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"}) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - / - sum by (node) ( - node_memory_MemTotal_bytes{job="node-exporter"} - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: 'node:node_memory_utilisation:' - - expr: 1 - (node:node_memory_bytes_available:sum / node:node_memory_bytes_total:sum) - record: 'node:node_memory_utilisation_2:' - - expr: |- - 1e3 * sum by (node) ( - (rate(node_vmstat_pgpgin{job="node-exporter"}[1m]) - + rate(node_vmstat_pgpgout{job="node-exporter"}[1m])) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_memory_swap_io_bytes:sum_rate - - expr: avg(irate(node_disk_io_time_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m])) - record: :node_disk_utilisation:avg_irate - - expr: |- - avg by (node) ( - irate(node_disk_io_time_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m]) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_disk_utilisation:avg_irate - - expr: avg(irate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m])) - record: :node_disk_saturation:avg_irate - - expr: |- - avg by (node) ( - irate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m]) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_disk_saturation:avg_irate - - expr: |- - max by (instance, namespace, pod, device) ((node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"} - - node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) - / node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) - record: 'node:node_filesystem_usage:' - - expr: max by (instance, namespace, pod, device) (node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"} / node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) - record: 'node:node_filesystem_avail:' - - expr: |- - sum(irate(node_network_receive_bytes_total{job="node-exporter",device!~"veth.+"}[1m])) + - sum(irate(node_network_transmit_bytes_total{job="node-exporter",device!~"veth.+"}[1m])) - record: :node_net_utilisation:sum_irate - - expr: |- - sum by (node) ( - (irate(node_network_receive_bytes_total{job="node-exporter",device!~"veth.+"}[1m]) + - irate(node_network_transmit_bytes_total{job="node-exporter",device!~"veth.+"}[1m])) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_net_utilisation:sum_irate - - expr: |- - sum(irate(node_network_receive_drop_total{job="node-exporter",device!~"veth.+"}[1m])) + - sum(irate(node_network_transmit_drop_total{job="node-exporter",device!~"veth.+"}[1m])) - record: :node_net_saturation:sum_irate - - expr: |- - sum by (node) ( - (irate(node_network_receive_drop_total{job="node-exporter",device!~"veth.+"}[1m]) + - irate(node_network_transmit_drop_total{job="node-exporter",device!~"veth.+"}[1m])) - * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: - ) - record: node:node_net_saturation:sum_irate - - expr: |- - max( - max( - kube_pod_info{job="kube-state-metrics", host_ip!=""} - ) by (node, host_ip) - * on (host_ip) group_right (node) - label_replace( - (max(node_filesystem_files{job="node-exporter", mountpoint="/"}) by (instance)), "host_ip", "$1", "instance", "(.*):.*" - ) - ) by (node) - record: 'node:node_inodes_total:' - - expr: |- - max( - max( - kube_pod_info{job="kube-state-metrics", host_ip!=""} - ) by (node, host_ip) - * on (host_ip) group_right (node) - label_replace( - (max(node_filesystem_files_free{job="node-exporter", mountpoint="/"}) by (instance)), "host_ip", "$1", "instance", "(.*):.*" - ) - ) by (node) - record: 'node:node_inodes_free:' -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/prometheus-operator.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/prometheus-operator.yaml deleted file mode 100644 index a8a8915..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/prometheus-operator.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{- /* -Generated from 'prometheus-operator' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheusOperator }} -{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} -{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus-operator" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: prometheus-operator - rules: - - alert: PrometheusOperatorReconcileErrors - annotations: - message: Errors while reconciling {{`{{`}} $labels.controller {{`}}`}} in {{`{{`}} $labels.namespace {{`}}`}} Namespace. - expr: rate(prometheus_operator_reconcile_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: PrometheusOperatorNodeLookupErrors - annotations: - message: Errors while reconciling Prometheus in {{`{{`}} $labels.namespace {{`}}`}} Namespace. - expr: rate(prometheus_operator_node_address_lookup_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/prometheus.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/prometheus.rules.yaml deleted file mode 100644 index 0480c83..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/prometheus.rules.yaml +++ /dev/null @@ -1,139 +0,0 @@ -{{- /* -Generated from 'prometheus.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml -Do not change in-place! In order to change this file first read following link: -https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack -*/ -}} -{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} -{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheus }} -{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} -{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus.rules" | trunc 63 | trimSuffix "-" }} - namespace: {{ template "kube-prometheus-stack.namespace" . }} - labels: - app: {{ template "kube-prometheus-stack.name" . }} -{{ include "kube-prometheus-stack.labels" . | indent 4 }} -{{- if .Values.defaultRules.labels }} -{{ toYaml .Values.defaultRules.labels | indent 4 }} -{{- end }} -{{- if .Values.defaultRules.annotations }} - annotations: -{{ toYaml .Values.defaultRules.annotations | indent 4 }} -{{- end }} -spec: - groups: - - name: prometheus.rules - rules: - - alert: PrometheusConfigReloadFailed - annotations: - description: Reloading Prometheus' configuration has failed for {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} - summary: Reloading Prometheus' configuration failed - expr: prometheus_config_last_reload_successful{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} == 0 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: PrometheusNotificationQueueRunningFull - annotations: - description: Prometheus' alert notification queue is running full for {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} - summary: Prometheus' alert notification queue is running full - expr: predict_linear(prometheus_notifications_queue_length{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m], 60 * 30) > prometheus_notifications_queue_capacity{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: PrometheusErrorSendingAlerts - annotations: - description: Errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.Alertmanager{{`}}`}} - summary: Errors while sending alert from Prometheus - expr: rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.01 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: PrometheusErrorSendingAlerts - annotations: - description: Errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.Alertmanager{{`}}`}} - summary: Errors while sending alerts from Prometheus - expr: rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.03 - for: 10m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: PrometheusNotConnectedToAlertmanagers - annotations: - description: Prometheus {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} is not connected to any Alertmanagers - summary: Prometheus is not connected to any Alertmanagers - expr: prometheus_notifications_alertmanagers_discovered{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} < 1 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: PrometheusTSDBReloadsFailing - annotations: - description: '{{`{{`}}$labels.job{{`}}`}} at {{`{{`}}$labels.instance{{`}}`}} had {{`{{`}}$value | humanize{{`}}`}} reload failures over the last four hours.' - summary: Prometheus has issues reloading data blocks from disk - expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[2h]) > 0 - for: 12h - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: PrometheusTSDBCompactionsFailing - annotations: - description: '{{`{{`}}$labels.job{{`}}`}} at {{`{{`}}$labels.instance{{`}}`}} had {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last four hours.' - summary: Prometheus has issues compacting sample blocks - expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[2h]) > 0 - for: 12h - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: PrometheusTSDBWALCorruptions - annotations: - description: '{{`{{`}}$labels.job{{`}}`}} at {{`{{`}}$labels.instance{{`}}`}} has a corrupted write-ahead log (WAL).' - summary: Prometheus write-ahead log is corrupted - expr: prometheus_tsdb_wal_corruptions_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} > 0 - for: 4h - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: PrometheusNotIngestingSamples - annotations: - description: Prometheus {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} isn't ingesting samples. - summary: Prometheus isn't ingesting samples - expr: rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) <= 0 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - - alert: PrometheusTargetScrapesDuplicate - annotations: - description: '{{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has many samples rejected due to duplicate timestamps but different values' - summary: Prometheus has many samples rejected - expr: increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 - for: 10m - labels: - severity: warning -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml index 109820a..c3a9ffd 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml @@ -396,7 +396,7 @@ alertmanager: ## image: repository: quay.io/prometheus/alertmanager - tag: v0.22.0 + tag: v0.22.2 sha: "" ## If true then the user will be responsible to provide a secret with alertmanager configuration @@ -613,6 +613,14 @@ grafana: enabled: true namespaceOverride: "" + ## ForceDeployDatasources Create datasource configmap even if grafana deployment has been disabled + ## + forceDeployDatasources: false + + ## ForceDeployDashboard Create dashboard configmap even if grafana deployment has been disabled + ## + forceDeployDashboards: false + ## Deploy default dashboards. ## defaultDashboardsEnabled: true @@ -665,6 +673,10 @@ grafana: enabled: true defaultDatasourceEnabled: true + ## URL of prometheus datasource + ## + # url: http://prometheus-stack-prometheus:9090/ + # If not defined, will use prometheus.prometheusSpec.scrapeInterval or its default # defaultDatasourceScrapeInterval: 15s @@ -742,19 +754,6 @@ kubeApiServer: tlsConfig: serverName: kubernetes insecureSkipVerify: false - - ## If your API endpoint address is not reachable (as in AKS) you can replace it with the kubernetes service - ## - relabelings: [] - # - sourceLabels: - # - __meta_kubernetes_namespace - # - __meta_kubernetes_service_name - # - __meta_kubernetes_endpoint_port_name - # action: keep - # regex: default;kubernetes;https - # - targetLabel: __address__ - # replacement: kubernetes.default.svc:443 - serviceMonitor: ## Scrape interval. If not set, the Prometheus default scrape interval is used. ## @@ -775,6 +774,15 @@ kubeApiServer: # - action: keep # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' # sourceLabels: [__name__] + relabelings: [] + # - sourceLabels: + # - __meta_kubernetes_namespace + # - __meta_kubernetes_service_name + # - __meta_kubernetes_endpoint_port_name + # action: keep + # regex: default;kubernetes;https + # - targetLabel: __address__ + # replacement: kubernetes.default.svc:443 ## Component scraping the kubelet and kubelet-hosted cAdvisor ## @@ -1250,6 +1258,10 @@ kubeStateMetrics: # replacement: $1 # action: replace + # Enable self metrics configuration for Service Monitor + selfMonitor: + enabled: false + ## Configuration for kube-state-metrics subchart ## kube-state-metrics: @@ -1353,6 +1365,16 @@ prometheusOperator: nodeSelector: {} affinity: {} tolerations: [] + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 2000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + # Use certmanager to generate webhook certs certManager: enabled: false @@ -1540,7 +1562,7 @@ prometheusOperator: ## image: repository: quay.io/prometheus-operator/prometheus-operator - tag: v0.48.0 + tag: v0.48.1 sha: "" pullPolicy: IfNotPresent @@ -1556,7 +1578,7 @@ prometheusOperator: ## prometheusConfigReloaderImage: repository: quay.io/prometheus-operator/prometheus-config-reloader - tag: v0.48.0 + tag: v0.48.1 sha: "" ## Set the prometheus config reloader side-car CPU limit @@ -1567,6 +1589,13 @@ prometheusOperator: ## configReloaderMemory: 50Mi + ## Thanos side-car image when configured + ## + thanosImage: + repository: quay.io/thanos/thanos + tag: v0.17.2 + sha: "" + ## Set a Field Selector to filter watched secrets ## secretFieldSelector: "" diff --git a/charts/kubezero-metrics/update.sh b/charts/kubezero-metrics/update.sh index 9f3d1cc..47c86c5 100755 --- a/charts/kubezero-metrics/update.sh +++ b/charts/kubezero-metrics/update.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION=16.1.2 +VERSION=16.12.0 rm -rf charts/kube-prometheus-stack curl -L -s -o - https://github.com/prometheus-community/helm-charts/releases/download/kube-prometheus-stack-${VERSION}/kube-prometheus-stack-${VERSION}.tgz | tar xfz - -C charts -- 2.40.1 From 62a12b7f79618d3f85e33cdd9ad426344545b349 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 1 Jul 2021 12:36:35 +0200 Subject: [PATCH 15/45] fix: update grafana dashboards to match new prometheus labels --- .../charts/kube-state-metrics/values.yaml | 2 +- .../dashboards/jsonnetfile.lock.json | 8 +- .../kube-mixin/controller-manager.json | 12 +- .../kube-mixin/k8s-resources-cluster.json | 30 +- .../kube-mixin/k8s-resources-namespace.json | 24 +- .../kube-mixin/k8s-resources-node.json | 8 +- .../kube-mixin/k8s-resources-pod.json | 30 +- .../kube-mixin/k8s-resources-workload.json | 16 +- .../k8s-resources-workloads-namespace.json | 16 +- .../dashboards/kube-mixin/kubelet.json | 3895 ++++++++--------- .../kube-mixin/persistentvolumesusage.json | 12 +- .../dashboards/kube-mixin/scheduler.json | 32 +- .../templates/grafana-dashboards-k8s.yaml | 20 +- charts/kubezero-metrics/values.yaml | 9 +- 14 files changed, 1918 insertions(+), 2196 deletions(-) diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml index a9853ba..f3bde2c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml @@ -162,7 +162,7 @@ collectors: - statefulsets - storageclasses - validatingwebhookconfigurations - - verticalpodautoscalers + # - verticalpodautoscalers - volumeattachments # Enabling kubeconfig will pass the --kubeconfig argument to the container diff --git a/charts/kubezero-metrics/dashboards/jsonnetfile.lock.json b/charts/kubezero-metrics/dashboards/jsonnetfile.lock.json index 3f85154..2926b20 100644 --- a/charts/kubezero-metrics/dashboards/jsonnetfile.lock.json +++ b/charts/kubezero-metrics/dashboards/jsonnetfile.lock.json @@ -8,7 +8,7 @@ "subdir": "grafonnet" } }, - "version": "55cf4ee53ced2b6d3ce96ecce9fb813b4465be98", + "version": "3082bfca110166cd69533fa3c0875fdb1b68c329", "sum": "4/sUV0Kk+o8I+wlYxL9R6EPhL/NiLfYHk+NXlU64RUk=" }, { @@ -18,7 +18,7 @@ "subdir": "grafana-builder" } }, - "version": "970b0dd2fc5b896406eba258cad78b7e0b0f18bf", + "version": "136b7e4fa204d6b4f9f3bfc5d8ace8834d2b4aae", "sum": "GRf2GvwEU4jhXV+JOonXSZ4wdDv8mnHBPCQ6TUVd+g8=" }, { @@ -28,8 +28,8 @@ "subdir": "" } }, - "version": "39a9cda705b5201c35105bd1f24c83923fa839ef", - "sum": "8GmlcgCeC46NSHfRBebyqUOJCG1YvqvuGMj98u+Pge4=" + "version": "baf5e20d3275ebed214eaa26a0377ec3bf2b8897", + "sum": "U1JPaRwc6xDBkE30bbEpT2j0vAJvrm5GXhCIZqLLLWU=" } ], "legacyImports": false diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/controller-manager.json b/charts/kubezero-metrics/dashboards/kube-mixin/controller-manager.json index d504a28..7d27b58 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/controller-manager.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/controller-manager.json @@ -134,10 +134,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} {{name}}", + "legendFormat": "{{cluster}} {{instance}} {{name}}", "refId": "A" } ], @@ -228,10 +228,10 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} {{name}}", + "legendFormat": "{{cluster}} {{instance}} {{name}}", "refId": "A" } ], @@ -322,10 +322,10 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name, le))", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (cluster, instance, name, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} {{name}}", + "legendFormat": "{{cluster}} {{instance}} {{name}}", "refId": "A" } ], diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-cluster.json b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-cluster.json index 7e3ab69..c598fbd 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-cluster.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-cluster.json @@ -199,7 +199,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{resource=\"cpu\",cluster=\"$cluster\"})", + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"cpu\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -427,7 +427,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"memory\"}) / sum(kube_node_status_allocatable{resource=\"memory\",cluster=\"$cluster\"})", + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{resource=\"memory\",cluster=\"$cluster\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -514,7 +514,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{namespace}}", @@ -759,7 +759,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -777,7 +777,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", resource=\"cpu\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -786,7 +786,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"cpu\"}) by (namespace)", + "expr": "sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -795,7 +795,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"cpu\"}) by (namespace)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1139,7 +1139,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_requests{cluster=\"$cluster\", resource=\"memory\"}) by (namespace)", + "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1148,7 +1148,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", resource=\"memory\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1157,7 +1157,7 @@ "step": 10 }, { - "expr": "sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"memory\"}) by (namespace)", + "expr": "sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1166,7 +1166,7 @@ "step": 10 }, { - "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", resource=\"memory\"}) by (namespace)", + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1871,7 +1871,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1947,7 +1947,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -2035,7 +2035,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -2111,7 +2111,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-namespace.json b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-namespace.json index 8b69944..94a2d3b 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-namespace.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-namespace.json @@ -46,7 +46,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -122,7 +122,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -208,7 +208,7 @@ "thresholds": "70,80", "timeFrom": null, "timeShift": null, - "title": "Memory Utilization (from requests)", + "title": "Memory Utilisation (from requests)", "tooltip": { "shared": false, "sort": 0, @@ -384,7 +384,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -597,7 +597,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -615,7 +615,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -633,7 +633,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1608,7 +1608,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1684,7 +1684,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1772,7 +1772,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1848,7 +1848,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -2181,7 +2181,7 @@ "decimals": 2, "link": true, "linkTargetBlank": false, - "linkTooltip": "Drill down to containers", + "linkTooltip": "Drill down to pods", "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", "pattern": "pod", "thresholds": [ ], diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-node.json b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-node.json index 4d71170..ec28f93 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-node.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-node.json @@ -45,7 +45,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -242,7 +242,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -260,7 +260,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, @@ -278,7 +278,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"}) by (pod)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"}) by (pod)", "format": "table", "instant": true, "intervalFactor": 2, diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-pod.json b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-pod.json index af45cdf..c7587e5 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-pod.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-pod.json @@ -64,7 +64,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{container}}", @@ -374,7 +374,7 @@ ], "targets": [ { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -392,7 +392,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -410,7 +410,7 @@ "step": 10 }, { - "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)", + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}) by (container)", "format": "table", "instant": true, "intervalFactor": 2, @@ -1126,7 +1126,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1203,7 +1203,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1292,7 +1292,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1369,7 +1369,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1615,7 +1615,7 @@ "expr": "ceil(sum by(container) (rate(container_fs_reads_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m])))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{pod}}", + "legendFormat": "{{container}}", "legendLink": null, "step": 10 } @@ -1691,7 +1691,7 @@ "expr": "sum by(container) (rate(container_fs_reads_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]) + rate(container_fs_writes_bytes_total{container!=\"\", cluster=\"$cluster\",namespace=~\"$namespace\", pod=\"$pod\"}[5m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{pod}}", + "legendFormat": "{{container}}", "legendLink": null, "step": 10 } @@ -1876,16 +1876,16 @@ "unit": "Bps" }, { - "alias": "Pod", + "alias": "Container", "colorMode": null, "colors": [ ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, - "link": true, + "link": false, "linkTargetBlank": false, - "linkTooltip": "Drill down to pods", - "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", - "pattern": "pod", + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", "thresholds": [ ], "type": "number", "unit": "short" diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-workload.json b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-workload.json index 4725d41..3437a4b 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-workload.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-workload.json @@ -45,7 +45,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{pod}}", @@ -242,7 +242,7 @@ ], "targets": [ { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -260,7 +260,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -278,7 +278,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1306,7 +1306,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1382,7 +1382,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1470,7 +1470,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1546,7 +1546,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-workloads-namespace.json b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-workloads-namespace.json index d3d92f7..3c6f46f 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-workloads-namespace.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/k8s-resources-workloads-namespace.json @@ -68,7 +68,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{workload}} - {{workload_type}}", @@ -320,7 +320,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -338,7 +338,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -356,7 +356,7 @@ "step": 10 }, { - "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", "format": "table", "instant": true, "intervalFactor": 2, @@ -1477,7 +1477,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1553,7 +1553,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1641,7 +1641,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, @@ -1717,7 +1717,7 @@ }, "yaxes": [ { - "format": "Bps", + "format": "pps", "label": null, "logBase": 1, "max": null, diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/kubelet.json b/charts/kubezero-metrics/dashboards/kube-mixin/kubelet.json index 95a690c..dac28b9 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/kubelet.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/kubelet.json @@ -10,2145 +10,1870 @@ "hideControls": false, "id": null, "links": [ ], - "refresh": "10s", - "rows": [ + "panels": [ { - "collapse": false, - "collapsed": false, - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { }, - "id": 2, - "interval": null, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { "links": [ ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kubelet_node_name{cluster=\"$cluster\", job=\"kubelet\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "thresholds": "", - "title": "Running Kubelets", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - }, + "fields": "", + "values": false + } + }, + "pluginVersion": "7", + "targets": [ { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { }, - "id": 3, - "interval": null, - "links": [ ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Running Pods", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { }, - "id": 4, - "interval": null, - "links": [ ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Running Container", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { }, - "id": 5, - "interval": null, - "links": [ ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\", state=\"actual_state_of_world\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Actual Volume Count", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { }, - "id": 6, - "interval": null, - "links": [ ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\",state=\"desired_state_of_world\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Desired Volume Count", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { }, - "id": 7, - "interval": null, - "links": [ ], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "span": 2, - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": "", - "title": "Config Error Count", - "tooltip": { - "shared": false - }, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "min" + "expr": "sum(kubelet_node_name{cluster=\"$cluster\", job=\"kubelet\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" + "title": "Running Kubelets", + "transparent": false, + "type": "stat" }, { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, + "datasource": "$datasource", + "fieldConfig": { + "defaults": { "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (operation_type, instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} {{operation_type}}", - "refId": "A" - } + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 3, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Operation Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" + "fields": "", + "values": false + } + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Running Pods", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 4, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Running Container", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 5, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Actual Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 6, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Desired Volume Count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 7, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Config Error Count", + "transparent": false, + "type": "stat" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_type}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 9, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} {{operation_type}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Operation Error Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" + ] }, { - "collapse": false, - "collapsed": false, - "panels": [ + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 10, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} {{operation_type}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Operation duration 99th quantile", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_type}}", + "refId": "A" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 11, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} pod", - "refId": "A" - }, - { - "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} worker", - "refId": "B" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Pod Start Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 12, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} pod", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} worker", - "refId": "B" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Pod Start Duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" + ] }, { - "collapse": false, - "collapsed": false, - "panels": [ + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 13, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Storage Operation Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_type}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Storage Operation Error Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" + ] }, { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 15, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Storage Operation Duration 99th quantile", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 16, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{operation_type}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Cgroup manager operation rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} pod", + "refId": "A" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 17, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} {{operation_type}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Cgroup manager 99th quantile", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} worker", + "refId": "B" } ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Pod lifecycle event generator", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 18, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "PLEG relist rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 19, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "PLEG relist interval", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" + ] }, { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 20, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "PLEG relist duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 21, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"2..\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "2xx", - "refId": "A" - }, - { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"3..\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "3xx", - "refId": "B" - }, - { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"4..\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "4xx", - "refId": "C" - }, - { - "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"5..\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "5xx", - "refId": "D" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "RPC Rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 22, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 12, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, verb, url, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}} {{verb}} {{url}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Request duration 99th quantile", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" - }, - { - "collapse": false, - "collapsed": false, - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 23, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} pod", + "refId": "A" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 24, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "CPU usage", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "fillGradient": 0, - "gridPos": { }, - "id": 25, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "sideWidth": null, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} worker", + "refId": "B" } ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": false, - "title": "Dashboard Row", - "titleSize": "h6", - "type": "row" + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{operation_type}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_type}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{verb}} {{url}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 77 + }, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 77 + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 77 + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } ], + "refresh": "10s", + "rows": [ ], "schemaVersion": 14, "style": "dark", "tags": [ diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/persistentvolumesusage.json b/charts/kubezero-metrics/dashboards/kube-mixin/persistentvolumesusage.json index 72ac8b9..e9ab9c2 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/persistentvolumesusage.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/persistentvolumesusage.json @@ -54,14 +54,14 @@ "steppedLine": false, "targets": [ { - "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "Used Space", "refId": "A" }, { - "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "Free Space", @@ -161,7 +161,7 @@ "tableColumn": "", "targets": [ { - "expr": "max without(instance,node) (\n(\n kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100)\n", + "expr": "max without(instance,node) (\n(\n topk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n topk(1, kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -235,14 +235,14 @@ "steppedLine": false, "targets": [ { - "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "expr": "sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "Used inodes", "refId": "A" }, { - "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "expr": "(\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n -\n sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n", "format": "time_series", "intervalFactor": 1, "legendFormat": " Free inodes", @@ -342,7 +342,7 @@ "tableColumn": "", "targets": [ { - "expr": "max without(instance,node) (\nkubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100)\n", + "expr": "max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/scheduler.json b/charts/kubezero-metrics/dashboards/kube-mixin/scheduler.json index f882227..d72172f 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/scheduler.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/scheduler.json @@ -134,31 +134,31 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} e2e", + "legendFormat": "{{cluster}} {{instance}} e2e", "refId": "A" }, { - "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} binding", + "legendFormat": "{{cluster}} {{instance}} binding", "refId": "B" }, { - "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} scheduling algorithm", + "legendFormat": "{{cluster}} {{instance}} scheduling algorithm", "refId": "C" }, { - "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (cluster, instance)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} volume", + "legendFormat": "{{cluster}} {{instance}} volume", "refId": "D" } ], @@ -236,31 +236,31 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} e2e", + "legendFormat": "{{cluster}} {{instance}} e2e", "refId": "A" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} binding", + "legendFormat": "{{cluster}} {{instance}} binding", "refId": "B" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} scheduling algorithm", + "legendFormat": "{{cluster}} {{instance}} scheduling algorithm", "refId": "C" }, { - "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (cluster, instance, le))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "{{instance}} volume", + "legendFormat": "{{cluster}} {{instance}} volume", "refId": "D" } ], diff --git a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml index dbcaac0..e66a9bc 100644 --- a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml +++ b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml @@ -20,32 +20,32 @@ binaryData: cluster-total.json.gz: H4sIAAAAAAAC/+1d6W/bOBb/nr9C0HSLFhuPLR85ChSDnE2BaSfbpDOYbQODlmhbiCxpSCpHi+zfviR1URIp24kTOzHnQyd+lJ/Id/5IPtI/NwzD7PddP4wINt8Z3y42YwqC/0QuggIN+H5AAHEDnxF/UhIlei4m7BH+yUiovGUQuR756NNGazOnOoAAHETIhrTBPEXBBJIxjLApPAN9MPBYO0ERFOhj15FQXTvwDwIvQIwhGg3Am9am0bYs+k+vt2lYb0XWPpjwF+/lYzFeG3seRKTQBXIb8uccgMeDACDHTNru+P8v6L93XCjQcUmpt+bIh+SjQyl+5HkxBYFwfB4EHnFDSm9xIhvOQeATFHhMoEPg4fj7buG7nutfCloIgQ/587HIU4GbduB5IMRQ5FRocCotI+Q6p0GuzFjIJYVd08/trkC4SQeQfL5ln1PpZLz5GNrZx7zXFxkNwRACIgxVoH4kEHH1yJu/BNcFAfMmPA6uz11SsRyTJETzIEII+sTYB75z7TpkbBafOXN/8OfGW0JDYgkouI5tIBljJnjguQBzA+SSzEUwAJxS6Aqzp9+hPyJczK0CHeKKhqZ6izl0Pa8gaUb4gIDj0oGKmlJre7ekbas9RduWXNud7KMHR9B3im+iYhr5e/hc7trgalQePDPdWF/ySHA0CcmtvOm/EAXVlgm4kbxj4voSKnJHY3ImjTfMzCRU+uxf3KSKVsmaAsSGkI2m1HQIsV3lR2h48iQduwJeBFOzquiBxgqJFTHqddI3S6ReljySyuKMemrB4002mtPA9cmngIuDE0zBs20os+gQUqv1CRhVI1LIuDETjdj7e0V6tftUaA5EkMf3oRfkAlREEAyRC/EfVxBRi4elIWJVf3Fl4JgA+7LSG0xgGELndyrTShsBaAQJFvKhmBFZtrgJ+TCY3vu0b/YbHE3euDTUwTc0jRFAuaI+TR/XAbqkKdiG7hXsD24JxH1uET9tL6I9QO+/m6+SP7+bmyyr8YG9/99389d/fzfvvr2iooSIWsu7VzSJB17EgunF27fG4NZ4kz3/VkyOLHgEaMIFSuPhBPZjSRYfSfkeA5vwnGsVmmPXP874/PyZvezursgJwSEP4uZekc4kzHWTEe+Sv3I1kjEd1TjwnJJ6WbePaZgs2wSjn43dIak2lHLDF6oLIxga+0zoxpdYBY6QD7Is/lOMCQDxBFuOCrHrt8sO3E+zius77pXrRNTXK86cPsPBQ96BG3DjlqL4ILIvY7srhZ5J4rIVRaZAqByq4timDDrfRD2l8SyjXVSGcAtuYI075Oa2H5bMzAMD6FU6yBqC0T7gMKdoeHF0rzweh/dW0cDyCJ6b2Obqd3FDFPP6wZACoQaHdDUO0ThE45DF4RAqIx9PXKKByPKAyHmiA6KxiMYiq45FbOrmE79G2ATe8HGcU28oDoSrN2uaKpKU0Z/sa8YvewpmWfO8DPfrGe7PzfCgnuHB3AwP6xkezs3wqJ7h0dwMj+sZHs/N8EM9ww9zMzypZ3gyO8Msxyg45u01OWVmuJ2DqCFN2unq5W7rX+b8cHvqGq+lWOTtVVBfYarxCKDPANj4waD1DNAM2yjgsip0ioW0Ewgc/o1iS5wlBWHRYFaShhMjZZ6ZKjKZC9jdenUpiU/p5GHSZhO9RCaVIG6ns8BcuqldwRwQ/U3/a3z61Dg8NE5O3k0m73AppzjQdieAr9q3izmFKq+amZOGfJfDPETUTA0nuPbNymNfEZOqWWwIAcU3yJcPWYG0CvCETrocWHpb5Lt8vPS7iEz35Ezolc2C6mLQeqhCnltnUYcfTQbUxaTqYBDmIcqQAeK10sf+MvWxR2e1dEqtnUMO65atjLV3jsNl6iOduacuYZyCeE68lqo4WqwqwnupQnCI9dbG8Spoo+wYxiEK2GLmeqrkw4o6yHpr5WSZWvksX0ZYBdkXV5XvJXqnuTPYBjuDTnvL2QZbQ2vY6ra73dbWVmdrx97aBsPmZUQl6UMCcSPZhXD9USNbPWmEgYN/C9Doo/Peeo3gkCnnfaeFX18BlD/2/lW/b0Nhl6isa8VyzUOVXZ5qVncNXlChCN9ILe3MYAIk26WyTZt27abNvfdpNp//vtcS5Lq/OLmCq5E210SsB08j1rUz18OnCQOpuYYxLFoDwR49cXxdH8keL8dknRjJr4GAPyzJdNdHwif3KJJ5cDHMGQFE3ABNUXgsAFURQnogprDDJ56HKTQs7jiMaqt0S3IepmYmKKnmVFV01lV11lR2znokS1LhWVvlqZSnbPtZWvEpk20uX0ugFWXD5bwtsetqN2orQdXVoLUVoVOqQusrQ9XVoeoK0SlVoupK0SnVolMqRuurRusqR5XVo1VdSqtI1UUFisICZXHBDFWldZWlU6pL6ypMa6pM68sZ1NWmUytOa4sT5AUK6urTaRWoykWOcqB7DrPHGQtSZyhKnbMwtW7RQ5F7xfxbyMHT17XkObo2TxdzdboNOeUEjbJyta56VVHBOlMVazWwyKtZFRWttVWt9ZWt6urWmgpXdZWrvNJVrHatDrVa9Sp1QXVpaW15aV2JaU2ZqarUVFluWhnXMx3ChkxltbsBGgPKiLOAwB0NAjUI1CDwZYLAp1/r1ijwIShQWZCmgaAGghoIqhZOn/6KnEot6SKuyFnq3USW4pj47opdTpRX7564mKaN26e4nKh8zn/1rwWYru+2XN/CeO57L0CJ+BjXApSJPLCUiIu8FOBh5/uVJ73aDz/pRUGfD+3CeeeXf8a/eBZNXzX0Ik/4J0ui0hz7HE/0k+JZvfrz/KUx5PhWn9h/3C4+6PagF4gT2gpcaFkaJ2icoHGCvgpo6UAhXTXTSEEjhRW622dZZXUdRVmd1V5oXZ1sd/R5b6oWd3NqCus6U/ZUrc5CN1UlDY+9pyprkGCnR91RXdymqOwwXhVbLWZPtIqz9MaoFI899r7oip1V0Rujs22MznYrxIvcDyXly5xm2Q2VjLMKARewzxk+/33OcH0K3tYYnHVbU8BZV4MzDc40OFtZcLaM864anc2HzqZeFKUBmgZoLx6grUAhWtn5FlN+9uRLlqpNxp5eslwMKu5NQ8VbGhVrVKxR8bNZsnzCmzQ0OH7Y0qX8hkiNkDVC1kuYGqzJwNruFLC2rcGaBmsarD2fJUyN1p7TUqYGbBqwacCmAdviANuOBmzPHbDVe3wMJPY9IL39v5h5/hoDYrjYOD84Nb7AFCv8Vo0TUfxDAGNCQvyu2QS2DR0X+L/awaQJWeoOkYshbg5ozGgm2KMRQ46GF2DcQCl3jClywA3gOw0nCj3XpsmvQR/0g2sPOiM4oexw05wxMWu0+izRKgerPtUMw6m0E6R/bof9xALP4AhLgakShhpNQ87xj4jMzy0GtfG1vE+PadP3vhBIWwgs2AgiwsjA8wzMrgnGcMQdXuPbJePbONTx30N57jh35Yei8e4T4t1djXc13s3x7q0xCWhkCJBx9vdnA+WpaTroHblkHA045A0zz2hyzEFRDs0+EDUpvKXCaFqtducX/jd9miW4Rre1Y7V3Ols9jWzXD9ke3dD/HZye3foJFloQvL03YNYQd7EQl8USBcwVQowGuRrkrjnIXYE61SOEGJa+T5nqRtJzM/kBS9baacW82GN5nDGxPYYT8CdEOO5/vO5KQ9ht3AsHoMv4ezQL51Zs5r+jaWZvI3ASelQQ/ijzSgqOMBFsv3Dfjcd/EbV6TQOISCC5EYaR+3YQcQzbaZVbYs2bVkuMXwLqLcxWCLzh9tmbFGPKVdIj1iCb8swy8+BouVSy6/o02Tlwz5NddSP3A3MS0bgreTyJU2aeKMWXByGjlONNOXRDL0ZicqicSqfTqtw1mwmItalCnvptspu85apQKWPmV9UNzBor30WbNmThQUjL5j8RRLeJEDZ7k80CO8Hp2gXqCN6Ufv3KxJdu+BV5FHDZEkUnSVSsDadOyJ0G/yftgllsLUMITpM/nGXhGBqJbRGG5zEv3qdyqfk6eXH7kb1YJv55fbjGq7pqS+/OY+nd9TXxKcbnwCGgOlZYYNoqNUNJolBYUWotguHK7UWqvFBq4LkSrWlKTAUpvP5eMUEQ5Yq6YzInnUe6/E39eCbwhuGifhg4fdcfBptGwu7tfZ0ndo/W47pHPJLpvrGRguS7GO65XGQJ0BvGk1TTD64bWT40eQpgNLPwtdClczSUfzkRST91VDH0mT0BBIt5oQBPTCvPAWI+MC3xQ6cltgh3jrWFvy0nNu6LdAxsYaEaktVvERlviYzFt7S74gfhjsFtR+xv2peC+H4EfFHI/Hp+kGDzdNLwOftNe6NpHAi2bEZ8mdkcDrc6PdBq9wb2EILtgd1xnO6w19rZ3W2BDtyNH77K5gOtjbv/A4FidTvKpwAA controller-manager.json.gz: - H4sIAAAAAAAC/+1dW3PbthJ+96/gsOfM2DNKDnVLrM7kwXGaTKZx6yRq+9BmNBAJUaxJgAFAW65H/e1nAV4EkpAtO4klx3jxWLu4LPb6cUWRV3uO404mEUkzwd0fnT8/dXIKw5+ziGGNhgihAomIEkm8AhIQ44gLNQQ+LtUwHEQCTWMM1BmKOVbEkGDxNgASyeI4pzCUzseUxiJKge4p4jwK8DElgtGY1+ZHtblxRM40wRiegaBzILh9j7s5jV6oEUrKXFag+jSOUcprstUYQYuTIoLj1VL6cvlM5M/xOEowzYQm44pPY8peIv8sZDQjrfVXY35HcYbXs+siKPoPvdHIHzxzO3UyC6dov9d/3nG6vVHHGXgdx3t6ODpojvshGAxQH7ka9VNt3wAJxGnGfCmWe8pogsUcZ7y2kDujLEHy6C6hBNd5IcpCXDlLRU7Qojxt1/MaYiURKZlNFp/TC5OGgCXm0gVoHLxDU1x3HtOoE8TOsNKpYBnWBi3r8rMoOKXK3esM5Y+9OokIzM5RbHKCusNqekjTiITjy1SpYh2vbfq6PmE0QYky0rnUnCOoI/BCNAzuFGy5VY2x7Gy2OEMk3GDxXn3xte4FXvAKPOyUguZ42xVcqUXFPKGB2t+nhGBfQJC2xo2lRAbFp5SLWbSQs10T4zUkm4/RP2r5offfxhhILMa5in7tVKWqE8goN5tuBnGlogdkbytV5Adzf/nfkYFJq4kb6pxDOmt5LhDZGbioIVJnURwfy/Sj7C8TS78LeaV7CH8ORzKxdA9biWUmBTIHoNxFXy9frgcpqjvqtxbS431tjKpiA2tmCWmbSiAWYrGBFfAiVTLxLNnP0is/zjiE84u/3P8U//7ldpy/6RQoZ9kUP/HzKhVj9iRBBIVywPKgbaJVdhRQIyYcswjz9rAyebxGvlDK6bWGxDjEJHhdrddeBNxS1Vj3aFN3qPIhN6guEqqIu7+lDUZVtK+a5kJsVUHXGyxPdy6H9BZjDpCivr7KI3p0HTajS43YLLqolNN9ccvAKlPZrWJLTfqlyJVQxVbztPPXsAOKI8SPy+reKDFTxIxlDMoyn7/DJBRzlTRbPLxu2kbVPFKx220R3zAURJiIZlm+oUb263VQ+XDbc0ARITni4wI3yprcSAXoPFyTUvyMsVwuwzSoMWumSQOZOSwK5+B7wRpJiqRk4sCcP6JA2aVZiVTcCLQuLyrfuRGOyOzJDXsrxkWxc3cT5NGqra3q46YYXIUIFJpBaSpnS5/I5PLDNs/shWCqADOscu0spo3QZzjFyFjI88T56zlm4G7YcCIoYj5eFxZF2WtSBcByo5SQ8NMUB+/ymtjm36msMCTw/gVlZ58znOEJCgI+UT5xt2rTcSICJyA+fvEvTCw/QBn6c5h8OjhwppfOfkntOBLE3Wt9uroqN18unasruf9y+dWLVsML5CFe54Cq5UKS93EezYSZWVS7P8BAzntpIecoCJwPYLQ7lb9mbqBMGK5oVNxPyooYkSA6j4IM1WqOuXqq6+e6ZAu0iHhbrmnmn+XOakhKSRH/UjlN+FXgfsOs9TmwSmSqI2A+wiVabHJNs3JNmhpcMpYXekbxJJOGLxFvXVVpNcE4LS8KRpZ25FtdPD2QU+jRttekVjFmTtAF9S2kCNUeMrM/0ItaB6iSZ1yEXr3vUgXkK4A0U4oYxCLIXueXCHH+TGMUAcJg9J5moW/ZBHoEQG5ggZwFcjsD5Ho7BOQCnIq5xXC7ieFeSeNYALdlAAfbMXGf4Mf7Bvhtlw9h4ZuFb9fCt6GFbxa+Wfg2j7igUHmTyecMERHFeN97Ohp1HAOuK9BdlmdFgFWA4AI+yavxveG9jhPjAwv6bgX63oEhiX9pYd+2Yd930LWzPTsL+h4q6Ht2F9BnyoKboD4juPtC2GfMyN8U911/64IFfl8G/AZbb9sBAhETP5bBpG5xhY/lt7B3xW4dH0woab2nTyskd5+ArbdY3AKedXZIY/0taax/rcZe7rDGBlvS2OBajR3vsMaGW9LY8FqNvdqNi6afQa/O0elb50OudXu/g73f4RHf7/B479N8bi8V7KXCLl0qHO5ui9iAVu6pTdxxQKly6umvH8f1rrFkdJyMxdtoGMvNVbMY9n8IvWLIg6KCPEW32BmNxNx5X9jcYqDH1Dv2Hnrj2N4qYLvGXwsKHtpbBeytAvZWgQeBA9/8ZGHg3WHgG2xRoEWBFgVaFGhRYBMFjmxD0DYE7b0DP8rHvVAfcw4QTzoJYL0EJ5RdTqaXAvM7Art1t4Fu6z7P3cdqJ0rpFpBtGZApp/8Ovpzc8XPYryfraKQhn4UjFo48VjiiWk8lJvHTrOo0fcmzZK79WYoFJetAyfHpb07GpVtbXGJ/YfwIf2H8iBFJ1yISi0gsIpF1hk5CymgmpAFtQ+S+vryqVG7Bx2MDH9/qt667fY7v+TurveIELvfnOEG/Y8ZzKbuDnCwu860CxM7yVxxA8Vp5qUqtjGDZ1arWEjhJYzguCdsvbNhreoOGNmpYrnxQcoBnKItNz2rWuSZoqF7rUE8ua3yqetS8huk0Lk3Lt0/o2dT9nGF2WXTq29hPez1Et0YNcfPZ7pVdtO33GsfRFIbiuHxTQeMYmip1PWyAVEtdadUK0i4U0gAfmZ6qXinSLYqtvlYCFokMc0otG6bcpGK13STPovvS5yYpDSYRmdGOUyx3YFZ+7yblFyVINxK4uNIwf1/u79a5LUElzTy4MG1+Eo2RcTzOF9Lh6X3b21tn73oNWxc3N1m6BDZ3NvVXbjgtV7c2fR/usldWpfwlOAquVDm3esUDvXjSLUFR+faGvASspqURwCG2mlyoZFICWh2ZuMOVL7ldT/vQ1z90k9X/Q+3/rv6h7+mcFXRze9r/3SBPiJ/KM0gIrjnSjbvoCz/TF9Z36Q30D6vXfbjPA13eUpaa+v6hJH9rwfi4KJNVs67yQ+ck98N8QKau593nPexhbzjFs6E3GuHhzBuMpt5wFviD3ggHg3zweVWYvb3l/wHmMGkaPWkAAA== + H4sIAAAAAAAC/+1d3W/bNhB/z18haBuQAG4nf7XxgD6k6VoUa7a09baHrTBoiZa1SKRKUomzwPvbd6Q+TEl04rpt4jR8Maw7fhzvjnc/nWnpas9x3MkkImkmuPuT89eHTk5h+GMWMazRECFUIBFRIolXQAJiHHGhmsDlUjXDQSTQNMZAnaGYY0UMCRavAyCRLI5zCkPpfExpLKIU6J4izqMAH1MiGI15rX9U6xtH5EwTjOEZCDoHgtv3uJvT6IVqoaTMZQWqT+MYpbwmW40RtDgpIjheDaUPl/dE/hyPowTTTGgyrvg0puw58s9CRjPSGn/V5g8UZ3g9uy6Con/XG438wRO3UyezcIr2e/2nHafbG3WcgddxvMeHo4Nmu++CwQD1katRP9TmDZBAnGbMl2K5p4wmWMxxxmsDuTPKEiSX7hJKcJ0XoizElbNU5AQtytV2Pa8hVhKRktlk8Tm9MGkIWGIuXYDGwRs0xXXnMbU6QewMK50KlmGt0bIuP4uCU6rcvc5Q/tirk4jA7BzFJieoO6ymhzSNSDi+TJUq1vHapq/rE1oTlCgjnUvNOYI6Ai9Ew+BOwZZT1RjLzmaDM0TCDQbv1Qdf617gBS/Aw04paI63XcGVWlTMExqo+X1KCPYFbNJWu7GUyKD4lHIxixayt2tivIRg8z76Vw0/9H5otIHAYuyr6Nd2Vao6gYhys+lmsK/U7gHZ20oV+cLcX388MjBp1XFDnXMIZy3PBSI7Axc17NRZFMfHMvwo+8vA0u9CXOkewsfhSAaW7mErsMykQOYNKGfRx8uH60GI6o76rYH0/b52j6pkA2NmCWmbSiAWYrGBFfAiVTLxLNnP0is/zjhs52d/u98XX/92O84/dAqUs2yKH/l5looxe5QggkLZYHnQNtEqOgrIEROOWYR5u1kZPF4iXyjl9FpNYhxiErysxmsPAm6pcqx7tKk7VPGQG1QXCZXE3d/TBqNK2ldNcyG2yqDrDZaHO5dDeIsxB0hRH1/FEX13HTZ3l2qx2e6iUk732SdurDKUfdLeUp1+LWIlZLFVP239NeyA4gjx4zK7N1LMFDFjGoO0zOdvMAnFXAXNFg+v67ZRNo/U3u22iK8YCiJMRDMt35Aj+/U8qHy47TmgiJAc8XGBG2VOboQCdB6uCSl+xlgul6Eb5Jg13aSBzBwWhXPwvWCNJEVQMnGgz59RoOzSzERq3wi0Li4q37kRjsjoyQ1zK8ZFMXN3E+TRyq2t7OOmGFyFCBSaQWkqe0ufyOTwwzbP7IVgqgAzrGLtLKaNrc9wipExkeeB87dzzMDdsGFFkMR8vG5bFGmvSRUAy41SQsBPUxy8yXNim79VWmFI4P0Lys4+ZjjDExQEfKJ8Yrts03EiAisgPn72H3QsLyAN/TVMPhwcONNLZ78YbtW240g0d6uJ6qpc33LpXF2VgqgLKcty+cUzWcM15IJe5iir5VeS934ezYSZWaTAP8FqzltpNucoCJx3YMmtcmIzYFAmDLc5KhhMyjQZkSA6j4IM1RKROaWqm+q6ZAu0iHhbrmnmn+UebIhUSREUpHKamKy4GTD0Wh8Yq+imygTmJVyixSY3Ois3panBPWN592cUTzJp+Bzx1q2WliiM3fJMYWRpS/6kO6p7sgp9t+01qdUeM0ftgvoa9r2qGZnZ7+hFrSxUyTMutl69GFNtyBeAc6YUMdiLIHudX8LG+RONUWwQBq33NAt9zcrQA0B3A4vuLLrbGXTX2yF0F+BUzC2wu0fA7oW0mEV1d4zqYDombhMReV8B1O3yIiyms5juWkw3tJjOYjqL6eYRFxQybzL5mCEiohjve49Ho45jAHsF5MvyqAgQC2BdwCd5Nr59ENhxYnxgkeDWSPANWJf4lxYL3jUW/Abqe7a6Z5HgfUWCT7ZBgqYouAkUNCK+z8SCxoj8VcHg9ScfLBr8PDQ4uPMCHyAQMfFjuZnUCVm4LH/E3RbQdXwwoaT1Hj+u4N1tgrfeYvEJ8KyzQxrr35HG+tdq7PkOa2xwRxobXKux4x3W2PCONDa8VmMvduOm6RfQq3N0+tp5l2vdnoywJyMe8MmIh3vM86m9VbC3Crt0q3C4u3VjA1q5pdpxxwGlyq6nv70f10vJktFxMhbfRfFYTq6KxTD/fagVQxwUFeQpqsXOaCTmztvC5hYDPaTasXffC8f2/ICtGn8pKHhozw/Y8wP2/MC9wIGvfrYwcHsY+ApbFGhRoEWBFgVaFNhEgSNbELQFQXt24Cf5tBjqY84B4kknAayX4ISyy8n0UmC+JbBbdzb0dvHa6pzn7mO1E6V0C8juGJApp/8Gfpzc8XXYnyfraKQhn4UjFo48VDiiSk8lJvHTrKo0fc6jaK79r4oFJetAyfHp707GpVtbXGL/dvwA/3b8gBFJ1yISi0gsIpF5hk5CymgmpAFtQeS2fryqVG7Bx0MDH1/rv667vY5v+TervWIFLvfnOEF/YMZzKbuDnCwu86kCxM7yNyRA8lp5qQqtjGBZ1arGEjhJY1guCdvve9hreoOGNmpYrnzOcoBnKItNj3rWuSZoqN4KUQ8ua3yqelK9huk0Lk3Ll1fo0dT9mGF2WVTq29hPe7tEt0YNcfPR8JVdtOn3GsvRFIbiuHzRQWMZmip1PWyAVEtdadkKwi4k0gAfmR7KXinSLZKtPlYCFokMfUotG7rcpGI13SSPovvS5yYpDSYRmdGOUwx3YFZ+7yblFylINxK4uNIwf1vO79a5LUElzdy4MG2+Eo2RcTzOB9Lh6W3b21tn73oOW7dvbrJ0CWy2NvUXLjgtV0ebvg132SuzUv4OHQVXqphbvSGCXjzqlqCofPlDngJW3dII4BBbdS5UMikBrY5M3OHKl9yup1309Ytusvo+1L539Yu+p3NW0M3tad+7QR4QP5RrkBBcc6QbZ9EHfqIPrM/SG+gXq7eFuE8DXd5Slpr6/qUkf+nB+LhIk1WxrvJD5yT3w7xBpu7n3ac97GFvOMWzoTca4eHMG4ym3nAW+IPeCAeDvPF5lZi9veX/fzSxfHxpAAA= k8s-resources-cluster.json.gz: - H4sIAAAAAAAC/+1daW/bOBr+nl+h1XYHyTZpbCd2DqBYNE4zLdBOM23awaItBEaibSG6hqJyNMj+9iWpi7ocx0cs2W8+xaREUu/FRy8fkfcbiqIix3Epoqbr+Oqxcs+KWKFl+pT9+v6T/XzY5pdhw6To0sKslJIAi7Khg+l7g5U4gWWFJQR5owvXtajpsfKWKByZBu67DiWuxbsYIMsP77dM58oX3YifBA8I9kesQN1r+WpY5t6IK8SwwsGxUt21LOT5WG5NVIywORzxkavtVsu7VZMKDznYSluSWxP1yDKR33ctlwgpPGzLlZeI+PmuRIWB/NEH7AwpH3S7VajDVbdR5LsB0fkDqOfEtTEd4cBXM1cNTMvizWYLXWIj8YAeZvc7NHBMmr3PNAp3mQ7F5BpZQjB29nILD7FjJKpPJXI9LBs8l35ACOu5otZGt1U1plNR44/cG8mwpBrKbNOquIs9UJBKWKrLao9Zmbgq37qouDGNUHn5Gskwk2Ju5ucuE+ZH1xCq4wUK8pVfmLhZqUbaQUNcagEeb4Ugwwx4N91iXbnhMLEbmGDC+x5Ybk7xPiYm9j9dY0KYx5WM3/eQjqvslVVy9XSyhRTpV6VD8Sn2PGx8YDIsraeIDDHNelze68SF+NYTz9NWdhRmc5sEUbzpMAFruhdoPtZdx/A1YQf3Nit+/YNZuIV/qNuKbgVsGISVvIj+/aE+fH+habwNLTb6n1tb6na+19SPqGljLRRd8TLTYRIQpl60TcmvzpBOXZKXXqSygQiR6hs1U/Ug/cpqiY54FHQtgwtPPWhtH7ayeuYjPmNBQwq8mbovI3NAyyupCOBq//yr8pWaLMqLwJ9rPgnf93kvRQQbVS7sEhpH/LyPavTOE/2ajmFem0bAXLrSXeNrfdMZWpiJP2flt+jW9IuDuwz0q9De8o/NI0/kr1w6OSWrDrJx+V3VUSmJPGKCLH+OO3SLJzD/1BBZd4QWTdBCl9gqHSCvdIcnSMyD7UJdGIhLbwsjcatQLj1x1li3a/sQpVXRc+TnhZzXbZQ83kqDgg5M/DDx127i9wN7kwdhMUg+6R9fBZdY81xDY3M/RawjorEpUfgG++dvpnvqH7Pb7ksBwJayq/A2RSsCSPBJJPA1ZFmujsRbzH3cHruV9cjARHlTAByKwOFzpAGl79q2SW0eEQBAAIAAALHyAGIPAAQAiFoCiHGQwTLZNOWXgoVtJQ8EAD4sGD58ENoA8ADgAcDDWoGHfQAPAB5quezAp/tjMdXb2HbJnfYR22+ukWnxuV67vKN4gnRD7v4LbkHhvfcADiYBBx+F7GBdApABIIP1QgZdQAaADGq+LhHO7M+8NBF2CumFpyEIWKAAJAFIYi2RRA+QBCCJlV2giOEA4IhnwRGwUgEoAlCEjCI2cg7J3NfDKO9MUel75vxhGq+0+rN7k/l2JBnQReSG2S87Eud8h5ERznvZui/mL1E/6oU+E+niKd+MdLqr8s1IqwiNDgAaFaBRC6BRCI3a5dioILzFJFk4gEkzLRmQdJzCJf45SOAz2WY/CuHZFvHBR1XK5fJOSfM4s2CdR8FM6FRnSXv390m/Dw9qxeVMlleV8ZmJW+hsGmyUM4M5fTXCFVBLHCQ+PQQIBBBohSCQbE8ZPwTsM2HiSECfQ4A+kBV6IvR5elrIp3fWJGFcuA5/oIviFBMaO04hxH/Z387Hjzunp8q7d8e2feyXQBMPURZonOo24zlyZBoGdtQnBexkuOeuUdK3zmNAZB6lAVaPg0RWN9M+q4F100YiNBUnJSvEUqXJKF53IeDpiYWcq3IXi69LII16SlhYUQz3xlGoq3ilMuC3fCXi6/pXu8buYRd1e53WwaFuDA7Y3xFGqI0MY39P17HexrtXh/5OnJ3zdxKM+J9rRHbSWPf6Rfr/b7wqxrcxuhWFyd2vX2iaji1La481kG88hij/fFNiJNXgMWNETmBfMo8s1IuUfDKvT2djf7nkynIRGNpNtSBy1oYODwaXLeOofYT1FkbdQa+7v9fu7ON2T9/voG7O2pKGl2N3J/W0u/JXm2XbXafK7sYY1CyGN9bWJtFuv77ajUkBoOBZFHxafwUr/wIVz6Lit4tTsbx2P72iw2U5UPIsSj6rrx9Hq67gxTMp+Pc6e/EfMXZsho7X+nVyrLE51YqsQzipuXlJgtx99e/d6WXoU2I6w6fJsHoNaSZ2EzNsTOawEie2mlwA3yi/RKeOYyRNuh63PZmgdDdw6Cbf4y5d7Ixfx7nwjrNSPCZYrIWMk2Z8+7aSyrVpcj6Zt5zrtaZcAwn3FyLhuW7Z1CyBnjbeZGUC6SNqm3zjjGYp8e0ilDjPfUiaJc6zdfCJ1Vbh72oDGEh/Bky19WIgEeT4XJPlekxgekkVMJSAobSaDKWcnwJPaRKO9hEQlYCjXUuOdoqFoi1uiF8Fg5JL/8EqgIE9T/wTb5HDoa6yebPrKjrSRzmRAiH7+eGO2OcJ4M6awp3QLQHrPIWTnesJsA6QsoGUDaRsIGUDKRtI2UDKzr/tABusprzs8M1nJgUDNbvm1Ow56hh4nSvMzs7smwR6rilBew7eDBxt4GgDRxs42sDRBo42cLSBoz0due+51pRXlYE9H/KutKkskLDnaZNzpVg3Vku1ZVk3VqJnzbf7lddR7ZnUST6OybX8VQ9Y1cCqBppR/WhGxeUS4BtNwjfKlUXhXhwBaavARQIuEnCRZuQi9UMvYAFKx+Y1Vk6QY4TWBMsEMywTLJAVdOL5M+r6gqM0BuZB2fNR9kkdlf2Z6UBxB7FjG8o5CpE0qLqepBxvZlXHfk1B27Wn53hzd2zllLgcyoDK68rVWYSDg9ZrztyZXuvA5gA2B7A5VonNYfJ9baRlGwdTTjTQSDiVa4LlF26DU7HOkhre/36or17+UB++MwvkzWpxduzn1toTQsbJmkZz6LoK++Q5hR0btheClfWTdn8ppr224j5dpnEbIRJfP6m/XaqRr63Yz2q/81uUb/4jVF0dz6EEvgLwFYCvMLHPAmuhwFoo2REut/gM1ATYEk6iJvSWuCNcHRMQsHvceM7nOF4GbBv3/PiIL8kDOloCOtpeCWywB9gAsEGTsMFSE+YADsaBg0eIfIAOAB1A7uT5cidFL4R8yUSYaB8wEWCiumEivp8E5EuaBoneMGPjG+33Y52l2Cjz2exxwp8FyASQCRIqzQUPXQAPAB6aBB4gobIC6EH6FAMABAAIyLksKecyqcdCSuZpqKoHqApQVRMpLMvhvgOsGktimWQvAkBOgJwg9dIokHAAIAFAQiO5LIAS6osSHtvGBoACAAVIsTxfiiX2y7wvQiZlIpB0CCAJQFKTMynL+cAasNI0GZXS7cAAMAFggsxKo0DDEYAGAA2NzqwAamhUhgWAAwAHyLTUK9NS8EnIuKRXpVuG7rQnhVWdFsAqgFV1g1XsDd7aZNiK4Rz5RL7NHNIa+BrByEjWrLIn+m0rpafZfu/aP7eUl0qxrRtiSuzjyRvbAgA2PQB7/+n8y+ZnrsWXfwn5bwHegu3uIFXT4FRNpw2YAjBFDVM1k8KJzFdI8wEVUzYJyGKGrWBGxA2Go/OACnwRwgtAF5DNgWzOsrI5X1go4p8mvf8ECZynnPrbge1zi4hqPU72DaeanKKZO7DC/ZxwWQN6MbA9wEnBizopOM1dbNXqxKidNhwOPJdzwISCy9JSoOEanQg8s4aFC7O3N9B0zQ8EnkXT0eugF70ObsHxoLMo+bSOB3xLKi552Qcd1+fg37nomLtxHLVB1TOp+qyOqobTfuG0Xzjtd4VO+10YnWMtz/N9XJpTE1rW8cTehnCN1vF43wWv267lGb4TR49VFerb5hnqsxMM4EjfyY/0LVnbrAXFAM7zBYIjkBBKSQjKjnJq8reiy0D08ggnYSMat+rrI2yjb5j44djC01rC5U5+h4HIlWiMedUwtUz1KmDvsA5mgVpN2qLY9iz2kM4w8TT2Zu9TyZ5TI5DW8TNkBopvhU0YeIACK7fGLFxIri2jm/KFUJy11ApTit1YJjVIta7HRZmPtOrfASZ3/C6vjPzAwz2P0VnDZKVDfJubH5KAJXW/kXscSWDIsr5Fz597jEdEWSHDcuFNwO+IBSxNhix+s9nYwG+sErpElfRtpkGz5PJYK9H8/hSViJ60MNJuOiyAa7oXaD5mMCJ+30kgxFa51jqPaS2az2TtMt8QqvH/jAeiZmsLI+Zl5RdHNhE+klQR+PgibEiOUGEU4lHpIfRBU8gu8r5BCAdUx73ZacesPDaNR2Vq5jbPZJMhSW+ORJJ8rCvPS2pX2imo3ZJ+7Mk/2nb6f1f6vy3/2GvJNdLxWh3p/7YRmuvP+Bk4YVSyhkd7kRvuyQ3LvXT25R/Sp3kHhjzeeCwZ8f1yBXVE/XrRjwJmAqtc2wsoVj7HGUtlV+lLtq0Gghmlsph22BsYbaOltztt1Ont7+9f9vZ6aG/QbR2hw/Di6yRWtzYe/g+LaviQMCwBAA== + H4sIAAAAAAAC/+1daW/bOBr+nl+h1XYHydZpLMd2DqBYNE4zLdBOM23awaItBFqibSG6hqJyNMj+9iWpizrsOLYTy/abTzEpkdR78dHLR+TdlqKoyHU9iqjluYF6rNyxIlZoWwFlv77/ZD/vG/wybFoU9W3MSikJsSgbupi+N1mJG9p2VEKQP7rwPJtaPitvisKRZeKe51Li2byLAbKD6H7bci8D0Y34SfCA4GDECtT9ZqBGZd61uEIMKxocKzU820Z+gOXWRMUIW8MRH7mqNZv+jZpW+MjFdtaS3JqoR7aFgp5ne0RI4b4hV/YRCYpdiQoTBaMP2B1SPmitWarD426jKPBCYvAHUM+J52A6wmGg5q4aWLbNm80XesRB4gF9zO53aehaNH+fZZbuslyKyRWyhWCc/OU2HmLXTFWfSeRqWDV4Lv2QENbzmFoH3YyrsdwxNcHIu5YMS6qhzDbtMXexBwozCUt1ee0xKxNXFVsXFdeWGSmvWCMZZlrMzfzcY8L86JlCdbxAQYHyCxMvL9VYO2iIKy3A560QZFoh76ZTrqs2HCZ2ExNMeN8D2ysoPsDEwsGnK0wI87iK8Qc+MvA4e2WVXD2tfCFFxmXlUAKKfR+bH5gMK+spIkNM8x5X9DpxIb7xxfNoyq7CbG6bIIq3XSZg3fBDPcCG55qBLuzgzmHFr38wC7fxD7WhGHbIhkFYyYv43x/q/fcXus7b0BOj/7mzozaKvWZ+RC0H65HoypdZLpOAMPWybUp+dYYM6pGi9GKVDUSIVN+ouap76VdeS3TEo6Bnm1x46kGzcdjM65mP+IwFDSnw5uq+jKwBra6kIoCrvfOvyldqsSgvAn+h+TR83xW9FBFsjnNhj9Ak4hd9VKe3vujXck3ryjJD5tJj3TW5NrDcoY2Z+AtWfoNurKA8uH5oXEb2VnxsHnlif+XSKShZdZGDq+8aH5XSyCMmyOrnuEU3eArzzwyRdUdo2QRt1Md25QB5pTc8QWIe1Ep1USCuvC2KxM1SufTEeWNt1PYhKqvi5yjOCwWv26p4vLUGBS2Y+GHir93EH4TONg/CYpB80j++DPtY9z1TZ3M/RawjorMpUfgG++dvpnsaHLPb7ioBwI6yp/A2RSsCSPBJJAx0ZNuegcRbzF3SHruV9cjARHVTABzKwOFzrAGl5zmORR0eEQBAAIAAALH2AGIfAAQAiBUHELbFJi2AD0uDDx+E/AE8AHgA8LBR4KEN4AHAQy2XHfh0fyymegc7HrnVP2LnzRWybD7X6/1biqfAC4X7L7gFRffeATiYBhx8FLKDdQlABoAMNgsZdAAZADKoeVohmtmfeWki6hTSC49DELBAAUgCkMRGIokuIAlAEquPJBa9RgE4YhYcASsVgCIARcgoYqvgkMx9fYyKzhSXvmfOH6XxKqs/e9e5b0fSAV3Ebpj/siN1zncYmdG8l6/7Yv0S9aNu5DOxLh7zzUirsy7fjDTL0OgAoFEJGjUBGkXQSKvGRiXhPQ004gAmw0c5THScoSP+OUgYMNnmPwrhKEm3+Bcf47BS/1bJ4Nc8YOdBNBN51Vna3t1d2u/9vTrmcibMy7EBmslbKG0WcFSwgwV9NsI1UEsgJL49BAwEGGiNMJBsTzk/BPAzZeZIYJ9DwD6QFnok9nl8Xiigt/Y0YVy4Dn+gi/IUExk7ziDEf9nf7sePu6enyrt3x45zHFRAEx9RFmjc8W0mc+TIMk3sqo8K2Olwzz2zom+Dx4DYPCoDrJEEibxuZn1WExuWg0RoKk9KdoSlKrNRvO5C4NMTG7mX1S6WXJdCGvWUsLCimN61q1BP8StlwG/5SsTn9a/2zL3DDup0W82DQ8McHLC/I4yQhkyzvW8Y2NDw3uVhsJuk54LdFCP+5wqR3SzWvX6R/f8br0rwbYJuRWF69+sXum5g29a1iQbyjccQ5Z9vKoxkPHjMGZEbOn3mkaV6kZNP5/XZbOwvj1zaHgJDux4viIK1ocODQb9pHmlH2Ghi1Bl0O+19rdXGWtdot1CnYG1pw8uxu5N62l31q82y7a41zu4mGNQ8hjfR1qbRbq++2k1YAaDgeRR8Wn8FK/8CFc+j4rdPp2J58X52RUfrcqDkeZR8Vl8/jpddwYvnUvDvdfbiPxLsuBo63ujXyYnG5o5XZB3CSc3NSxLk3qt/780uw4ASyx0+Tobj15BmWcNL2UzMsDFZwEqc2GvyCQhHxSU6dRIladr1uMZ0gjK80KXbfJO7bLUzeR3nwjvOS/GYYLEWMkmaye0NJZPrqsn5ZNFyrtmicg1E3HsSES9006bVEujp6tts8t3qBmvx7XLd4mGa72qJ82wDnWLddPi7ugIkpD9Dptt6kZAIcgOuyWo9pki9ogpISkBSWk+SUsFPgao0DU/7CLhKwNOuJU87Qz/xNjckqN7bpqGkl/6DVQAJe5H4J9kmh2NdZft6z1MMZIwKIgVO9vPDHbHXE8CdDYU7kVsC1nkMLbvQE2Ad4GUDLxt42cDLBl428LKBl1182wFCWE2p2dGbz1wKBnZ2zdnZC9QxUDvXmKCd2zsJ9FxTjvYCvBlo2kDTBpo20LSBpg00baBpA017Nnbfc60prz8He0Eb1AMNe5E2WaKTbqSW3i7d8oFpXX+7Byb1MzOJ0nwck2v1qx6wqoFVDTSj+tGMysslwDeahm9UKIvDvTgG0lGBiwRcJOAizclF6kVewAKUga0rrJwg14ysCZYJ5lgmeEJW0IkfzKnrC47SGHYHZS9G2Sd1VPZnpgPFGySObSrnKELSoOp6knL8uVWd+DUFbdeenuMv3LGVU+JxKAMqrytX5ykcHLRec+bO7FoHNgewOYDNsU5sDrGvjbRs42LKiQY6iaZyXbD8on1wxqzjZIb3vx/qq5c/1PvvzAJ5s3qSHfu5s/GEkEmypvEcuqnCPnlOYSeG7UdgZfOk3VuKaW+suE+XadxmhMQ3T+pvl2rkGyv2s9rv/Bbnm/+IVFfHoyiBrwB8BeArTO2zwFoosRYqdoQrLD4DNQG2hJOoCd0l7ghXxwQE7B43mfM5iZcB28Y9Pz7iS/KAjpaAjhprgQ32ARsANlglbLDUhDmAg0ng4AEiH6ADQAeQO3m+3EnZCyFfMhUmagMmAkxUN0zE95OAfMmqQaI3zNj4Rvu9RGcZNsp9Nnuc8mcBMgFkgoTK6oKHDoAHAA+rBB4gobIG6EH6FAMABAAIyLksKecyrcdCSuZxqKoLqApQ1SpSWJbDfQdYNZHEMs1eBICcnh85+YCcIPUyO0g4AJAAIGEluSyAEuqLEh7axgaAAgAFSLE8X4ol8cuiL0ImZSqQdAggCUDSKmdSlvOBNWClWTIqlduBAWACwASZlZUCDUcAGgA0rHRmBVDDSmVYADgAcIBMS70yLSWfhIxLdlW2ZeiuNi2sajUBVgGsqhusYm/w9jbDVgznyCfybReQ1iDQCUZmumaVP9GvoVSewfe94/zcUV4q5bauiSWxj6dvbAcA2OwA7P2n8y/bn7kWX/4l5L8DeAu2u4NUzQqnaloaYArAFDVM1UwLJ3JfIS0GVMzYJCCLObaCGREvHI7OQyrwRQQvAF3Ap0mQzVlWNucLC0X806T3nyCB85hTf1uwfW4ZUW3Gyb7RVFNQNHMHVtguCJc1YJQD2z2cFPxUJwVnuYudWp0YtavB4cALOQdMKLgqLQUartGJwHNrWLgwe3sDTdf8QOB5NB2/Dvrx6+AOHA86j5JP63jAt6Tiipd90HF9Dv5diI65GydRG1Q9l6rP6qhqOO0XTvuF037X6LTfJ6NzbOR5vg9Lc2ZCyyae2LsiXKNNPN73iddtN/IM36mjx7oK9e3qGeqzEwzgSN/pj/StWNusBcUAzvMFgiOQECpJCMqucmrxt6J+KHp5gJOwFY9bDYwRdtA3TIJobNFpLdFyJ7/DRORSNMa8aphZpnoZsndYF7NAraZtUez4NntId5h6GnuzD6hkz5kRSOv4OTIDxTfCJkw8QKFdWGMWLiTXVtFN+UIozlvqGFNK3FgmNUi1ns9FWYy06t8hJrfiY64q8gMP9zxG5w2TlQ7xTWF+SAOW1P1W4XEkgSHb/hY/f+ExHhDlGBlWC28KfkciYGkyZPGbzcYmfmNX0CXGSd9hGrQqLk+0Es/vj1GJ6EmPIu22ywK4bvihHmAGI5L3nRRC7FRrrfWQ1uL5TNYu8w2hmuDPZCBqvrY0Yl5WfXFsE9EjSRVhgC+ihuQIFUUhHpXuIx+0hOxi7xtEcEB1vetdLWHlsWk8LlNzt/kWmwxJdnMskvRjXXleUjvSTkFaU/qxL//QnOz/jvS/Jv/Yb8o10vFaLel/zYzM9WfyDJwwKlnDg73IDXflhuVeWm35h/Rp3oEpjzcZS058vzxBHVG/XvTigJnCKs/xQ4qVz0nGUtlTepJtq6FgRqksph12B6ZmNg2tpaFWt91u97v7XbQ/6DSP0GF08VUaq5tb9/8Hdtob7zQsAQA= k8s-resources-namespace.json.gz: - H4sIAAAAAAAC/+1dbVPbuBb+zq/w9fbukF0oSSBAO9O5U6BsO9NuWaDdubPtZBRbSTw4tivbBMpwf/vVi+3ItpwXEhInPnwiki3L5znn6NHRkfWwpWk6chw3QIHlOr7+WnugRbTQtvyA/vrnO/35uMMuw6YVoI6NaWlAQszLeg4OPpi0xAltW5QQ5PWvXdcOLI+W13lh3zLxqesExLXZI7rI9sX9tuXc+Pwx/CfBXYL9Pi3Q9+u+LsrcIb+Cd0t0jpYarm0jz8dya7yij61en/Vcb9Tr3p2eVHjIwfaoJbk1Xo9sC/mnru0SLoXHHbmyg4iffRSvMJHf/4idXsA63ajn6nDRbQHy3ZAY7AX0C+IOcNDHoa+nrupats2aTRe6ZID4C3qY3u8EoWMF6fssM3eXjXvYMRN4R29921N1kEk4JIS2XlA7QHdFNZZTUOP33aGkPFJNQPXPLrjrFtnhSIpSXRohqkn8qmzrvGJomQKgbI2kfEkxU+UL13KCT67J4WEFGvK1n5i4aUFHCKAeVqLssVYIMq2QPaaVr1MrBxW7iQkm7Nld282A62NiYf/zLSaEWpWi/76HDFykk7SSwbOfLgyQcaPsih9gz8PmRypDZX2ASA8HaavKWha/EN95/H38cLDtULm2HTTAvKdtzzXbBnUOiD6EvE7+axte2A59Ktu2j2mp6be5mrymTbQJCvCDYYe0f+TNN/1F9O83fUdLGmblyY9v+mNN29PY42/CTuahbep0uDHSf35QZQv8Gdve0eIWaAXtN3uavpOVwchyA2vA3ooBmb/Mcige3PDylsKraWeoUZwjI3CZTJu5S6gb5U5Zf6unqh6lX2mdCfrM77q2yaDUj+o7x/W01rEen1M3Jbn6VN1V3+oG6sqADxn66cUX7Utg0XGFDzXadpc2p8UCr2UelwwgD1kfggg2ixyMS4J4zMl6kHZw7/F+WI5p3VpmSB1OoTOJr/Utp2fT7qGMDd6hO8vPd64TGjfCGrJiYH4x8iZMWhnQdaZK6ruKfWbiF/kQrX6Pe3SHpzDOkWLSx5Egr5I26mBb2UFW6fZOEB+JG7k6MUwobxPjRD1XLr1xWnl3SvsSyqroPbKjVsYKtxSvt9G0pAm0BGgJ0JLZaIltDSwgJUsjJULcQEmAkgAlqQAl2QdKApSklJRkRAUGeOCS+/bQJTd0AKDDdNDu3Ad4Zk6QtPgvWsOutAYUE/FjaRES8TLAR/J85BOXjKAkPyFOAqQESElFSckBkBIgJUBKlhUfAUoyDSWBKAkQEiAkGUKylTFMasYeRlmjiko/UCfAjUhdfekOU+k0SYeuI3NMJ7skRvoeI1MMoem6K+snr+8fCpuJsJgljabZ2pQ0mnqeZbWAZeVYVn0zWdZ4J8TVlrX1I6Ri1naTCXfeHRlMudmlv5w3D161TvNXJBqrJAKROuYdaN8yTexcCXpRcDPLpBtl1ykvSbRYXSshnWcgKSo5kxfPCVDQg7HiO391vF+viPgKKdw0fL/RVBL+nBlv1rpo517bpk+bh5FPpNwC7fOkvYcH+sTHR73gQirZm0LeQIXPEZzJbhJJG8hGRMxy4nkJN6WZZzWMA9NCyrdN+lOa48Qu7eX8i8Ezy3WyW90AAQuXt0rxFjndJwt3uplnhhwsaF2eOZNSzi55jjtMLGFiuUETS1mfxtkhzCvHx/f5tPIQppUQvJ+RzM8evfeDe3umue11fqARyo5HTOK/9G/306fdszPt/fvXg8FrX0EnPBRQd+MUtxmPlGJOpj9tJqkmAaNJZKQjSl9rxJ4iDdBTX9jEhjVA3D81VXPCG7UBRZXXfNp1YqNJ1yX0Rj8j1Llopjt0dOWlXwizWX0sNF+Z9Wq/vFXAU0zeUvA54aBDbSFXz9csk3H16ehejg+yAMBTAXxSfoC1fwPE80B8+nwQy2kITwf647hoH4A8Fchn5bVjAS9Y8XwAvyuzFV+45nqgWxSXf0ZwX+6Ze4et4wY+OMQHuGUcddBBHdWPDg8O91+1cPeo09zfuzn2d+NgoL/rueZ/bhHZHU0l37wY/f8rq4rjjnHUkReOwo6joCOvoA2+edFuG5hOOsfpmqfCsQx+pOTaJYlw7+Vve0+XoR8Qy+nNJsPiMOvmrdjwb4g8Q/ZUNiauj8uvWvTiwvK+K7A2Qj55DiGvXLWX+xGJtQH7dNkWtbAtsWsj4rOq2lP1oH6nr8H67F9szblc67MEOT5DUo1jwtAUVbB+C+u3G7p+m7ZTWL+dJi/4CBZwIS8Y8oIhLxjyggunDovfB6hN2AgICcHPlBC8kN2PkBM8Jid4tRLelLTgeC8qi0Ro28M9VzOQ0cc1yBJe8SyTu3uYZVZ0limbJUw0Z0kUPoZ5JiQKQ6LwnInCSv8DGUrlyhUWFGEugCFduOTpwgvEGHINNzhjOMIakoZLnTS8AGuGvOGNzxtOx2Qur65qAPY8YJ+X2qAjmE/zYTcAekag/1gHoK+GyAOc58L5fTlxhv0exbDCfg/Y77FG+z2W/kVe2OexlNMRqr3VY2VavfRTMGCXxzI+7F3tjR5ltqZKovxuKSgTf15c10ik50sRKU85qo5Q/1iKUP0h8qoj0/f6eiTawRYv2OIFyXdrkXwHu7xmTr57lS6KPDxrqzHQITEPEvMgMW/eL3gKK9AusYGtW6ydIMcU2gQrSuXM0jvx/DmxvmYMjc7nAezSp+s9HexLioHmdmPDNrULJFg0QF3SnL25oY7tOgC0S5+55y3csLUz4jIqA5CXNZHvOQwcUC95Rt/TUa94+o8WuJrnmj6kAUEa0KakAVnsS3zS0oqDA7bI2yZiLBervOLDfROXWv6X+WDfP1QHWfPtOEz2vVbxNKAiaQfRMFpdcZ8sU9yxcnuCsVRR3qcrUe8KC/xslQpuCkpeRbm/W6miV1jw56X/Hm0UfP5TgFfGs0MhcQESFyBxYWqbhQyGXAaD4ju1mTLIU9jgD9XOnqdwuMKvg5YzGFH6j4OuiD+NT9CALyounxuxtXlgRitgRjsbwQsawAuAF6wTL1hx2ByIgZoYTMjmA2YAzABiJsuLmeStEOIkU/GhJvAh4EPrGCdZ1TIrEKKCSMk0me9Ai4AWQcBkrQjCPhAEIAhrGTABhlBOhjBpwxSQBCAJEDtZXuwktsusLUIEZSqCdAAECQjSOkdQVpW/CzxptkiKctMpkCUgSxBRWSvC0ALCAIRhrSMqwBjWJrICpAFIA0RYyhVhydkkRFpGV42+SbHbmJpSHQKlAkpVNkpFZ+/2NuVVlNuIAx+2M/yq67cJRmayQpU9N0JJrMbwqtbge037Xcs/ZUgsKXd4EY+pAVF7ClH78PniavuSYf773xyTGvAy2GsN4Zx1DuccAfcA7lHCcM5k2pHaUfTc5GPhDwMG8qR9S33ihr3+RRhwHiJoCLAQiA5BdGhV0aEr6oTYmeIfPkNAaJZjahrHwLxyzKsaZ9GIoSYDNDUHWniQES5twMg7tkc42+a5zrYZxThqpfrE8W4DjrNZyIerOcCq8BUgXKIzbOZGmJswndEB0iU/wmYepKPpoBdNB2twoMU8IJ+V8UgqCWLFZB8wLs9RNQvBmJlx7LUB6rmgPi8j1HA+jZaEj+GUGjilZlNOqVlqlkglT6YZJ+FnyJCp4mk0G5nqVMVjblaweFzJ022m8EnVE/S7zVDoEmVDwFE50x+Vo1iOLUVWBJyTA7mbkDehzJvQdrUzi83gOiF/yoQ0iq2o37pv9PEAfcXEF30T3/IQK7TsDhORG94YtareSDP1m5DOtx1MXbietBXggWfTl3R6iaXpNu2UpM8jJZBSD1L5FwG+4zph4i4K7cyyODchuVaVScvWbnFaUwtUKTZjOQ9DqnU9Jsqsp9V/hJjc84iFKl+DuXvmo9OKSUt7+C4zPiQOS3r8VuZ1JIEh2/4avX/mNSaIskCGauFNkZISC1gaDKn/puO0id/aigyPIukPKIKW4vIYlWjknwUS/qS28LTbTEfblPG0LafrJpyipgarOQmsaBiTQaUmwRHx/4qfr6drcx1lZeqLI1UQbyJVhD6+Fg3Jjmkt9KP+zPqRMMDFaIhyF/ejtI17MzRnKx7GHoXTtrgwI3fdFfxRd9zhbiPOPKW8LyrTU7d5FmVPZHRzJJJkU7tMZPSW9FWtRl36sS//aAxG/7ek/xvyj/26XCN977wp/d8whf5+j9+BpUNLyjHxKXLDh3LD8lOaB/IPaTvrkSn3N+5LSnw/XZ4epX+5Po1G2ISHuwMvDLB2GUfltT3tz1gLte0LN04s0kOeBqgft1DrsFk/OjbM7hH9e4URaiDTPNg3DGw0hIHot8koX996/D/z/w4F3xUBAA== + H4sIAAAAAAAC/+1dbVPbuBb+zq/w9fbukF0oSSBAO9O5U6BsO9NuWaDdubPtZIStJB4c25VtAmW4v/3qxXZkW847iRMfPhHJluXznHP06OjIetzSNB05jhugwHIdX3+tPdIiWmhbfkB//fOd/nzaYZdh0wrQjY1paUBCzMu6Dg4+mLTECW1blBDk9a5d1w4sj5bXeWHPMvGp6wTEtdkjOsj2xf225dz6/DH8J8Edgv0eLdD3674uytwBv4J3S3SOlhqubSPPx3JrvKKHrW6P9Vxv1OvevZ5UeMjB9rAluTVej2wL+aeu7RIuhacdufIGET/7KF5hIr/3ETvdgHW6Uc/V4aLbAuS7ITHYC+gXxO3joIdDX09d1bFsmzWbLnRJH/EX9DC93wlCxwrS91lm7i4bd7FjJvAO3/quq+ogk3BICG29oLaP7otqLKegxu+5A0l5pJqA6p9dcNcdssOhFKW6NEJUk/hV2dZ5xcAyBUDZGkn5kmKmyheu5QSfXJPDwwo05Gs/MXHTgo4QQF2sRNljrRBkWiF7TCtfp1YOKnYTE0zYszu2mwHXx8TC/uc7TAi1KkX/fQ8ZuEgnaSWDZz9dGCDjVtkVP8Ceh82PVIbK+gCRLg7SVpW1LH4hvvf4+/hhf9uhcm07qI95T9uea7YN6hwQfQh5nfzXNrywHfpUtm0f01LTb3M1eU2baFsEBfjRsEPaQfLmm/4i+vebvqMlLbPy5Mc3/amm7Wns+bfhTeapbep1uDXSf35QbQv8Kdve0eIWaAXtOHuavpMVwtB0A6vPXoshmb/Mcigg3PLypsKraWeoVZwjI3CZUJu5S6gf5V5Zf6unqp6kX2mlCXrM8bq2ybDUj+o7x/W02rEen1M/Jfn6VN1Vz+oE6sqAjxn66cUX7Utg0YGFjzXadoc2p8UCr2Uel4wgj1knggg2izyMS4J40Mm6kHbw4PF+WI5p3VlmSD1OoTeJr/Utp2vT7qGMEd6je8vPd+4mNG6FOWTFwBxj5E6YtDKg60yV1HcVO83EMfIxWv0eD+geT2CdQ8WkjyNBXiVtdINtZQdZpds9QXwobuTqxDihvE0MFPVcufTGaeXdKe1LKKui98gOWxkr3FK83kbzkibwEuAlwEum5CW21beAlSyNlQhxAycBTgKcpAKcZB84CXCSUnKSIRXo475LHtoDl9zSAYAO00H75iHAU3OCpMV/0Rp2pdWnmIgfSwuRiJcBPpLnI5+4ZCBQAqQESEm1SckBkBIgJUBKlhUfAUoyNSWBKAkQEiAkSWlimNSMPYyyRhWVfqBOgBuRuvrSHaQSapIOXUfmmE53SYz0PUamGELTdVfWT17fOxQ2E2ExTSJNs7UpiTT1PMtqAcvKsaz6ZrKs0U6Iqy1r60dIxaztJhPuvDsymHKzS385bx68ap3mr0g0VkkEInXMO9CeZZrYuRL0ouBmlks3zK9TXpJosbpWQjrPQFJUciovnhOgoAcjxXf+6ni/XhHxFVK4Sfh+o6kk/Dkz3rCF0ZsHbZs+bh5KPpZzC7jPk/YeH+kTn570ggupaG8LiQOVPodwKsNJRG0gGxExzYknJtyWpp7WMBJMCynhNulPaZIT+7SX868GTy3X8X51AwQsfN4qxVvkdWcW7mRTzww7WNDCPPMmpZxe8jR3mFnCzHKDZpayPo2yQ5hYjg7w83nlIcwrIXo/JZufPnzvBw/2VJPb6/xAI5QdD5nEf+nf7qdPu2dn2vv3r/v9176CTngooO7GKW4zHinFpEyfbSqpJgHDWWSkI0pfa8SeIg3QrC9sYsPqI+6fmqpJ4a3agKLKaz7vOrHRuOsSeqOfEepcNNMdOLry0i+E2aw+EpqvzHq1X94q4Ckmbyn4nLB/Q20hV88XLZNxdXZ0L0dHWQDgiQA+KT/A2r8B4nkgPn0+iOU8hNmB/jgq3AcgTwTyWXntWMALVjwfwO/KbMUXrrke6BYF5p8R3Jd75t5h67iBDw7xAW4ZRzfooI7qR4cHh/uvWrhzdNPc37s99nfjYKC/67nmf+4Q2R1OJd+8GP7/K6uK445x1JEXDsOOw6Ajr6ANvnnRbhuYTjpH6ZqnwrEMfqTk2iWJcO/lb3uzy9APiOV0p5NhcZh1A5ds+HdEniF/KhsU10dlWC16dWF5nxZYGyGfPIeQV6/by/2QxNqgfbpsk1rYrti1EfFZZQ2qeli/09dgifYvtuxcriVaghyfIanGMSFpiipYwoUl3A1dwk3bKSzhTpIbfARruJAbDLnBkBsMucGFc4fF7wXUxmwGhJzgZ8oJXsgOSEgLHpEWvFoJb0pmcLwflYUitO3BnqsZyOjhGiQKr3iWyd09zDIrOsuUzRImmtPkCh/DPBNyhSFXeM5cYaX/gSSlcqULC4owF8CQMVzyjOEFYgzphhucNBxhDXnDpc4bXoA1Q+rwxqcOp2Myl1dXNQB7HrDPS23QEcyn+bAbAD0l0H+sA9BXA+QBznPh/L6cOMOWj2JYYcsHbPlYoy0fS/8qL+z0WMoJCdXe7LEyrV76SRiwzWMZH/eu9k6PMltTJVF+txSUiT8vrmsk0vOliJSnHFVHqH8sRaj+AHnVkel7fT0S7WCLF2zxguS7tUi+g11eUyffvUoXRR6etdXo65CYB4l5kJg370c8hRVol9jA1h3WTpBjCm2CFaVyZumdeP6cWF8zhkbn8wB26dP1Zgf7kmKguZ3YsE3tAgkWDVCXNGdvbqhjuw4A7dJn7nkLN2ztjLiMygDkZU3kew4DB9RLntE3O+oVT//RAlfzXNOHNCBIA9qUNCD+JT5pacXBAVvkbRMxlotVXvHlvrFLLf/LfLDvH6qDrPl2HCb7Xqt4GlCRtINoGK2uuE+WKe5YuT3BWKoo79OVqHeFBX62SgU3BSWvotzfrVTRKyz489J/jzYKPv8pwCvj8aGQuACJC5C4MLHNQgZDLoNB8Z3aTBnkKWzwh2qnz1M4XOHXQcsZjCj9x0FXxJ9GJ2jAFxWXz43Y2jwwoxUwo52N4AUN4AXAC9aJF6w4bA7EQE0MxmTzATMAZgAxk+XFTPJWCHGSifhQE/gQ8KF1jJOsapkVCFFBpGSSzHegRcunRR7QIgiYzE4Q9oEgAEFYy4AJMIRyMoRxG6aAJABJgNjJ8mInsV1mbREiKBMRpAMgSECQ1jmCsqr8XeBJ00VSlJtOgSwBWYKIyloRhhYQBiAMax1RAcawNpEVIA1AGiDCUq4IS84mIdIyvGr4TYrdxsSU6hAoFVCqslEqOnu3tymvotxGHPiwneFXHb9NMDKTFarsuRFKYjWCV7X632va71r+KQNiSbnDi3hMDYjaLETtw+eLq+1Lhvnvf3NMasDLYK81hHPWOZxzBNwDuEcJwznjaUdqR9Fzk4+FPwwYyEz7lnrEDbu9izDgPETQEGAhsHcJokOrig5dUSfEzhT/8BkCQtMcU9M4BuaVY17VOItGDDUZoKk50MKDjHBpA0besT3B2TbPdbbNMMZRK9UnjncbcJzNQj5czQFWha8A4RKdYTM3wtyE6YwOkC75ETbzIB1NB71oOliDAy3mAfmsjEdSSRArJvuAcXmOqlkIxsyMY68NUM8F9XkZoYbzaeB8GjifZpPOp1lqfkglz6QZJeFnyI2p4jk0G5nkVMUDblawbFzJc20m8EnVE/S7zVDoEuVBwCE5kx+So1iILUU+BJyQA1mbkDGhzJjQdrUzi83gbkL+lDEJFFtRv3Xf6OE++oqJL/omvuIh1mbZHSYit7wxalXdoWbqtyGdbzuYunA9aSvAfc+mL+l0E0vTbdopSZ+HSiAlHaQyLwJ8z3XCxB0U2pkFcW5Ccq0qh5at2uK0phaoUmzGcgaGVOt6TJRZT6v/CDF54BELVaYGc/fMR6cVk5Z28X1mfEgclvT4rczrSAJDtv01ev/Ma4wRZYEM1cKbIBklFrA0GFL/TcdpE7+1FbkdRdLvUwQtxeUxKtHIPw0k/Elt4Wm3mY62KeNpW07HTThFTQ1WcxxY0TAmg0pNgiPi/xU/X0/X5jrKytQXR6og3kSqCH18LRqSHdNa6Ef9mfUjYYCL0RDl/u0naQP3ZmjOVjyMPQmnbXFhRu66I/ij7riD3Uacc0p5X1Smp27zLMqeyPDmSCTJdnaZyOgt6Xtajbr0Y1/+0egP/29J/zfkH/t1uUb60nlT+r9hCv39Hr8DS4SWlGPsU+SGD+WG5ac0D+Qf0kbWI1Pub9yXlPh+ujwxSv9yfRqNsAkPd/teGGDtMo7Ka3van7EWatsXbpxSpIc8AVA/bqHWYbN+dGyYnSP69woj1ECmebBvGNhoCAPR75JRvr719H8DpzEx3xUBAA== k8s-resources-node.json.gz: - H4sIAAAAAAAC/+1cbW/bNhD+7l+hcRuQDO5iOUmbBeiHNWmaAs2W5qXAsBYGLdE2EUnUSCp2Gni/fST1YkqiGidOF9uhP0l3FI+85+54Jk+6bTkOgFFEOOSYRAzsO7eCJIgBZlzc/f1F3E7bshnyMYf9AAkqpwlStGGE+HtfUKIkCFIKhfHogpCA41jQO4o4wj46IBGnJJAiBjBg6fMBjq6YEqNuKRpQxEaCALY7DKQ0MlYt1LDSwQmqR4IAxgzpvSnGCOHhSI4cdHc78QQUjBhGKJj1pPem+DDAkB2QgFClhWlbZ/YhZVVRiuFDNvqAoiGXg3Y7NR5qeoxDRhLqyQmAU0pCxEcoYaDUaoCDoN4tlvp2S6QADVHkF+DN5nQ9NImX+ksoRRFv4IZw0sTBUQOHjchYMw2Nw4V1BQ1PXcMgmelI45X1L+xEtar2rhhj7Cv1d6oczbQKsjTUU4IjfkJ8pXxJcCBzviJKyuqPkcAn4nCIjBjGshcKfZxIMbt1nhl6oXYfUUSl7EFAeFkmQxQj9uc1olT4jGH8LIYearI4wZTwuN0ylUPvyqQ8xlEcI/+D0KFxqBzSIeJln6n6jWqIJrGaD0vCjUjotRfBEKmR9mLi9zzh+lAIofvFVc+Lk17ChG57DAmqz3rKTPZFFz0KObr1gkSMj77+DH7KLj+DtiN7f/2voMmLz2C66fRvnA0hZBO0q6MaEBpCFQo4DqUcqdp6M4EUosIOj6DHiZxGt9Yk9a+jor/bWyFxOgUNDYVCr7SYWHYToXMFXIkx1e7KePORjIgk8A22IKd1JGKHQZbinY/wgJuZXMVxcHB66VxKFMpWyIv4fVt1ckiR3xQBCOVVN8xdvMdvYiURRz6+xn4iIkKjt+dt1VpSHtkETjCrj6ufeFeppdaVDsLM06VKKpABaajmp5rjWRGz1OJonsINnKA5HGdmokIc5QaDgn0UNNlSQIZvoFoD3RovDeHGx9IY3qmb5mzGZeNsL+0kzC6WzqO6olS8rFWlFr4lQnSMYNVxMup7ESxUrmRmn5FxKR8qBnSRuZxuT9/yw5R3jr8q/uhl6i0ZFM8zDapnQV2bBdWyINdmQd/MgupjuSsNYvwmmCeYK9eRE7qoLzSpsaNZCvGX+L04OXlxeOgcH++H4T4zpCYx5CLcRM195iul+IPlowjcK2wXwzUnAXl8ITSzEWOs9fJIUQbooRP2kYdDqOKTIQtL0yqDA2XMC5WvvgngXe2K9AYcUhFcHJ+MI2Bsekmlz4JvQvNJeq/z4+8GeJqTtxJ8URL2hS/U+EmEtXX14eieoX9EfOHMArwIwG+WH2DnZwvxIhAffD+Is3VU3T4c6A84xNaPFwP5cHn9OIXXevFiAL9dZi8+Jb5F90HoxibNLYPnLjmemgq3fv1l6+E6ZJziaHg/HTZvbK7N5rI6FDNsK4t/nGpvob4Z8LBd57oMigZqtwkY/ns0bDS351f2VdKvqLgn7ERt14iLNOGcT5NtJ39QtBPgrJRu33wP3T6VITtbjoW2gPbg/3abQOV3z0Czh8/MaZ4NsG/BChxpfkwE4Mt1pElhxCSSZhyLFMvAskee9shzTY88y35qjzznqfzatmeetvJrlSu/QhQSetMbE3qFo6HI0Xivf8PR3MlT0dMPoqUtAVs0XzpRcKQHwM7GeIs4HvRGaNNWhD1xeqScwqZHzzQ90t3SZkj3KQrbsQmSLQqzRWELFoUZ4489r1yuurA0RVgIYFsatuSlYY+Isa0rWePqsAxrWyC21AVij+DNtkZs7WvEynsyZ+fnmxbsRcA+WmqHzmA+qG+7WaDvCfS7VQD6fAxji/NCOB8vJ862ttfW9tra3qU+Pnzg6aGt8b1XVVsKki3zfWqb/r7lviuI8upU/K6gcg+fqQs9J4zfPj3GlK3/Yn/09GpWZUBrr+h3T69oNobx2uv5GKxGHZx9dcC+OmBr41aiNm6utwda2VABE6tZCD8hytLhuDspWdbxyCd8SK/SL85yOJwZI5CJH41QtimY9sVRGAdiXtGw/sHcVhV3raCsVFTH0USZgY8GMAkqxU7Ka3Ruy2Bf6ru6ZeNssJ7cc/XiOo1L4vzrv3pYBeKPIb1Ru3OmIjzta71uiTpEk8piUMQoTXyrMh1NYTAIPmXzr0zjDlU26NCsvDnqDHMFayufCNliffbR74Ghbq9J+6FAEBua56hkK/59IFGSemlwnf05wdGAtJ2su00zWN27wMpWLh1U4RIKEfYxlw/K3NpAJc3cODOFdCYaI2HoIu1Ij0UrYR+dR7KP8vJZmIdM/h7HNoxp5jTNM9fDXFr5cpV+tFxlMUWMHqQZolDo+IWbv0cg8ruMBkqPxVhkSXT2cKaSXp4X6wkL2J2ZBXA72s22fuOGs+td7drVb7Y7Omf2tgPoateunxrtl3wO8vUWzS7ulKJ3/FLvWJfS3dFvZic04JWvjzcfS0l9X4mqnAWXFwfZslq8+UfCOOHIOcs2Qpiz5fwhDNDZOCU+S80QJKqWG3Q7HejtDfz+oN9/tdPf/g0OBnt7rruHdnzX63pp4+tiVe+0pv8BvRGm/tReAAA= + H4sIAAAAAAAC/+1cbW/bNhD+7l+hcRuQDO5iOUmbBeiHNWmaAs2W5qXAsBYGLdE2EUnUSCp2Gni/fST1YkqiGidOF9uhP0l3FI+85+54Jk+6bTkOgFFEOOSYRAzsO7eCJIgBZlzc/f1F3E7bshnyMYf9AAkqpwlStGGE+HtfUKIkCFIKhfHogpCA41jQO4o4wj46IBGnJJAiBjBg6fMBjq6YEqNuKRpQxEaCALY7DKQ0MlYt1LDSwQmqR4IAxgzpvSnGCOHhSI4cdHc78QQUjBhGKJj1pPem+DDAkB2QgFClhWlbZ/YhZVVRiuFDNvqAoiGXg3Y7NR5qeoxDRhLqyQmAU0pCxEcoYaDUaoCDoN4tlvp2S6QADVHkF+DN5nQ9NImX+ksoRRFv4IZw0sTBUQOHjchYMw2Nw4V1BQ1PXcMgmelI45X1L+xEtar2rhhj7Cv1d6oczbQKsjTUU4IjfkJ8pXxJcCBzviJKyuqPkcAn4nCIjBjGshcKfZxIMbt1nhl6oXYfUUSl7EFAeFkmQxQj9uc1olT4jGH8LIYearI4wZTwuN0ylUPvyqQ8xlEcI/+D0KFxqBzSIeJln6n6jWqIJrGaD0vCjUjotRfBEKmR9mLi9zzh+lAIofvFVc+Lk17ChG57DAmqz3rKTPZFFz1MIUe3XpCIAdLXn8FP2eVn0HZk96//FTR58RlMN53+jbMhpGyCdnVYA0JDqGIBx6EUJHVbbyagQlQY4hH0OJHz6NaapA52VPR3eyskTqegoaHQ6JUWFMt+IpSukCsxptpdGXA+kiGRBL7BGOS0jkTwMMhSvPMRHnAzk6tADg5OL51LCUPZDHkRwG+rXg4p8ptCAKG86oe5j/f4Tawk4sjH19hPREhodPe8rVpMyiObwAlm9XH1E+8qNdW60kGYubpUSQUyIC3V/FRzQCuCllodzVO4gRM0h+fMTFSIo9xgULCPgiZbCsjwDVSLoFvjpTHc+FgaxDt105zNuGyc7aWdhNnF0nlUl5SKl7Wq1MK3RIyOEaw6TkZ9L4KFSpbM7DMyLiVExYAuMpfT7elbfpjyzvFXxR+9TL0lg+J55kH1NKhr06BaGuTaNOibaVB9LHflQYzfBPMEc+U6ckIX9YUmNXY0SyH+Er8XJycvDg+d4+P9MNxnhtQkhlyEm6i5z3ylFP+wfBSBe4XtYrjmJCCPL4RmNmKMtV4eKcoAPXTCPvJwCFV8MmRhaVplcKCMeaES1jcBvKtdkd6AQyqCi+OTcQSMTS+p9FnwTWg+Se91fvzdAE9z8laCL0rCvvCFGj+JsLauPhzdM/SPiC+cWYAXAfjN8gPs/GwhXgTig+8HcbaOqtuHA/0Bh9j68WIgHy6vH6fwWi9eDOC3y+zFp8S36D4I3dikuWXw3CXHU1Ph1q+/bD1ch4xTHA3vp8Pmjc312V1Wx2KGfWXxl1NtLtR3Ax627VyXQdFAbTcBw5+Php3m9vzavkr6FR33hKGo/RpxkWac82my7eQPinYCnZXS7Zvvodsns2Rny7HYFtge/N9+E6gM7xlo9vC5ec2zQfYtWIFTzY+JQHy5TjUpjJhE0oxjkWUZWPbU0556rumpZ9lP7annPNVf2/bY01Z/rXL1V4hCQm96Y0KvcDQUSRrv9W84mjt5Knr6QbS0VWCL5ksnCo70DNjZGG8Rx4PeCG3aorAnTo+UU9j06JmmR7pb2gzpPnVhOzZBsnVhti5swbowY/yxR5bLVRqWpggLAWyrw5a8OuwRMbalJWtcIJZhbWvElrpG7BG82ZaJrX2ZWHlP5uz8fNOCvQjYR0vt0BnMB/VtNwv0PYF+twpAn49hbHFeCOfj5cTZlvfa8l5b3rvUx4cPPD20Vb73qmpLQbKFvk9t09+33ncFUV6dkt8VVO7hM3Wh54Tx26fHmLL1X+yPnl7Nqgxo7RX97ukVzcYwXns9H4PVqIOzrw7YVwdsbdxK1MbN9fZAKxsqYGI1C+EnRFk6HHcnJcs6HvmED+lV+tVZDoczYwQy8aMRyjYF0744CuNAzCsa1j+a26rirhWUlYrqOJooM/DRACZBpdhJeY3ObRnsS31bt2ycDdaTe65eXKdxSZx/AVgPq0D8MaQ3anfOVISnfbHXLVGHaFJZDIoYpYlvVaajKQwGwads/pVp3KHKBh2alTdHnWGuYG3lEyFbrM8++j0w1O01aT8UCGJD8xyVbMW/DyRKUi8NrrM/JzgakLaTdbdpBqt7F1jZyqWDKlxCIcI+5vJBmVsbqKSZG2emkM5EYyQMXaQd6bFoJeyj80j2UV4+C/OQyd/j2IYxzZymeeZ6mEsrX67SD5erLKaI0YM0QxQKHb9w8/cIRH6X0UDpsRiLLInOHs5U0svzYj1hAbszswBuR7vZ1m/ccHa9q127+s12R+fM3nYAXe3a9VOj/ZLPQb7eotnFnVL0jl/qHetSujv6zeyEBrzy9fHmYymp7ytRlbPg8uIgW1aLN/9IGCccOWfZRghztpw/hAE6G6fEZ6kZgkTVcoNupwO9vYHfH/T7r3b627/BwWBvz3X30I7vel0vbXxdrOqd1vQ/jChRb9heAAA= k8s-resources-pod.json.gz: - H4sIAAAAAAAC/+1daW/buBb9nl+h0fQNkpmmsZ04XYDioUmaaYEWzSRpBw9NYdASbQvRNhSVpUHmtz+SWkxJlPdFcm6/NCJlLvfcSx5eXlIPW5qmI9f1KKKW5wb6G+2BJbFE2wooe/r+gz0+PuevYdOiqGtjlkpJiEVa38X0o8lS3NC2oxSC/MGl59nU8ll6QyQOLBMfey4lns2r6CE7iH5vW+51IKoRjwT3CA4GLEHfbwR6lObdijdEs6LGsVTDs23kB1guTWQMsNUf8JbrrXbDv9PTDB+52B6WJJcm8pFtoeDYsz0ipPD4XM7sIhLkqxIZJgoGn7Dbp7zRzUYhD5f9jKLAC4nBO6CfEc/BdIDDQM+81bNsu1isxeXdzCTZuI9dMwVv2Kebvqp6Lr+QEOzSklwH3ZXlWG5JTjDwbiXVkHIo0y675Fc3yA6HMpLysvJneiLeypcuMm4tU4i/kc+RVCtN5op65lku/eyZQvg8QUOB9hMTLyt+HzN8XIr6WImhz0shyLRCXk27mKeGnondxAQTXnfP9mi2zgATCwdfbjAhzGay6ppX2aHa8rII/odJkmZVKLEVT1T362nr4HX7uPhGrGiNQgY33KExF7GVdU+dK+HTKuQGFBnXCvBz8I/qtm051phOn75+td+oRaelp6zWBj4ycNk4wzK5UTZb2dS4moLJBBT7PjY/sUYqFZQi0sd0AtXDd76QcBA62y6zpo6LHCxa2vE9s2OwAR+xSsib9K+O4YedMGAW1QkwSzWDjhgc3rAiOgRR/JAW8fZKf5Y+XOnPNVYkT2P/8SfDDlk/CE+J/7zSH3e07r22nda2o8DcIw4SswO1HN4Ibm3F15jxYsKGplNkUKFFrRINOE3Le3hI63181EteZzK/libLnFpgX2A7lR3IGFy5POk67Obk32FzqphrOskY8aAS33NtUuEn5bEkBigT/JW7c+WuUtrlg101BB2NSrUXc9ngOrOQSwc4OuDEz7NNxZTNe3XKKJKiLpF3MbB6VJ1JBV3Vj8++al/5sJOdbGk6yD/kuQwi2CwjOh6hxXkjYjIdeu+LGi3XtG4sM2TEp5TUJO8Kypxt2R26s4Jiu7qhcR0NzUWh605MaLhIcpDpXMvUvyqnbSk1E2sAdRfu0d0kJGWooaw6QhUKhbrYLtMl2+sfIUH1m4W8iKkqfxZR1eL0LvV4qhFgjZ1Qm1jUjzE0YiufmtoWG0R9jPKGE6d+ZGOFWBKqs8+928yyL23QZWxysj6NssMo78L6KfIHh5G1xFDAai9e7bUWttpT2HmkfaoMWOuteq1XM9ZvuQbBbFTbzjJ8o8d4/YB4lJm22WGitbyE6U9O8JMSf2GJpYz/e9v5sZOn/dremLatvEWbsBCZkLqNVh6xKk+s0SAWtQw2XJQtzUsX12V5ni/4lGJuFmMPHzdetNqFzPuYbTHhMCI5abcXwkovIzOx3D5Q0zVT03gKCF1r8dyuCey0HuxUYY9AUQsUtchQ92E/osBRm8BRR3LUYlvGkdSA3ttTbYpcFiecSNnxkDD9j/3b/fx59+RE+/DhjeO8CRREzEeUjTlueZnJjDmwTBO7+mybGWo/VYE5KQdcIxkpsgDN2mETG5aDxPjUUnGwa7UBxZmXYjVxZKNx76U0Rz8hbHDRTO9W4eHkr34l3Gb1kdB849ar/fpOAU+5fzEDnxs6XWYLhXzBCtLJdXZ0z0fv0gHAEwF8VH2Atf8AxPNAfLw8iGWiPzvQn0ZtPAPIE4F8Ul07juAFK54P4PeVtuLETwYYz4SxUS6/KlhxxVGVBLn34ve92WUYUJJ3H46VYblXc53BP/OES0wX/CPiWBXedrY0Ldkmm80ZX6yD4J7wTemKRcr8kSnrCP6pm+CPliH4SpqAtqeBUkymFMertsblRIjVTewnYIugElmVeK/XIJjwr5CpSrV2bAlyA46kGseULSqyINgQgg03dDs3a6ewkztJsOEBbOU+lXDDFRwtSzVVSR2e5MGz2omkvsfShizbwY5H7ju3Hrm23D6j/LTTvad4XrpdCAq1HGbU0QOcR+NJq/U+RDDDkbTlry7XJ+lNOZX2WUgwCvjRtv++uNiBGOA1rxjFlAArxie6YpQNEhaN04T/tmHNCOG/EP47Z/jvGEIAkSnViQOOiMJcMEMocMVDgReIMcQRbnA0cIw1BARXOiB4AdYMMcEbHxOcpWDnQMHmBPu00gYdw3yMjAEGoOcC+s86AH1xi3zAeS6cP1QTZzjLAWc51ofqBp3lqNGOORziWOg+LpzjWIX6j9L2tZzXqC3+tT+yUVvJn9TS8qaaeNZwXKO26vB+JepAgoUpgPbLWy2+wjGbdPblpIbyP12J/A3uJAEElAj8uRIEglvkAwBKAD7o9Yj1g3NjcG4MogBrEQUIR8emPzp2mE2Kh3heWNPRIUSwECIIt9jHIYKH67zEnh/2l9iWiylferKFnoGtGxytPeO74sexr39V9Ovf9FKA78864mqBTmIayVXxLHvFp4NYjeu4Hn7RvOo8Akk7Qq4ZWRUcoVgveTry4QDFWqjT801gEC+BQQCD2BQGIdbWjkWBQlSZQlzGKAGHAA4B7pe1u1+KVgi+l0mY0ytgTsCcNs33wprPpzKgThX1vjBpal5Pi70wpnYW4QUECggUOGFqSyVeA5UAKrFxThjgErXgEok7hgKdADoB/pi1+2MSu8zbInhlJqFS+TTgUsClNsctYxKPVw6Uql7uGe0kwg14FfAqcNPUl1s0gVsAt9hYPw2Qi7r6a4BfAL8Av021/DYFmwT/zfCt4d0+u82J2VcLGBYwrKoxLLbQt7cZzWLcRlAbbTtHt3pBh2Bkpttg+UsxlDxrcprVdhixWimhOue9Wf1HdyYR9C2xpLjv2kv6b9Gd+jPXj1/OLoCawhl7cH7V2fm1D/QL6FcFHVzjmVfmNNgyWMEToF+TMa+NkvSm0K/LAfHC/uAspEDCwD8I/sE1+Qcv2LjDL8X/+EXb1U4sfvN0N+S1bJ95JkviA7v2mxaNOjvgNlyU2/AAeCvw1iq7DaVLcFfhPMwyKu0PbRV+tHW60TZmB5j70bYF//8jP0sAoQOvGnjVauhVgw85AzupqldtEmKycI/P5PRkyVUDR5nP2SSYSkRUgKeA4wkcT9VzPKUf8QN/0yjepqBth0DbCrSt+SRoWzQL5YBm5sASD3LCZQUYxTHvcWoamIt8THhgsW/jiGBA7+1J5o30M5CXxTlttq85Sh9sVJeZTMoDyzSxO+NXK4cOkmp9dHa3Wbevzr6r5mdJBcAq3xcgPC3CRxVGWJgwW/oB0otA+riaSMcrRT9eKcJnwucC+WR5IPO149wQK/wAgPGUGL+vOMYigiEetQHquaA+rSLUZ55ZD1SVH0CcG1SNetxNHIwE98WeuXfYftXEB4f4ALeNl1100ECNl4cHh/uv27j3stva37t+FewmH6INdlmZ/71BZHfolHj7bPj3bzwr8WYnvmyROPRmD13ZIoP7sp91Oga27ZG65qvwrAI1qLiWSSLce/H73uwy5K44tz+dDMv9+kvc5VnTJksFPnT6Tl9OIPVoqS89AKfqYj9ai9g3Mtaq6lAfrxHqtW4hV0D2J+sc3Z668N9vruJXNmyjArCfVv3T6cfR3qQ23DeGD6hDGCoEeNQvwGNcUMdW3G49MAbYQd8wCaK2RYdmov1i/gsTkWtRGLOq/lAz9euwyxbCmB+TTMui2PFt1km3n1qabrNGSfo8VAIpECITDULxndAJE/dQaOc26YUJybmqoGC+k4yzmlqiSokZy1EhUq7nc1HmR1r9nxCTe+FFUUWP8OGej9FZxWSpfXyXmx/SAUuqfivXHUlgyLa/xf3PdWOMKEtkqBbeBAEyiYClyZCN32zWNvE7WxFvUiZ9hyFoKV5PUIl5wDSQiJo60Ui7zXW0wwhCx3J7XsowdtRgtcaBFU9jMqjMJAQiwV9J/Xo2t9BQnqZ+OVaFqCdSRhjgy6ggeWCqhX40lqwfKR9cjIYoL758lG6+BM3ZFM3J+r8XrTPyZanZZcujWLdshh5tJXToMZr8LSHaeNrvResQ3fVud5tJqDVbP8RpeuZnvsVYOBn+OBZJeoGsTIj1tvRdjGZDetiXH6RbmfW29HdTfthvyDnSV1Bb0t9NM9LmH0kfePy/pDBja5ELPpQLlmtpHcgP0tWRL025vUlbMuL76YmgP/3r5XHM1NL1nOf4IcXaebLjpO1pyVaeHop4Vn3cllX08k1KEBtbj/8HRf4tecv0AAA= + H4sIAAAAAAAC/+1dWXPbthZ+969g2d47dhvXkmw5y0wfYiluMpNMXNtJ506c0UAiJHHMrSDoJR73t18AXASSoLWLpHzyEhOgsJzvHODDwQH4sKNpOnIclyJquo6vv9EeWBJLtEyfsqdv39nj4wv+GjZMivoWZqmUBFikjRxMPxgsxQksK0whyBtfuq5FTY+lN0Ti2DRwx3UocS1exRBZfvh7y3SufVGNeCR4SLA/Zgn6YcPXwzT3VrwhmhU2jqUOXMtCno/l0kTGGJujMW+53mo3vDs9yfCQg61JSXJpIh9ZJvI7ruUSIYXHF3JmHxE/W5XIMJA//oidEeWNbjZyebjoZxT5bkAGvAP6GXFtTMc48PXUW0PTsvLFmlzezVSShUfYMRLwJn26Gamq5/ILCMEOLci10V1RjukU5Phj91ZSDSmHMu2yCn51g6xgIiMpLy1/pifirWzpIuPWNIT4G9kcSbWSZK6oZ67p0E+uIYTPEzTkaz8wcdPi9zDDx6FohJUYerwUggwz4NW083lq6JnYDUww4XUPLZem6/QxMbH/+QYTwmwmra5ZlZ2oLS+L4H+YJGlahWJbcUV1P5+2jl63O/k3IkVr5DK44U6MOY+trHvqXAmfVi7Xp2hwrQA/A/9T3bZM25zS6dPXrw4btei09JTWWt9DA1w0zrBMbpTNVjo1qiZnMj7FnoeNj6yRSgWliIwwnUH18J0nJOwH9q7DrKnnIBuLlvY81+gN2ICPWCXkTfJXb+AFvcBnFtXzMUs1/J4YHN6wInomQRQ/JGX8caX/kjxc6S80ViZPY//xp4EVsI4QnhL9eaU/7mn9e203qW5PAbpLbCSmB2ravBXc3PKvMevFhI1Np2hAhRq1ClTgNCnv4SGp9/FRL3idCf1ami0zeoE9Ae5chiCDcOXwpOugnwGgxyZVMdn04kHiQSW+F9qswo/LY0kMUSb4K2fvytmktItHu2oIOhyWai/motF1YSEXjnB0zJmfaxmKOZv36pRxJEVdIu9ibA6pOpMKvqp3zr5oX/i4k55taTLKP2TJDCLYKGI6LqH5iSOkMj1674kaTccwb0wjYMynkNXE7wrOnG7ZHboz/Xy7+sHgOhyb80LX7YjRcJFkINO5lql/VczbEm4mFgHqLtyju1lYykRDWXWEKhQK9bFVpEuWOzpBgus3c3khVVX+LOSq+fld6vFcI0CJnVCbWNiPKTxiJ5ua2BYbRD2MsoYTpX5gY4VYE6qzz93b1LovadBlZHKyPj1lh2HehflD5I+PQ2uJoIDlXrTca61suaew81D7VBmw2Nv0Yq9mtN90BgSzUW03TfEHQ0bsx8SlzLSNHhOt6cZUf3aCH5f4E0ssZPzf2vb3vSzt1w6mtG3jLdqGhciM1O1p5RHL8tgaB8Sk5oANF0Vr88LVdVGe6wk+pZibxdjDx43fW+1c5n3EtphwGJGctdsrYaWXoZmYzgioacnUNJoCAsdcPbdrAjutBztV2CNQ1BxFzTPUQ9iQyHHUJnDUJzlqvi3TSKpP7625dkUu8xNOqOx4Qpj+x/7tf/q03+1q79+/se03voKIeYiyMccpLjOeMcemYWBHX2w3Q+2nyjEn5YA7iEeKNECLdtjAA9NGYnxqqTjYtdqAosxLsZo4sdC09xKao3cJG1w0w71VeDj5q18It1n9SWi+cuvVfn6rgKfYv5iCzwnsPrOFXL5gBcnkuji6509v0wHAMwF8Un2Atf8AxMtA3FkfxDLRXxzoj0/tPAPIM4Hcra4dh/CCFS8H8LtKW3HsJwOMF8J4UCy/KlhxxVGVBHnw+68Hi8vQpyTrPpwqw2KvZqnRP8vES8wX/SMiWRXudrY2LdgnW8wbn6+D4KFwTumKVcryoSllRP/UTfAn6xB8NW1AO9BAK2bTis6mzXE9MWJ1E3sXjBF0IqMT7/QaxBP+FTBdqdamLUGOz5FU45gQRkUWxBtCvOGW7uim7RQ2c2eJNzyC3dznEnG4geNliaYqqcOzPHxWO5HU92jahGXb2HbJfe/WJdemM2Kcn/b69xQvS7dzcaGmzYw6fIAjaTxps+6HEGY4lbb+1WV5kt6Wg2mfhATDmB9t9++Liz0IAy55xSimBFgxPtMVo2yQsGicJwK4DWtGiACGCOAlI4CnEAIITqlOKHBIFJaCGaKBKx4NvEKMIZRwiwOCI6whJrjSMcErsGYIC976sOA0BTsHCrYk2KeVNugI5g4ajDEAvRTQf9YB6Itb5AHOS+H8vpo4w3EOOM5RHqpbdJyjRjvmcIxjpfu4cJJjE+r/lLaXcmCjtvjX/sxGbSXfraXlzTXxlHBco7bq8G4j6kD8lSmA9tMfWnSLYzrp7HO3hvI/3Yj8B9xJAggoEfhzIwj4t8gDAJQAvNfrEesH58bg3BhEAdYiChCOjs1/dOw4nRQN8bywpq1DiGAuRBAuso9CBI/LvMeeH/aX2JaDKV96soXeAJs3OFx7RtfFT2Nf/6ro17/JpQDffun1eG292DTi2+JZ9oZPB7Eay7ghftW86jwESTtBjhFaFRyhKJc8nXhwgKIU6vRiGxjES2AQwCC2hUGItbVtUqAQVaYQlxFKwCGAQ4D7pXT3S94KwfcyC3N6BcwJmNO2+V5Y8/lUBtSpot4XJk3NHWqRF8bQzkK8gECV/Tk7IFDghFmYSrwGKgFUYuucMMAlasElYncMBToBdAL8MaX7Y2K7zNoieGVmoVLZNOBSwKW2xy1jEJdXDpSqXu4ZrRviBrwKeBW4aerLLZrALYBbbK2fBshFXf01wC+AX4Dfplp+m5xNgv9m8tbkbp/95szsqwUMCxhW1RgWW+hbu4xmMW4jqI22m6FbQ79HMDKSbbDspRhKnjU7zWrbjFhtlFCd895s/qM7swj6lphS3HftJf236E79meuHz2cXQE3hjD04v+rs/DoE+gX0q4IOrunMK3UabB2s4BnQr9mY11ZJelvo1+WYuMFofBZQIGFwzg78gyX5By/YuMMvxf/wWdvXuia/ebof8Fp2z1yDJfGBXfuvFo46e+A2XJXb8Ah4K/DWKrsNpUtwN+E8TDMq7TdtE360Mt1o6/oye4netF2xCvgtO1cArQPfGvjWauhbg885A0epqm9tFnqycr/P7CRlzVUDU1mF40nwlZCuAFsBJxQ4oarnhEo+6Ae+p6fYm4K8HQN5y5G35rMgb+EslAGamQNLPMoIlxUwyI95j3OTwUwUZMwG832bRgd9em/NMm8kn4S8zM9pi33ZUfp4o7rMeFIem4aBnQW/YDlxk1TrA7T7zbp9gfZtNT9RKgBWecAA4XkRPqkwwsKE2QIQkF4F0p1qIh2tFL1opQifDF8K5O76QOZrx6UhVvgBAOM5MX5XcYxFNEM0agPUS0F9WkWoO8XfrwdsZ8B2UCy/KkzIFUdVEuTB778eLC5D7gBzRvPJsNibvsYdlpI2OCrwqdG3+npCmZ+W+tpDYKou9pNSxL6V0U5Vh7pTItSlbt9WQPbdMke35y78d9ur+JUNmagA7KdV/3h5J9wR1Ca7tfAJcwgBhbCK+oVVTAul2InarfuDMbbRV0z8sG3hsZVwl5b/wkDkWhTGrGo00Uz9OuizhTDmBxWTsii2PYt10hkllqZbrFGSPk+UQAo/SMVgUHwndMLAQxRYma1xYUJyriogl+/f4rSmFqhSbMZyLIaU63pclNmRVv8nwORe3Lqlitngwz0fo9OKyVJH+C4zPyQDllT9TqY7ksCQZX2N+p/pxhRRFshQLbwZwlJiAUuTIRu/2axt4LeWIsqjSPo2Q9BUvB6jEvGAeSARNfXCkXaX62iPEYSe6QzdhGHsqcFqTQMrmsZkUJlJCET8v+L69XRurqE8Tf1ypAphT6SMwMeXYUHywFQL/WisWT8SPrgaDVFePfko3T0JmrMtmsMgX6POyNeVppctj2Ldsh16tBPTocdw8jeFaKNpfxiuQ3THvd1vxgHObP0Qpempn3kmY+Fk8uNIJMkVrjIh1tvSlymaDenhUH6Q7kXW29LfTfnhsCHnSN8hbUl/N41Qm7/HfeCx95LCTK1FLvhYLliupXUkP0iXN7405PbGbUmJ74crQu30L5ediKkl6znX9gKKtXMc2p2vHWhnkerrgYgi1Y/br5r46Bgf4fbgZR8dNVDj5fHR8eHrNh6+7LcOw5dvEoLY2Hn8P5Me+81R9AAA k8s-resources-workload.json.gz: - H4sIAAAAAAAC/+1da2/bOBb9nl+h1XYXycCZ2Hk4TYBi0SQNZoB2J5NJu1hMCoO2aFuIJGooKo8G7m8fPvSgJCpxHk4k9/ZLLVLm6557eXh1It+uWJaNgoAwxFwSRPa+dcuLeKHnRoxf/fmVX8464jbsuAwNPcxLGY2xLJsEmP3q8JIg9jxVQlE4PSPEY27Iy7uycOo6+JAEjBJPdDFGXqS+77nBRSS7kZcUjymOprzA3upGtiojV/IOOSw1OF46Ip6HwgjrrcmKKXYnUzFye3OnG17bWUWIAuzlLemtyXrkuSg6JB6hchVmHb1yiGhU7kpWOCiafsTBhIlB97qVOlz3NYYiEtORmIB9QomP2RTHkV24a+x6XrVZV6x3r1Dk4QkOnMx4+ZwuJ6buxfrFlOKA1dT66Lquxg1qaqIpudKgodUwji6v5luXyIvzNdLqiuvPcSLvKrcuK65cRy5/t1yjQSsrFkA9IW7APhFHLr4osFBkfcOUFJc/xNw+AUMTbLRhKFqhyHFj0c1Otc5ser7sDqaYir7HHmHFPiNMXRz9dokp5T5jGH8UohGuQxyvFObpbRZLGRpdmBYvYjgMsfORr6FxqAzRCWZFnyn7jbwRX4dyPlHsr54HoijgqzsIkI/leAchcQYjHgAQ74ruZ58GozAexBFf4UGEeakTDSRY9nlDA4oYvh15MR8lfXduv0k+ntsdK2tYlGcX5/ZMdP6TRYLVrLDDu15TY5pQEocDD4/Z6hWhFx5BTsdKPw3YTYjX8pYHWQVvYP8iHqpJkCsxAYo9NMTeA0eXdyYq0s96uRyEqBT/y+msWcMba1XNwe6U131MqI9ksGOuL9ZQgKd6G8ciptzTjtGIEWGozcotKoIcZ+3d3vI+ZzO75kYOmQst6hcDAUeVhGahYqZdFRHNpiLmE88xoF1M65hHR0Nfsu6PqTtm5komdyr78OSz9VkgrOhnLNuhbsthDFHs1MU4Qlk50KRBTNpO9OgGjnvpOjGPebXxLL1X7pbFkV2jazeqjmsYjy6UL1YX3faTWCaWpGQyWwDQ/K36iJ1FZbn9m6dwg67xHKEhhyjvjjIDoIQn1WHJI5MDJHf5XqVObVLGr6ldqluFZj7jIjg7jZ2E2cXUPMp7ZsnLVsqlmW/xTSjEqOw4SemvPFhINmiuPiVXBcaXDegscTkdT3f5oar7w/0m66d95S2JKX5MolfleZvA8yo8rwc8706eVx3LfUQvYjfePMFcuo6Y0Fl1o1FgxzmF+D//t/7p0/rRkfXLL/u+vx8ZqEmIGA83QX2b6U7Jj5AODuwHhe1suGYSkMYXQhOMGGPtKI0URQM9dsIOHrk+kvHJwMIUrTI4UFJ5Jhn5gYfuuy+jN/YR5cHFcjhxtY23fqbCZ+07TfNFeK/1z/cG89STt4L5gtgfcl+o1MeBq+2rj7fuKf6LxxcWgYGfYuCD5hvY+heY+CkmPlyciZN9VF4+3tAfXd8FP36akY+a68fKvODFTzPwhyZ78Qlx2mHd6hlm4cb9ecPZ6O+87eHtPt7GO6PdIdruou5uf7u/tbeDx7vDza2Ni7fROjemPDZG6yFx/nOJ6Hp+lHz3Jv/8b1GV5iHTLKQszNOQeRJSVvAG370ZDEaYHzrvwlposmMT4kjD0aUt4cbPP208fg0jRt1g8rA1rE+zQjK/Ncl8+ZjVkMbnJ3yZyzEGrkdk+at9UDyW2T3bcNarSex3HgqnbP1z6KTRjn9QNP/B1khb4BUch4Ce10XPweLQA8Ho+eG0Ac75Aznn4euEdk+e/AA7rcbOEQT2JQzs4JpL4Jof7BZIcX6PubM2S4pDURAJS5rtmB3GDVUg1QGpzpJKdYp+ClKdeTTZW6DVAU12gzXZOfHzsU/ojeRYbjDhRJoNhjcMP5wBZk3+g1eJAtfnplEXIMYGMXZpZ/kkcQd67EaQPOnxQPJ+UJJndEXgeXNIsreB5oEkGyTZT5Rk11IB0II1R5WtKMKTDAzC7IYLs5/RxqDqXGJtdmJrkGc3Wp79DN4MCm1QaINCGxTaoNBuj0IbUvsgzW6DNFuhEwC0tOpsCETtlGWDY4IyG+AD4myI68ukyga3BGH2/LIc0GaDNhtkO62Q7YA8+8GynaJKJI3woq2eb4OkByQ9IOl56lsWlRdYp3iE3UtsHaDAUWiCJ4nN1PcchNETbX0mGBo/poCxGy/0ebyxT7kNLDJOHduxTpBi0WDqhqp9nmzq1K8ZWLvxmp/w2R3bOqJEUBkweVMlQItwcLD6c1j9uIlWB8EXCL5A8NV6wdeqeHDkirccrebPiwLMxHOSAVWbuHoap16KdO+DmO+llyH9yUEomh+k+bGva+eBeHBkVZ4cmZ4aLfSh0ffap0bf53xslD80+hFVXneChyV0ANCzHOg5eFH0pKEnVEQS4NN2+By+TvAB/CwJfo5eNfw46hwLMGo7jD68bhgCHC0Jjo4b/2rM5HHafxUUm/iGJJBigRQLpFhz+yxosiqaLMMrM/sgvKoIr+CVmYnwqv9Kb8yELOsLMUN4TaaZDd4toIN3Zb480xPaKeB5r8DzOsvAcnaB5QDLaRXLgceBQHMWTnPu0Y4DzwGeA/msl8tnVb0QcljzsLu3wO6A3TWO3XG8QQ4LyN2rkbv33G+4u1qHKfpylifW9oRbN/uzE6B9QPsgvdVaArQHBAgIUKsIEKS3gAE1ggFpf4UJJAhIEOS+Xin3db+vQlLsQZywVAakEEhhW5Rd8EdIwAoXqO2a511KQP+A/kEOrFV8pwd8B/hOOzVeQHiA8Cye8Nz3RkHgPMB5IOX1cimv1C/Lvgj5rbn43ibwPeB7rc5vwdstgPa9fJ7L+JJZ4H7A/SDf1Sr+swX8B/hPu/NdQICAAL1K3gs4EHAgyH81K/9V8cmaPNhKMmo7Gk2xj75gGqmR9bZVsfiRN/ENB9EL2RjfxiY5Km0R0infk7D8LQfVFsN+6PEpBpPMyzh9iJiG5RwCGnUqEEiGryUiHDxGsVfa1qX76LUmPip+/g0XcVoDpNSFdSKp1ZJQmKscVu2/Ykxv5M8YmAineF2jCMhFWPLSCb4uvd8xC1Za9yul6WgLhjzvSzL/0jTuWcqaNTQv3hycOl1gbRvksZlv4Q5+7xkYat3q+9yCruH21CoJKXiISWRPAxVlVzPa4QZj0rGS5tbMxtq8z1jJFqUblbuEtEj0e9q/XaytDFSUmW9OoKBmolXEET5TDelhqRX46C4YHxk3fB6EGPnoTCOkgJxlQU56Xng0cBb5Q/Kz/DwDiFsWxMk1ayLatFN14VA9K52qlwOKKymDnynG6krrJFx1rE7KdkCu1nvp4ZEfaJMyu/C10OWEm+ZfTpYky6PoJzh7R3sw3etqF1v6hfYL3vaO9rmnX2x19RrtLSeb2ueeoxziazoHkd7QIHdvL3rDfb1hvZfNbf1CO3PsOvp407EUlu8bkdky+/PZYXK8yF5ITPww5oea0/R3yqwN6396vLZjmbu1Ua+/jXbH3a2tvfHeHn67N8JoZzTc3sV7Q9zv7aqbL7OjTXdl9jcLUC4nKdoAAA== + H4sIAAAAAAAC/+1da2/bOBb9nl+h1XYXycCZ2Hk4TYBi0SQNZoB2J5NJu1hMCoO2aFuIJGooKo8G7m8fPvSgJCpxHk4k9/ZLLVLm6557eXh1It+uWJaNgoAwxFwSRPa+dcuLeKHnRoxf/fmVX8464jbsuAwNPcxLGY2xLJsEmP3q8JIg9jxVQlE4PSPEY27Iy7uycOo6+JAEjBJPdDFGXqS+77nBRSS7kZcUjymOprzA3upGtiojV/IOOSw1OF46Ip6HwgjrrcmKKXYnUzFye3OnG17bWUWIAuzlLemtyXrkuSg6JB6hchVmHb1yiGhU7kpWOCiafsTBhIlB97qVOlz3NYYiEtORmIB9QomP2RTHkV24a+x6XrVZV6x3r1Dk4QkOnMx4+ZwuJ6buxfrFlOKA1dT66Lquxg1qaqIpudKgodUwji6v5luXyIvzNdLqiuvPcSLvKrcuK65cRy5/t1yjQSsrFkA9IW7APhFHLr4osFBkfcOUFJc/xNw+AUMTbLRhKFqhyHFj0c1Otc5ser7sDqaYir7HHmHFPiNMXRz9dokp5T5jGH8UohGuQxyvFObpbRZLGRpdmBYvYjgMsfORr6FxqAzRCWZFnyn7jbwRX4dyPlHsr54HoijgqzsIkI/leAchcQYjHgAQ74ruZ58GozAexBFf4UGEeakTDSRY9nlDA5cihm9HXsyHSd+d22+Sj+d2x8paFuXZxbk9E73/ZJFgNSvs8L7X1KAmlMThwMNjtnpF6IVHkNOx0k8DdhPitbzlQVbBG9i/iIdqFuRKzIBiDw2x98DR5Z2JivSzXi4HISrF/3I6a9bwxlpVc7A75YUfE+ojGe2Y64tFFOip3sbBiCl3tWM0YkRYarNyiwohx1l7t7e8z9nMrrmRY+ZCC/vFSMBhJbFZqJhpV0VIs6kI+sRzDHAX0zrm4dHQl6z7Y+qOmbmSya3KPjz5bH0WECs6Gsu2qNtyHEMUO3VBjlBWjjRpFJO2Ez26geNeuk7Mg15tQEvvldtlcWTX6NqNquMaxqML5YzVRbf9JJiJJSmZzBYANH+rPmRnYVnu/+Yp3KBrPEdsyCHKu6PMACjhSXVY8sjkAMltvlepU7uU8Wtqm+pWoZnPuAjOTmMnYXYxNY/yplnyspVyaeZbfBcKMSo7TlL6Kw8Wkg6aq0/JVYHyZQM6S1xOx9Ndfqjq/nC/yfppX3lLYoofk+lVid4mEL0K0esB0buT6FXHch/Ti9iNN08wl64jJnRW3WgU2HFOIf7P/61/+rR+dGT98su+7+9HBmoSIsbDTVDfZrpT8jOkgwP7QWE7G66ZBKTxhdAEI8ZYO0ojRdFAj52wg0euj2R8MrAwRasMDpRUnklKfuCh++7L6I19RHlwsRxOXG3jrZ+p8Fn7TtN8Ed5r/fO9wTz15K1gviD2h9wXKvVx4Gr76uOte4r/4vGFRWDgpxj4oPkGtv4FJn6KiQ8XZ+JkH5WXjzf0R9d3wY+fZuSj5vqxMi948dMM/KHJXnxCnHZYt3qGWbhxf95wNvo7b3t4u4+38c5od4i2u6i729/ub+3t4PHucHNr4+JttM6NKY+N0XpInP9cIrqeHyXfvck//1tUpXnINAspC/M0ZJ6ElBW8wXdvBoMR5ofOu7AWmuzYhDjScHRpS7jx808bj1/DiFE3mDxsDevTrJDNb082Xz5oNeTx+RFfJnOMkesRaf5qHxSPZXrPNhz2ajL7nYfiKVv/HDtpuOMfFM9/sDXSFngFByKg53XRc7A49EA0WgCeNsA7fyDvPHyd2O7Jsx9gp9XYOYLIvoyRHXxzCXzzg90CNc7vMffWZqlxKAoiYUmzHbPzuKEK1Dqg1llStU7RT0GtM48uewvkOqDLbrAuOyd+PvYJvZEcyw0mnEmzwfCG4YczwKzJf/AqUeD63DTqAvTYoMcu7SyfJO5Akt0Ikic9HkjeD0ryjK4IPG8OVfY20DxQZYMq+4mq7FoqAHKw5gizFUV4koFBm91wbfYz2hiEnUssz05sDQrtRiu0n8GbQaQNIm0QaYNIG0Ta7RFpQ2ofxNltEGcrdAKAllafDYGonbpscEyQZgN8QJ0NcX2ZVNngliDMnl+WA9ps0GaDbKcVsh2QZz9YtlNUiaQRXrTV822Q9ICkByQ9T33RovIC6xSPsHuJrQMUOApN8CSxmfqegzB6oq3PBEPjxxQwduOFPo839im3gUXGqWM71glSLBpM3VC1z5NNnfo1A2s3XvMTPrtjW0eUCCoDJm+qBGgRDg5Wfw6rHzfR6iD4AsEXCL5aL/haFQ+O5FuOVvPnRQFm4jnJgKpNXD2NU29FuvdBzPfSy5D+5CAUzQ/S/NjXtfNAPDiyKk+OTE+NFvrQ6HvtU6Pvcz42yh8a/YgqrzvBwxI6AOhZDvQcvCh60tATKiIJ8Gk7fA5fJ/gAfpYEP0evGn4cdY4FGLUdRh9eNwwBjpYER8eNfzVm8jjtvwqKTXxDEkixQIoFUqy5fRY0WRVNluGVmX0QXlWEV/DKzER41X+lN2ZClvWFmCG8JtPMBu8W0MG7Ml+e6QntFPC8V+B5nWVgObvAcoDltIrlwONAoDkLpzn3aMeB5wDPgXzWy+Wzql4IOax52N1bYHfA7hrH7jjeIIcF5O7VyN177jfcXa3DFH05yxNre8Ktm/3ZCdA+oH2Q3motAdoDAgQEqFUECNJbwIAawYC0v8IEEgQkCHJfr5T7ut9XISn2IE5YKgNSCKSwLcou+CMkYIUL1HbN8y4loH8vT/9CoH+QA3s83+kB3wG+006NFxAeIDyLJzz3vVEQOA9wHkh5vVzKK/XLsi9CfmsuvrcJfA/4XqvzW/B2C6B9L5/nMr5kFrgfcD/Id7WK/2wB/wH+0+58FxAgIECvkvcCDgQcCPJfzcp/VXyyJg+2kozajkZT7KMvmEZqZL1tVSx+5E18w0H0QjbGt7FJjkpbhHTK9yQsf8tBtcWwH3p8isEk8zJOHyKmYTmHgEadCgSS4WuJCAePUeyVtnXpPnqtiY+Kn3/DRZzWACl1YZ1IarUkFOYqh1X7rxjTG+l3JsIpXtcoAnIRlrx0gq9L73fMgpXW/UppOtqCIc/7ksy/NI17lrJmDc2LNwenThdY2wZ5bOZbuIPfewaGWrf6Prega7g9tUpCCh5iEtnTQEXZ1Yx2uMGYdKykuTWzsTbvM1ayRelG5S4hLRL9nvZvF2srAxVl5psTKKiZaBVxhM9UQ3pYagU+ugvGR8YNnwchRj460wgpIGdZkJOeFx4NnEX+kPwsP88A4pYFcXLNmog27VRdOFTPSqfq5YDiSsrgZ4qxutI6CVcdq5OyHZCr9V56eOQH2qTMLnwtdDnhpvmXkyXJ8ij6Cc7e0R5M97raxZZ+of2Ct72jfe7pF1tdvUZ7y8mm9rnnKIf4ms5BpDc0yN3bi95wX29Y72VzW7/Qzhy7jj7edCyF5ftGZLbM/nx2mBwvshcSEz+M+aHmNP2dMmvD+p8er+1Y5m5t1Otvo91xd2trb7y3h9/ujTDaGQ23d/HeEPd7u+rmy+xo012Z/Q1WMOtNLdoAAA== k8s-resources-workloads-namespace.json.gz: - H4sIAAAAAAAC/+1d6W/bOBb/nr9Cq+kOktmksZ2rCVAsGqfBDNDOdNK0i8WkMBiJtoXIokpRORp4/vbhIcmURMVHDkvJ65dGpERS7+LvPb4n365Ylo2CgDDEPBJE9oF1y5t4o+9FjF/99Y1fjtfFbdj1GDr3MW9lNMaybRBg9pvLW4LY91ULReHwlBCfeSFvb8nGoefiLgkYJb6Yoo/8SD3ve8FFJKeRlxT3KY6GvMHeakW2aiNX8g65LLU43uoQ30dhhPXRZMcQe4OhWLnd2WmF13bWEaIA+5OR9NFkP/I9FHWJT6ikwnhd7zxHNCpOJTtcFA0/4GDAxKLbrVIfrnqMoYjE1BEvYH+iZITZEMeRnbur7/l+eVhP0Luda/LxAAduxrzJO10OTNML+sWU4oBV9I7QdVWPF1T0RENypYmG1sO4dPkVT10iP57QSOvL05/LibyrOLrsuPJcSf5WsUcTraxZCOon4gXsI3El8UWDhSLrB6YkT/4Qc/4EDA2wkYehGIUi14vFNDvlPjPrOdldTDEVc/d9wvJzRph6OPrjElPKdSYvrkWRnYitGOt7zMlsbVgUf+cUZXlRSnWGyGl/Ou5s7+90y3dkElvm4kQcW6UOrt4uDj7LpVc9LEzAxCwYb8mk2NyrcbpT6o0Yci4MYlQQpFkI6Hsjbwr5jvffbLVeCPm0q7wmRSFycJXt453CULQ7+dZkmpIaRwyHIXY/8EUalYYhOsBsBnXA16HkURSPVs8Cywq4lvcCNMJytb2QuD2Hb0SIT0QPsr96Thj34ohrei/CvNWNetJoHfBhehQxfOv4MV8jfXtmv0r+PLPXrWxg0Z5dnNnjs+AXiwSrWdM6n3hNrGdASRz2fNxnq1eEXvgEuetW+leP3YR4bTJqL+vgjx9cxOfqBciVWDzFPjrH/pwrK0wmesX/cslr1vmNVbmss6As8H1CR0hutswbCdpJES7dxm0hptzSHyOHSRXqVAjvcTbe7W06+3jMdXJyKRczHtsVI3AJutDgSEHIcSgldS77kEmUg3xEVyUXOEiRm7c0GXNzICH8EFGXX6Zj8ZbUdL/m4sg5svaUBJ++fTwDAivTvkzyVm0uCxO30jyzoYDSxHcNIEi83TEHnYa5ZN/noddn5k4mHQC7++mL9UUYzDx8YdkWdVtEh4hitwo6EsrKO6PChlLfxYxe4HqXnhtzKFkJE9N7pROSX9k1uvai8rrOY+dCbSxlotujBCIKkhRYZguBMz9VDYQzsCu9KvMr3KDrWWDfRFL5dJQZBEpsDlWy5JPBIZLOU7vUp7C/8TEF/ssARnvjuTR/iS9hVjH1HlNA0EqxNdMtbsBDjIqKk7T+xm2GdLLN3SfkKudIZws6TVROl6e79FD1ffZ+yP7hrtKWhBUv038uu88dcJ9LXkH7ebrPD+a0lNcyzWuJ2I0/lw9/Wt5olLDjCZL4P/+38fHjxtGR9euvB6PRQWSAEyFi3NwE1WOmO6XyPe3FPOaTOAi8YGB9Im6Vv5yIidHcOqmxyPNo0Xd2seONkDRRLZP7e2HWoaTzVHqYhz6adl+GcOwjyu2L5XJ3zDbe+oUKtbXv5M5XocDWT+8MHKrGbzkOBvHonKtDqT8OPG1rXYzBZpS3bO52msbdw/py9+TuaCEweCYGd+vPYOvfwOL7sPjo8VicACV5uTijP9wVtgYmz8Tk9/XVY8Ve0OL7Mfi4zlr8vyTE3QwWV50yPSKHX2+6m6i9u432+q2trf3+/j5+s+9gtOOcb+/h/XO8297bvHgTbaQh32gjPTb47yWiG5OowdtXk79/Fl1piDkNMMvGSYR5El+WHemob1/1eg72fdkoo85JQ69zpzReVXK6DuYmlUPrVEwG9mYRe5M7rqonm2vOWY2Ym69/2VychhGjXjCYj4bVpyrzHkQ7JA7Y6nKOdO8+0L3z+EsmehkOviKGZNjTaP0XOBcrz0FxXwbCbUNM5J4HjJAT8LQ5ATUQosPHEqKM8hOBSVFHLz1Dn5sP2lG1OqMGmVmCzHTB8NRfiDZBEZ+9Ih4tw3irDB2QmEZKzHsw3c/EdIMaNlgNj+0GpC/+KVIy65W+SFEQCU6a+ZhFNAxdkN4I6Y3PNL0xr6eQ3lhKbzSUB25BfmMpvxHKA6E8EMoDoTxQd+ssa+J6jPCI0BuJ9L1gwB041ju/YXh+HyQb8l+8SzR4I24k1MVYzFrhkUB1IFQHzlIdqAQVCgQfr0BwuRR+LjWCHyUVoUywFn603MrAj36hfrRRFcGVLrnSZU96GzzpkoMAlYJQKTif8wyVgs+8UrAS60HKcH2KBRUGvBeDoV6w5vWCD8hjKDa6H5drXTKY8BqqBmtdNfgA2gyFg1A4CIWDUDgIhYONsDdQOAiFg1A4WM/CQcgWgLrButcNZkfXkPi+BLHpgumBykFQRSgeBKGB+kGw300uHwRFXKYi1r6CMImuQxEhFBFC8mMjkh+hjnDu5Md8rl1q4cVY7ZENiZGQGAmJkfdMjOwqLbBOsIO9S2wdosBV0gTHZ/XMkjwMo3vy+lQgNO6eALNrnzG5OLNPOA8s0k8V27U+IYWigdX1TJwM783qVK8ZcLv+CZQPrtjWESUCygDLa5pL+SgKDlyve3blwlyHrMqrwGLECo01Ww3LrjRn00FmJWRWQmblS8usXBUHiJ74HOvq5OQwwEwISI8qWKdOZNXXW6cey/1d+GrrX9z+iOF7acT025o6RLRKp4imM8RHPUL8e7Ec0LUGnBk+eN7nnYLCEjAIktI8STl8UklJTUqoXAYQlSaJSnc5RgVkpYGycrRUs+KqSASITJNE5v1yzQvITANlpv5f108OOn9XYlfHLwBCkhwkyUGS3Mw6C9lypWw5w1f3dyElrpQS14KUOJUSt7ukT51DtNOEAkXP6/88IC58xC+cN/9L0HdnN8LnoJ8e7InENoB6S4B6688B6OwB0AGg0yig83JP6wDpPCXSmZLbD1AHoA5EtZ4uqlXWQohkzQLw3gDAA4BXO4DH5Q0iWYDvlonv3nHV4RprdVMBnAA9QeU0T/sgKw8C+AfwDyJdjQVC+wCEAAg1CghBpAuQUK2QkFY1C2AIwBDEwpYUC5tRYSFSNhdALLQBQgSE2JSkrxdYYwQQ8UnTvmb5BhbAQICBEBNrFORpA+QByNPM9C/APIB5ngTzTPsYJMAegD0Q/Xq66Feql0VdhCjXTJCvA5APIF+jo1wv8FMXgPzqEO0yfiIY4B/AP4h6NQoCbQEEAgjU7KgXYCDAQMuKfgEMAhgEUbB6RcFKOlkRDVtJVm1HzhCP0FdMI7Wy9rZqFr/SJ55wEb2Qg/GdbDCRSltYdcq3JSx/jEONxfAo9PkrBoNMyziCiJgmyxMR0NBTDkMyfC0lwsV9FPuFnV2qj95rgqTi9/twXk4rBClVYR1Lar0kFOwqmlX7e4zpjfwlEhPmFF91FAY5L5a8dYCvC5+BzIyVNv1K4XU0giHf/5q8f+E1ppCygoZm4s0Aq1MCa3sht818F3fxO98AUquoP+Ic9Ay3p1xJcME8LJEz9ZSVXc2Qhxf0ybqVDLdmZlZnGrOSLUpnKlcJyZHoz3R+O99bWqhoM9+ciIJ6E60jjvCpGkg3S/PJB4oZMdBZNPccEkvJ2WoVe5ShtNutHO+nqm3ok5uRuKNKc7MbFpY/rv1e4CW2NM/zJwah4+IPsdt326HHUJPCr5/MpyN1otc8ynjhhV+o//kmcAyUKaPJxqjqkkz5Y8toJhQPY8yNgjjWJPF5GPmVFG+OFb7yJDETZNVXfp0dkKuNdurq2NLQizY791jocXhIJw8nJMkcf93fsHe0w1Td+ttb+oX2g+H2jvZ3W7/Yauk92kc7OtrfbVfJ77f0HYRHrgnH1Fn0gXf1gfVZOtv6hYaQ91x9velacuT7QWR4x/5y2k3AcPaVXTIKYw7BT9JfQrM2rd9TKbRW0xqMSAmkHcvIo43e7PXPW+5+ex87LYx2+rs721vtzjZu7zrbHbSjbr7MUHlrZfwPKXnTzNnvAAA= + H4sIAAAAAAAC/+1d6W/bOBb/nr9Cq+kOktmksZ2rCVAsGqfBDNDOdNK0i8WkMGiJtoXIkkpRORp4/vbhoYOSqPhIYkvJ65dGpERS7+LvPb4n360Zhok8z6eIOr4XmkfGHWtija4TUnb11zd2Odnkt2HboajvYtZKSYRF29DD9DebtXiR68oWgoLRue+71AlYe0s0jhwbd32PEt/lUwyQG8rnXce7DMU04pLgAcHhiDWYO63QlG3+tbhDLEsujrVavuuiIMTqaKJjhJ3hiK/c7Oy1ghsz7QiQh91sJHU00Y9cB4Vd3/WJoMJkU+3sIxIWpxIdNgpHH7A3pHzR7VapD1c9RlHoR8TiL2B+Iv4Y0xGOQjN318Bx3fKwDqd3O9fk4iH27JR52TtdDXXTc/pFhGCPVvSO0U1Vj+NV9IQj/1oRDaWHMulyK566Qm6U0Ujpy9OfyYm4qzi66Lh2bEH+VrFHEa20mQvqJ9/x6EffFsTnDQYKjR+Y+HnyB5jxx6NoiLU8DPgoBNlOxKfZK/fpWc/IbmOCCZ974Po0P2eIiYPDP64wIUxn8uJaFNlMbPlY3yNGZmPLIPg7oyjNi1KiM76Y9qfTzu7hXrd8RyqxZS5m4tgqdTD1trH3WSy96mFuAjKzoL0llWJ9r8LpTqk3pMi61IhRQZBmIaDrjJ0p5Ds9fLPTeiHkU67ymhQGyMJVto91ckPR7uRb42lKahxSHATY/sAWqVUaisgQ0xnUAd8EgkdhNF6/8AzDY1re89AYi9X2At/uWWwjQmwicpT+1bOCqBeFTNN7IWatdtgTRuuIDdNzCKL4znIjtkjy9sJ8Ff95YW4a6ci8Pb24MCcX3i+G762nTZts5g2+oCHxo6Dn4gFdv/bJpesje9NI/urR2wBvZKP20g72+NFl1Jdv4F/z1RPsoj5251xZYTLey/8XS94w+rdG5bIuvLLED3wyRmK3pc6YE0/IcOk2ZgwxYab+FFlU6FCnQnpP0/Hu7pLZJxOmlNmlWMxkYlaMwEToUsEjBSnHgRDVuQxEKlIWchFZF1xgKEXs3sJmzM2BmPAjRGx2mYzFWhLb/ZrJI+PIxjIJPn3/eAYElrZ9leSt2l0WJm6lfaYjjqV919agIP52pwx1auYSfZ9HzoDqO6nwAMzupy/GF24x8/iFpnvUXREeIoLtKuzoE1reGiU4FPrOZ3Q827ly7IhhyUqcmNwrvJD8ym7QjROW19WPrEu5s5SJbo5jjMhJUmCZyQVO/1Q1Ek7RrnCr9K9wi25mwX2ZpLLpCNUIFN8cqmTJ9YfHSHhP7VKfBP/axyT6LyMY5Y3n0vwVvoRexeR7TEFBa8XWVLeYAQ8wKipO3PobsxnCy9Z3n/nXOU86XdB5rHKqPN2nh7Lvs/ND9I/2pbbErHiZDnTZf+6A/1xyC9rP039+NK+lvJZpbktIb925nPjz8kYjhR1nSOL/7N/Wx49bJyfGr78ejcdHoQZOBIgyc+NVj5nslNL5NBdzmc8iz3O8ofHJt6sc5lhMtObWSoxFnkeLvrONLWeMhIlq6fzfS70OxZ3nwsU8dtG0+1KEY54QZl8Mm7ljpvbWL4SrrXkvd75yBTZ+eqfhUDV+y3HQi8Z9pg6l/shzlK11MQbrUd6qudtpGneP68vds/vDhcDgmRjcrT+DjX8Dix/C4pOnY3EMlMTl4oz+cF/cGpg8E5Pf11ePJXtBix/G4NM6a/H/4hB3M1hcdcz0hBx+vW1vo/b+LjoYtHZ2DgeHh/jNoYXRntXfPcCHfbzfPti+fBNuJSHfcCs5NvjvFSJbWdTg7avs7595VxJiTgLMojGLMGfxZdGRjPr2Va9nYdcVjSLqHDf0OvdK43Ulp+tgbhI5NM75ZGBvFrE3ueOqerK55pxViLn9+pftxWkYUuJ4w/loWH2qMu9JtOVHHl1fzZHu/Qe69x5/iUwvzcFXSJEIe2qt/wLnYuU5CB6IQLipiYk88IARkgKWnBRQAyk6fiopSimfSUwCO3rJIfrcfFDOquUhNcjMCmSmC5anAVK0DZr47DXxZBXWW+bogMQ0UmLeg+1+LrYb9LDBenhqNiCD8U+elVmvDEaCvJBzUs/HNKih6YIMR8hwfKYZjnk9hQzHUoajpkRwB1IcSymOUCIIJYJQIgglgqpfZxiZ6zHGY5/cCqTveEPmwdFe/5bi+X2QdMh/sS7e4IyZkZAXEz5rhUcCBYJQIDhLgaAUVKgRfLoawdVS+LmUCX4UVIRKwVr40WIrAz/6hfrRWlUEV7rkSpc96V3wpEsOAhQLQrHgfM4zFAs+82LBSqwHWcP1qReUGPBBDIaSwZqXDD4ij6He6GFcrnXVYMxrKBysdeHgI2gz1A5C7SDUDkLtINQONsLeQO0g1A5C7WA9awchWwAqB+teOZgeXUPi+wrEpgumB0oHQRWhehCEBgoIwX43uXwQFHGVilj7CsI4ug5FhFBECMmPjUh+hDrCuZMf87l2iYXnY7XHJiRGQmIkJEY+MDGyK7XAOMMWdq6wcYw8W0oTHJ/VM0vyOAgfyOtzjtCYewLMrn3G5OLMPmM8MPxBoti28QlJFA2srmfiZPBgVid6TYHb9U+gfHTFNk6Iz6EMsLymuZRPouDA9bpnVy7MdciqvPYM6huBtmarYdmV+mw6yKyEzErIrHxpmZXr/ABRfI51PTs59DDlAtIjEtbJE1n5+dapx3J/F77a+hezP3z4XhIx/bYhDxGN0imi7gzxSY8Q/14sB3SjAWeGj573ea+g0BgMgqQ0T1KOlyopiUkJpMsAotIkUemuxqiArDRQVk5WalZsGYkAkWmSyLxfrXkBmWmgzNT/6/rxQefvUuzq+AVASJKDJDlIkptZZyFbrpQtp/nq/j6kxJVS4lqQEidT4vZX9KlziHbqUCDvef2fR8SFT/iF8+Z/Cfr+7Eb4HPTywR5PbAOotwKot/kcgM4BAB0AOo0COi/3tA6QzjKRzpTcfoA6AHUgqrW8qFZZCyGSNQvAewMADwBe7QAekzeIZAG+WyW+e8dUh2ms0U0EMAN6nMpJnvZRWh4E8A/gH0S6GguEDgEIARBqFBCCSBcgoVohIaVqFsAQgCGIha0oFjajwkKkbC6AWGgDhAgIsSlJXy+wxggg4lLTvmb5BhbAwOXDwABgIMTEFoc8bYA8AHmamf4FmAcwz1Iwz7SPQQLsAdgD0a/lRb8SvSzqIkS5ZoJ8HYB8APkaHeV6gZ+6AORXh2iX9hPBAP8A/kHUq1EQaAcgEECgZke9AAMBBlpV9AtgEMAgiILVKwpW0smKaNhavGoztEZ4jL5iEsqVtXdlM/+VPv6EjcilGIztZMNMKk1u1QnblrD4MQ45FsXjwGWv6A1TLWMIIqSKLGcioKCnHIak+EZIhI0HKHILO7tQH7VXB0n57/fhvJxWCFKiwiqWVHr9gLOraFbN7xEmt0LvdJiTf9WRG+S8WLLWIb4pfAYyNVbK9GuF11EIhlz3a/z+hdeYQsoKGuqJNwOsTgis7IXMNrNd3MbvXA1IraL+mHHQ0dyecCXGBfOwRMzUk1Z2PUUejjfwN414uA09szrTmBVvUSpTmUoIjoR/JvOb+d7SQnmb/uZYFOSbKB1RiM/lQKpZmk8+UER9DZ15c8/yIyE5O61ijzSUZruV4/1UtQ1c/3bM76jS3PSGheWPab/jObEtzfN8ySB0UvwhdvN+O/QUalL49ZP5dKRO9JpHGS+d4AtxP996loYyZTTZGFVdkSl/ahlNheJxjLlWECeKJD4PI7+W4M2JxFeOIGaMrAbSrzM9/3qrnbg6pjD0vM3MPRY4DB6S7OGYJKnjr/ob5p5ymKpaf3NHvVB+MNzcU/5uqxc7LbVH+WhHR/m7bUv5/Za8A/fIFeGYOos68L46sDpLZ1e9UBDyga2uN1lLjnw/fBHeMb+cd2MwnH5l1x8HEYPgZ8kvoRnbxu+JFBrrSQ1GKAXSjETk0URvDgb9ln3YPsRWC6O9wf7e7k67s4vb+9ZuB+3Jm69SVN5am/wDnjkd7N3vAAA= kubelet.json.gz: - H4sIAAAAAAAC/+2d62/bOBLAv+evELR3QAv4en6m8QH90KbtYrHtNpdme8BtC4GWaFkXWVQpKo8Nsn/7DamHqVdi1/F7vgQxSVHkzHDmp5Eo3R0ZhmlZXhDGIjL/ZfzxrZWUcPo99jjVykgQMEGExwJZeAdFUOh7kVBN4Oe9akYdT5CRT6F0TPyIqkI3oOIXB4qC2PeTEk7CyQVjvvBCKG+rwonn0FMWCM78qHC8VzjW94JLbWCcjmGgEygwe+3ITMrYtWqhRpmMFUpt5vskjApjK1Q4lZqQBNSfdaV3lxxJ7Am98KaUxUIb46ye+Yy/Ifaly1kcVPqftflC/Jg2VxeHoMp/6g6Hdv/YbBWLuTsiz7q9ly2j0x22jH67ZbRfnAyfl9v95PT7pEdMrfRb4bwOESRiMbflsMwzzqZUTGgcFToyx4xPiZy6GbCAFutcErs0N5a8eEpustl22u3SsKZekFWWq6IJu66TEFSJiTQB5jsfyIgWjaeu1UfCL6mSqeAx1RrdF8fPPeeMKXMvVih77BaLAkH5FfHrjKBosJocwtAL3IvbUImiqa6q+qI8oXVApkpJV1JyhmCGoDeipHAjrZanKlTct+brnJPAnaPzbrHzRvMCK3gLFnbGQHJR1RRMKUVV+ZE56vw2CwJqC1iklXYXckQ1gg9ZJMbejTzarKt4D87ms/en6n7Q/nupDTiW2mNV+YOHKlF9BI/yuOrGsK7U6oGxV4UqkomZv/3zdU0lyw+cU+YRuLOK5UIhvwQTrVmpY8/3T6X7UfqXjqXXAb/SOYE/J0PpWDonFccylgOqX4DyLHp/SXddcFGdYa/Skb7eG9eoCjbQZzwNqqoShLtUzKEFehOqMUXx9NllDC6ECisAu7Ok8d/ZfhzB6n711fxb+u9Xs2X8j42gJG391bx/XtXQzDkKCBFWRLlHo2qzzHe8J7ZQsulWmvjUpYHzPu+v2glYpQqx5ut5rSF3h1GN5DyhYrh5HgcBOCLj12SeJe8v8gh+V9Yd4bNw2qy9xPeZEZzCpxHwRbF/5VT0pXZSXmqqxXxLjclxmq8WXGWZX1tooamDfksdJ4S02XHa/BEkECSkInsIEggSCBJ7CxI8CaFWyJxoPpZoGV4AwTCw6au/oGH2Q0KG8encaOjcsiEyiKXPsE6MubvLznx/v3akOQN9IM4gziDOPCnO9BFnEGcQZ/YeZ8DuBIGB89VBTX4KRJvF0OY0ExzyDfIN8s2T8s0A+Qb5Bvlm7/jmSnZGrSkJiAvEIZggvpUULks4LUPGSgptgR9i6Fb9tNjYumbcdw4CTV6rmRtflEABUIDnEE4QThBOnhROjhFOEE4QThBOFoCTjE0cGnkQaA8RTt4mU0c6QTpBOlkdnbxEOkE6QTrZOzrhAAzF52bB8saea1HOGV/2ts0fg+m35/vPIKdKZMY7KTIkkCchkKNSe1BdSEnZ76Wlv4CZqP1N9dXn7LqwhSlfbxepAosbh2ZoSaLJiBHuGNCDWazPJD051ipSNXFofaRpcZW7mIjvkeg0g6ZS5B4RXksHQDvR5AMNXDFRsahSR5sOmwuSPOUSO5XCnzlxPBqIMu08gh4nRbxQC766lEAQbvA6ukh3sEnUKXlYcuU2eGo75jwZV81hELobDpMWW1/DPXcCi9FpGEnq6+tq4Jj/eI7SSznAK0ciSFO4UYvpUcqTQSmqObequE7P3JkH6CrIUgnqZkjBVAJB3HrWD+XR0ibiqHJzLsxIqXoYqMqhnKoQNvZZyRfW+olEtirKfLqCyAZSrpkRsIFNm5ZFShPFLB24YvuydpAQKsOQOh8S0qjWLx+sOYQZGTxZmDq/KMkt1Mfscsh+OGIbo1vjWd6xJR3bLMhvLJ4bd3fFMa0gxJdMQk7nfQKtFXuSdZ8n3ljUV6ZB5FM2YuMctPdDXFD2EYyLmgtGtf6tLAZ5geNdeU5MCsG4HivUjt7iyG7IjRdVxzWK7cvEamuc0zT1A1IuZbpNL6tqjmr2hblDU3uU66dwS27muWSc2ScLa+zSl9fRtcOTlcx9Q6LKRasWG2oPS4JDbZU25YWuTXdkFvpCezSldAD8MkR+QX5BfpmDX1TO4akxJittGUV4QIz5MYxJshwIM4cHM+3dJ5n2shiDeSnMS6VcVxofgh2C3QbBrtPdFNlNvEgwCLtT63tMgOl8+qz9YjhsGfMgnxOnIBRRmwVOZCVReeX01zJ8+hwR8McQMFOaMRyKiZFpHXFwwzi4D5mtnctrIRAiEGZA2EEgRCDETF8Z++QLbuD8XFRp74HN4UvA3ua4Dqa6AMi1lhDoNZNPde6/RJN5PiDUN9tBx2fMMT5LI8fUKKZGDzI1erh3eEu5FwQ/BL9DBL/5E4FrJ8INp/tWgoWLibuJF1eWb92owHePGt+mikFyPKQsanvXU6h4Qx3zp0+G0b3NYrT8iNG7aShum6v/SzlDBkcG3/7kK7AhB4nO7rWvD7Vnp5RBvWWkb8AI/dj1gu245y4HpooKY9uFu/CfE70auK8E842YbzxQUOojKCEoISitCJTWtg0F8WgNeIT7VRCSDheSML2G6bWMGgdIjUiNuNllxfe450i8PXTPddGXqy0Cl1u06WWfCPMtbofBG7l4IxdJE0kzJc1jfB4SYRFTjOXn8Gz5XvQw/xTAIjdknwwLN/Dqm93b6nyqFGWkippJz+CYRsQ0It5rPSSWeYksgyyDezvm3mzwGOSsJfmFr3h5Cu7BfBbmszCfhfmsXctnQXi1uRem+lCbrHxvTO1b26cGvQImMMBxS40xvmJ6xI9XID0iPVZ3pMJKsDj1gSrXlwbbnw9RPf0+1A/vfjYSfWCOC3NcmOM6rBwXfqICKQUpZYEXamj4krHBfr9RY7cQJv8+MWIM5qwwZ4U5q8Njui5+nQKZDh/Y/0GoO4zXpO0W1Dn4hjSEOoQ6hLoDhrof+sJEnQOch+pq4W1JrKt1xivlOlWJYLd3YJfzG/CHsGxfLifAt++gdbHQyzuabiG2bFCkLOu+eJHT2zpRrXtzs8KPSaxcbL0Nia33oNjebLvY+hsSW/9BsZ1uu9gGGxLb4EGxvd2Oy6jzs1N8Aw7e1sc34ODl00FfPuF3WjAnjpdOc+TEa3BtuZz4Yht5QLCjlhFzf+N7eORI1D8wmF1ImJ8nysJPNGPqHFPnyH7Ifjn79TB1jqnzbeK//qbwL+TMplEEYCeNBAhvSqeM31qjW0Gj5R5xwCcZmsDso5IxAtiGAUzZ+DoZprZqeQ7b8nngDpsSffSRPpA+kD7kdY3ML2UIYodxnk56qu9tIIM0vt7l7HcjjqQVI4ZsOA80kdLb9VzQFk8CAaQEIAMEEAQQBBAZZ5jlMs5iIRWI6Y7VoMbPuYSRNQ6NNVaU8tjyeezzHaijdAZmZE/olHyhPEpGmXyJFHz0bXIqh/BL1RG4ZXdmpcqV8oDKnFXel6DT0IfpBm6+6ky50Uuz7Zk1aHBRQDdBb5RxOHRMYr8UwdRy0mvrSFB9eKroXBpsKlvSOsJptUy9cK7sTc3vMeW3adq9inrScUs/XLRQKHWptE5Tb5rpRTv9UWk6msCI739J51+ahiZKXQ5zgGkmKy1agduFuOnQ134NT+WCNNPYqvc1BY14NcdkUq455DERq9NZiRdV+zqtkDmWF4xZy0i7e14v/O5jwk9DkK4kMHEl4ejf2fnNYm1loLKsvnGq2mQmWkUc0YukI51G163vdpO+izGsad08pukMbJZStdzCy+NAYVf+5t0Hs0rll8ndz55S2g8rOcqC0X3icD0l7tTVjhO+MwN2/Y9OxkLAZ2mZWTgs9ICC+OzgVCT5y290IDEHMxMyO23tR0//0ZnO/h9o/3f0H722XjMjNrOr/d9xEj/4LZtDYgK5/Tx6Fr3jY71j/Szdvv7Dmf3/0tHHm42lIL4/mbp4MX+/OE2jYxabf02MLymN1aW62ev0TsakMxg4g2Fn8PJ4OB451DkZnhC7PRw7w6TxVR6E20f3/wcJJ2tEDu4AAA== + H4sIAAAAAAAC/+1dW2/bOBZ+z68QhH1oAW8R23ESLzAPbXrBYjrTTNvpAtsWBi3RNieyqFJUEm+Q+e17SOpC2VIc144tJydP0SHFy+E533d4EX1z4DjuYMDCKJGx+y/n6/eWkQj6I2GCWjIShlwSyXiohDcgAmHAYqmzwOOtzkZ9JskwoCAdkSCmWjgOqfy3D6IwCQIjESSafOY8kCwC+aEWTphPz3goBQ/i0vus9G7AwgurYREJqc7/VbfJtAzkPpEk5onwVFvcc8GnVE5oErutLMeI0cCHCkdsnHfJvEpHJAlkXJIu1JyLpySKWDiuSJET0OGEB/58Ueot7uuWkWHMg0RSt1VOjyWN4ky52d9tqfgkZEr9bshD6uYJtwdzeUHbzD/n5Ta4E3g8KYpzr+D5yHq+zsYlfZ6p54Wi9dB08sdFDbk8KhuNFno84OK3VAOXJEjs7hvryFKJoMRO/CuJJRvN8uREcjuZC0ZDY6hVyYL6iUc/VLRJtYoEXmFJxbCTWP7O5e9ggK6VUh5qbUzqZbc0kKZzuTnfMUhRkIxZ+IWKOG36SWGpkogxleWmlVpOryOh3omT6bOLZEgDKgchaGgQkim98QLQGRW/fHP/kf77zW05f/EhSNLc39zb5+WGj7iYEm1ekk3pIKag2LichYVQFHTwLfEkF7YhGL3RMQ39t3k55ZcFHWlMcF8umm5hPZJJDSbuxyQMwcmcX017LTeWgoRxBGYSShs1TNos0m/HYBGmmlThCBMbhImj+8FEF2GigTAhjGMNIu7H90OKlsNCcKjQo7/8DRmzBwUhzoePTk3hA48noVy7hocGqZubrLbb240A1jnoFcGqOWB1ej+wOkKwajBYeTBNICyEgh8MsvIqnipwnWUKQPRqDnq1O/eDrx7CV4Pg6xLsCbx8SkIyBkCRXJJgYITrAljLUQ5HIS/ARALF6scBHw2uuAj8vUKel7oHzhetGMAfgF3EngZhz/H9sOcYseeJYE8GPT6NGahyn7HntekCgk9Dwadzz7XoEwSfBoGPADwoL0h72sgHVAgu1p1Tfe1Nvz/fH4gx/u28UV3fLMCQgJH4TNmxtqNiIIdExAvl+SSevKfhWCq/ax+W5LQq+3L0AmuEokqCd4L4zHTncHX3L81zqraiTqrd/7Rwfz2M5ZpAT+PwZfw53aGUIqFWseRyPN955YiJSEdlLvuUXFdkn7KwQirYeCI/Mb+iVgDlqwop5P0P8/UQZTufmTkoBq+oI/duVdSCdgAN88SS8CqtpX0XcKomnHNwnAzqlKAwgIiCdQDSjemC9UTqLWUKSVyalmr5orGBqn0qqAaQUcAtDxE0otohbXW4xr8/XAKegMbmWg0O5dEqSwdP8i4W6lY0F1H/PShl0Qd/GvUE+LlCIg5KMvvnJgarBr957Lsb+pzhzHmWFzxQQFGg5VaB0bm5KbfjJ7HSjlK+2hg6pW8BeeaHX8k/TdhILiakoPsha5XzkViRDXhRdvjgxvZFQGC/whu5kHMYpL1tkEEzC312yXyYvboLnpfl0ZFE0YBrcs3mgHCYeBfGyOZcPgvQVHftCELt7i7mrkaUHB7yKK5o4oxc0ztsuzAdHs2ZTEDAUheaoBL4+BWJqQ0sFmouZDewuSC2unJQEXg2uZWZbT9Vwl5Yqaxh7D4yNjJ20xlbT1c2TdyZtOWUqROJu0zcZsKE9I30jfT9UPTdWXb0s31UTd9Wf5C/kb93wN8TFksO6Dwd/EgIMHdAnx2+6Pdbzn2I3U9S6oupx0M/HhgQf3CObzkBfY5EXyb6bDCcfl9OnGw0kfTXI/19oHwk/MYtsHfaNYTfRsJHwm/UhF2dLof6hFyk8zvObq7B5tslbujePZi69RNKu+LigorHqTXTtxrFvdpuiHPOfeeTMlBcxcBVDAxqdrgJURvVdDCqwahmP5Yxth7u7GCxYiMxz2oqrQuGHmxFaOtKbWZI9DpVOIZFuM6DIdH213lOayKi7vYiInXzy5tpJGfVSf+lgmMYhWHUSotDQP0CFFJs9mwvWiqqVCTQctJPcMyp+t1t+qjGaFGpPU3ZBvpkxsvBA5u4VoKBQSPWSuoigyOMDDAyeEyRwdaOd2I8sEY8gOdAMSrAqGDH50C7vZqgoIdBAQYFj3DX5R7rCHftEKz6NfsqocOOD5HuW/zwGo+X4rYDxhHN2HY46tTEEcd4EANDgkYdL/XGgidRfoPWKtsIG2P/LX0X2swvRM70ADjpABRacQSuBeBaAHL4znYIakn8BEkcSXw/TlMuY/etTO7xk9C7CB/n6zhfR64v7A4Q1hMsyu4VVUdpAzai3swLqEMvgQIccHrl2FzseJbfrwkQ8J5GDBAa9hEpGOJAUPWThtub4jf/8trNH/x//+adY/SM83ecvyOn73L+XkfPeCkj0vO+fA1p8XZGjo/v273mcXfWGeRvnI8jd2/9HF7vuOa3uPE+RqTuPaTux/vZffOo28cv7pG6kbp3Rd3H3RrqXulmxXnyW8bdCzT9k+Q9n7Z59rZ/nQzpe2cL48BAcuAFyhGApH/A2MiVPpGr/dVID4ZAyTovXmztR9w619cbvEPxwVXT3aJqurWqedVE1RxtUTVHtao5a6JqeltUTa9WNa+3G9h+PD/Db0JxHwkD2t0FtCc1v8XbwUs1cS2q4WtRFaS83lrUakdAQVXDlpOIYCenP1Xt+h9oQFMWqj6aQcAfA8ElK2T4LTH86TKCr/ntzk4XV6xwxWrXJB8J7tE4BvpWYwg8PqVTLmaD4UzSeL0NpKe3T/Sb1h3S7Ho0q01vD6i2Ie184nR7el+6PUK6RbrdNd3qaXPGuV6U5LPkTV2f+PRI9+z8TyeJlXEh7643vZ0oTezBFLcZ7XzivFvcNbSMeHtIvEi8uybeMR+MueCJVPrGee1qFPsu1xxyLHLszjj2ILVy5QfKxPUBnENjk67gV4W9u7E3oVPyhYrYfOdvrt0HKJkZi/aJuDDvAToWI6e9X4RUTezz2iSdRgEB8x/n1uaqo8DWeN9UUVNJtZJea8X6dESSYE612rTs1KqR0tfzzjlR9ZhkZm0HClYq15cflMEBxD8SKmbpkuB8QFHSebskHVM1yq6dNXNYq/p5jykFAMGXtP9z3bBUaethafhT6Mr+ZpOFAPU+fRlUsXSmSDelA7usKYwIq3gn03LFK8tUrKsbGETRJ/r1j+ixcMRbTlrc82rld5YpP4Vae5DAxLWG4z+y+t1y6kJDlaw6czq0pidWQhLTz6agyjhnS+N9WDfec1he4zfLRjrj5bWGWn28IZJQRwv5DUJ3LgDM35JwW+yTPw4rOchA/tYALtPqTqF2ZEIWN+RX/2xn/A/hRypzS69FzLvQUWr6cqqS/ONWm6TdnhXOtA+th6790J4W//es/9v2Q/fQTrGilI71f9s3OPg964Mxgdx+ltZiF3xsF2zX0jmyH4pfAnVPfLu9WVtK6vsf1zG2++fns5QdsxDwV2N8RproiZ3bbXdPR6Td6/m9frt3ctwfDX3qn/ZPiXfYH/l9k/kyJ+HDg9v/A9gbKcQ1tQAA namespace-by-pod.json.gz: H4sIAAAAAAAC/+1daW/bOBr+3l8hqN0ixTpTSz6SFAgWObcF2k6mSbvYbQODlmhbiCxpKCpHg+xvH5K6SIqU7CTNMVY/tDVJvSTf8+FLUrp+YRjmaOQFUYJj853x/bSTliD4Z+IhyJWBIAgxwF4Y0MJrUkQKfS/GtAn7ZWSlrGaceD7+EJBKq1OWugCDOEyQA0mFeYTCOcQzmMQm1wYGYOzTeowSyJXPPFdR6jlhsBf6IaIE0XQM1rodw7Ys8tdg0DGsNzzpAMxZxzvlXIzXxo4PERaGgK8i1s4F8WwcAuSaWd0N+/eU/H3DmAJdD0ujNacBxB9cUhIkvp+WIBDNTsLQx15EyruskE5nLwwwCn3K0Anw4/R5T3jW94IzTgoRCCBrn7I8Z7jphL4PohjylIQKt1IzRZ57FJbCTJksCeyC/Lb7XMFlPoHs9xX9nXOnoM3mYBc/y1GfFmUIRhBgbqpc6QcMEROPuvpLeCEwmFXFs/DixMMVzTFxVmjuJQjBABu7IHAvPBfPTLHNsfeTtZsNuYpME1B4kepANseS8cCZwRNvDsOkMhmH6uUucM6mKEwCVyWbEH0DfgLVVTFnWqTspb215fSHvKIyhbd7G0TR7a2O0Se63/1tc0tQ+pduvw96IFfhUgKNxmi60PHmgEmukLk5CdGcCY5wbQ5HMUQe5J6ZgmQKRaWag8t8llaXV565F+QVfDEVpcwRKosZcUiz0Hc/gjEUjUZu8QmgM8j4R1Whop5a1d+SVN+yG1TfqtCeQW86wzyt1Bp65c+AqPc58GV1EU0941wUecH0JNVCS1UuKkk5H87dnVMeGzg0MLzEnGoYWRU3D24mGmIIBNMGYnZJrKJ0RBf2id4dhYQLsagQVB2OiasQ+G5SFrHWn0Lm/4lpBAF0MPFoQpsTOhyJo2EkxqtUfz3ou78rapg1+47I0FQygIQ5ruhUmLYLJyDxsUwsnWw6xfxPR6r3AlGnRIf1ivI8jgAxUKlJEnjMAnejmB/XjTCuTElElUr5cg4RSsPptfhMYUFVLohzy31UGibR2foUQRhI42TK7sLL6iQ5baESE+puOsv0ewV9n7gLbceWtuNBIZY7dI84Paz0bWv73tD0rVUy9mDh8yoGVvqfKIzxxKO9m6ZceEjwRh7lBt1/cPUIVp9hZdpHmB/4RKCF3v9MSEihz1EJi54Cp8Zqfn67I1WExQM1TiSueAlSgs6IA4Wyrfu+iA57BBla1ib5a3OLBktrUwiWE9pzNbBQyjydlIxNwq211RMI8LGrIhuGFQmdZB6IvMYATSGuYSW8jFjfcTJf8wg2gmvEDWJAhoVGBG9ehOiMYHYHeudwNL7CMB5hgnD9a8dPYhJttn+Yr7L//jA7hVvZ/v8Pzsn8MG++v8qD07tXxBGQkVI3efrmjSilegiQmUCMQSB7ZCH+HQIHM44KJmr6cAoD97DoQqRLlJJhP3OnTkEEN8bzmQz3MNVKIVDQ8uOZN8HVCgk8fiG8N8KJsUuZbHxJWe5WAWMKg4piZr68LW3ytsRq620ppCsHc3tBM8rdTKMlsYafs9DupHNsgW4LdMWC5ZBuv0W6LdJtkW6LdFuk2yLdX4J0MRFJPPdwC3UfDuqeZDzHq4F2merWjDAfDEXEmtGwqka0kRNi+NB4qZtaUb0swd16grtLE9yrJ7i3NMH9eoL7SxM8qCd4sDTBw3qCh4sTjEJXQ4vW1Khu8xqGRgABOk84AyTIjLPAhZcFjVs/lmbvZ8BjfRjLezK0MN2C4QesWhYUQLkcShUo0wIDxMZPiEIubhPfChFk0WTihyWEN2MHhYxZwqBoQHsPgcueEGtChEVmEf8gccOFsaMNhyzofITBNJ1xV47v3ORifOXXLXWA74FY7XkYXst4UglF5UJbwvTEy5cR6L/kz/qnT+v7+8b79+/m83exFOy4lbItxjIivCqeyCrKzU9zHxE9NdzwQoTQrNlX5FfDYARIzEGBesoiihdnlgenmee6EmAvlhXkWYSbzbdgerF3WE39rIYI1GFqETEEyXxMTEspBmF1t4QQVKBkpeSw+5hyyAFibgrGEXDOII5XUxR79yuK6Fai4AxitaWx/xSkIRuGsY/CKFpVX3XwRA1ktaVy+JhSOQqfINfFE4a3Yrr7dgNYm93hhgO3+j3Q7wJoD/qD/nBj4Fhbk8nAeXuWEB4GEMN4PUtrecF0nawC/xWi6Qd323qN4ITKYrvXjV+fA7Re5rLKRBarIA9tvxqNHCgnHzlpVxaedxW0jJyriamnvqdtjK+MNcIXfcaPHe3U5PoqWlLJ9dm3y/UJ5WR2Eb/M1tvUo+dNnyQ7d38lO3PljNI4sgr83HsQ9Vwhhu4/pIK6KdBZBb4ePKiirhBjDxsZW0UCd96KOsYA87nuHKCkDNDt4DzejQhrS50WHz6xKxH3ehWCofq9HKOXxy3MMUCVQ0zsbo0qI07Loar5gvsftlDwbwRcDzJ76P6C4/+2Zvtjo9zTYFYldkUYNQ124pPs5pC8BQ/Op4rSfLuyWkNvEh3MI3ylvjP1P7opUqlJD+zIpNJjOnIpoue8jtNTNHJddgpAoh6Txv/J9nfEZZzJXKSCkPLYx83i20j2YttIS5630u3bRJBoIgkL06p/iShJqnZJLGyERflBMLG5fpdK7Q7SEwi/Z+eapHk27TPx50gwiV2VjS7izUkc+5geLxEH+jdYz+lPcCx1TOP6mvRyc3PrFVz9GY7v9xE4s8yj8q5bkcm45s0YIBYRZUNOdz5t2VBHeTTwAtc799wEFAcvbqqnQ+j9x3IAl+DSk5zvOElT1RVvMc9ME4s7f/kZTam12hkVnuX7aWWIV+CybrO11B26MyIqBT2bq8pS+eF0F8SwokWpx600V5yM5OfRiFef0hBzxV5VeFA5NK3DB5stPmjxQYsPnlyGcnUAQr4P1iKEFiE8FYTApWwEr8NnbISK+0vY2JqEzZYiYVOzp6mAN1qIUwdzaqDOYnBHCXlqYY+WoSVT5es7MgZScbfgcK+rvb7DOC1NXoGLFsBGenxUj5EacFI9VtLjJT1masJNemfVgJ/qMFTd9RpZJio8pcdUGlylxVaL4qs6jNWAs+qwVg3eqsdcetzViL1q8Zcag2lxWBMW0+Ix5TWvp7rVuSA4WwCgLQzS6oCaBqzVX2arPWWhBm+1AE7K8ix0AFMF5+ognQbWLQTtqo5EDfE0MK8W6tXBPT3ka/CkVehXnUIVAirtqFTVKFKoqA5r1eGtGsylv1arwV6L3Ph8DlPgbW2xk2YtKqtkpxaHZVYLy1pY1sKypwnLHuXEVIvLFsVljbcxWmjWQrO/PTSrZLMf/pyTbHx3eeHr4yUpe5rdRMtus5T3lKXsN8HhXguHWzjcwuHnkaV8yGPZLSq+bbZSff+yhcYtNG6zli1KU2YtG2Fav4VpLUxrYdozyVq2OO2ZZC9bqNZCtTaL+VhZzAOEKCK8TRLzRTZyM3upCK3tdVNatBn3ibHYmcE5+AZRnI7f2kyL6XsJ85dBp8+RSFlqrVm+0MQsesNwHvmEEcF0kW/GlYhK+e7M7O3jmvdn5rUqcJ1/O45PqKpVrXj5O4eKudryNeu81zX/TCC6Ykqt/KJdyXNLKJ3CS+miNffluaJ7OecrXBTxv3HvFlfeBuFXDwt9fy/jFX+g2wtIIHbhjvqN0WpGzokwPEXznMFZaF+Gu6ynUepQ16i6jUisHXnBJOwYGbk3ar7bTXzPohGvIES7GXPjP/L+TbG2MlBapm6cSTWdCVeRxPAkJaS81aIWtfnbP3kaIGGvEq/cEiLFIydMmBL0unJN6hVNqyvIvsECKc/X4yvC6LnGCvkWSktcRAOJKXuBl7lPUep3OZl40zEK3PjGrPcOgsZLtwVuqfCqTxwsp/K/ZPLLGMmZF31F/vFV4KhueqUmZD0HE5JE94AmNNBZzuAOBtOkwPfkssvVlUaFX+iQH1nM+uliXJ1HyLnT68qgsGQQrdMhaH1vCkSuEYVOGAt3VTcxa6bti1S9UKFPpT8gTOgM5h2B3HMz4eKrQK0VPwbwUrF/WRuusaq+XtP7y2h6f0VUXPqiNEs7FGul4rsy4cV6YfP5p2PSxV35WOQREIDKhzOejfLB8OI1B9w6ktd9wQWbVqnnvM6bFv+j1+VruGucNvd/K/u6wWk+B5p3q6qdvhee8JAnzPdi9/kf3D3tDZcfbz4WgX0/Q5b7NL+e7GXL23zd/bl4P6fx1vicoyhj7Sh04xRLmQnbdjA3xxtgc9yzh+4GGE6sSbdv9/vd4bA33HSGG2CSNj4v1tbdFzd/ATu1gJODfQAA namespace-by-workload.json.gz: H4sIAAAAAAAC/+1dbW/bOBL+nl8h6HpFcudcLdtx0gDFIa9Nge1er0m7uGsDg5ZoW4gsaSkqiRvkfvuR1BslkZKdOH6JuR+68YxMkTPD4cMZDv2wpWl6r2e7fogD/VD7cd2IKAj+GdoIcjTguh4G2PZcSnwgJEJ07ADTR9gnLaYyTj+0HfzJJUyjkVEtgEHghciEhKF/Qd4Y4hEMA517Brqg71A+RiHk6CPbElBt03NPPMdDtEE07IPtZkNrGQb5Z2+voRk7fNMuGLMXH2Vj0d5qRw5EONcFPPHZcxYIRn0PIEuPeY/s/9fk30cmFGjZuNBbfehC/MkiFDd0nIiCgD+68jwH2z6hNxmRDufEczHyHCrQAXCC6Pt27ruO7d5wWvCBC9nzkcgTgeum5zjADyDfUo5hlThDZFtfvEyZkZALCrsjn1sdjnCfDCD+PKGfE+mkbbMxtNKPWa+vUxqCPgSYGypH/YQhYuoRs796dzkBM1Yw8u6ubFyyHB3HRP0kRAi6WDsGrnVnW3ik55+5tH+x50ZdjhFbAvLuIhuIx5gKHjg2CJgBMklmIugDRsl1hdrTb9AdYibmZo4Og5KGameLPrAdJydpSviIgGWTgfKakmv7fUHbRqtG24ZY2+30owOH0LXybyJiGrpHwZV4aoPbYXHw1HQjfYk9wdnYxxMx678QeWXOGNwL3jG2XQEV2cMRvhT6G2pmAip59g9mUnmrpCwP0SGkoymwTmFgltvDxD05go7dAieEiVmV9EB8hcCKKPUu7pvBU28KM5LI4pLM1NyM1+lovni2iz97TByMoHMz24Qii/YhsVoXg2HZI/m0NWqiIX3/Xp5e7j4RmgURZP594HiZACUeJIDIhsG/biEiFg8LQwxk/Q1KAw8wMG9KvQkw9H1o/UZkWuJhgIYQB9x6yK+IdLW499kwqN57pG/mdhCOt23i6uA2WcYwIK2iHlk+7jx0Q5ZgE9q3sNefYBj0mEU8mE5IeoA+/NTfxH/+1Bt0VWMD+/A/Qk8//dQff7whIoWIWM3hG7KYe05Iner1zk/3b5rnatvpsw3fswh1iLzQ7zlwgLdpFxwPWI3kjx51hTta+pVeyiDfPbwJ+5D+0fPuyBgOEXRAH87a34aWtEk5//g7T2Gvp+3Q/5Oh7Wj9iZb2cod0nptZ1BF6aMyMg/j2MexFVpF/JJHNOTAxww9Gjh25sfO0nYeHtDPa42O+KQQHbEXSj/J0ai7M0FLiY/xXZpN4RFQz8hyrYKu03+fE5xcNnNIvR/YAlxmFhe4rMSzNG2jH1IK0r5E9WdzilkKSB97BAcTQQtHFRX6sVfRGvWSJtF3LvrWtkDiukmdKnmFIKOvAPbi3C0tSPzRvoklU8KPj2P+UNJmguqLfjRy11IP+4PWUOOeUdl0awgTcw4q5ndnbsV+wMzYVSh2kDG94DBhmy1tetFSVHo/WqmbewLLlKDOxxup3cYsX8+ZhqhyhAlR1FKhSoEqBqvmBKiIjNxjbWKEqharmgqquYoPCClgpYLXqwMokvmrsVggbw3s2jisyG/IDYepNWbUiSRr6Tr+m/eVI0ljKnrXB4+oGj2du8KS6wZOZGzytbvB05gbPqhs8m7nB8+oGz2du8GN1gx9nbvCiusGL6RtM1hhJgym7YkWZeueQ4cEBwR9JVPl986/67DuH2ti7IQm+75UAbG7X9AL4VQOB9ovuEqZAmYGJPCarXKeoQ7uAwGLfyHOiNZITFnFlBWlYEehn61JJJjNh1IlTtSCx3anYSZp0zxrLpOTCzWRDm0k3sSuY4aH/kP92P3/ePT3VLi4Ox+PDoLCiWNC0x4BlU1r5FYUor7wux4ws+6SfImKmmkUgpV567BuiUtXzDB8QdINc8ZAlOCsHTsj+0YKFt4WuzcZLvotw/TxOhV5K4pTjWpuhCvHKOo063HDcJ1NMqA4KYJ6jDBEc3ih9HC9TH0dkgw6GUE0OMahbtjI2fnKcLlMfyb49mRLaFxDtiDdSFWfzVYX/JFVwE2KztXG+CtooTgztFHk0LruZKvm4ohNks7VysUyt/CEMIqyC6PMh5SdJ3nq33zroD/b3TdMwul2rNWgb7XZ/0No76HQP9rum+Y6mI5ALMQx243yK7Q53k9jJPz00/GR9MN4iOKBa+dBuBm9vAdrNMhNZWoIxWPKBpR7YxzRN8abXMyGXECvagjiY81xbKO5EyykFdb5nJTNRLP1dyEEFGAiS3KL0VKsyPfXkhFRDZSs33EaO52cj4Hao3MgrNJGTxZiIciNrbCOni1lqEjfiR7sMZSRrZSRnC8YjykrW0UrOl+NKrChgoYxlrYzl45JcirKWdbSWiyec0Hz2ScxLDDB//iaJ8kQCkJ2AS+pkcwdM+DLZHGN+VbLGe/FJna6gTLYiECmoi5DVRlTVR1TUSExbqS2olaisl5DKU3T6SVg7IZJtKt9Wk6PlZcPkvC+w63I3Kmsq5HUVlbUVNfUV1TUW8joLea1FTb2FvOaipu6ipvaiuv6iqgZDWodR1qWwHkN+pk1yrk16tm2K+oyqGo2aOo2qWo2Keo3q03Tyuo3a2o3Ks3Hi83HyOo66Wg5pEL3o6FQUbLHYYcrSjinKO2Yt8aiKqkuABA8mcoCiPgkkBhyVoCMPPJIjPTWFtdIakKo6EEktyFT1IGUvKa4LkdSGVNaHVNeIyOtEKmpF5PUi4poRvm6kPNRy/YjQn8iLNCoLNaqKNSoKNmRFG9LCjdK41nQIWyKVVWbWFaAVEadBtAcK0SpEqxDt60S0KmmnIO3SIK30pLpCtQrVKlQrC2kv/k7DUpHJPO40XOplki1JmPz9it0mmZX1XHwKyLoxWcRtksW7jF7BdZLtA7G+ufFMcfdR8TaKusuPipyF3n5U5M18/VHtTUbiIujaSvDWdJXgmU7L+woCZF1o5m5DeWXXGXGjTzYN+Vp1dUXk5l5mlLxtta8yiiPWQtSwjlcX4fy1BNUXFxXGkCF2dTXRy3bxWXc+vgLkU7r0UQp9DAV9FPRR0EeVRinsM3fsk4Q2FfhR4GeF7mVc1qnUtiTcZrTmeixVlI/fkHOpnZosvtF+ShpflHyYJo8v4i41kS/iPzmTP1UynkeI8my86EqFMlqcLhmft5VpkOMmZuQLUhIhzJdOyKsKyfXNyEvg55rl46e7pexVpuFx8XLRaZLwgnGWQe0c0uv++qfX/c05NLohcFN4arQWb3YU3lR4U+HNVceb6rIFBTgXCDhr72JVmFNhzlePOVfgSGdx8s3nIOei48odye+SGHsqrjynuLJRg/O7CucrnK9w/rrFldUFSAruLz6+LL5WXmF+hflVnFnFmVtPwZ/7Cn8q/Knw59rFmRUAVQB0KfFmhUEVBlVx52XFnc8QolD3KWHnrbjnevzTT5TbbkZt0ccyR6IH5giOwXeIgqj/xkFEpj+6zH6GCqCb6HsEA2RWq2e/PaWnb8Nw7DtEEO4wnYUEswSYs/UHUQWU8BfCLTgAoYMlPxCecEW7BgYkCyFwsaklU5yH+xzX86lSi15X/zOEaMKMWrQt4GRu5KhDeF+4sDnVHPf6YpQ+V2fmfI/HXxgGJ0peDlNsYhJZ8VUTtksWagseOaK6Lokgx0QZtuDxRMDx0j+LdNmbepFD3U7Bhe0OvIYWN7cjlnurTu7xasQbCLFuJtzg38n79Ty31FFKEz8cazUaCccIA3gVNSQsiptK1SDEnqjIkJB7phcyI2g3i5zIK+pGM6f7mhlIZb4bTIigx5JZyD8hnInTWCCZyrZrx+4zr/XnHPp9bGRAdUev9g4vYfHpy59s8y8y+llmyY3tf0PO5cQ1RZWi0Rwy1ByqXsV8x5uMi7cY8gtZ+sBLzKDF7tUeC5u1Zcw7ZjlPnXIrJK55T1S12FVO1D3ZGrf3jKVtQSafxXUkhr8l26PpAXSigKA4lplIp90sXceXCojyZHtd+dtEN7eKVSFTxtSvqhqYMZK+i7C2RPtEoRchQmjsjRu55tZtrU2CYGoWL2WLJBL/rHO4YlZ15JbemcXSOxti4ltJROgxim3YTEtxVGMQhVx117vbTee8zsyc0vTc13yboHWUfTmWWS/pDK9efY+L+PC2n3PBupHZOW/zusF/aDd5DnerQYv727Ai3V8nY6Bh8rLZyd/CN9zlG+bf0urwH7gLWfYtvr9JX3Li++Wx/Iv+7eokDkQlEbLf0x89195pvydIS9tOfhc+wlN6yNKfer/fb4H97h7ottoAtg+MdrPV7Jr71vsOMLrNQfTwbRoJa249/h/ptNL4j6cAAA== persistentvolumesusage.json.gz: - H4sIAAAAAAAC/+1bX2/bNhB/96cQiA1oBneTnLixB/ShzZBhQNtla9o9LIVBS7TMhSY1isqfBfnuO5KSTElUY7ddlrYG2iK6O56Od8e7H6/KzSAI0GxGeVaoHP0Y/PluaCmS/F1QSRwa5lworKjgmngDJCAymisjAo+3RowkVOE5I0BdYJYTQ0w5Ub8kQOIFY5YicbY8FYIpmgE9NMQlTciR4EoKljfW08ZaRvm5Y5gkCzB0CQS0H+bI0sSlkTBWWluBGgvGcJY3bGswkg4nw5ywtSpXneFjRnF+JJiQxiu3Q5c5xzJvazSMBOfLF4SnSlsdhR0e6VumcC4KGesdoBMpVkQtSWH3XEstKGNabYf4s8QJJVxV/q6ZqaTJifBswPh91CAxkhKe1BngOiLlz/LTMvZKFg3btcRF6mfEhZTWLA9zha96GJT7GZKmS/UaUsnnQ+DnS3HpX5nDoj9oYqJSZZvDVpD/rEfpBWaFiZpW67Ca/oTMrYU6jMvyzVGb4yR7TdbmnQjK1UthNmoIzTzICCQKVzj1OgJlerXOiEKrH3d5/hyESCVEEqnfuWBCoRY3I1h53IdyIinJf70gEpKNeHaUZzgmfYcCmDra0yZR4fjc581ckSwjyQtwqncPCsuUqOapbp9sI0iuMrPRR2c8CPJiFVxStRSFekQ5vJzHZBhw8P9e8Oi8mBNG1OxCsGJFZsBV+SzGsCeqrmfza0Xym5gVYJl8eoa+KX88Q8PgLzEHSrleEzheEeMMLVg/aA5ENIeCC0G1r4kZpistZR/P0O2eNvTxh1mLLzBl+vjeq7nwBw3bfl8IuTJphBTV9pnc6YpBkhIJJ+8Yx0rI9tFxytVxre8NFPngtbawqw4aiWlS6BlqsG6Hm2XJ5+Ly+/T3sSTkTn8/b/nbeWrWCLXUnV6wxFM/tOXH0BJ9xUfzXi/pQvmZyrQs9Nb4yFobvMl14WwK1nDlpt1RsOxih4opZKfjVg1jpq4z82rKE3pBkwL6S2/zqGQNdGpadoWvaN61a17E57bOebrZqmwc2jet2CCdU/5V/c2z7oAGDPq3cI2vyAZld52O5lh0MwfODGFeAzVTpM+xAXnd9LRowrvM4omwQ3d2vFVV+Dw24R61gWdrDbAb43hJTiFfoML5TlKscfBzaMqpFAX3Hggr81bnSj+7myJIpnP8aBwOg+hwNAwOxsMg/H56uNdOXCM22j8EudEU5EItN5n2yGk14wP7F9Ttod7CsyHyrkNeYq8mO8WFwWI3HYRb+SMKwy7KrZjhxkexLpMvdIrlPYWplnqJ5TmRd4LX998S9pukslP4sqQH0a5wllGentoqF/XxNigfZfWyFSlQIlDkSnXP30Xl9K2OdaVcYp5uoHy0YWeDJPgJcuykAt6tTOji/VhwTmIFbacjd6ot8jg+E7laUF07EPIxjuHq/Zr+Y9SPw29bMtCtvWsN/b1Ljatewv16g8pvG3j3NmMz1m4MvfrhmYcp6oUb+ry8VOy3ifKc2ZtD66DqO7S56pv46wqyH0GhiSbwz2SqS0g06VSaRcH67oz6La4+q26ki9x0v6OoPO5GUT9G0GDySOMY3g3V1tceSMouoC3x7Bk3l6IHcu+prz0PBWabi80PZ/zB+Oc7XVI+Kfgf3Qn+t7hibQb50QQOxzRE/yl+vxOA59CKGNGhbOo3Nd+thJN2JTQSm1VCoe1ET7csglXb2aoOmkWvyr5WDeMG7XX1Gv+Qp6T+AklihsR+9u/isjEHrivbaRnE5vS1Du1POF/OBZZJABpQk195e/nEYZShkiA9cCK5GwV/1Cj4YDcK3o2Cv+5R8PZDPqqZ+ayAmvIFDvjMQNVu8dNPVD9w7m7Neejz9vvPi3tOjcAMf+/MjYc1/bXm7sa/D2L8ywUnn/v09wHvYTf83Q1/P8nwd7wb/u6Gv7vh71c0/H0IiLZ/wHp/F4CvabD60dB4N1n9Eierg3IHKAeAuMJv9ZkxVkYHlqyu7asSKK7281yF03WgzFGUnOiPJGpdiqwyBtvlafdj40E7sM7YsAFZq4AmZIEL5sspl+tDwOaT5Cb86oHuNTZw0KLDFVn15bSLfNDfBZHXBjP6UKXzaXPUoKak3YzruDivH7S24zgMM1Yhy9Y2HFe6ftgAA1e+csoX1DYovAl55muDtSNRWZxdXSuICPWsqbzsWXKXi83rZvaeusn3msOgfMmePySju0JSXv3d0EHiG7/nv1VWoSa3Y76m+YXLgNv9OQxov6dWkVt07zsLwq2z4FXVebfJA+5b9KkzYTMgcesgiV3K3EvKnNT4zEKUI43PtkkfG/T/P3f6QehtDwr9MjJsUM2e7C8PmWFk3e/r+6C4fBxVI8/qqmfhx3pZRuNz819R5eLSJbMKXbvAEI3X6Yei0HnYdx+idSqhsfNz5D7shy5nPZhFI+fnKLHN+F21B30fcDLtzre4ip+4it23jA7ch/VsAB0mrr2VLQ33/SPMpRe9OT0qIVoFENdnLLCHrPwVq8JMftA0ms6nIzwhk/AgmifjJ4d4MSUJnkejeHIQxlb4ogaF4eD2X7zU1kR1NgAA + H4sIAAAAAAAC/+1bX2/bOBJ/96cQiF0gOXh3JSdu7AX2oc0ihwXaXm6b9h4uC4OWaJkXmtRSVP5skO9+Q1KSKYmqbaSbpq2BtohmhqPhzHDmx6lyPwgCNJtRnhUqRz8H//1jaCmS/FlQSRwa5lworKjgmngPJCAymisjAo8PRowkVOE5I0BdYJYTQ0w5Ub8lQOIFY5Yicba8EIIpmgE9NMQlTcip4EoKljfW08ZaRvmVY5gkCzB0CQR0FObI0sSNkTBWWluBGgvGcJY3bGswkg4nw5ywtSpXneFjRnF+KpiQxisPQ5c5xzJvazSMBOfL14SnSlsdhR0e6VumcC4KGesdoHMpVkQtSWH3XEstKGNabYf4T4kTSriq/F0zU0mTc+HZgPH7qEFiJCU8qTPAdUTKX+YXZeyVLBq2a4nr1M+ICymtWR7mCt/2MCj3MyRNl+odpJLPh8DPl+LGvzKHRf+hiYlKlW0OW0H+sx6l15gVJmparcNq+hMytxbqMG7KN0dtjpPsNVmbdy4oV2+E2aghNPMgI5AoXOHU6wiU6dU6Iwqtftzl+XMQIpUQSaR+54IJhVrcjGDlcR/KiaQk/9c1kZBsxLOjPMMx6TsUwNTRnjaJCsdXPm/mimQZSV6DU717UFimRDVPdftkG0Fym5mNHlzyIMiLVXBD1VIU6oByeDmPyTDg4P/D4ECJ7OogGgYHV8WcMKJm14IVKzIDMZXPYgybo+puNr9TJL+PWQEmyl8u0Xflj5doGPxPzIFSrtcEjlfEeEUL1g+aA6HNofJCdO1rYobpSkvZx0v0cHh4qG3+4ZGG42tMmT7ST2w5/EHDdjQWQq5MciFFtYUmo7pikLpEwnk8w7ESsn2gnCJ2Vut7D6U/eKdt7KqD9mJaF3qJGqyH4Xa58wV6/yldfyYJ2ej6Vy3XO0/NIqKWGgoIlngKjLb8DHqmrzpp3rslXSg/U5mehj4YL1lrg/e5rqxNwRrP3LdbDpZdcFExhey05KqjzNRdZl5NeUKvaVJAA+rtLpWswVZNy27xLc27ds2L+MoWQk+7W5WdRfumFRuks8q/qr+71i3SoEX/Fu7wLdmiLq/T0RyMbubAqSHMa6BmivQVNiiwm54WbniXWcARdujOjncqEF/GJtyjNvBsrYGGYxwvyQXkCxQ730mKNVB+BV07laLg3gNhZT7oXOlnd1MEyXSOD8bhMIhORsPgeDwMwh+nJ4ftxDVio6MTkBtNQS7UcpNpj5xWMz62f0HdIeotPFtC8zrkJThrslNcGLB234HAlT+iMOzC4IoZbn0U6zL5WqdY3lOYaqk3WF4RuRHdfvwacdQklZ3ClyU9kHeFs4zy9MJWuaiPt0X5KKuXrUiBEoEit6p7/q4rp+90rCvlEvN0C+WjLTsbJMGvkGPnFTJvZUL3QhALzkmsoO105C60RR7HZyJXC6prB0I+xhnczd/Rv4z6cfh9Swa6tXetoX90qXHVG7iAb1H5bQPvXndsxtqNobc/vfQwRb1wS5+Xt46jNlFeMXu1aB1Ufck2swATf11BjgDbRdEE/plMdQmJJp1KsyhY36VSv8XVZ9WNdJGbHnUUlcfdKOrHCBpOnmocw7uh2vleBEnZxbYltL3k5tZUIdznci+qb0UfNexzIG9z6/npkj8/j/1DV5tPei8YbbwX7HAR2+42gCZwbqYh+luh/UZsnkOXYkTHsqnftAO3SE7aRdJIbFckhbYT/bJjfaw60k4l0ix6W7a8apA3aK+r1/gHRCX1N0gSM2D2s38XN40Zcl30LsogNie3dWh/xflyLrBMAtCAmvzK28sXDqMMlQTpgRPJ/Rj5UWPk4/0YeT9G/rbHyI8YBVItlc8KKC5f5RjQTGDtJj/9CPax43tr1xc0tv8c2fLECROYwfHGjHlek2Nr7n50/CxGx1xw8qVPjp/xHvaD4/3g+JMMjsf7wfF+cLwfHH9Dg+OPTkCfGttunsk+4fXgW5rFPhos74exX+MwdlDuAOUAGVf4gz40xsro2JLVnX1VAuXWfg2scLoOlDmLkhP9yUWtS5FVxmC7PO1+2zxoB9aZNDZAbBXQhCxwwXw55XJ9mNh8Ad0EZD1gvkYLDn50uCKrPtR2sRD6syDyzqBIH850vqSOGtSUtNtzHRfn9YPWdhyHYcYqrNnahuNK1w9boOLKV075gtoGlTchL32NsXYkKquzq2sFEaGeNZWXPUs2udi8bmZvrtt8FToMypcc+kMy2hSSchjghg4S3/g9/3dlFWpyO+Zrml+4DLjdn8OARnxhFblF96mzINw5C95WrXeXPOC+RZ86E7ZDEg8OlNinzJOkzHkN0CxEOdUAbZf0sUH//LnTj0IfemDo15Fhg2oaZX9XyYwn635f3xDFzQ9RNQStLn8WfqyXZTS+Mv97VS4uXTKr0LULDNF4nX4oCp2HI/chWqcSGjs/R+7DUehy1qNaNHJ+jhLbjP+o9qDvA06mbXyLq/iFq9h9y+jYfVhPC9BJ4tpb2dJw31/CXIPR+4vTEqJVAHF9xgJ7yMrf6CrMLAhNo+l8OsITMgmPo3kyfnGCF1OS4Hk0iifHYWyFr2tQGA4e/g/h46Wk5DYAAA== pod-total.json.gz: H4sIAAAAAAAC/+1d6W/bOBb/nr9C0GQXCdadsXzkKNAPSWc6KNB2sk2mC2wbGIxE20RkUUNRSTyB929fHjpIipSd9EiCqB/S+D2Kxzt/fLSY2y3P8ycTlKQ5zfyX3ufznqQQ+FeOCFRoIEkwBRThhBNvGYkRY5RR3kR88gqq4FzkKKZvE8YMejU1AhRkOCchZAz/hOAFpHOYZ77SBibgIuZ8SnKo0OcoslBRiJPXOMaEd0hmF2Cn3/MGQcB+jMc9L9hVu07AQgx8VK/F+6d3FENCtSnQZSraRSCbX2BAIr/grcT/5+znSggFRogas/VnCaRvI0ZJ8jiWFALS+RnGMUUpo/cFkS/nNU4owTEX6BTEmXweac/GKLlUtJCCBIr2UuSlwP0QxzFIM6j2pDGiBmdGUHSCa2VKIRsKu2afByOFcFMuoPi85J9L6VR9izUMqo/1rM8rGoEpBFRZqkJ9SyER6rGzP+JrTcCClc3x9RmiDcvxaUH0X+eEwIR6xyCJrlFE577e5hT9LdrN9xRGYQkEX0sbKNZYCx6Ec3iGFhDnjcWE3C6PQXg5IzhPIptuMPkE4hzaWZniWoz20+DwMBztqYYqDH4w3GeGPjjseSNm+/2fDw41o/8pGo3AEJQmXGtgrTP6EQzRAgjNVTr3p5gshOKY1BZwkkGCoPLMDOQzqBvVAtyUqwz6qvEsUFIyVDJXpSkRros5C0hzHEfvwAXUncZs8R6QSyjkx02hYZ5O0z80TD8YrDH9oNH3HKLZnKp9SW8Y1h8TZt5XIDbNRXf1QnJpipLZmbTCwEbXjaRejxLurriMPYo9Cm+oYhpewVLWoazE0RkByWxNZ4O6s4bRMVv4ldndCWZSyHSD4OZwykKFJnefi0i0fo9F/GeukSQwpCyiaW3O+HQMieJUz1fSfhGMoz8sHOHNcagLVGoGsDSnkM61ZUdwCvKYmp3Jxcollv96Bh8luk3pAWubyzxLQQhfetspjnyjYZ4g4YfHaabObqXNrjAV3bCkdK4gITKp3urPVH7UlIW+wjJSyWRJLl/MCISJMU9h8hG8aS5VsRmuN4236t1l3CWMYxY0nAMHzoHHlXK+YngCI/fYA+fY+46xnaYmHqwiX8PN6iiU4oxOER/d903iG4Y6ylw37v9D4RPYfEbQnI+IaPCeAQx3FJqyxMKf4xrW4wWVLut/+OXIYODqgZZQkjViBaOQSxZGoenxcaxjxCHDh0FwwH4cHPKUGRxoKXPKR26mF96z2o/sZsCSbnA41DpQM1hDNwIxsn7yRaLLmgIyg7RFlPAmFWNn+WIHMYQEd1gwpIBNi0wY6rzG5JIh9xCiKzi5WFKYTSjDufFtGOcZyzmvvvjbxa9f/F4VXF7974sSahjHY7FGENn/X/zV5+0yY73cZnGBTZzHzvPdXV1p7big8IiMgsQM01pSfANCKgSseawfwxlMojfVEHq/zEYFIPSP2uxFi2qq2Nl030gj1bIHp5/O0ZQ2GQai/MhU4eGpd8xl7n2UGoiaKFJio4osvFl1rQPVtQS33bUw3074rzb0qjLqrHUs0fBDke9DucYO/XboVyfcDf6OOvjbwd8O/nbwt4O/Hfz9jvCXMpVkC0Q7/Ptg+PesUAF9HhD4ISvvgaP0Pn5kpfdvWnIHMQLZ63KLUOdu/wKQBi4WZzjvYDKjQsB9jQ5tzddtC3g41U43OOF3AiIEhWv3v0eZObDrea9GzyJA6EMxQc2So+ysOKEy4zm4mlmopbE3OfzE6rdFSpf2s7n/QoKbHIkBza4k8jOphG8dTiUkM3lFSjF6z1jj/wi7MkOqLyK/pSMrhlipuxCYmXbMidfFMIPWDct9ITzPPDYrTSGzRJbjZs34kvIuudnlme7x5d5Cb86UGkECRQKdxpj6a8KBzF9/FCDZWGfmmq8FlFAQXpoCZSk4TWH0TmIVfaJPrQrmXSy9HdbmPnDgTjn/9paNslo5U79G5xJWE8RaQKBo9/6AoCh9WY9Yq/PvW9WrAREJ0vRrTKi+dZB+OymTA9tcoCsU5aDK4qsm1ODH7vUEbsANMmLxRR5eSlMzgsei8FS+YssXCIzW9thUBZrP540pLsFNW+With2+t9WNgld/bAAyxrNjkMGGFckA3Ghu2Xur61hbD3lMUywN+7mihWZZzgEX9ju40MGFDi48bNWgwws1XihLBR1g6ADDYwEMSkFHC0JqPUdjfLtyzsBRzjmwlHPsUnahHSfiaUM9Lchn02/zWhBQKwpyCrQWqnleZEIim3RrCQfOkyIh6UNLRG3Oox0pudFSO2Jag5rakZMbPbkR1DoU5Y5Va9BUG6JqO8gxVWJDV26E5UBZTqS1KdpqQ1xrUFcb8mpBX+0IzI3C1iKxVjRmR2ROVLYOmTnRmfVA8S5FHTZ/noUfAqZtCNU2gGsbQ7Y22OaAbu2nqA4Y1wblWuGcUQIqzoLKb0F5J1Jb5hFnE9y1ATwHyNsI6DXjih3wOUBfK/BrA39uALgmsDaBYHMJTUBodavaVNPUYqIu5NWGvloQmPtbHQ4ktslXDZ7CElRfW4uEO4xmL11tDNKM1XcorUNpHUp7NCitqqV1MO0JwDTlyzodUuuQ2jNFao1S94//ipTpfF/zTvIDVjAdJ49B0JUwv1EJczhYg44HHTru0HGHjp9GDTMimA/egeSnVMv0fpVa65Byh5S7mmYH2jaoaa5FbcMOtXWorUNtT6Sm2cG2p1nb7JBbh9y6GudD1Th/I4QDxPuUOLeKmfMwxGMH5w77si/eTLkjMwvncAE+QZLJ+QcHkkyXMSyvLJDPscRZW61/mV9AwiI+FNdJyL4oXKQxE0Qy2+TS0xpg3dpeRy5uynC8klxybVi7vPxULbfaTa26qEQByQq3vhJEjbr+XzkkS2HU1itZa5kHGnUGb4yX0ZWrU6vhzYqw9spJ/Em5AcP6Xom6mdjoAtlCVup3wVHC8nIEj+z3GtgFuWDKQJbmpYCLTH8X6YqRJjKg7nBzm7BcO0HJFPe8ortdu9wH6+ReZCPVQJh1C+Fm/y7H93VuY6KcZm9caFWuRGHkGTyTHVnfj7Gr2v/5X2ofIBcXXjTeN2LkSYhzYQTDvsmRUdEP+pru13ggl/mLbMkEvXB4odrC6ombWCBzZZSgInzqWv+abzWuel4FI3f99uigWbzxosE9Db4a+94m/10WfxcnuUTpnyQ+XSah7Z0x6UJB50JtLuTwm0fnLO7910pswO7kQN8oZeiXWP0437mTLJ6lPxmK/IH+NHZlovFXJKAfZM91tcJh1luunZSfwVjWuuxlulI6w765yaoFxHmuHal7NMsO16EKlzI2HqptYcHcORZjbdl2c9YYwYTQGy96WndPzYWrGyE7L36IjYxN/Hf14RavGrktfXQXSx89ExM3/sSIKONVtYfqNkF8/aLy+fLCQFksqR9LEQMGpH64kNmknIyqXn+s1GVU29dCsB/Udq7avB+oH4Z9laO8UT1Qfg+KP6pyXq6B17GbZuceRe14T+1YHWUwUj8oNyjsR+p8y7lo4vsbi6MF/8+z10W5qKxjfZAIDCUz7xfvpAB1fi4O8vx9EBz09/ZDeDgaglEfwMF4NB7t7Y/D4HA6HYey8VVVnupvrf4PBQMVW4dnAAA= proxy.json.gz: H4sIAAAAAAAC/+1dW3PbthJ+96/gsD0zzoyao3uszuTBcU4ymSapm6jtQ5PRQCRE8ZgCGAC0pXp0fvtZgBeBJGjLcR1JMV4y4i4ui93F7oclHVwfOY47mYQkTgR3f3b++txKKQx/SUKGNRoihAokQkok8RpIQIxCLlQTeFyrZtgPBZpGGKgzFHGsiAHB4o0PJJJEUUphKJ6PKY1EGAO9rYjz0MdnlAhGI17qH5b6RiG50ARjeAaCzoHg9trcTWn0SrVQUqayAtWjUYRiXpKtxPBrnBgRHG2G0odLeyJvjsfhAtNEaDJu+DSi7AXyLgJGE1Ibf9PmDxQluJldFkHRf+iORl5/6LbKZBZM0XG396zldLqjltNvt5z205PRk2q7H/x+H/WQq1E/l+b1kUCcJsyTYrnnjC6wmOOElwZyZ5QtkFy6SyjBZV6AkgAXzlKQF2iZr7bTblfEWoQkZ1ZZfE6vTBoClphLF6CR/xZNcdl5TK3eIXaBlU4FS7DWaF2Wn4X+OVXuXmYof+yWSURgdokikxOUHVbTQxyHJBivYqWKJl7d9GV9QmuCFspIl1JzjqCOwEtRMbiTseVUJca6td3gDJFgi8G75cEb3Qu84CV42DkFzfG6K7hSi4r5jvpqfo8Sgj0Bm7TWbiwlMig+plzMwqXs7ZoYryDYfAz/VsMP2v+qtIHAYuyr6Dd2Vap6BxHldtPNYF+p3QOy15Uq0oW57/99amDSouOWOucQzmqeC0R2AS5q2KmzMIrOZPhR9peBpdeBuNI5gX9ORjKwdE5qgWUmBTJvQDmLPl46XBdCVGfUqw2k7/fGPaqSDYyZLEjdVAKxAIstrICXsZKJJ4vjJL72ooTDdn7+yf0x+/nJbTn/pVOgXCRT/FPM6HL1yV0/qVtlExAFpIUJxyzEvN4sjxevkCeUPrq1JhEOMPFfFePVBwFPVGnVPd3WA4oQyA3aCoXK2+7vcYVR5OnrqoUQ2yTNZhulEc7lENEizAFFlMdXoUPfUCfVDaVabLehqJTTfX7HvZRHrzttJ9XpfRYeIXFt+mnrL8EFFIWIn+UJvZJVpogZMxdkYj5/i0kg5ipO1ni4qdtWCTxU27VTI75myA8xEdVMfEta7JVTn/LhuueAIgJyysdVqKg3uQwaOF7CWCqYiQuJpYkTkgYOC4M5eJ/fJEsWiiRaqHKg05+hr0xTzT9q6wjUFA2V+/Bbd48MmtwwuWJcZVN3tgEctZRaSzpujMFdiECBGYvGsrf0i0QOP6jzzJ4I1vIxwyrEziJa2f4MxxgZ83caPH+9xAxcDhtWBLnLw01bI8t2ZSkh+ngXRiEhzMcx9t+mmbDO/6pkwpDAxzJlqIwx4SviTdKfLIFQOPETps5UkCcA3/h84sExQWydgFpOSGBFxMPP/wdt8wfITH8NFp+ffNP0JJf6j6eoir2lzK9SxFRzFsn7OA9nwszMctsHqXbnI9jB+VAVeNtEVw0BlAnDcUVt70me+0Lih5ehn6BSdjHnSXU4Lku2RMuQ1+WaJt5F6pOG2LPIdrlUTBVbZaDe0Ks51BXxSh33zUtYoeU2B5aNF9LY4H2RPMUZxZNMGrxAvHZk0mK/sVsa/Ns1urbeOx2LDmEJ+h6z2ORnp/812MSwE7aBJoZu90UmJvjxoMDkxuqIxSUHikvmIRcUcsxi8iVBRIQRPm4/HY1ad8YqafY5SLByfZ1Pv14fBmhJMctbsBHxVs5oJObOb5n5LIjZMYg5eAhzWADmqLIDzZE6o76B2KDClpn9gV6V3kwV8oyzzVd+H1RsyZeAbaYUMd+BEdwyPy9jzYcaI9sWDFofacZ5yJdTjwDRDWy1yVab9gnVDfeo2kSwuKLsQoI4CffAIQNbcPrW2O19agTnfGMEW3mylSdbeXpUOGVoK0+28vToMUpT5cn5WvDyD1egnOnKOc6pLSfCtih1N2Bjy1O2PGXLU7Y8ZctTKex7ZstTtjy1T9Cvv/PyFEASMfEiuZnUX5nAI58oz7g/jGt5YEtJ6z59upOSVHe5vANwa+2j6no7Ul3vRtW9OATV9Xekuv6Nqjs7BNUNdqS6wY2qe7kfJ61fQJXO6fkb50Oqfls/foT1YyPr4ErId1uFrSKXjxMn9jhhjxP7dJw42d9KsgG23L9+3ARjQIuy7fmvH8flYrJktJyERbuoJsvJ12vn+hrmP4SKMgQ+UWAcW0q2pWRbSralZFtKTrHfyH5BYL8g2Bfc1+k+MuDXWMByMuj3+j8W+X098nuNLfCzwM8CPwv8LPCrAr+KfLbqZ6t+j/UjAkBjHuYcYJ10EsB3C7ygbDWZrgTm967ire23nk347J3SswVhOwZhys+/g5eOe74O+9qxgkA6FoFYBGIRSPY3p8c5DPHipCgo3fGDqBv/DsXikCYccnb+u5Nw6ckWiuy4HjSX2jv0mtAeL8KCkAoI6VoQYkGIBSEyz9BJQBlNhDSgLXs84GupQssWbzw2vPFApY89X8f3/DbqKFuBy705XqA/MOOplJ1+ShardCofsYv0HiHIVxsvVdGUESxrV8VYAi/iCJZLgvqtSEdVb9AARgm+5VcT+HiGksh0O4LONaFBdXdSObg0+FRxn4sG4zQujfMrnvRo6n5JMFtlJfg63NPuYOqUqAGuXqBS2EWb/qiyHE1hKIry64Aqy9BUqethC3Ca60rLVhB2IXf6+NR0dUmhSDfLr/pYC7BIaOiTa9nQ5TYVq+kmaRRV/yHGJKb+JCQz2nKy4Z6Yld+9TflZCtKNBC6uNMx/y+d3y9yaoJJmbpyZNl2Jxkg4HqcD6Yj0W9u73WTvcg5r2je3WToHNvcy9cP93yfrzTdM34f3HOVJKr14TqGXIgQX1yrRq586OUbKb0xKM8KmWxyCJtmmc6aSSY5vdaDiDjau5Xba2kNPf+gsNr8H2u+O/tBr65wNknO72u+On8bHz/kaJCLX/OrWWfSBh/rA+izdvv6wuWLLfebr8uaylNT3NyXptUHjsyxrFh/0S9dLaYk6xLvDXhd3hwMft7uj4Unf67en3Y43fTaceqg/G/XTxpdFam4frf8PZcHkVKRwAAA= scheduler.json.gz: - H4sIAAAAAAAC/+1d62/bNhD/nr9C0DYgAdzO8iONC/RDmq5FsbbLWnf7sBUGLdGyFllUSSpxFnh/+47Uw5REJbbrxnnwSxHd8XG8O979jlLpqz3LskejIIoTzuzn1l9fWimF4q9JQLFCQ1FEOOIBiQTxCkhADAPGZRN4XMhm2As4GocYqBMUMiyJfoT5Ww9IURKGKYWieDokJORBDPS2JE4DD5+QiFMSslL/oNQ3DKIzRTCKJyDoFAh2t83slEYuZAspZSorUF0ShihmJdlKDK/GiVGEw+VQ6nBpT+RO8TCYYZJwRcYln4SEvkTumU9JEtXGX7b5A4UJbmaXRZD0HzqDgds7tFtlMvXHaL/TfdaynM6gZfXaLav99GhwUG33g9froS6yFeqX0rwe4oiRhLpCLPuUkhnmU5yw0kD2hNAZEku3IxLhMs9HiY8LZynIMzTPV+u02xWxZkGUM6ssNiUXOg0Bi0+FC5DQe4fGuOw8ulbvET3DUqecJlhptCjLTwPvlEh3LzOkP3bKpIhjeo5CnROUHVbRQxwHkT+8jKUqmnh105f1Ca0jNJNGOheaszixOJ7zisGtjC2mKjEWrdUGpyjyVxi8Ux680b3AC16Bh50S0Byru4IttCiZ74kn53dJFGGXwyattRsKiTSKjwnjk2Auets6xmsINp+Cf+Xw/fZPlTYQWLR9Jf3arlJV7yGi3Gy6CewruXtA9rpSebow+8PPxxomKTquqHMG4azmuUCkZ+Cimp06CcLwRIQfaX8RWLoOxBXnCP45GojA4hzVAstECKTfgGIWdbx0uA6EKGfQrQ2k7vfGPSqTDYyZzKK6qTiiPuYrWAHPYykTS2b7SXzlhgmD7fzib/vH7M+/7Zb1DxkD5SwZ4ycMor6XhIK+OKhbZhkUOaSGEcM0wKzeLI8Zr5HLpU46tSYh9nHkvS7Gqw8C3ihTq328qhcUYZBpNBZwmbvtz3GFUeTqq6qVEF0mzmY7pVHOZhDVQswASZTHl+FD3VRH1U0lW6y2qYiQ036x5n7KI9haW0p2+pCFSEhey37K+kuQAYUBYid5Uq9kljGi2uwF2ZhN3+HI51MZK2s83NRtpSQeyC3r1IhvKPICHPFqNr4hNXbL6U/6cN1zQBF+dMyGGVwUqbgSAdC53xBJ3ITSVC5NN0gtDd2EgfQcGvhT8D2vQZIsFuk40OfPwJN2qSYguW84agqH0nduRCEiaDLN3JJxkc3srAI4aim1lnTsGIOrRBz5eiwai97CJxIxfL/O03shmMrDFMsQOwlJZetTHGOkzd9p4PztHFNwN6xZEeQuFzdtiyzblaWEyOOeaYWEMB/H2HuXZsI6f6NkQhHH+0W2GOEOZIP0CeLgyEuorKggQwC68djIhSKBr5V+WlYQwZoiF7/4D9rnD5CX/urPvhwcWONLaz+n3mqqurrKp10sLFj4GqmrtbGCx0HkPS7NZiu+Rrsvt6ZdxXVR6BMa8OnsEal6uXyrWP41ej/Zmt7PBb59pJEjXfs1an61Ee6tJBKxhNdpKVbLQoL3aRpMuJ6ZAeZPS+f4CNbbCD1XoQWhXHMOImHDKAfUYvufB16CSpBVD77lqVtZsjmaB6wu1zhxz9Jkp8E0sww+CMVUi7bstEDTqxlCFThIniPql3CJ5quchCydksQaZwzF8ZBWPMEk/kvEamcxCqTUdksxZbtGV9a7Vhy4D0tQ95gpeJ5bPVPwmILn0Rc804BxAjlmNvqaoIgHId5vPx0MWtYGlVCagNYDNGvgmZYV4oMHVg6to/7GOulh6/27FUvr6H61KuphG+J2Sql1rLJCjfWwTXIPC60QTBi5l9ZgwKfW75mJTeW148rr3tdd96vq2qtsQT28zKhvIUbIoKZnfyQXpe90CnmG2e4rfx1T7MlXUJCNCaKeBSPYZX7+Qm96qDCybUGh9Z5inO/5qc4jKEP7m5Shuvi3Sh2qLTe/sRDVxuLvWole/9balKLfVor2dv7uDZAIH7mh2Ezyo0Z4ZCPpGds5LG+5YE9B6zx9WsC628Rwnfn8O75ZuzX1dXekvu616nt5X9TX25H6eteq7+S+qK+/I/X1r1XfHSm4fgV1Wsenb62PqQnM+y3zfsu833pUhcWhKSxMYXGXCouju/uOSwNatnOU3IhiLNCl6HH626dh+WhZMFpWQsNdnC2LyRcL6+oK5l8stv7l+tZxDoQ/XmCcd+Zo2Rwtm6Nlc7RsjpYlAnxmvnAyXzjdFfTndAz8q8O/N78Y9Lc5+nuDDfgz4M+APwP+DPirgr8jc/xnjv/MdwXPxeUfxMWMAbITTgIQb4ZnhF6Oxpccsy3997tdfQF69zHae6lsA8R2DMSks98mltGyvh2P3fF1mJeQZRQyMCjEoBCDQuRVbBzv51DEjZPiXGmL30alB0kGizRhkZPTz1bChDsbOPLI4Ej7IWAR8zXUxkCkIp9BIgaJPFYk4pORTyhJuDDgdv5nrIEcja+oCk0bzLHrd1FTob37fwRyx9fxkN9M7WUrsEVQnKE/MGWplE4vJfPLdCoP0bP0WnvIWUsvlRGVRljgxmIsjmdxCMuN/Pol/XtVb1BARgnC5bfkeniCklB3Ua/K1SFCeZV/Obg0+FRxvbgC5RQuifNfHFCjqf01wfQyO4+vQz7lJwGcEtXH1fu8C7so0+9VlqMoDIVhfjt9ZRmKKlU9rABQc10p2QrCLuRPDx/rbtIuFGlnOVYdawYWCTR9ci1rutykYjndKI2i+8LnRjHxRkE0IS0rG+5Ar/zOTcrPUpBqJHBxqWH2ez6/XebWBBU0fePMtOlKFEbC8DAdSEWlt23vdpO9yzmsad/cZOkc2Gxs6u2cLy2WB0wPw0v28mSU/t6JRClFqC1u8ycXT5wcC+UX9aeRf9ktDgAF0WXnTCWjHMeqgMTuL13IdtrKQ1d9cJaX6Nh95W9Hfei2Vc4Ssdkd5W/HS+Pgl3wNAnkr/nPjLOrAh+rA6iydnvqw/GUH+5mnypvLUlLfvyRKb6ofnmTZsXJDTBbm7EQW7XYHH44PUXfcG3vexOl1no27qN9HyOk6jnvYT3O1fV6k4fbe4n8f6M1nH2cAAA== + H4sIAAAAAAAC/+1dW3PbthJ+96/gsD0z9oyaI+riWJnJg+OcZDJNWjdR24c2o4FIiOIxRTAAaMv16Pz2swBvIAnakqL4EuHFI+7isthd7H4L0uTNgWXZk0kQxQln9gvrr8+dlELxlySgWKGhKCIc8YBEgngDJCCGAeOyCVyuZDPsBRxNQwzUGQoZlkQ/wvydB6QoCcOUQlE8HxMS8iAGelcS54GHz0jEKQlZpX9Q6RsG0YUiGMUzEHQOBLvfZXZKI1eyhZQylRWoLglDFLOKbBWG1+DEKMJhOZQ6XNoTuXM8DhaYJFyRseSTkNBXyL3wKUmixvhlmz9QmOB2dlUESf+hNxq5g2O7UyVTf4oOe/3nHcvpjTrWoNuxus9ORkf1dj94gwHqI1uhfq7M6yGOGEmoK8SyzylZYD7HCasMZM8IXSCxdDsiEa7yfJT4uHCWgrxAy3y1TrdbE2sRRDmzzmJzcqXTELD4XLgACb33aIqrzqNr9QHRCyx1ymmClUarqvw08M6JdPcqQ/pjr0qKOKaXKNQ5QdVhFT3EcRD54+tYqqKN1zR9VZ/QOkILaaRLoTmLE4vjJa8Z3MrYYqoKY9VZb3CKIn+NwXvVwVvdC7zgNXjYOQHNsaYr2EKLkvmBeHJ+l0QRdjls0ka7sZBIo/iYMD4LlqK3rWO8gWDzKfhHDj/s/qvWBgKLtq+k39pVquoDRJS7TTeDfSV3D8jeVCpPF2b/8u9TDZMUHdfUOYNw1vBcINILcFHNTp0FYXgmwo+0vwgsfQfiinMCf05GIrA4J43AMhMC6TegmEUdLx2uByHKGfUbA6n7vXWPymQDYyaLqGkqjqiP+RpWwMtYysSSxWES37hhwmA7v/zb/jH7+bfdsf5LpkC5SKb4JwZR30tCQV8dNS1TBkUOqWHCMA0wazbLY8Yb5HKpk16jSYh9HHlvivGag4A3ytRqn67rBUUYZBqNBVzmbvv3uMYocvVN3UqIlomz3U5plLMZRLUQM0AS1fFl+FA31Ul9U8kW620qIuS0X264n/IIttGWkp1+yUIkJK+yn7L+CmRAYYDYWZ7Ua5lliqg2e0E2ZvP3OPL5XMbKBg+3dVsriQdyyzoN4luKvABHvJ6N70iN/Wr6kz7c9BxQhB+dsnEGF0UqrkUAdOm3RBI3oTSVS9MNUktLN2EgPYcG/hx8z2uRJItFOg70+TPwpF3qCUjuG47awqH0nTtRiAiaTDO3ZFxlMzvrAI5GSm0kHTvG4CoRR74ei8ait/CJRAw/bPL0Xgim8jDFMsTOQlLb+hTHGGnzdxo4f73EFNwNa1YEucvFbdsiy3ZVKSHyuBdaISHMxzH23qeZsMnfKplQxPFhkS0muAfZIL2CODjxEiorKsgQgG48NnGhSOAbpZ+OFUSwpsjFL/8H7fMLyEt/DRefj46s6bV1mI1Str3XnHWTL2i1sm5uchHgArSxQT7rbK31aRB5Rt1WpoZbVP5qZypXnByFPqEBny/2Xf+lTqxCJ7cY42xnxrgU8NgEntISqUJu0f3rrbB0LTmJ5bxJy7tGZhO8T/NgxvXMDIR/Kj3mI5h0K0RehyuEcs3ZioQikxyki0BxGXgJqsBgPaCXJ3lVyZZoGbCmXNPEvUgTqAYnLTJIIhRTLwSzEwhNr3ZYVmAreTapX8I1Wq5zulI6KIk1jhmKIyeteIJJ/FeINc53FJiq7Zbi1G6Drqx3o+DwFJag7jFTRL2wBqaIMkXU3hdR84BxAjlmMfmSoIgHIT7sPhuNOtYW1VWagDZDOduAnI4V4qPvuMTaxCattdceGuObFWCbGGS9ymwPrXM/5dkmplqjbttDOz3B4i0Eu0butTUa8bn1W2Z3U809cDX35Gu5p1XJHdS2oB6yZtR3sPFlpNOzP5KryvNEhTzjbPdVn+Ip9uRrKPKmBFHPghHsKj+/8Tg/VhjZtqDQ+kAxzrd8pGgPStvhNqWtLv6tU9tqS9ivLG61sfibVre331035e3XlbeDB79HCEiET9xQbCb58CVcson0jN2cyndcsKeg9Z49K7DefeK53nL5DW/23Zv6+g+kvv6t6nv1VNQ3eCD1DW5V39lTUd/wgdQ3vFV9j6Tg+hnUaZ2ev7M+piYw98zMPTNzz2yvCotjU1iYwuIxFRYnj/e+mQa07OZ8uRXFWKBL0eP810/j6nmzYHSshIYPcc4sJpeHzDD/arXzJ+x3jnMg/PEC47w3R8vmaNkcLZujZXO0LBHgc/PUlHlq6rGgP6dn4F8T/r39j0F/26O/t9iAPwP+DPgz4M+Avzr4OzHHf+b4zzxX8EK8pIS4mDFAdsJJAOIt8ILQ68n0mmO2o//zu1+cVj4B+vgx2gepbAPEHhiISWe/TyyjZX09Hnvk6zA3IasoZGRQiEEhBoXIV8ZxfJhDETdOinOlHT4blR4kGSzShkXOzn+3Eibc2cCRPYMj3e8Bi5inobYGIjX5DBIxSGRfkYhPJj6hJOHCgLv5d1kDOVpvURWaNpjjoe9FzYX2nv4RyCNfx/d8Z+ogW4EtguIC/YEpS6V0BimZX6dTeYhepK/fh5xVeqmMqDTCAjcWY3G8iENYbuQ3PyZwUPcGBWRUIFz+Nl8Pz1AS6l4orHJ1iFB+cqAaXFp8qngNugLlFC6J8y8jqNHU/pJgep2dxzchn/LpAqdC9XH9veOFXZTpD2rLURSGwjB/i35tGYoqVT2sAVBzXSnZCsIu5E8Pn+re+F0o0s5yrDrWAiwSaPrkWtZ0uUvFcrpJGkUPhc9NYuJNgmhGOlY23JFe+b27lJ+lINVI4OJSw+y3fH67ym0IKmj6xplp05UojIThcTqQikrv297dNntXc1jbvrnL0jmw2drUuzlfWunfUfl0veQgT0bpd1kkSilCbfHVAXL1k5NjofyDAmnkL7vFAaAgWnbOVDLJcawKSOxh6UK201Uu+uqFU75Zxx4qvx31ot9VOSVis3vKb8dL4+DnfA0CeSv+c+cs6sDH6sDqLL2BelF+gcJ+7qny5rJU1PcPidI36o/PsuxYe0NMFubsRBbtdg8fT49RfzqYet7MGfSeT/toOETI6TuOezxMc7V9WaTh7sHq/10T1QPHZwAA workload-total.json.gz: H4sIAAAAAAAC/+2dX3PbNhLA3/0pOJxMx75TW1GybDczfXDS5i4zvbtc7LYzTTwaiIQkjCmCBUHLqkf32Q8A/wEkQFGOLNsxXhJzlwKBxe7ipwUk3R04jjseoyhOaeK+dj5d9TIJgX+miEBJBqIIU0ARjrjwjomYMEQJ5beIKyeXCs0kRSF9HzGl16ukAaAgwSnxIVO4HwheQDqHaeJK98AITEKupySFknyOAo0U+Th6i0NMeINkNgGH/Z4z8Dz2z2jUc7wjuekILMSDz6uxON845yEkVOkCXcXivgAk8wkGJHBz3Vr8f8X+XQujwADRWm/dWQTp+4BJojQMMwkB8fwS45CimMn7QsiH8xZHlOCQG3QKwiR7PVJeG6LoWpqFGERQ3J+ZvDC46+MwBHEC5ZYURdDQzAgKPuBqMjMj1yZsya4Hx5LgthhAfr3i14V1yrbFGAblZdXrq1JGYAwBlYYqSd9TSMT06NUf8VIxsFAlc7y8RLThOS7Nhe7blBAYUecNiIIlCujcVe+5QH+J++YnkiL3BIKXmQ/kYywND0IEEuGAwpKVCSZASJSucH/6BUYzKszcV+QwaczQxmhxpygMFUtzwT8ICBAbqDxT5tn+oTbb3mDDbHv62R6WlyGcwShQn8TMNIvOk0t9aIObWX3w3HWz+dJngp8XMV3pVX9AgpuaBbjVPGOBIo2UoNmcXmjzDXczjZTd+7twKdUruQoTPoRyNDXVTzDxm+1Rlp5CTcduQJjCwq0a88ByhcaLuHSZ982Tpde1iGS2uGCRqkS8y0fzAaOI/gsLcwiBK0W2D3UeHUPmtREFs2ZGinlr3EVT/vyRKm92nxktgASK/D4NcWVAQwZJIEEw+c8NJMzjYW2Iiam/SWPgCQX+daM3CYVxDINfmE0bOgrIDNJEWg/lFZGvFrexGAaf9zHrm3+YpItDxFIdPGTLGAWsVTJmy8cSk2u2BPsQ3cDxZEVhMhYeceeHKesB+fGz+yr/87Pb46uaGNiP/2Py8uqzu/70ipkUEuY1r1+xxRyHKU+qV0efo785OHIOy3t7MQ6YdEZwGo9DOKWHvAshBkGv+GPMU+GRU75kXCrYa19fpxPI/xjjJRvDawJDMIHb9rfnFG0KTXEhK0QveHP8fzbCI2eycg5571n3pdjiqRCThXAPlt0XcJz5hXpLYZ13wKeCIDxFnSWyd2U7d3cOe5KzXqutEDgVy5F7rsq5rwgvK4Xr/K/KIemczcsch0HNUXmX37GEX/duLr+YoyltKmqr3EfmVQ6eOm+4+zgfM2cKpJWt5JE7ObsBIlChnt+yJDaop6JxsT6iKEA3KEhZ1mqkpeIegUFVB27BLaqtR5PUv84iqJZEF3nyaUxigXT1pJtlaWP6/CTPU5GZS9lVYwgrcAtbArtytTdxzcVEHDQ6yBV49gYIYFOdLlunGrdnC1VfdbBqLapcrPf0u3ggm/nlAZUiaCGqY0tUlqgsUe2OqJiNomSBqEUqi1RfjlSXuTdRS1WWqp46VUn1QQWg5PKgothdddAzlAdHmvKg3somJDRhYRsatuBh1wq1BhNbUdFoTx0yarFRZ9vKvp4kU20j7HyiScDNbrTipBkpW7FyA1q246UZMc2YuQE1zbi5ATk3YGc7erbhpxFBm3OpRVEzjhqQ1IilHdC0DU83IGobpragajuumpF1I7a2oqseX80IuwljjShbT3QGpGWBZ4uED0O0Ham2A9luQbdthGugXHmBV2i3jXjbqLeVfFX6PWcRxEJ6U0HRiL9tCGzA4E4o3EyReiQ2YHErGrfjsRmRWzDZjMp6XJaRuTnUJjprk4mZT1sZtY1TW1jVxKtGZm2M65kO4UA3Zab3DJZmdYXQzjh7anHW4qzF2a8TZ22F1vLs4/CsrpprkdYirUVaGWkbmy37P8VZhO9OT3E+6vFZz7D/f/bEzs+WFnf++T5hS8ZqH+dn6wc4voYDtAP9fP+wzXmP+ibcpgMfdc1eT3zUdVsf+dh4ekMojMc3FH+W318Mup3eqKa0+Z6CQWwEfWUT+Cs7wiGNvnjDoBjUnol90Qc42IOe9vGNvFStJYbneFyDj7jzYY3aGCpat8cxnvAh16+AepqnXA3YIw3Ico/lHss9ti5qwWcn4FPUNC35WPKxB1GdgaHO5nk7PYmq24V/IUdRB4P2vXu1oa6b97pdhy679zrto27f6/T33r/vtAUv46F5D75x2kSLit224GtT3AEbX+I+fM1KOrx86G34LmU2NhK+IFvefHL78E32fGa78MXue3GO1PmQ+doL2HyvUWy3rXfNOJtEu4NN9fj5b6rHL+ec6AthTe1B0Y2wObSwaWHTwuZTh82ytmlp09LmA9OmdMrTAqcFzhcKnE/gFGc9+HZzdnPvFWXTlvaxrSjvqKI83AD5Iwv5FvIt5D+3inJAMO+GZX3L+vuqLDs/ZT5ngd8Cv60wW/bsUGHeBJ8nFj4tfFr4fHYVZkuflj73X2m2AGoB1FacH6vi/DMhnHPvU3A+yHvO0xDPHVw77Gdt8duk3+dK/DlcgN8gSbL+e2eZmK7C/BfFyHX2OgYAlde6PNUTtl4xTy6fRuEiDpkholmXH1yrQFFhfApvhccEcArSsIYFIrxkre4tQ/HDa3LxW+9q5e+rSawvaXFc/HqcnHXdP1NIVsKptT8HV9ncU6QzyJ3W1f9sW/n4en1e+WBZ+Fs+/towJFOut/zxutxW8iclUMRW6QCeh7oPchkMuWCTgTS3FwbO1/1trCueNM4S6mFJFiia4p6TN3ekt/tgk93z1Uh2EObdwrjJf4vnu6q20VEu09+cz2o2EkmRJvAya0j7KTj9VLvf/V1uA6QU6z5VyMRjH6fCCYb9uibLiq7XV+Z+QwRym3+brJihF4YolO/QRmIXD2ShjCKUp0911r/koO+6V1HqkdueHRSPr30M554OXz773i7/IIPfJkiuUfwrCS9Wka/7ZGgWQt5zCKHa1O0xhAxx8yDB8pDvydbVe6+tYmlHq0fx7HvH0mPYxobaXkMtgHGIV4v6N4PKyFje8NzCz1gSWddqIo8RnMKBnmJg3s9quw7bvg3btrAdmdhy9AVIuSfPr4qpBv8/MNVG3ASGWRVev4FQWGfYb3znZWkgrjPVmMxP0303sn4qTJPR+VFtA/Pmxmcx1YGuPqNNJswIvdGipzT33Fbeou5so/hRShM6828bwy1RdWz29ONtPP34hbj4QVGJXWc1RSRmKa8mTrOtDjfCy2/LmHeFm3OZq7wsRuxtMqlenNtsXHRGnl53JFVaZd9XUrDrVX4u+7zryRfDvqyRvkFkIP3tBdncXxVj4DtTTbczP0Vu+ERuWH7K4Fi+kL756DSQ+1v0RTHfX1hserq/Xr7NC8BFZfrfWT0CRTPne+d3+b2Zm4pzBu7p4GwyPT31fc87OQkG06E3HE6mg9HZ8cnZ6YnvZzfflFXn/sH6/+UubNfagwAA diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index cb46f63..18a9d46 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -155,9 +155,6 @@ kube-prometheus-stack: default_timezone: UTC sidecar: - # We require at least 1.12.0 for the SCRIPT and relative folder names - image: - tag: 1.12.0 dashboards: searchNamespace: ALL provider: @@ -219,8 +216,8 @@ prometheus-adapter: default: false resource: cpu: - containerQuery: sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}[3m])) by (<<.GroupBy>>) - nodeQuery: sum(1 - irate(node_cpu_seconds_total{mode="idle"}[3m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>) + containerQuery: sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}[5m])) by (<<.GroupBy>>) + nodeQuery: sum(1 - irate(node_cpu_seconds_total{mode="idle"}[5m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>) resources: overrides: node: @@ -242,7 +239,7 @@ prometheus-adapter: pod: resource: pod containerLabel: container - window: 3m + window: 5m istio: grafana: -- 2.40.1 From e4379c07cfd50bbffe260071e096309f6552a012 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 1 Jul 2021 13:36:35 +0200 Subject: [PATCH 16/45] fix: fix build automation for metrics --- .../charts/kube-state-metrics/values.yaml | 2 +- .../prometheus/rules-1.14/node-exporter.yaml | 2 +- charts/kubezero-metrics/update.sh | 3 +- .../{metrics-zdt.patch => zdt.patch} | 43 +++++++++++-------- 4 files changed, 28 insertions(+), 22 deletions(-) rename charts/kubezero-metrics/{metrics-zdt.patch => zdt.patch} (78%) diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml index f3bde2c..1df6254 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml @@ -162,7 +162,7 @@ collectors: - statefulsets - storageclasses - validatingwebhookconfigurations - # - verticalpodautoscalers + #- verticalpodautoscalers - volumeattachments # Enabling kubeconfig will pass the --kubeconfig argument to the container diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml index 468e1db..28660b7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml @@ -275,4 +275,4 @@ spec: {{- if .Values.defaultRules.additionalRuleLabels }} {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/update.sh b/charts/kubezero-metrics/update.sh index 47c86c5..d3dde44 100755 --- a/charts/kubezero-metrics/update.sh +++ b/charts/kubezero-metrics/update.sh @@ -7,10 +7,11 @@ curl -L -s -o - https://github.com/prometheus-community/helm-charts/releases/dow # The grpc alerts could be re-enabled with etcd 3.5 # https://github.com/etcd-io/etcd/pull/12196 -patch -p0 -i metrics-zdt.patch --no-backup-if-mismatch +patch -p0 -i zdt.patch --no-backup-if-mismatch # Create ZDT dashboard configmap cd dashboards +./build.sh ../sync_grafana_dashboards.py metrics-dashboards.yaml ../templates/grafana-dashboards-metrics.yaml ../sync_grafana_dashboards.py k8s-dashboards.yaml ../templates/grafana-dashboards-k8s.yaml ../sync_grafana_dashboards.py zdt-dashboards.yaml ../templates/grafana-dashboards-zdt.yaml diff --git a/charts/kubezero-metrics/metrics-zdt.patch b/charts/kubezero-metrics/zdt.patch similarity index 78% rename from charts/kubezero-metrics/metrics-zdt.patch rename to charts/kubezero-metrics/zdt.patch index 4b64f1a..b056dff 100644 --- a/charts/kubezero-metrics/metrics-zdt.patch +++ b/charts/kubezero-metrics/zdt.patch @@ -1,7 +1,7 @@ -diff -turN charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl charts/kube-prometheus-stack.zdt/charts/grafana/templates/_pod.tpl ---- charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl 2021-04-22 00:36:35.000000000 +0200 -+++ charts/kube-prometheus-stack.zdt/charts/grafana/templates/_pod.tpl 2021-04-25 11:16:06.807876896 +0200 -@@ -197,11 +197,16 @@ +diff -tubr charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl charts/kube-prometheus-stack.zdt/charts/grafana/templates/_pod.tpl +--- charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl 2021-06-26 19:54:57.000000000 +0200 ++++ charts/kube-prometheus-stack.zdt/charts/grafana/templates/_pod.tpl 2021-07-01 13:31:20.750609396 +0200 +@@ -198,11 +198,16 @@ - name: FOLDER_ANNOTATION value: "{{ .Values.sidecar.dashboards.folderAnnotation }}" {{- end }} @@ -18,9 +18,21 @@ diff -turN charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl charts {{- end}} - name: {{ .Chart.Name }} {{- if .Values.image.sha }} -diff -turN charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/etcd.yaml ---- charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml 2021-04-22 00:36:35.000000000 +0200 -+++ charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/etcd.yaml 2021-04-25 11:10:33.021207900 +0200 +diff -tubr charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml charts/kube-prometheus-stack.zdt/charts/kube-state-metrics/values.yaml +--- charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml 2021-06-26 19:54:57.000000000 +0200 ++++ charts/kube-prometheus-stack.zdt/charts/kube-state-metrics/values.yaml 2021-07-01 13:33:01.713947294 +0200 +@@ -162,7 +162,7 @@ + - statefulsets + - storageclasses + - validatingwebhookconfigurations +- - verticalpodautoscalers ++ #- verticalpodautoscalers + - volumeattachments + + # Enabling kubeconfig will pass the --kubeconfig argument to the container +diff -tubr charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/etcd.yaml +--- charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml 2021-06-26 19:54:57.000000000 +0200 ++++ charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/etcd.yaml 2021-07-01 13:31:20.750609396 +0200 @@ -54,34 +54,6 @@ {{- if .Values.defaultRules.additionalRuleLabels }} {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} @@ -56,10 +68,10 @@ diff -turN charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yam - alert: etcdGRPCRequestsSlow annotations: message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": gRPC requests to {{`{{`}} $labels.grpc_method {{`}}`}} are taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' -diff -turN charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/node-exporter.yaml ---- charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml 2021-04-22 00:36:35.000000000 +0200 -+++ charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/node-exporter.yaml 2021-04-25 11:10:33.021207900 +0200 -@@ -30,7 +30,7 @@ +diff -tubr charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/node-exporter.yaml +--- charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml 2021-06-26 19:54:57.000000000 +0200 ++++ charts/kube-prometheus-stack.zdt/templates/prometheus/rules-1.14/node-exporter.yaml 2021-07-01 13:31:20.753942729 +0200 +@@ -31,7 +31,7 @@ summary: Filesystem is predicted to run out of space within the next 24 hours. expr: |- ( @@ -68,7 +80,7 @@ diff -turN charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exp and predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0 and -@@ -48,7 +48,7 @@ +@@ -50,7 +50,7 @@ summary: Filesystem is predicted to run out of space within the next 4 hours. expr: |- ( @@ -77,10 +89,3 @@ diff -turN charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exp and predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0 and -@@ -259,4 +259,4 @@ - {{- if .Values.defaultRules.additionalRuleLabels }} - {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} - {{- end }} --{{- end }} -\ No newline at end of file -+{{- end }} -- 2.40.1 From 2d9e45ae462da6d3b29657e7b1b48785cc46f459 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 1 Jul 2021 15:09:51 +0200 Subject: [PATCH 17/45] fix: patch for apiserver dashboard cpu metrics --- .../dashboards/kube-mixin/apiserver.json | 14 ++-- .../templates/grafana-dashboards-k8s.yaml | 2 +- charts/kubezero-metrics/update.sh | 4 ++ .../zdt-apiserver-dashboard.patch | 67 +++++++++++++++++++ 4 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 charts/kubezero-metrics/zdt-apiserver-dashboard.patch diff --git a/charts/kubezero-metrics/dashboards/kube-mixin/apiserver.json b/charts/kubezero-metrics/dashboards/kube-mixin/apiserver.json index 9830c36..1c940dc 100644 --- a/charts/kubezero-metrics/dashboards/kube-mixin/apiserver.json +++ b/charts/kubezero-metrics/dashboards/kube-mixin/apiserver.json @@ -968,7 +968,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_adds_total{job=\"kube-apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1049,7 +1049,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(workqueue_depth{job=\"kube-apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1130,7 +1130,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"kube-apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name, le))", + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name, le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} {{name}}", @@ -1224,7 +1224,7 @@ "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{job=\"kube-apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1305,7 +1305,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{job=\"kube-apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])", + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1386,7 +1386,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_goroutines{job=\"kube-apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", @@ -1490,7 +1490,7 @@ "multi": false, "name": "instance", "options": [ ], - "query": "label_values(apiserver_request_total{job=\"kube-apiserver\", cluster=\"$cluster\"}, instance)", + "query": "label_values(apiserver_request_total{job=\"apiserver\", cluster=\"$cluster\"}, instance)", "refresh": 2, "regex": "", "sort": 1, diff --git a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml index e66a9bc..b85c334 100644 --- a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml +++ b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml @@ -16,7 +16,7 @@ binaryData: node.json.gz: H4sIAAAAAAAC/+192XLjyBHte39FmbbvtGa0cF8c4QeJlDy67kVuqWfCywQDAosSLBDgYJFa3bf97TerAJDYQRILATAfZlokQKAqazvnZFbWtzeENKZTSVmaht74C/k3fCbkG/8/XFGEBYVvG5Pb6c2nj+8v736+/HzbOHYuy8I9ldn1G01dUOORmvr64ozqoiYtDUlV2C3rC8brkj90JhiCrpqaSNfXlrL5ICnXM3Z9GfJQ6/oHu1iu1/IbvsP/fzu2qqTR301JoyGVct6/FBQo/erREn/pg2A+uMrjGOBv3q+fqaY79bLefBz+jgdNmAuKEHiL/+vVe/wXXG8anHZOB/GvC6+SJiwfQ1/l/nrjKoU1mxTZYEqwqcJe2TptnjZ3qJsuKQ8y1Q3BCL7yNuRasJarLiMoigr3wlXWZ6zXN2RJN1Y9aF0ouPKnPz0K+uPf6St7lnr/Xyoaf2k1e8PVu+Cee1OSjWv2tpbrW5cBww0D91BFuJfZdUMzqev7R2kW8q0kqspYlVWNPVB7uBfeNo9Ju9WC//V6x6R15H60Y57zdX3J/yHnMtUMTxHW7a0/3quCNmvY177zf397YzdVg84kw1faxoNCDT6KW8N+0/qGdbc7VZUNaQnfW1/yRlRMWbY+GVQT7Pmi1W91+81es9XvD/v8qiwpTyGjmdWdFZN+gV8rgkzYfa7JRnjgP/pt/Y1kyNYAkIxH8z44L3kfYGp8hns0jKX+l7OzB/6jU1FdnGlzTZDkzpk9mE9WhtLDO3KKkvpnhc2Kav2Kl9X++2xdxjPWMv4xwMdYiJFFVZaFpU5Zc80FWafrWT6pM0PDS7MbdT2mrH7sGxMv8LnddX3xxekj9mc2zJpOB/ROAu1+a70+rCqwNqNGl1QwXB3Na9x/mJL4RMY3n8kZeU8X8P+JpD8Fba2pL+HtKgriI72TFlQ1A29JNo9vmbww9VfC5ixK1DkRZJmXTFRhJSOG+sB+ra1/PJeoPBurylx68Np3RueCKRteq1sNyWcJ95fw9ULl00rDeIT3PKqy04V99rYeYeqGumDP8H6/EJZLmHF113TpnzTdrdY89n+tsnmh8ddG4IIBI4Zd+nB2HnLRap5W4MKzIJu8UqxBGp6r312ffvNVgnW8VrPp/VZS/CXmT/2FveK9bTv+Gs8tLmtGWly411XZNKivYg3doMugKYPGdLWpNfP3YOpvDdrHpAszf/N0NDgK2MxlHFZo39Xvx9u+st0ZwDvbI3hnk71zOIp957CX/o2scr2u9R9UMvZ9o8D7PJ9/i+zppiLxfrekMH4VY92HXLc1VIAUMMNZQPO3wAwVOft1fbNfJ2Hya0VMfs01GncWyPXUpy69eMb6UpOgNs5C23iEz19V+MLde2HSnJki/Rjyez7jyWKwcwId0I0PqvHBN968I4xPWbqHEaxbS3fWl1BTw1h6uXPG1DvGPXT/euS/672gPQHks6FJwIQWnfjFj7Fda7IGU663qh5L0C9L3ivfvn0rqqZi2P9XYHRPxaU51Sks+jN9agDUkr9JCszsikj/+p/Gn9gt/2kc/1e9Z5/gn/80vh+R+1fyFn53dHRETojw/PBWNxf8SzZfHL0FeESjHs7u+OsP0kymPxwnvejffwKCBM8C4gdYBEz/2xF75Y9s5juCFTDbungb2oawvmZj/dkuy5UgGnyse+b0hkwfqDK7UrUFX8x93Uejc4s1epcIPo9ybNF84x/+IUCLrbVsBQ6u/hYvLMf6T972CCxJ0D8ALz/QI8QDiAcQD9QcD3Q2wwMtIPwICPYPCNjazVdOWRVmvQ1WyzOS8aprreZem85Xq6cB6xU8G1pez2Z5zmYBvn3VyTuwGCxxC4Z/jsq+ErdwKcalGJfig1qK+xsuxSNcicu2ErdKsxSXZZ1t7X2h/aCy9VOQZOZEIp/O35MFXajaaznX0hkVpYXAu3czcpHFlavwlatZ8MrVrOLKNUpcudi0xAb2R7sEAYeftbT1cWkrherM1yZrupy+p4s7tjJN718NqieuT+SE+H59pVG64Y/ZIpnq5Wn5qTcCIttl8zje6lBqsF3A9ufOErapCVZqewo7HhXP7y/S4A62un9mcQR7Qxvs7eT2RVgiVUeqXmHA0yoY8LQrSdVb7Q3d6C0ENKUDNGyWToFo2M/TQJqtXx+OaYrh87+e35RhYf2kqga5usW1FddWFBNqLia0epu6pLu4uJZgcfXw1rkkU/0VxvBiyrXXzda44wVTxJcqrGnw3Rl8MdfZ9PYH+KTB1D/Xg8TW9SZd+kozfNH25LeQtdheA7NdjvnsciGITw8aWCYYh8+v/2LPJyGXvP0jeY7aZN6MnM4DttkaTHDcBRZY3FONud9Xrvc0yGINC3aZJ9fdC0at5tpTZLWopwiwSP+y4mnuSF5Yp50LTd98EDJNGIkTibHhJBI5ybf9mzD8k/xww0l+Pcc7g8zfhUPmfhuR3fkQkvv7mInN2cnEp2ViqITDsOCM7Sp1tLrnPEwTlIeEh7VjZgFo+Ql09hs2c+ne5mfXbqj2iTf1WkBnNuK3h8I0/unOgpceay5V3ZhLX7xbK+0vr2CpvIW5ll3sNf/sug4zXOA3/LvIn3B7vBeWMe0w1/iwCsOXEbi4YahB3Bs0pr6EXg3dxje6YEWRvdvdOi2Yg6CrQncdsUmoNfTMU3P2nuDgYU92P8d6TJvNZ6OO5wHu8Rno/3zzGzzHXPg3um66MucZbu0ajY1GWeKt3bTDbTB3HPbYO90762fYjk8+MCN7ML8a34PDiF10543gbcF68hudzcqiqWkrjI5Lv3vpX3nUW1F44JYDSGIuGbQrBQao6frv34PZbu7iMnZ75xwSWDw+CN+iPep6R3OmKCLilb3wV+aINdKjCh1hRcVhBYcSNhfmWGJjBf0eKOx2v0yAFu0CwcNn3yKRK3KIGPADb8UODl/EarybA4wgTEmHL5rxegNTbtI5L7KCF1wkO1SJoRPho+2gxIASA2KB3bFAflJ8RbaZeaDDoHk8ajbC9furW8KXBNQfDkp/SMIH5+8RHBQLDgL6QwQ4GPQQHJQSHAwQHFQDHOwWgN7Y0+LdCI8tx0UbF23fos1CI3HVLnjVbm+2aq+pP67auGrjqr3zqr1trHqJlm0eun5Y63Ypsun2orLpdtJk070QdEn0ZNP9QI3tc+oKsiToYweIuKrKs/PBL/94eX4x7Izcw40n7LlWXwQrJvqPw1HzqtkO3KHyFLrshtbVYDhuuW+4nlm1+GOz177oBS6RE/IrPB06JYF1nei8edgnmPgfYfGhSsSPH0yqW4UanY+7Q3cXa0jR75TWlQlWV9J+51curloXnj0nDUWy+s8fx63euDXwTEnq3HB+eNnuTroe+8CAtgKF/ng1vmxPLj3XuG7IL/aaw773h6Zum7TXGvXGl41A17oXND2kt+uP76jyYPD+2/R8T/UdBocLFLYj0zqtOqikzNVsgKE3bmoFWHbCixJfdLqeL/6mCTMWD+8Bu5EzwcA3E7TaCTNBPyykY0aVW0u3Dc0AMBisERoPFPOWA8bvg3Ku39l54f1rqPD8EPKtM2kGr1ibXgLf8k0v/m816eER1oYwedlZkX3p83W4+Vdpxrthu+dB2CzQ1zvV2d9OoFcFr/BYvZD3hm4c+O7GuFT3h82wL1/sUrXi4fAOADF014XADgFY7UuI3JBgbTYRHgKRPklbFbgTgfVlk7245/0+2NGgM8yoZs0sc1l1YQDLo/DRPZTC13q+jqyyudmLw3HYfrPVepGI+VfP5KtFxMMGlxft/mSLh/ECWmFuUc+014Atn/lZd2WM9z2xed7tDMbbPvGjJwm975H9Seuq307A5yINm/cB54lP/j7FQOaSzt5ZeN7bQTaGyk4WZQcWb5BJ+T/2igfIeZdsyrlnX2zHButG9qYM82DsYNQfGFD4obIGDQ6lbbJi5GBOCyAWatBd7BY2764sN96H5f73n8bpjwCEix3dO9nu0z/0CMtNMrFcorH+4GSVtz9YY9j+YE2Sq49Oh3Q+ar+v77SoR+k7a8jqtrL4ZWZ9Nf98/sWYLIiBVsa6SqsWeY5CWtArSzD06Q4L+ok+2FDW94PbR2kerVTwXP+MDLpkidXxVN/cnEHQ6CyENVjcoOmH+VNH3JCUmfQszUxBDvJh1/lw7kPYvghfJB8mvzfFJwvj+LjGKiOCO9h1LUn7OUso81nxkhAm+ip82Tq4vt2J6m/ePZP2Vmz79EDf1+oDNEsgY4LNARuBPm2xwIbvW1d9dwvab3d3qIrX6ImVCdxu1SXwdYiQHDZ4XoO9h/PwMALOvn9Hn1el9pzttrEad77kQiug/vbo0qtdXZjzueVagqut7mTk7hqNMXNduvmC/9rMlpNag65H9Rqri4VkGM71gBLFEr64qY3r0s+CNnuBscx2lmnmEp5BTsg5j25jW4yZ+9h4FAz4HyVPVFOoTAA8KoY0l+BWQWcbkO3fsa3tBnlRtSdJebCktqur86srj2CowAwqPVul6Q273YFHibsB/szlEd0ubq953vLfMI29A0o8XdX3snk1mnjesM4JEy4T3kKvtYhvs395MfBfmq7b6LI56LUnnhtYArWIx8Kl2NfCddezgxolv2FVr/ZVb9DzKJGfFdN5upcRl1dutLw0xNSFB3qQkmMgD9dOmuOwnppjx6M5Hp6G6G3qEouIwQgjv0Lnn4TXw6vpA8OW1BWZSCa6AHzqJD8Ra4GNVPYsZ9J2j+aTblaC5vkz+/M+WiKdXE7Or1IYqwzCYmaRcwVKWRG9OENlcF9ZiVl6I/evLSS56Y9/8rzZxq8lyMOb3JqeTENZ65L7MWjeukXSTJqhPLl7fy7CCME5P0N9cZ/pPPM2XjDl5tZa4T7lr/cWJUEFbGcFLCq0d3dxK0IPSqNtoWwVIlt9ouKzNY1MgdO33fj2OPQuWY2SYNg9M01drh/Uv5w0L5vhN9nPCWo1/BYgIq4CBZWT9U32c8bj8473ljtNUPTkqrlvi6ybdVNS5Vx3RdbOuiepeq67IuunbdAmmrtNmlbU3ajbbZ6H32Q/Z9QfdLzRgpq7TZoRhdb8bRKMOjS8bdKMaBNjkzYxPG0SVTkj0CbB2hmeNomqnhFoE2+UTPGSW4i6plCDSbE8oI8sqUb4Sj4HMojhfeyGzi5aW7dIrY2RocvF0niNuPYvqqnFS3QWvFkVOzwC0Pejvah3qNN5paez0x/5xHX645kvyp99yyARNxJ9EJiD5uSfqaLVvNXIRFVahwXYU9sUlhkKRbWXh81SQoYEBfw4zJsVsfWQfPs2o8+SSL9/z01iCpqIt+1CMspvI15Sl5FII8OTlopjkB/sZRdw23wOyzBSySyo5FLfB0fbkksulznS3bLzyD26mTtRQJhtryJ8qSLMOe6cWL5O+KMTHujgzkh/yO7nMExMGUL0blJMxsmtXrtGTumNEG9gNzk6rPcGhPeLU7M4UcQCQX/43w9WLrEfsjk+JPDU3N0B376t86Slwb37RHR8Fbnlqwg/zQsRXQaIzjmbKQtUl0f0awk8BN5yFgfs3BvxPctQkfvw282ojfi9kI34McpHiKcj9c71pN3rCTvYE3exx+9kT9jNnryjPWlXe+LO9rjd7b5BFcEH4jhBDC/YrBtGhqGGcYRG8IyzEPQfxwDiWEAUEwhYKYoRBM3pZwax7CByVK5HZttXjJewL78ET370ZDAKFjOOJKyHcycEPQTL6aML/lPOV4wh5MKaNIRctKblsAt8Yg65EMMeolfM+O300Vvq47fVx7EM7zrtWUj87RRGOKJJRwTxiCYfGxCQSBKSSESCtYnchb8JJ4njJTHcJJ6f2FdD0sMkspdYBhMZwplEZCLJjH/9TLNDL/t94hsSFA9JsbBhzA3hLCY8sMk6a+eEwIIjUl2nOqFfqGhy9CApzt4YDrADD4ugOlF0J2R1yKphst1rvkujbGFztrMcLP6BfSGTZbjhWYWSzH6xd7MzvFYVs3+AsjKzwz+zNFYf793qme4TztnqGxKSaHNP9m/ubBMv5G1wXlqfya/PPjJji+piKVMjrndf7t/cGeYNyNvW2u9gaGADz5JozSDwCLZtVI+x8NXeLZx1doacrXxrFddjabsKcXb+2/7tzMSIyliZFRZszHJMEx3mZIMtiDyBIlGBgAh8MnEcjC+PVCGaqSj2wimQZ0kzTKC0X2F1pQp8UpWFXwr1NtDPe28gLnZVpYH+xgrrbSCnAVbW57kO2YQvEF63QMvFNMf15s3h+uSldRH+izgfRqwfI9aX4fVnQM39CYSDHow4L0aEJ2Mjb0aQsod7NSI8G7HejTgPR7SXI0G3CXo7glUIej1CB1+cT8EbL+KSMYI3hfsY4vwh0T6RSH/DJpPJxvUJGnyjeoT+LNxnEh0REz8MX8N7WIQjJdaZ4n1R7EbgKJ9EdOIQfjk2eQi/IyaByPp6RBIR64b4RCL8nuhkIvxysQlFLF9ObFIRfktSYpHVTdPEu+ITjPBbopOIrC7HJRKxbopMJrK6zIAe+8eTSCP+J3E5RtY3RecZ4fcEc42E3gDFYw9aFU8RdB3Gjv1L/158dBvV020U+DILv1G7e5COow46jopwHAU73d48R3HbPEJBmXu7R+hS/OPpWRh/CUsmEo+dSuKZ2mNajd0TOfh+vF0GCV8GAAYmdvrlGhPtnntgq5LvLAI8Rk6cuykEDGhDZd47uAkGxj3/VzuxwsWF5VKWRMGaWop056Vqn9LYd11yn5UNlSwAsN5T44VSZSXGCMqMLB9fdTC57MBFYTbTuOOpUL/efvr3LiqkU7i1hTmteqIU+jasQ+ITo1tLaAnduvIoPFOwPJh9Tg1WK8IOmyI6YxD3psG51is1rFsW6owTskLdeztOa6Xp9qzMwWnFRXSZZ4m3GSMjRIfFUTRM6OVciZQMnagv3LVK3vLbdCJLT5RIrNLHhJFk7fWYQOMdFeoG3HmlKk3DOAPlhp3sN+MbgKwmeMv/FFV4pGIcWY1TqM8vo1xQ+zSuXWww74Wswqxj7TIgb+npwylMOtpsJulPyba9ytW2KVJM7dO2SUJKkU690qSt2usU75Ld1jsLC/Xdue3oMLwVwataB89NLa6t+85GF7ecYKMfb69+vGCKM68fL+p6GtcXevBK7MFzMpLAysC6dkxmq9DbnVxhoTt2Vqk8Nnx44H7v00vmeEEXynY7b1q9lC6U4eG4UMrk7ECXxtYujbCTaHrDQSu4XLmdH3dhua7WE2NcvqsM/B5Bo2Tu+CgqH1bOQqYr49MJ+WTVoFDJvbCkWUUa8s6uQjbieRl5kC/RFlKhfVCh8Kl52I3r9/5kXj7KxGVw0yBvT1i2E0khb386ahTHPbbmUBEm6MWZANnWjmyrZCFiHQwRyy1ELI7gBJOSbUF8Wr3+4QSPReYu88eIhT0Xg8Qqy6jKSVmyzF3meJSyyrBWLFSPTIu2M9spI0b3pU5DjI7uCnRXIICuB05ul8IP0B6k3UrRHu0VDrtOskA3AcLdvPdEfKLCLIVX4HjzN+kzYRr6pujjY7d/xX3SK7yHge/yCjHpFfbxVyleMUusxdWw2xmneQVNfEXALb51c7cSXmFvTU3zinZSNS7Ou53zVO0tdJLecdWd9EbDVN02yVT21ts0r2gnWcraJFvyV3SSuq21eTjN+E5qi1G/PRmmGhlikqHs7expXpFkKDtLb5pZKslQ/d64N7lI9YokQ12NRp3uZapXJPao8363n6rT0lZip+W75nM11PjyfJyquWknsS0u25N2dcMGWFz+VAM8pE+dfGezzZzdtnr21//BF+wp1kc93Au+gWLW3TZKAMq8ytE221O0ADffiyaxEIE922/L4IBfeaE3MuBFDQTI64/+g5mSpcdQ5oja4+7ao6TGuv2vPxI2E9lefz6qyu74RzUy4+BpSSW8CwcP1A32P3T919j1n1rT7LYPzcOPCWBQ7AySGLakhuZ7yUPszF/rzF/qzF/pzF/o3EXnrCJr3SY6OyXh2tPmWA9puzVFloh/DhPgq4WVI9z1xZJfgyqH3BC2CRLbouI8mqXF/8zSS3LtBRgalxAwniccpsViqnyC7kf92IDzi2Vi3E/l+He4GfpxZljEWWGvdUWKjhS9Suf+jVIy9FZ7gBQdKTqG3++RREnq1EbR7gM/dgbwpHAEH75qp8L39Y3F5xDekGTpK09riti9NNi91ey24gaKPe+ZihR3ZsifTWjcEgfvR9S8HbtPthxwvRwutcAJ8KGnynrO+QEqZ6cLOyMfqAH/Zw7x9XntzsDToJolPgu+FXUWfD+bs+Dx3BU8d2V9U/K5KzkdolKVo1Cy3v/STMNEB820TLTTx7NESs0325nxTdbx34WpMtsRzfKSyGKP23Dm/xKmx3eKtk7H/vIoiY8EMKiV4B7WGiZ5s3aWrVztp4RcG0TSCdxOCYXhLsFQZznb7ym7UxagjDOeqt067HNpaktVp/s7IOK8rNY/99nePn/ANj1Pac2NvLI/OwDC1IEkwt8Me6zNbSq8pYR7XZVNg7LrlLnfBO21vu4222y2Gc/ICmgVRNpFc2HKPCYEg1hxAz2GrCJzROaIzLHCzDE0zDgD6thD6nggrspqUcfNA0ZX68f0/NY+HXLn4FPrWe8k+F/UQZOBvdFBVmLNEc2KBah6Dsly2bSE7MzX5OuV3j5faAm8x6ZlsqyKAlvYVYUv/1aitb0RXlcHK61dne6/NqtjV3suZ3ZlieqEtbqwvZ3rQXAtixWXGA55LfJa5LXIa5HXIq+trUd0mJrWjlpIa5HWokc0hUeUZaWu2lmcnsKzM2fhn5N7dnjczKEwQE8kx7P67tNnIku6sTc2uCqvoKhKZY3NCs9wCfzzCthE575Qfii7dVR4mMmP4StRNmcARoixWM518lZ/BKscZcMYd3dFV7Hbn2/Q6Tfv8uMizFzFDn++cXe3jS1TQTdOnBCBEx418BZa4Ch8CMS0yaSe0QBkQg3BF/ON4gmKJzuLJwk1KVY8qewe2fO1AjAadbvN8zj1pDdsdZvjvagnE0kzXt1pTaLUlYvB5OL8IlbqsHOG+m55LzCkbQkYvfZFr5WjFuJXMiwBIQcl487mhFGm59eZuBRT79U95CeyKlPcraubwtW2Xxbcg/B5LZ/4U7iifLJxWEB6/aTTPEj9pEJbmDGkvKy+c55YhvGvEjp4V2XzR5VLuk1a5FdyTxklWeUFYncbKpm5990VLZSsyn23WJbZrFC8tWU537t/JVefby950L1BF0tVEzQr6ZLVDvc2lNqX7MExVAlNyssV0ktfBMngjFklMEQDvZR5whJ66rgWXHo9kpn8YCFh3BCPNBpjEDAGAWMQMAYBYxDSxSCM0nPoIcYgYAxC5ULrw7LcdFudkPw+q8D5W+a5/dl8oGzC1YMEyEKcjjPSdvRy3Gq5ftfBw+RFMh7JIzyKLNmzNg2uP96pTsNS16lSGwYsobyEPNIqGLTfZ33dXJJCFtb3vEUIP4F9tb/+mVq7vBdwD50dE92ErwHIyNI9kHdpj9vleacsoZF5uRwbB0YG/3hPjRdmVB2QtibIZKmpbBM81Y8BOZq6yxfOECWj8Pr+YkG8g78og2eWDLDIuaugsBFepZvFrNh5JtsWWRWfE6gVg7Jbw2qGM94CR07b2GE80GYbtUE9wkRuLYOw/mj7XzE+ZA/CVjhoasamJ6+6BBZR50FsikcUyzDoBINOMOjksINOMhDMWhh0gkEnuGtnB/nl9rNiJ5wrozqwKhwQnxtB46yHLTzHludIFBTmF/LkKFQVdy4H3dTo/kSXT1ap2HxTRuu6ihdu3wWbpjzmXWtafMuCXvs0DdyBh+ERGB6B4RHI+JDx1ZHxIZnLLvih1UxL5ga4gQCjH3ADwU5kzp7Jxo+m8lS1PdLusgMbecfsoAPFUxVDAiBv6uQeLj8xhvJs3UoAbgvr+O45W0z3RfXs0vNVp6KWt1bME2L9q0tfqdvWNqdmJt+bc921UFfUxjzMwR0EG96VeRxm3fch2CZBco3kGsk17j3AvQe49wD3HqT1pabn363eCHcfIP9G/r0D/76A9UUso6vPKpgvnpdtLLc4tXUCOLm37kreWH5eCwJiGQVDRJF/IP9A5x4699C5h+wi3rnXSk0u2iP07iG7qPaxYasctKGnfEFZFZ2BAm4t+sAh38k/GxXeossSFhe+p3Gb9Mru8q1JjqQQYZVqeaNdcDkfWQ2FKbMJg+YDmqjZG515nO1qTyHf9lz/M6id3oM+KuSIyBHRR4U+KvRRoY8qtY8qPY3sN9FHVWoWifv9ykrm/s5XklujnGmmXaWDxdD65MQecoOdkrtHSWdBcmzF09Y72PZG66DPi0uzcolloNjjm89s3x9wPPaXE+G5SuEzewUCIInwxSvjfLIqzPhWQaAM5iHQP7v3nTHroJcQGSAyQPQSopewjF7CcPIY6iUMvxW9hBl6Cdup6V03W3oXNjWhmxDdhPUkeGtPHVtbSsXuvEUD7vHzyivH3E1M6FwCtLZkT6aMMo73So01Kdkby1sX/ZP+PCupVVnRvFZlMZ7WtjKBiByuLGAQsfNuHJOSOaw2a9M/Crqdw1iY0WNybxrQCM7NMID5DaxJVjftbTvgut63prYsaZOwokV3dOFefab8I5/L2bUzlt/4TH/Vz54XZ4o2ZY7rDDPolphvr134Y+YDgUkZ3a57Jd0JVLUR0S2rR7obDaTcSLmRciPlrqpLNTXnHrSQcqNPFSl3SsrNJ8ByUpFAapcVC4G/SxAe+7NDdKB4JfRMu4vnEDpmV27O2vtC173o1l9dJGboDUVihsQMiRnumERfqI+XddLvmCzwMNiweYtlwrtcLPnAjbr+L6qpEZeR16ErtbK8DhYsKhqwqLT+VkJG4iqdZ+OI5WayjykVdOB5kp7EUvKldquStt9XLk7WVfbUVr4oxsrdp+paufv31Fauhzd0ZREk20i2MfQYN59uSbZx8yluPkVPaR6MvDPAzadIqJFQ70CoPyss06hR0gMRXaXzLIam63t7l6Rz/OQP/HhE/cUiKKpp8IhbgTwLUCrjlf1aozB7Kvvzq75nKV7zPzFjB3u/f8dLBra+tV3SFt2zCsyClBdOulpAFhyKLNi1t0dEfwULLQjbnFp736u7WwrKjNhmwy2pyAuRF6IbFt2wtXTDWreubgpXEtBXm6GvtpueGbYxuy1SQ6SGu1DDD1e3QA/1snJDKB5xiufJkcouWLRFt3dWMp4CbwbzO/snrU2s1v5Li9jwe+DdrF/U/ZwNsBD6sJCrIFfJiau88b0O5o8lFfxDMzAs4R9JmauN9Q32wNCgbm9creKUuCGqsiws9cAITYaBobit4VvBOVxrd11f+KAah2nt9htfn7HAV7u/kuUbS0GhsneAxNG4MmDkUkHhdictEh70DwYJI9QtF9Td/CAH1TTqe4CDJhj0LQfYzwt4rTFdPiwfJCURVP/7T9Mp++3UAcm/HRWJs/kmI4DVJ9bmKvhLhX4lWD2zSBdKqP2gx1TCgMwRZVuQ/bmRCevhLLE2qV0r5Ix8NA3kHRXIIMK76NuTI2gyGO5vfzoqcNMXMpHivSYHDbdDg5JS4+12G+E2wm2E22WC2/rLsgJwm5fSxor6KpAJhu1+sTYUqwpY2ypmwHyqaRwI0Gb+aUTbiLYRbVckRgn3ruDelSyjonDvSmFemU56r0wPd64giawcifzTnx5hhP+d8qBV9f6/VDT+0m+1hkE4sKKbNw9zwZRXIvhC+C/blqHMCHQ1+Mu6GCeL2yO/GQLVXZwyS5Ia6ElFuIS4IUpPs9I2ZrFuIihfRczqlJQFtHGjMoawjT0virDnzsYkUK9qt4+krNqHd/Vt22dcG6mBXLFa67jpaR8SQ/gC3OnEjYYkMWIebM9yxRxGVLqbotKoUmyoUuBOKvsW3EmFO6lKLVSE+rNTKxWd5gB3UqFUgTupdqb2qrqYPsH43wvJySwtINSCsFpQjUjKs30EnU7qu2Xq48f35O+8wuhMLQvT6XUGqZiOGHKaXfmpDtQaqc52VGfHjVi/8Bm7cvuwOhH7sEadKu/DglVe1CQOVQKrWMk2aXXTBo32mxUD2dWICkW3XTD28xdBkzh4+/E0JAZUZFOAFc0wal4121UL/uTAm6GZL1Oqw7/spOop2EnVACqLqjLLMZfBYyTnzAOQXzrVI7x6LITMrmHx+e0sg6vzuU6Nutn5DopCrKoxG99T44Wdt81O4Zad5HfMAwo2hhEL9SUiy4xXfO55qxFgKq9ld38PWHFhLrbp7BV3ufGOd/uqiJqqSF9hnE809hMko/uN7I2hl5FdsoJhvDlxZdw5V3eCFL6tLjVDGrWQISFDqp23wIXcZFVd8j+nsIowzGZU6NSgm0dYEk7sdNGsJoS9nwiz/5q6UV8nAcdoN+/ekfNgRRGclW7bVfGYBvdZITYrt3jdSQvNWv0hQjOEZiheZwkG9VdFnDJnpFmlsyOvdUv8JKz4j45uZahEIBqFZmQZe618vORti/yVvFL9mDThD0U92pdqPdfo7yZVxNepBVYXMCCmPJ6/QoZ/x9Voy/arCpF1heqbAyGgkt7yQYNIHJE4InFE4pVSSTvpVdIuQnGE4nVWSQ1JfMrft505QLu1SrwKH7CgGqvM3sI1DEEqLGQje6bBfqJwKgjA99xQF5JIOBZ8e3d+fWQHbNQc9r6XdBGBLsYDIMgtKIp8NfxurbgvNgqZ1FG5iPFueMR4Zw1L8eSOlLr6ICWW77cRyucA5XHjZKlB/lJTRX16bwUSlOrYOZh0RKrrFDC8HebwIkgGOw2bnTN+ffaR6dyiuljK1KDFY3rLcJqpKFCkkhpOUggroOUGMIRYM1Ucn68rjXp0ybLl9JrNw9tDCrVu4R7Smkrc5VKx0wLfTquHyBeRb1mQb/7JQFf4jQMGHvVBKyTFfvtmgTny/Xt9A329eI4inEM4h3AO4Vz9ZcxWWjjX6mJ4MMK5QxIy1xngYEJ+0qe81xSa/W2jrfk7SXi8Rvbe+0NAe+SKVRjhXmngXr+bCu5Z/feM9eCqQb5+FyEfQr4iFLwMMN8IMR/GoaZAdptvCXsvfDn98Sz62KAqnl5ri4HTZ0kzAAxMFzw95fT+1aB6NkCy2ORNazxl14hYNSK69JXyrFqsZoU6zR0TA8bkJxI6NoapY0M7lyUjlrA6btFnXOFZkGTubN/AwjmdH4Q92WfncaF23rw3l9DW2XbwSV0oqZWrGCnpfuO+Z1SM6HErrhl1vSwsE7kjcsfMU2Gnj/7oIHVEd8EeSGWY6DZojuKOtr2e6ESWFpIRl4vkqt0d9ca1Iai+aJWlVKl9gx/MxT3VGJZkjbefMG3LcAyarz5VKdNfTKevTfQ2q6PdVVhu73eB6iLixpgfjPlBEF9PB1B6FN/EGG5E8WVB8d1RvxMD489Of7S38oU6kqBOis6mWW5V+sDPIj/5Z6OKjiWOQXV23jI/mtLeLujk/9hDpFLm8HR885l8+0bEpUm+fycnzjEhRF9C1yV2hYlAbOhdKAMIawK75x1OE9y/OrZnDeHeQ2s8SjqBH9d+VyjhzW/au2B18snulWfkV8scSDXKQjW6w3481ah2+peIKiPPQJ5RBM/opPcWYMJD5BnV8BbcAe4QZgfvMDAsM1RI8T6X2WGP7KRNu+z7dhtUz4QJfb82WRntaqLrAF0H6Dqod+7HVUBe5ZI/9sKTP7b7I0z+mFEY1DAlr8Et08hqMmA1Gl1SIRSyVHAztajCyvvFmOovkiE+0n1K5duI4VapiVPqPTkcWOeqwUZ0nvZdM5eGXl8S4fSYW7vHkDMSUW1kEgdyrhFG/aOQX2ohPy3eHSDeRbyLeNeRe2VVmLUKTOLd3eCASWFGWovilW9mil75TNFbFLqXet0rSmiLVqwxxtVG47baxyqK2Ls0Kn47VYhKVVX8dr+NKj7i+fzl61ZaPN/uYQKoPBA9Rt/4gmrOTn8ca5IhiYL84+lZXFTNZbs76bY3jao53ioBVdKrr4bdzrheKaocVdnWJesQt/7tG2Gwygpahw+SMldrndR+LSuTCTUESUaEi+oyqsuIRsuWj3SYejtqH+EoCsy5ANXq7PZkS58uSyKt3y7D+mK0W2er4Lr1CP1CRZPFo9+/EiqIj8wICN0qs2VwLyAPtw0iHqxLjsG0cLDVayEcRDh4QHCQI0GwjqYuX6ertMnTe8nQS3VW+KVVRFdmZ0MlmqDM1AXMo3xTFfyAAihlk2NtQZ9tBsR05dk21h8e4rax/gghHUK6AiS+dnqJD/dMIaY7OInPOfdDXJp1SiF2B2+z8oXVF+QxAdNY1ZMdYWLqdtIA3Qo6tHe84aYfTAeGuh6CwLrrep20GLCPycAw6DC/oMNQitxpJ6QWTg4PHI6aV812bcIDHUjKMlXNd8pSldtJciogLQJmpQQaXNSkZXYy4pbHTrKSlMU6H7e0SsX311+xmk4iaoroep8qa6dziCprp4Mq63YAO5CcK3Tjb2AT33tJFyuXrasfnq2r0+xitq6sAgr6qQMKUHxG5lEw8xi0+/HMYx8boyJKOkjLkWq2hYrHYzy+LFRlCgvTcirC/A1drEIZfVno7aO0ZJul4E+dKoCB2QdWneKzRLhNCb1+KsiCtsjfqnZ6s+AsWqTFnWFOzlmli09L4bd9ffqyY9lssluksekjANhD684/Szo8K8H+k9ztz2Ssw7A8rMIxlr6sturzs6DNXgSN8gWKBRCa8Dc0ssSshgpQWRSgQS/Wv2oPxBgNyNW8VdOBBj10tKKjtZBou37qaLsmCh4oeBQqeLT6vR7KCEGsKgJeYSeIzuizJMJHU5uy7b40d7Qan9U7u42+1kTAsCpDLAypSgpZJ2spXm3wWZwxhJwtnodHfBeDVv98JtWAv4ndgsRqQcT/ZcH/rf5geIB7p6Ha6AJG6F9EjGV65N9G5I/IPwXyL/MppeoLBfRsLpfy61RVWCPWBUUDvHPXjoE+q4L13WBzw+pLrPoixEOIhxCv4lF+QWdOJSP6BhHnb476GNGXEcxtNdMmMB/hkUQIc+sHc115IXlM9GyqqwxbTAVRpEuDztgxmwosWKwJapLU2xE5x+ua1ThfpNWs5Fa1ICPiXszonQrMIkRFFTL7jN7p8RnuuEB8ll8Awiqs4EqQZDqLiym4andHvXFjt/NjrhWAMtIzjX3B1WjYae74gvPExw86F1f90Y6Pn1BefsGQlIf4OowvBpM0dUh8xXh41R63G1mGcATGUz6yswPFTWWDfJzHPNQAPq4NX6rIgrjGyjswI4UhafmMSIvfy7GbAWe0pH0xfnbKexPHbsac8wWnVGaMWgPz3oWxmwElpYQDOnqdr8sGC0d4+Mwai9yyxkDx4RDFh0xdaag+7OQgswdj5VxjwwjX2KBZZdeYk1YJplcrCpU6yezfCnMYgGRBgTboR0Sdk+uzj0Sjv8NkALOoqC6WMmUn/SxZJANPIElgSBB4SWjs6lyi8mysKnPpIYSf07lgykawL3KxQzf40vDte3AUhXH6wNBtqB7vR+QsVi4/YSulDrXfY45ZCNG/qKZmpVFp0sOjcRsZl3TQKasFmWrGnYOvkmqTJHLJ5oOk/EI13Z4TBqedU2+SjHyEsNBESbtJZGFRLu1mJyEd4icqzE5/DNmkA1VVdLbc83ahDwKDyyf/bKSOvIEydePLpM+EaWiZXCLZ5UW7P8mkML2kwtwnFeby/GLYGWVSmH5SYcSkwvQvJ83LZiaFGSQVZpZoGWt/VxaFGSYVhiYWxkqsk0VhRokduJVQmN6w2x0MsihMt5lYmHaSaS7Ou53zTHpwt5VYmk5Saa66k95omElp2omDO6mhmue95nkrk8J0EgvTTmqnq9ZFM5PB3U2cg+87SeOpOei1M5mDu4lzsJjUTKN+ezLMZHB3k+fgpGbqt7qTUSeTwiTOwWJSM43Ox93heSaFSZyDZ0nN1O+Ne5OLTAqTOAfPkprpajTqdC+zKEwvcQ6eJY6m8363n8nQ7iVOwbSVOLSvRpNMVqdeO30zjS/Px5l04F7iDEw7iX3msj3J1qtZaJjhTNKfphqAfn26UjOyjCsMaPHdDUICLenk+3dyQhgfcQkthfosfVZ60SSDFmWm9nZm+pWXbSM7VXyP+ASaglx/XOpkHFrZDfwYodIJOjK2dWSEzqmtYT/Oqyepy7jsUNdMXBVm5O3JETkjfMCRtz8dVW0zEVhhgJuJahqpGe0tUOfk/pVNw7wPz+H3RNV4Lzaowk7KXbsFXA4D9BCkC1VN6yLodNBHgD6CCvkIdg+WzV3fL0C1L0CLL0BhL0A3L0ANL0DjLkK5LkKPLkBlLkA7LkARLkDnLUC9LUCTLUBpLUA/LUAVLUDrLEDBPFRdcsop1R63OqdRLi0+uG/REthneay4g7DJ2HOSIeugan46+5VMBENAPbM0emanH5sV6CJWzXSJQRUXNDuDJuqZh6NnCs9UEx4oYYOUxzOvIp4lXTfpzKdlwqd7SmAFeKazU3L3KMF9iiib8GR+G3+MvmRJou9f+Tfr5ykE/jLhlcrMfy97oCSyNMBwYXFaVZ20U44zFFPLpAOUSd0r66q4IZfZ6cWosaLGihoraqyosaLGihoraqyosaLGWlqN1ZbwWDhKCpGQnJHiAkuzTM++pbKrTYUXQTL2HouaW7PlGeladFp9V7u9JLZbHVTkc1u8+RWqSu78MivKyfuUk7vd+CNE4w5OhRqe1kJM7obtwkMxue5isqXzyvzNLFTWowMbj4JBXqhGQxVmDJFNd5xsau23h9ovar+o/aIii4osKrKoyKIii4osKrKoyOajyEqqJey9UIZT6Sy9wtcoUiCt79k9jrL2D85kb6WvKK2VR1rrtTpxqpKihh2lulLXhN/NE/1riY/5Ca9yF4W0wxHS1rvMuQrMIiYtDXilovE8tZ5ktGtdjctvqKtlGFLZSSurtXDrOepqqKthTCUqeKjgoYKHCh4qeKjgoYJXl3yaFhsrU5ZIviXdKlYJEmmWzz48i2aygeqy1/x9sKKoYe5Vwxz20iTPPPuoVy0WsBefLhQlzPpImDcrWYJJmNQ6zcza7z0zNbbN++VREh+9J2xFhgSSt/eCMuPaDDENSZa+CuxFvtO3jk7JxLpdFwxTs25RRdHU4NGPLCkn26zORzW8hYiyqvPt7K1m88/8UdZjdGsvOhRxVTAmZgiy/HpKyIVpeO4N3iwpZClocDuVj4luQiUFnXw6v54QQdOEV51ruWyu0hRyezvRj61y2YLvTIWHKqoBz5vLMGhY/SSNqbx8oMCiSWRpIRk6bpFPFyeZWtDt9FHPRT0X9VxUWVFlRZUVVVZUWVFlRZUVVdZ84ySrFR5JTsj1x32qr/A/UdBmFbWdXfr6qrNsxza55QnwJipTEQKyHmq0+9Ro+73Yo31ssmYqUoxu2fgzk6wqFm3a7w1Rqj3EaFPVNNiyMPNImgKXIYm1Ylh/6wJLF0JeBJ0YwhNVTsm1Imp0AcOBzpjiuf6xRsmD9GwdiiQsl5q61CRYV4gOPVU0nBun69SgM+p5ErzulcwlRdIfTzGINU0Qaze15tlFzRM1T9Q8UfNEzRM1T9Q8UfNEzRM1T9Q8c9M8FfWlHEezX38kUJb6bve+tritoFDV1HHTdxnFuEF7VK6AybxFuEEHD+I5EBEOda1UwXzpha1mC5WtqotTTCz+yoyIItWWK0272QnJLlKAnBVRmF5SYXIRviIK008qTC4SWURhBkmFyUVMiyjMMKkwuchuEYUZJXbgPAS68MJ0m4mFyUXKiyhNK7E0uYh+EaVpJw7uPOTBiMJ0EguTh5AYUZjEOTgXyTGiMIlzcC7iZERhkufgPGTMiMIkzsG5CJ4RhUmcg3ORRiMKkzgH5yKihhemlzgH5yK3RhQmcQrORZiNKEw7fTPtIOFGFCZxBj4MsdeOdczpIJ584zQndtnJquwliHnNM+dAPkcWrcx4EEkJrj8u9ZCeQ87Cqo/K+z6V93Yrfud+kvRevVQFUOMBau9bae9vfK+D2WpJBf9EsJ4EbmGOZPkB2GTQWF+2B5oGNXvjaj2nvGzZt7IJ+EZCsu4eqkA3fGIkF57brvXPH0xpHbIzeuPrW5aM3B6sHtVYCgqVvQOu/E4JUVoIvNAd9FbkFYXbaqd0VnT3m0kWvREYKruBF6KcrIvTBBj5VH+FZy+mwrMgydP7V4PqSRTh2ILqf/jfD5qqGnP9h+9xCMHeccdH8m7nom5FIxaqqRi8+TiVOGfV4gM/iCeooUliOIPJkaT5DT/XKN2X3YMTTUZmv4JKZUPbdrSqLn2tn1UDcVheq46rTYavVq1H+CRJhNCxuwEDjt6nggQ4HQHuDNuxBJiPuBgGHHW9zNFnUGekwBh+hoQuOfwsPaPD6LPy8T1kdZVldfxPTjHqROoYuyCsaoRVRK/vZgwGickHVscgo0IcvFccPEpzCl0jrvOWHAyPegiGEQwjGE7ybrTTYuH2sGLeDfReIM4tGufOZ1NoUGlhLhIDm3bGsztEhb0XvhB1SRWOUvX9+BfANIIsq6Jg0FmBxknG8x83McxFDZD7xF5PwS4Yw7UP6L4dIC8zFkeEjQgb5ebMIXZrhHoz6s2IwzPVm2slNa+EOmINkUPQmnfJ94Nac/GA1WouRK11Rq3eR5/x02w/KjLX5P84HDWvmu3ogbP36PnAdIMAOEOJuZM63GL/aawvF0vjFdEx6tflwc2BNsodNrOj01WY1AtEzlmA49VatM/4ecskU2hyVdvafsdznYFL+MJYLDOxZ4Y7on3Wts/05RWtt3BuR4FLyqqHkTNyGag3spJ9RsD0B2mScjZE1rmptgt5aYTkECko+qU/aGH0y1YsZ9fd0Ot5oGp7ojvNqD3R7V33RPtwLBWp9EyngGXYdDKFqjTdiQeP42+XVXfGOT/S1QRFX0jGpg8P3O99esmYabnYWzMle+s3D2b7sw4/+tVmSZ1ms6bOC9aJ34XJFduxr9IdAXTHJonQ3FV8+mDrJ7cUfRAMmKlO/lndFFYKNV5U7Wnqn3QzzL20P0LiStH0yarfntJbOVYOrD51M/OdXcH68r0PVlOyms7nkkjuX8mN1ZjI9fbrgVrG5rKyRxw7J5O8PTkCgg50/e1PR2VOcIVErRZBVOUKQkoN41vddmm8MGH9x+WGCbtcHhpQm7NIkUMgh3BxCLDXPpFtSopgOSz0sjCFqhrTLn+yNWvGBy6D9UUugFwAuQBygdIp+t30VKCDVACpAFIBpALhVGCmqcsKUwFW/LIQgYqa0vEIJNmyZjRg4q8tkgAkAUgCkASUzyGQAQtoIQtAFoAsAFlAOAtgZ4QBMtOzPdOtYC6wrkRZGEGlzbriBRvZtWbsYBxeZ+QIyBGQIyBHKJ2joJeeIvSRIiBFQIqAFCGcIixMAD6ioBsVZgirOtQ36ZIfyL4PrTLiWMSxiGMRx5ZP684AyHYRyCKQRSCLQDYcyM6luVphDMuKXxZ9u6KmXCnbCbasmaZ95a8tsgBkAcgCkAWUTs3upycBPSQBSAKQBGyUhK43CDmCHdmCxRY0gA57wLjZn0MQxiVY5Q5HC78KVBcR8B4TfvaGo7jhUT+oHGqEURPzfyKmLkBZTw2q2x2MIkdQfZiguiqwdR16LEDhqVZBdfbWABKhG4DX/JU5HKQ6DqkwYtXSHZmVIuU8yrMIJasqzw7SI0lMUIhIEuVZjNGI3jkny3q1N81BBQ4IsAaqi3AV4SrCVYSrJVA+U+PVfutgzsVBUFgNn32/34nx2X+4AgCisKKLT0SWQs/7ENmM4T70OYiZrFHcrBistBDlfLqywBRamJk8EUkWCRs9TWQXsPhjT8PMNOUdprzGiujPtQnbvQI0bVUVAXVZYhX6g2Gas0kjx3cqxJr34aT9wQhjExCh5y8oD9MLyk1E6IjQDy4AgGM4QVuWEuF++0YsZZRwafT80022OLeM4I1V8jKkkgjeSqeGXpYekyHUQqiVuRiaHmsNh4i1EGv5Cn0gWMtxVC8Mc3r/apQbcV2wAtYXa72/+4wYax8Ya0ZFaQEDKmg2T+++D+9+KwAWdR3hF8Kvuipdo/ToC5UuRF8Hjr70JaWz8uOvW1bM+uKvYPUQgSECQwSGCKzUAlgGEGyEEAwh2GFDsNVOjd9NCsu0zItXZjBGyDV7xhxsudpYTHjhiVX4+gK1f/BavgvWEvFaufCaoio0OecNAjYEbAckmbWaqQHbqInbjTeb83C7cc3i1raP+m132vHZINnJ6QAsD2Nnsq7ODcC806Wmivs+yZPdmAk0Ht98Jt++iUuTI+Mbp2p7SivvmHhm9av6GdgeMPXdsnJrtSC5CQGoyDH2uG2l3Ryk2bZiEQ6qEzY07TSb9kRYxVyb7eYQ97MgZSlAY07PWTot5CzIWXCvTZk5AQMB+lT/3aT0ax1x661dsxqHMNjA9aNpEHVO/mFCEyJ6rQl6rUC6I0SpiFL3J6y3UoPU5ghBKoJUBKnljNEwl99UsBa83mBw1FwCFi1XhIbkhGRwvPkRCiuwziHIhBeafL7ZQ0ok23p20vk8Y1rYg43IU55iE43ePL7qkgiGYgPRslasvlydlKLuXsDOAjBRT8a8ohiYkiN+fuN7HcwbSyr4R2tk8t/G+g57cGhQuTeuZnGKzHIOysJSD4zSZPgcCoQbPlzC8W+76/rCh3057u203vg6jR0mMuisvlgKCpW9gwS5xTbcopM25WoHmUUWzALJAZIDD7zVVfGJwcXp3fhmKsiyKuYftZ2ZPr0qM8snxf4F2DsjrEqhcbF5UwWPLSXF1GnFbMnLzI4tEJeOFYk9L8qv7ERNuJxN2MrOZl3QRX5GtU/ZDc6qeRkcagPm/gwIiMCfqvZKoKTEEJcxVh4XYGVVWz4KSsV6r1VoJhxYfyTPA5MCTGm8VMyMxgvz9dmj/wWot/JARFmNHfmXVff7WQ1GoP6oLhyiuoC7XtA5V+4QsrQMuuUWQpBCI4VGCp01hf48uXl3fXdZOernKTfjI7OlLBl0FwqYN7OGolbRvG7TlpFZsyLmyqxzMWo4ezZnGbHnUjMVqD8yFWQqyFSQqZTL1ddOT1T6SFSQqCBRyY+o2Ah0CqBzViHMd+sqtkujzoeelBv96Zhqs+zwL9L9hPAP4V99herU+K/dxq3OiP8Q/+Uc67Wgi6JSpmcGAFdlZjFK45uVkgq4z3gEWJS07+G8GBm10naFOmxt14uawOr3lop8K32liK73i64xVz1ia5RWvdC6k15a7SG0RmiN0Do/aH316fxvlYtQWBcaAOCVJjyUMfSDF9Ly81fNtHZ0QjBWYQ623mPYx6fzXyvXV1dlBnN+El526al1iQFhfYucEbAIspWyJKlq9Qbt8mWpypLZRNS6g1mqtuJAu+6yd0Z/5bbZt6O22Xdxm31WBLEdTRApo0KBVXhjn0wLiWMWxLE2mb3ApDdU+6QG+yeephG3UA4H/fjjND6axmEcpaHA7xgJuV5efoH/Kx9FA76pdsJcpxaMm1CRQgvNiMq/2dNhGl4jQ9/ao5W3sOOqoCzMCwZ6shEr7oL6YLUTub4h1wpwOpaB+GOwzkjt9knthsNRGmrH+rAIXVqFtrUOz5CUKp6b0RqOmkj20OFVQDBZboRmiHwG+Qx60crNC6YACF8EbSYpD9WmBet6AJ4FiDdffa7vXgUXor0Kry6C2X2C2VEv1WkabM4Dyy30qnkqRj08T6Ou4NV1rnpgYiqVUN/NC9e2WvuP8CoLcEWp/WCk9sMW0MXFcnqtvNcfKq+eszoARCbvqa5Dj9PJy6MkPhKY6gn0Qcl4JZotq5+SD6pBre0JxqOkEztwgkiKKJvQaYggy3BF1al9aUbuX4kEtrpWLjWNrVp7luRZs0FnrX672ZWAhgu0GzSM3XCCYdDFkjWDoRIdfp+iBeGFiU1YcYfA9fj92hWAtKncW70XTrevkrSPnAc5T6Fifm6kp9dEzoOcBznP4XEeCwVWnfVYtQiDzwHaQ+4BYcwovARmD/go6ORReGaOBQYYT/QlrAZzSSTUeuTbe2HGrxDxkYpPurnQjwn7TuaNekyoIZ4e1dcJwaseQhUQQyOGRgyNGLpqfoNebhAa3QYIoRFC7wyhjzd/0a0yOyys/nnGoPrECRioOlpfVQQA+/pvB6Dv2anAbA09uSbGdtfEY209ODPXSPtnKbhQ+q8GbVnFQSFvQd5y2MH6uZGTVnOE7ATZSf3YSbUQfM3Edoay1oiSh6GIqinPiKIa5J6SGZUB0GtWwIqgEGG5lCVRYB2vBCD/g3oDY73irWFXItAYK2+HCmYnS7iHvEjGI7QMkSUwpxKWICfHrFmhTfBOAqtmOyriDRzZl1lJLBvyv7Lv1ONGhj33k/h8b87rMJm4a2L3Yfg8p5rjfttAFJgUNF/cKrOaWN1dk1Crs+DC+p5VzuqLHs1ybabqtjvtMm6myj0VAFQc876hV3TPXtF+bmnPRig8oPCAblF0i+YgqtyJTFS5pYXv/mHvMsJHYz6SC6sjT+T1sIC+v6ZFx/ZWHxa+aO3zWRF/SbHQ/Cm5W20NWm8M0v1PYkLBOgE0hRreyxIsfzP4paIAUuGDdr+qDWtvGE17aPDsvbL+FmX+2GBrrhvF3Qw8hJV+8d4K1w1BUtgXqiKzeFc+7heSYWySQq/iDl92lhU6fNHhiw5fpGQJlEwXNYlj0CCnKZc3uJ+fN7iLuduQtKG3uMTEhiUbfsc9dawGULuXYjHvY+RoyQMQ+2rKDieFwul8/5blryS/m9QEQDwngn1gDVHt2/ceMupprommLuvfVLyWjL/884POHKHvrm/vLj+sjz19UFQttl0uimsX4Aa3r8oniw9Vmzh6qmKZ/wT+Ozsf/91F+HiL3GtUeCIz9UVZX9F1Th8BbMOPjqHH6MYZqwDD4a6fZ3PwUyqab1cxO6q/XWCA6/Ve2cXNqU+s6ABJP+bzlGIu7qnGZih+sLLzG/cvPBwdJjONLGCYkKVGnyXV1IG1b8XZJ40MLc5jIfYXhREqb61ELPKWnj6cWhtvmXVXe3GPYt3U1dAt0BddBdWi/MeKoUKBIelZO4YHuaWPx+2yqCHUz/EbGuLT68YfWPVe+DJWFYX8eBrivRXZ5MRu/ONw1LxqhkVJWRNGs2L+3AACHcOAvWSOx/zO0i2WKq8qxPQMhlpdLjx2dvE6R43j5GO2oISlfJTga41c3t6dXwC3/vlywtjC+N3H28sT8uv59V3xx0i7m8rusXVpKGcAnpB3EpAv5nZlrcLnXh+vc7ehK7uQKChEN5c80P7tRAJUK4msHf/TOGn9p3FUb8frOCJCAFnMPiNq+6N0pyjHRH2U+nyC7gDP1kLv7d69t7kxpxEevIXOWyRecYGwUNqDi4Rlnq7bV0VU1Sew3JUgyXRWYzekv6rsWDAFHiHNmG+L2NfKklQo0D6fqPh8EK3DKmo5Krdpk4t9tAmbNg6iTfh50N42SUgHNa4+SYXqj3llkaJWwtGGwcFILw/a9TbMzfXWRtcbMkAM3811X+K5yLjlxyWtesihqyIhbiweAfcoPFOyEGaUCGQmAbw3rDg2id3DYhGZo4SHKl5+uLN9XHOYgvn33Kc1sb4twf7CG0HX69Fw7pqkbrlP418m/pZzYn4TWq4GDq6JZZq7lWmQQ5SdQ1TBe4WsIYY1vPG9DiaUJRX8Y3U9Tj9Q40XVnsgHazJvrO+wx4cGlXvjahanyCzCRhaWemCgJpOGUPjf8IE6jvrbrh13fsTP0X6n88bXaewEKoNVApXGUlCo7B0n6LDLim71utF0ayNa1d3vGXrb86YDcJwhN4rhRgHz5hPiBxOAsKRTNstSBl6nM1Pj6TIBCsM6PdPrEkX27duqjt+/M32dV5w41a3vGXEfoM3I5RcW/EY1p9p3fvyIMLl4mBwDke3Bh6o6quoYtJUZCGz1BrVBgQjkqrG/pDUKC/V1B0RZieB+jN1dctXujnrj/cRM7QuJ6qYoUr3uANSuZfE7Rgz6xYCpfGV43g1rbmynzs6+9Rb5K+/eNd4CEgb/EfmXZR9IqzvsptsIkse+99zzqkO1e7gNJF/F3T3u4/T2N/ZD2YzHZil2tdO0ZvCGLj7ShfAL1XSLH7T71tfGq/WWmaA9WXcCNFuPgcaTeU81hRowLlbPN+hiKQNAUR5WBmqw9FGukePS6ddA+psXk7FZnc9KIcADpndeKjoXTNmH5vhIdl8NY2sNvjAEOoSzern9Ana23nM5BIV702naJMx1eQHvl0J+Zs9Qjcnt9ObTx/eXdz9ffr51/3ANo92LQeN3k2p8fC3DWJ6rZVuebx/oF9+K29CfpOVnTWbhqCHlc3qPq1Z+x4irCQVZ/sU2us+crsZ1G38DxspaT1KkELqaedP9X/V+mzb7r/f2pJbir5lay4sV6GCyJ00lZa4eE3jYUS4taK/b7qfAyOXNpP/DKVvDezVQCfZd+M12/7Bq6bpg6vTOepB7Nt1Tp4kz/TcP2mU5ny08fJRnT/tZ1Y2/bNPXWKGz6mwb1Rj7X7oV6d/Cydfffvp/yvOC/rt5MvrtJ8X6J2KJirx9hzWrvW139D0mqS/OJP1pRp8l0cP63F3yTRQ6dNsuTOLaxnjbmc8NrUJHzAbv3HCtFE3dAHb3xv1axiW+W6BI4la04dDc4oEwvl9OWg4bAoJmf9fw/GwpAQ/S1j+2x+gq3M1t+0bPFRfRaro+dNwfWov13z3X3y33h07TfWXN2Rpt19+tmVXj35w6MG0g2Cei3+J+cN/9YPdb2l33h/WepcZg5i6vUxaP+b6qXL5q3AMU1m2IHEWbrxw9s2FyIbmh/XM2m8k3v9rg93kFkHv9N9//P4On2tnk2wUA apiserver.json.gz: - H4sIAAAAAAAC/+2d62/bthbAv+evINQNSAa39StpXKD3Is2jK5C2WetuH9rCoCXa5iJLKknZ8TLfv/0eUg9Tr8RpndhJOGBtxJco8jx+OjxRL7cQsno96gWh4NZL9OVbLSph5HtIGdHKsOf5Agvqe7LwEoqg0KVcqCZwOVfNiEMF7rsESgfY5UQVDj0i3jpQ5IWuG5UwHIy6vu8KGkB5XRWOqEMOfU8w3+WZ/jTT16XeuTaxAHtEtf+i5hTNDMptGIl4cnpWd0TQp9MPaJsTNqE2QS6ZEBf5/b+JLeiE7CDsOcgXI8LQmAhGbY4cygMXzwiUe0iMKJRgPur7mDkIM4IGPkPUgz/HalGwi4KQBT4nHDq4s2dWLZmIgwXmfshsoj1FVEO4zWgg+69/mkNGnTN/sbmqcASXzdrieiqv21rBRbJ78fVMXseX83RstYHpONbYd+RaWGPMzh1/6i0mwWE3oaaxaCuoUNJkvfcFrMmiqZgFqlyQC2Gpwjn8GckEIwMQXjl5q1XnVlTmT0ulxHVxwDPymqlwCjU5gdOHi3pie0S6dEz8UOR2PBnZZ6+xfT5kfugVxl+0+RO7Iamuzk5BlT9pdjp2e8+qZYvZsI+3m60XNVjYTg216zVUf7bf2cm3e+K027iFLa30W+a+GVG2zpgPYjgiIbeyrYhNx1gtUStXkZH33/0pGmNvhgICI3oC+QMkDQ/hgqPtPgg6XGJHSf2UUQEKQD3UqoOIzzga4QlBfUI8qOdTwkAFeGjbhPMBrPhM9RpgLhDx/HA4+m92jpFGyFnENw89KrJNhjgckow6RMKLL5KNadTruRUcUy+pzFfxkT8t20wpyiMprb7rnOI+ydq+slbvQG2I2n7BQqI1mmfnv1DobIXSxuzOULCVbILdMnnN2lttHYKAesNupIaNqrqilGbXE1p7eKzkaSJXDgkfKZWu5ZtNklXPVMxryw3OsDdcYvBmdvBKTQApOAJlOPNh5XhRFCy5iqryXWzrwB15YMbBnhTadeWMShYezLQYUGlhLaus4gQ83Cf6jxp+t/5rrg3YwNK+qvzKrmqp3oHxu37rBmAC5Bhy7sVFFdGDWe+fH5RU+mnHJdc8dg7tfCE7BxEt0dQBdd1DaSnV/ksb2GqACWzswx/7HWkDG/sFGyiNR4UCyrvo40XDNcGaNjqtwkC6vlfqqGIlGDMce8WtEpgNiVhiF8hFoOaEAyrRgbBebEZf4gmmLu5Tl4pZq+5cQl3/1VcLu+5Xq4ZsN+Sg91DwS/zjV2te3KmFrRTg1XpwC0p4sVliQ06wLdQaNQtNXDIknnOSjlccBKRTkaJ1sKxUpGaRl6xgAg8H2jqgbViJHfQf1Ok8q9fzwi9SKL3M7yfwlHP9jsZYwsH+ubAFOOdVlKHR1W8/PwPVYjn18+U8rVc31LzE1t1I+VSn97ExxZPhop/2/BkOwi7F/DAhlZwP6mNW6ucsia+nxBuKkbKqhTpS1W3lZBLaI0QYA4Duhw4oIgK4dslAINf3z2F3EQZiCRmYkU6n/ivSVQ0NQwxWVBDC89xBlXnJPZgsfcOwQ6MXlvoN/HjWGkYKVhRe2Iuhd8C7+VczvQnsaXmNHTIWTaysFjxhVQ31KmoYHY5AAZyqucS2U+JNvgY6/UUdJR15h6m0V+Aq860kmF+rwNLK85Kbq4ppfOvGMoRUYICCl0zgEw/LOT+QvaVchHL43WJduTLAbjkAxMonDFw/Z4EYCQguBY7Isn8AJwEiR0qeCJytTaq0M3bP+9lCAS86pZMEdxMExDmNXHex/sbeDxAM/Ya2V+IE0dNIreG/nbt0iMreROZm5b4xt5dy6icRvhUEQdZ9GtGBKK+MneqxnOzryDauwqfmVd1nouQ9SqlxL3Gz1HPohDohzjiycpesgk7ZmV3gC8qL8+qH9nkkeyU2JoleyEXKQ1/8tlHSq9qkpXZJhdHKH2GGL5Z5k6ryb0u+8kYiKd9BS59BVvrD15gXXvg0R1DaLfIEpVXautzove4hPaquv1v50lRryw13XPoWTA2O8aWk+qM/zQRg0/l0Y2XOhrhSFT9Kg4kwgpWtTwB2tFcMyjFovaVto4m3bWy8DTta0O307adu7c1x10TaVhdp2zWRNhNpu5eRtpaJtP1spE3a18cVavsoPUox3maibI8oylaOHZWsAWCAOAG77SAH3MaIoFS/OJJvl/0ZssG633Ysbc/E0kws7fZiaVdbGaXI8i7Pm8+ePadFg2InDvHJ7t7BXvu1dSM6Wwzfum74k+bhYf3wR4dvXzd8q/mifdT50eF3rxv+uN48abdvQDrXRjRbpRHNgkyvPKDJw7G0fNvS9O1EfwFuRPb3ZYFAekpfXzIsyO54Kfy408jm5aUy4Wg+3/zApkKYT6dv0VP0MXZXJqC5loDmQh5BygN+b6OV9+Y5ykOR9589lwl5aRgqUycZESHziIOmVMQnwdB89+Jip4JDV4WhLwyGGgzdpCPd1rqOdBMCSqjn5yhIdn31v68WIGQ1E6HnaLV33Qz2SuZ+3/hLnTEb+lo3fa3pGLV+CwC2+Y/yUBksYisu8+hkjK/TAayKd4O60a+qSJspU+sUlf2rqAyjIZ0QL7Vgtwxf+wa+DHwZ+JLnqBGs9L6HWCloCfY4YZR00otV++WIcuGDYxynvTbuIOwew9BRvNwGh9aMQw8hEMVNPpzJhzP5cLngoPo9Uy06ePYBOPTsM/x/0D38/d+j49Pj7rHJjltddlzHZMeZ7DiTHfc4s+OUtX1c6XF/KQdj8uNMflwBQ5Zkj03JlsuNZEJlJlRm0uVMutx9Tpe7mkdMvpx7FdOYhDmTMGcS5h5gwtzNuXSt6XONhuFSw6XmCHd1+XMJFt1xAt1m0di9OjTWicyk0JkUOpNC92hS6KTVVCl0FZh2x/l0jaahMUNjhsZWnlC3OWdn9xmOTEqdSakzKXUmpe7qlLoNJMSVc1rrAXNaaZUBtQ0Htb01hs22ZShqe+qzcwAzcN/YcXgUpLr825f4dR72ydMU4CSHUQ+m6NkqQvZLclEJaF92x992dlSYLGlbQ9Ib33FcK7n5fI4uL+X97wXBwb6gP+TGoAMHHAfslQG4NQOcf7endLcR19rcR7jv8ayV00rb0IqhFUMrFbTikECMDKhsFKgcyT0xlLLuMNNIrt4955RNfghDKjlS2f0RUilRh2VApaTbz3JK2RHXrR5+Xfm7hAZSfg5Scqexd0gpxYOsbfkvLNVQCb7EEJM7C+tF3ue2saaGXLJj2OZGbHMK++fZM0M35hDNHKKZQ7THC3vmXxAwUamNAr72ungvYL78BorM+YbZe6I3JmOfzXr9mSC8iuFuhnDrYrTNR7N3aq0Njq0Zx5SsPwAk2/DnMCGnHIWYD8gbCjEUEn1tS5DtBEXsIExjSVem8vxAJMmwSBWLHJ59RiGX0mxwxJx9mbOvxwQi5mPqBkQMiCg/4/eGPvNDITfQhD9uFznepCttmOOxMcdtnUpt9nM85JOprfgJLG6PyBj/SRiPZhklAYOtnkW3cjA7VwOBeR4upFTZWeYRGcNKxxJkHLhY/qJ8qnXgKbnQZHshDRpkZBAu+b6pQwY4dMs+sarXlhGhNYo4QzMuFTKVfkBaQzmt1lffCshbU+t7SNgsDscXkU8abmmHsxIKpUOS/2Jzui/a7bdyj6MtGHbd5PvjucfQllJfhyUANVkrzVuB2QV/6JCDsm8lpwuZ/OK1PtYYdoSW9ElWuaTLdUusbteLrOh2xRduUle+U74Nzeu2IXZG+naBsKu15n8kM7GytYUpy7LyxvEmR8+kVYScdKOBdD69652vV+181ptVadB1e54gzqo3vTJjqRTrFolMD0NGthKnNI8MLx0vPpG++G67P33aSJgo+SR75AEW3QIKNMQWneMl6SU8q4OJtbsQIKtR1y5a+kVjvPh5V/u5oV+06nrNgtyspvZzw4ns4bfkGSSBa9Jz7V30gff0gfW7NNv6xeIb/tYLR59vMpfM8v3jq5cZ63P3MPaSiY8+OHuLIqmMKkL19m7VO8Tex7hBOp09Z28wsJ29/caLfh8PBm2n3+hHjSepP65vzf8P1OdmvlGiAAA= + H4sIAAAAAAAC/+2d62/bthbAv+evINQNSAa39StpXKD3Is2jK5C2WetuH9rCoCXa5iJLKkk58bLcv/0eUpJNSVTitE7sJBywNuJLFHkePx2eqBcbCDm9Hg2iWHDnJfryrZaUMPI9poxoZTgIQoEFDQNZeAFFUOhTLlQTuLxUzYhHBe77BEoH2OdEFQ4DIt56UBTEvp+UMByNumHoCxpBeV0VjqhH9sNAsNDnuf4019enwak2sQgHRLX/ouaUzAzKXRiJBHJ6TndE0KfjD2iTEzahLkE+mRAfhf2/iSvohGwhHHgoFCPC0JgIRl2OPMojH08JlAdIjCiUYD7qh5h5CDOCBiFDNIA/x2pRsI+imEUhJxw6+NNnTi2biIcF5mHMXKI9RVJDuMtoJPuvfppDRr2TcL65qnAEl83a/PpMXre1gvNs99LrqbxOLy9nY6sNnI3jjENProUzxuzUC8+C+SQ47CbUNOZtBRVKmpz3oYA1mTcV00iVC3IuHFV4CX8mMsHIAIRXTt5p1bmTlIVnRinxfRzxnLzmKrxSTUHg9OGSntgdkS4dkzAWhR3PRg7Za+yeDlkYB6Xx523+xH5MqqvzU1DlT5qdjtvecWr5Yjbs481m60UNFrZTQ+16DdWf7Xa2iu2eeO02bmFHK/2Wu29OlJ0TFoIYjkjMnXwr4tIxVkvUKlTk5P338AyNcTBFEYERA4HCAZKGh3DB0WYfBB0usaek/oxRAQpAA9Sqg4hPORrhCUF9QgKo52eEgQrw2HUJ5wNY8anqNcBcIBKE8XD03/wcE42Qs0hvHgdU5JsMcTwkOXVIhBefZxvTqNcLKzimQVZZrOKj8My0mVKUR1JaQ987xn2St32mVu9AbYjafsFiojW6zM9/rtD5CqWN+Z2hYCvZBPsmec3bW20doogGw26iho2qurKU5tcTWgd4rORpIlcOiRApla4Vm02yVc9VXNYWG5zhYLjA4M384JWaAFJwAMpwEsLK8bIoOHIVVeW71NaBOwrAjIM9KbXryhkZFh7MtBhQaWEdx1RxBB7uE/1HDb9d/7XQBmygsa8qv7KrWqp3YPyu37oBmAA5hpx7eVFF8mDO++d7hspw1nHBNU+dQ7tYyE5BRA2aOqC+vy8tpdp/aQNbDTCBjV34Y7cjbWBjt2QDpfGoUEB5F328ZLgmWNNGp1UaSNf3Sh1VrARjxuOgvFUCsyERC+wCOY/UnHBEJToQ1kvN6Es8wdTHfepTMW3VvQuo67/66mDf/+rUkOvHHPQeCn5Jf/zqXJZ3am4rBXi1HtyCEl5ultmQI+wKtUbNUhOfDEngHc3GKw8C0qlI0dlbVCpmZpEbVjCDhz1tHdAmrMQW+g/qdJ7V60XhFzMovSjuJ/CUd/2OpljCwf75sAW44FWUodHVb7c4A9ViMfUL5TydVzfUvMzW3Uj5VKf3qTHFk+G8n/b8OQ7CPsV8PyOVgg/qY2b0c47E12MSDMVIWdVSHanqtnQyid0RIowBQPdjDxQRAVz7ZCCQH4ansLsIA7HEDMxIp1P/FemqhoYxBisqCOFF7qDKvBQeTJa+YdijyQtL/QZ+PG8NEwUrCy/sxTDY493iq5neBPbUXOPGjCUTM9WCJ6yqoUFFDaPDESiAVzWX1HZKvCnWQKe/qKeko+gwlfYKXGW+lQTzaxVYWnluuLmqOEtv3ViEkEoMUPKSGXzioZnzI9lbykUsh98u15mVAXbLAyBWPmHghwULxEhEsBE4Esv+AZwEiBwxPBE4W5dUaWfqnnfzhQJedIyTBHcTRcQ7Tlx3uf7G3g8QDP2GNpfiBNHTRK3hv627dIjK3iTmZum+sbCXcupHCb6VBEHWfRrRgTBXpk71UE72dWIbl+FTi6oeMmF4j1Jq3MvcLA08OqFejHOOzOySVdApP7NzfE55eV792D1NZM9gY7LohVykIvSlbxuGXtUmbWaXVBjN/AhTfL7Im1SVf1vwlTcRSfkOanwGWRkOX2NeeuHTHIGxW+IJjFXautzove4hPaquvxvF0pnWmg13WvoWTA1O8cVQ/TE8ywVgZ/PppsqcD3HNVPxgFkyEEZx8fQawo51yUI5B6w1tG228bW3jbdjTgm7Hbz91a28OuzbStrxI27aNtNlI272MtLVspO1nI23Svj6uUNtH6VHK8TYbZXtEUTYzdlSyBoAB4gTstoc8cBsjgmb6xZF8u+xPkQvW/bZjaTs2lmZjabcXS7vayihFlnd53nz27DktGxQ3c4hPtnf2dtqvnRvR2Xz41nXDHzX39+v7Pzp8+7rhW80X7YPOjw6/fd3wh/XmUbt9A9K5NqLZMkY0SzK99IAmj8fS8m1K07eV/AW4kdjflyUC6Sl9fcmwINvjhfDjTiObFxfKhKPLy/UPbCqE+XT8Fj1FH1N3ZQOaKwlozuURpDzi9zZaeW+ewxyKvP/suUjIS8NQmTrJiIhZQDx0RkV6EgzNt8/Ptyo4dFkY+sJiqMXQdTrSba3qSDcjoIx6fo6CZNdX//vqAEJWMxF6jpZ71/Vgr2zu942/1Bmzpa9V09eKjlHrtwBg6/8oD5XBErbiMo9Oxvg6HcCqdDeon/yqirSZMrVOUdm/isowGtIJCWYW7Jbha9fCl4UvC1/yHDWBld73GCsFNWCPFydJJ71UtV+OKBchOMbxrNfaHYTdYxg6SJfb4tCKceghBKK4zYez+XA2H64QHFS/Z6pFB08+AIeefIb/97r7v/97cHh82D202XHLy47r2Ow4mx1ns+MeZ3acsraPKz3uL+VgbH6czY8rYciC7LEu2XKFkWyozIbKbLqcTZe7z+lyV/OIzZfzr2IamzBnE+ZswtwDTJi7OZeuNH2u0bBcarnUHuEuL38uw6I7TqBbLxq7V4fGOpHZFDqbQmdT6B5NCp20miqFrgLT7jifrtG0NGZpzNLY0hPq1ufs7D7DkU2psyl1NqXOptRdnVK3hoS4dE5rPWBOM1ZZUFtzUNtZYdhsU4aiNs9CdgpgBu4bex5PglQXf4fq68EZu0kEowHMLnBVcOyX7KKSzb5sj79tbakIWda2hqQjvuOQVnbzy0t0cSHvfy/gDbYE/SH3BO154DNgmyy7rZjdwrs9oLuNkNb6PsJ9D2UtHVTaFlQsqFhQqQAVj0RiZBllXRjlQG6HBZRVB5dGcvXuOaKs80NYSClAyvaPQIpBHRZhFEO3n0UU08HWrR55XfkbhJZPfo5PCmewdwgo5eOrTfnvKtWQgVxSfimcgPUS73OLRFNDPtmyWHMjrDmGrQvcqQUbe2pmT83sqdnj5Tz7TwbYWNRasV57VagXsVB+9EQmecPsA9Ebk3HIpr3+VBBuwLeb0duq8Gz9qeydWmZLYismMSXmD4DG1vw5bKCpACD2Y/EWQCyAJF/WEmQzoxA3imcRpKq0nR+IH1kMqcKQ/ZPPKOZSkC2J2MMue9j1mBjEfjPdMohlEOVnwt4wZGEs5AbaoMet0cab2SJb3HhsuHFbx1Dr/RwP+ShqI30Ch7sjMsZ/EsaTWSa5vmCmp8mtPMxO1UBgmYdzKXVO4z5hAZGRq9lYgowjH8tfhZ9pHThJLjTZnkuDxhc5esu+YOqRAY5900dU9VoTDDqjBDE041IhU7NPRGsUp9WG6msARWvqfI8Jm6bx9zLtScMt7XBeQqF0SIrfZJ7ti3b7jcLjaAuGfT/7wnjhMbSl1NdhATbN1krzVmB2wR96ZM/0NeTZQma/Wq2PNYYdoYY+2Sobuly3xOp2vcSKblZ8w2bmyrfM29C8bhtSZ6RvFwi7Wmv+RzYTJ19bmrIsMzdONzl5Jq0i5qSbDKSj6V3vfL1q5/PerEqDrtvzDHGWvemm7CQj0c2Tlh6GeGxk/ugysbl0PP/++fyj7OHZ00aGQ9n31hPjP+8WUQAhNu+cLkkvQ1mdSZztuew4jbp20dIvGuP5z9vazw39olXXa+bQ5jS1nxteYgq/Zc8g4VsTnGvvog+8ow+s36XZ1i/mH+h3Xnj6fLO55Jbvn1C9wjifu/upg8zc897JW5RIZVIRq3d2p94h7i7GDdLp7Hg7g4Hr7ew2XvT7eDBoe/1GP2k8mbni+sbl/wFvRsHeLqIAAA== cluster-total.json.gz: H4sIAAAAAAAC/+1d6W/bOBb/nr9C0HSLFhuPLR85ChSDnE2BaSfbpDOYbQODlmhbiCxpSCpHi+zfviR1URIp24kTOzHnQyd+lJ/Id/5IPtI/NwzD7PddP4wINt8Z3y42YwqC/0QuggIN+H5AAHEDnxF/UhIlei4m7BH+yUiovGUQuR756NNGazOnOoAAHETIhrTBPEXBBJIxjLApPAN9MPBYO0ERFOhj15FQXTvwDwIvQIwhGg3Am9am0bYs+k+vt2lYb0XWPpjwF+/lYzFeG3seRKTQBXIb8uccgMeDACDHTNru+P8v6L93XCjQcUmpt+bIh+SjQyl+5HkxBYFwfB4EHnFDSm9xIhvOQeATFHhMoEPg4fj7buG7nutfCloIgQ/587HIU4GbduB5IMRQ5FRocCotI+Q6p0GuzFjIJYVd08/trkC4SQeQfL5ln1PpZLz5GNrZx7zXFxkNwRACIgxVoH4kEHH1yJu/BNcFAfMmPA6uz11SsRyTJETzIEII+sTYB75z7TpkbBafOXN/8OfGW0JDYgkouI5tIBljJnjguQBzA+SSzEUwAJxS6Aqzp9+hPyJczK0CHeKKhqZ6izl0Pa8gaUb4gIDj0oGKmlJre7ekbas9RduWXNud7KMHR9B3im+iYhr5e/hc7trgalQePDPdWF/ySHA0CcmtvOm/EAXVlgm4kbxj4voSKnJHY3ImjTfMzCRU+uxf3KSKVsmaAsSGkI2m1HQIsV3lR2h48iQduwJeBFOzquiBxgqJFTHqddI3S6ReljySyuKMemrB4002mtPA9cmngIuDE0zBs20os+gQUqv1CRhVI1LIuDETjdj7e0V6tftUaA5EkMf3oRfkAlREEAyRC/EfVxBRi4elIWJVf3Fl4JgA+7LSG0xgGELndyrTShsBaAQJFvKhmBFZtrgJ+TCY3vu0b/YbHE3euDTUwTc0jRFAuaI+TR/XAbqkKdiG7hXsD24JxH1uET9tL6I9QO+/m6+SP7+bmyyr8YG9/99389d/fzfvvr2iooSIWsu7VzSJB17EgunF27fG4NZ4kz3/VkyOLHgEaMIFSuPhBPZjSRYfSfkeA5vwnGsVmmPXP874/PyZvezursgJwSEP4uZekc4kzHWTEe+Sv3I1kjEd1TjwnJJ6WbePaZgs2wSjn43dIak2lHLDF6oLIxga+0zoxpdYBY6QD7Is/lOMCQDxBFuOCrHrt8sO3E+zius77pXrRNTXK86cPsPBQ96BG3DjlqL4ILIvY7srhZ5J4rIVRaZAqByq4timDDrfRD2l8SyjXVSGcAtuYI075Oa2H5bMzAMD6FU6yBqC0T7gMKdoeHF0rzweh/dW0cDyCJ6b2Obqd3FDFPP6wZACoQaHdDUO0ThE45DF4RAqIx9PXKKByPKAyHmiA6KxiMYiq45FbOrmE79G2ATe8HGcU28oDoSrN2uaKpKU0Z/sa8YvewpmWfO8DPfrGe7PzfCgnuHB3AwP6xkezs3wqJ7h0dwMj+sZHs/N8EM9ww9zMzypZ3gyO8Msxyg45u01OWVmuJ2DqCFN2unq5W7rX+b8cHvqGq+lWOTtVVBfYarxCKDPANj4waD1DNAM2yjgsip0ioW0Ewgc/o1iS5wlBWHRYFaShhMjZZ6ZKjKZC9jdenUpiU/p5GHSZhO9RCaVIG6ns8BcuqldwRwQ/U3/a3z61Dg8NE5O3k0m73AppzjQdieAr9q3izmFKq+amZOGfJfDPETUTA0nuPbNymNfEZOqWWwIAcU3yJcPWYG0CvCETrocWHpb5Lt8vPS7iEz35Ezolc2C6mLQeqhCnltnUYcfTQbUxaTqYBDmIcqQAeK10sf+MvWxR2e1dEqtnUMO65atjLV3jsNl6iOduacuYZyCeE68lqo4WqwqwnupQnCI9dbG8Spoo+wYxiEK2GLmeqrkw4o6yHpr5WSZWvksX0ZYBdkXV5XvJXqnuTPYBjuDTnvL2QZbQ2vY6ra73dbWVmdrx97aBsPmZUQl6UMCcSPZhXD9USNbPWmEgYN/C9Doo/Peeo3gkCnnfaeFX18BlD/2/lW/b0Nhl6isa8VyzUOVXZ5qVncNXlChCN9ILe3MYAIk26WyTZt27abNvfdpNp//vtcS5Lq/OLmCq5E210SsB08j1rUz18OnCQOpuYYxLFoDwR49cXxdH8keL8dknRjJr4GAPyzJdNdHwif3KJJ5cDHMGQFE3ABNUXgsAFURQnogprDDJ56HKTQs7jiMaqt0S3IepmYmKKnmVFV01lV11lR2znokS1LhWVvlqZSnbPtZWvEpk20uX0ugFWXD5bwtsetqN2orQdXVoLUVoVOqQusrQ9XVoeoK0SlVoupK0SnVolMqRuurRusqR5XVo1VdSqtI1UUFisICZXHBDFWldZWlU6pL6ypMa6pM68sZ1NWmUytOa4sT5AUK6urTaRWoykWOcqB7DrPHGQtSZyhKnbMwtW7RQ5F7xfxbyMHT17XkObo2TxdzdboNOeUEjbJyta56VVHBOlMVazWwyKtZFRWttVWt9ZWt6urWmgpXdZWrvNJVrHatDrVa9Sp1QXVpaW15aV2JaU2ZqarUVFluWhnXMx3ChkxltbsBGgPKiLOAwB0NAjUI1CDwZYLAp1/r1ijwIShQWZCmgaAGghoIqhZOn/6KnEot6SKuyFnq3USW4pj47opdTpRX7564mKaN26e4nKh8zn/1rwWYru+2XN/CeO57L0CJ+BjXApSJPLCUiIu8FOBh5/uVJ73aDz/pRUGfD+3CeeeXf8a/eBZNXzX0Ik/4J0ui0hz7HE/0k+JZvfrz/KUx5PhWn9h/3C4+6PagF4gT2gpcaFkaJ2icoHGCvgpo6UAhXTXTSEEjhRW622dZZXUdRVmd1V5oXZ1sd/R5b6oWd3NqCus6U/ZUrc5CN1UlDY+9pyprkGCnR91RXdymqOwwXhVbLWZPtIqz9MaoFI899r7oip1V0Rujs22MznYrxIvcDyXly5xm2Q2VjLMKARewzxk+/33OcH0K3tYYnHVbU8BZV4MzDc40OFtZcLaM864anc2HzqZeFKUBmgZoLx6grUAhWtn5FlN+9uRLlqpNxp5eslwMKu5NQ8VbGhVrVKxR8bNZsnzCmzQ0OH7Y0qX8hkiNkDVC1kuYGqzJwNruFLC2rcGaBmsarD2fJUyN1p7TUqYGbBqwacCmAdviANuOBmzPHbDVe3wMJPY9IL39v5h5/hoDYrjYOD84Nb7AFCv8Vo0TUfxDAGNCQvyu2QS2DR0X+L/awaQJWeoOkYshbg5ozGgm2KMRQ46GF2DcQCl3jClywA3gOw0nCj3XpsmvQR/0g2sPOiM4oexw05wxMWu0+izRKgerPtUMw6m0E6R/bof9xALP4AhLgakShhpNQ87xj4jMzy0GtfG1vE+PadP3vhBIWwgs2AgiwsjA8wzMrgnGcMQdXuPbJePbONTx30N57jh35Yei8e4T4t1djXc13s3x7q0xCWhkCJBx9vdnA+WpaTroHblkHA045A0zz2hyzEFRDs0+EDUpvKXCaFqtducX/jd9miW4Rre1Y7V3Ols9jWzXD9ke3dD/HZye3foJFloQvL03YNYQd7EQl8USBcwVQowGuRrkrjnIXYE61SOEGJa+T5nqRtJzM/kBS9baacW82GN5nDGxPYYT8CdEOO5/vO5KQ9ht3AsHoMv4ezQL51Zs5r+jaWZvI3ASelQQ/ijzSgqOMBFsv3Dfjcd/EbV6TQOISCC5EYaR+3YQcQzbaZVbYs2bVkuMXwLqLcxWCLzh9tmbFGPKVdIj1iCb8swy8+BouVSy6/o02Tlwz5NddSP3A3MS0bgreTyJU2aeKMWXByGjlONNOXRDL0ZicqicSqfTqtw1mwmItalCnvptspu85apQKWPmV9UNzBor30WbNmThQUjL5j8RRLeJEDZ7k80CO8Hp2gXqCN6Ufv3KxJdu+BV5FHDZEkUnSVSsDadOyJ0G/yftgllsLUMITpM/nGXhGBqJbRGG5zEv3qdyqfk6eXH7kb1YJv55fbjGq7pqS+/OY+nd9TXxKcbnwCGgOlZYYNoqNUNJolBYUWotguHK7UWqvFBq4LkSrWlKTAUpvP5eMUEQ5Yq6YzInnUe6/E39eCbwhuGifhg4fdcfBptGwu7tfZ0ndo/W47pHPJLpvrGRguS7GO65XGQJ0BvGk1TTD64bWT40eQpgNLPwtdClczSUfzkRST91VDH0mT0BBIt5oQBPTCvPAWI+MC3xQ6cltgh3jrWFvy0nNu6LdAxsYaEaktVvERlviYzFt7S74gfhjsFtR+xv2peC+H4EfFHI/Hp+kGDzdNLwOftNe6NpHAi2bEZ8mdkcDrc6PdBq9wb2EILtgd1xnO6w19rZ3W2BDtyNH77K5gOtjbv/A4FidTvKpwAA controller-manager.json.gz: diff --git a/charts/kubezero-metrics/update.sh b/charts/kubezero-metrics/update.sh index d3dde44..fcbc7d9 100755 --- a/charts/kubezero-metrics/update.sh +++ b/charts/kubezero-metrics/update.sh @@ -12,6 +12,10 @@ patch -p0 -i zdt.patch --no-backup-if-mismatch # Create ZDT dashboard configmap cd dashboards ./build.sh + +# Patch for the apiserver dashboard +patch -p1 -i ../zdt-apiserver-dashboard.patch --no-backup-if-mismatch + ../sync_grafana_dashboards.py metrics-dashboards.yaml ../templates/grafana-dashboards-metrics.yaml ../sync_grafana_dashboards.py k8s-dashboards.yaml ../templates/grafana-dashboards-k8s.yaml ../sync_grafana_dashboards.py zdt-dashboards.yaml ../templates/grafana-dashboards-zdt.yaml diff --git a/charts/kubezero-metrics/zdt-apiserver-dashboard.patch b/charts/kubezero-metrics/zdt-apiserver-dashboard.patch new file mode 100644 index 0000000..4b6adfd --- /dev/null +++ b/charts/kubezero-metrics/zdt-apiserver-dashboard.patch @@ -0,0 +1,67 @@ +diff --git a/kube-mixin/apiserver.json b/kube-mixin/apiserver.json +index 9830c36..1c940dc 100644 +--- a/kube-mixin/apiserver.json ++++ b/kube-mixin/apiserver.json +@@ -968,7 +968,7 @@ + "steppedLine": false, + "targets": [ + { +- "expr": "sum(rate(workqueue_adds_total{job=\"kube-apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", ++ "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{name}}", +@@ -1049,7 +1049,7 @@ + "steppedLine": false, + "targets": [ + { +- "expr": "sum(rate(workqueue_depth{job=\"kube-apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", ++ "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{name}}", +@@ -1130,7 +1130,7 @@ + "steppedLine": false, + "targets": [ + { +- "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"kube-apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name, le))", ++ "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{name}}", +@@ -1224,7 +1224,7 @@ + "steppedLine": false, + "targets": [ + { +- "expr": "process_resident_memory_bytes{job=\"kube-apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", ++ "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", +@@ -1305,7 +1305,7 @@ + "steppedLine": false, + "targets": [ + { +- "expr": "rate(process_cpu_seconds_total{job=\"kube-apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])", ++ "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", +@@ -1386,7 +1386,7 @@ + "steppedLine": false, + "targets": [ + { +- "expr": "go_goroutines{job=\"kube-apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", ++ "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", +@@ -1490,7 +1490,7 @@ + "multi": false, + "name": "instance", + "options": [ ], +- "query": "label_values(apiserver_request_total{job=\"kube-apiserver\", cluster=\"$cluster\"}, instance)", ++ "query": "label_values(apiserver_request_total{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, -- 2.40.1 From adc6ba9c91a43b981688972cb764e609b2741f14 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 1 Jul 2021 16:40:23 +0200 Subject: [PATCH 18/45] feat: ArgoCD version bump to 2.X --- charts/kubezero-argocd/Chart.yaml | 4 ++-- charts/kubezero-argocd/README.md | 11 +++++------ charts/kubezero-argocd/dashboards.yaml | 8 ++++++++ .../templates/grafana-dashboards.yaml | 13 +++++++++++++ charts/kubezero-argocd/update.sh | 4 ++++ charts/kubezero-argocd/values.yaml | 11 ++++------- 6 files changed, 36 insertions(+), 15 deletions(-) create mode 100644 charts/kubezero-argocd/dashboards.yaml create mode 100644 charts/kubezero-argocd/templates/grafana-dashboards.yaml create mode 100755 charts/kubezero-argocd/update.sh diff --git a/charts/kubezero-argocd/Chart.yaml b/charts/kubezero-argocd/Chart.yaml index 64c2eb8..f5bd506 100644 --- a/charts/kubezero-argocd/Chart.yaml +++ b/charts/kubezero-argocd/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application name: kubezero-argocd -version: 0.7.1 +version: 0.8.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -15,6 +15,6 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: argo-cd - version: 2.17.4 + version: 3.6.10 repository: https://argoproj.github.io/argo-helm kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-argocd/README.md b/charts/kubezero-argocd/README.md index f972fd2..272c351 100644 --- a/charts/kubezero-argocd/README.md +++ b/charts/kubezero-argocd/README.md @@ -1,6 +1,6 @@ # kubezero-argocd -![Version: 0.7.1](https://img.shields.io/badge/Version-0.7.1-informational?style=flat-square) +![Version: 0.8.0](https://img.shields.io/badge/Version-0.8.0-informational?style=flat-square) KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application @@ -18,7 +18,7 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| -| https://argoproj.github.io/argo-helm | argo-cd | 2.17.4 | +| https://argoproj.github.io/argo-helm | argo-cd | 3.6.10 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Values @@ -30,23 +30,22 @@ Kubernetes: `>= 1.18.0` | argo-cd.controller.args.statusProcessors | string | `"8"` | | | argo-cd.controller.logFormat | string | `"json"` | | | argo-cd.controller.metrics.enabled | bool | `false` | | -| argo-cd.controller.metrics.serviceMonitor.additionalLabels.release | string | `"metrics"` | | | argo-cd.controller.metrics.serviceMonitor.enabled | bool | `true` | | | argo-cd.controller.resources.requests.cpu | string | `"100m"` | | | argo-cd.controller.resources.requests.memory | string | `"256Mi"` | | | argo-cd.dex.enabled | bool | `false` | | -| argo-cd.global.image.tag | string | `"v1.8.7"` | | +| argo-cd.global.image.tag | string | `"v2.0.4"` | | | argo-cd.installCRDs | bool | `false` | | | argo-cd.repoServer.logFormat | string | `"json"` | | | argo-cd.repoServer.metrics.enabled | bool | `false` | | -| argo-cd.repoServer.metrics.serviceMonitor.additionalLabels.release | string | `"metrics"` | | | argo-cd.repoServer.metrics.serviceMonitor.enabled | bool | `true` | | | argo-cd.server.config."resource.customizations" | string | `"cert-manager.io/Certificate:\n # Lua script for customizing the health status assessment\n health.lua: |\n hs = {}\n if obj.status ~= nil then\n if obj.status.conditions ~= nil then\n for i, condition in ipairs(obj.status.conditions) do\n if condition.type == \"Ready\" and condition.status == \"False\" then\n hs.status = \"Degraded\"\n hs.message = condition.message\n return hs\n end\n if condition.type == \"Ready\" and condition.status == \"True\" then\n hs.status = \"Healthy\"\n hs.message = condition.message\n return hs\n end\n end\n end\n end\n hs.status = \"Progressing\"\n hs.message = \"Waiting for certificate\"\n return hs\n"` | | +| argo-cd.server.config."ui.bannercontent" | string | `"KubeZero Release 2.20 incl. ArgoCD 2.0 -> Release notes"` | | +| argo-cd.server.config."ui.bannerurl" | string | `"https://blog.argoproj.io/argo-cd-v2-0-rc1-is-here-f7d21ff1aa64"` | | | argo-cd.server.config.url | string | `"argocd.example.com"` | ArgoCD hostname to be exposed via Istio | | argo-cd.server.extraArgs[0] | string | `"--insecure"` | | | argo-cd.server.logFormat | string | `"json"` | | | argo-cd.server.metrics.enabled | bool | `false` | | -| argo-cd.server.metrics.serviceMonitor.additionalLabels.release | string | `"metrics"` | | | argo-cd.server.metrics.serviceMonitor.enabled | bool | `true` | | | argo-cd.server.service.servicePortHttpsName | string | `"grpc"` | | | istio.enabled | bool | `false` | Deploy Istio VirtualService to expose ArgoCD | diff --git a/charts/kubezero-argocd/dashboards.yaml b/charts/kubezero-argocd/dashboards.yaml new file mode 100644 index 0000000..1324955 --- /dev/null +++ b/charts/kubezero-argocd/dashboards.yaml @@ -0,0 +1,8 @@ +configmap: grafana-dashboards +gzip: true +condition: 'index .Values "argo-cd" "controller" "metrics" "enabled"' +dashboards: +- name: ArgoCD + url: https://grafana.com/api/dashboards/14584/revisions/1/download + tags: + - ArgoCD diff --git a/charts/kubezero-argocd/templates/grafana-dashboards.yaml b/charts/kubezero-argocd/templates/grafana-dashboards.yaml new file mode 100644 index 0000000..82058fa --- /dev/null +++ b/charts/kubezero-argocd/templates/grafana-dashboards.yaml @@ -0,0 +1,13 @@ +{{- if index .Values "argo-cd" "controller" "metrics" "enabled" }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" (include "kubezero-lib.fullname" $) "grafana-dashboards" | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} + labels: + grafana_dashboard: "1" +{{ include "kubezero-lib.labels" . | indent 4 }} +binaryData: + ArgoCD.json.gz: + H4sIAAAAAAAC/+2d63PbtpbAv+ev4OXt3LE7sivKsiV3JrPjJn3tpo0bJ72z22Q0EAlJrCmSJcHEqtf7ty8eJAWSIKkHZVH2yYckAijhdXDO7xyAwP0LTdNHI9v1IxLq32p/fOqIlAD/FdkB5mk0RdPu+d80jyx8TFN1H7nY0TtJsm2xxCmKpniZ6KI5f/bHbPJnHIS257Icnac9dNRlTAM0QS4qlJJPTsvJZ0glDU+7p93q4tRNCpA/Uxblz7Qjz7GON2iYsqQZRmSO/GJZP+UzVi7HQgSFXhSYOF+YH3hzTGY4CovlXSvypCKNDXsyJIgUC7vJpG7XgwTfKUp4n0ktlkD/FmKPXNejtaG5TO5FkbpjhySdBcuK0JxxZDvkZ/ZLRmeZKnW5uivpM9hFY4flkyDCUvrMthSptum5rzzHC9gPBtMxOup2tJ5h0L/OzzuacSz/dNLoq2VbtH9pVw4OSKYKSwkJZ2MPBZYe5z3wfz+9iLtex5ZNcrXVpy4mP7MeN/rnw75IYvPhvec5xPZpRpcn8lFxI8cRnwgOeI3YFy96ZxeDweBs0D/nvUf72b2VVBAfX4X6MT3HQX6I2S9PkBOmHVXf8frExo71ynMn9jQdX/FVPEGRwzXg/YPUSR4VloCOiahX0kHp701p3rUXZn9rlpOHL/Rzry8l3CXdE39esM+F3+ZddzFMPy/741OaRmzCx0V/S+v52cZflk1NhjfwvqjnUX1vlbaun29dTeMMdeN6F+nn7NiLrvez8zAefJdgl01H/R9/oGDq2XM0xZ+OZoT44bfffIM+01YFoXE6tcksGkchDuKvnJre/Jvom7Nu7+JyMOz+R/jSMLr/+vyyn5k8c4/PP32OglvL++LqhZr7TjS13d/zZmXZ7wFyQx8FopbyRE6HhCsp5ZiYyJzh9/YcexGRJk6T0p2miqnEdYqcKHUCmVH7P6MGLtSlB6TpwZ5Fvm+70+zYidYuv11aAhqHnhMRrHey+SHBfijp3OTPfe6z1AYqrRi7uR8S+h45EY57M5f70FmjgICqnIqfH3bzP575/Km0CyPX5gJtESsKYuPzQvHoxsooP13PctO1t9p0PestP9MpFdCG52VUMY3n6O41ldxrj36HZRndbt0Up/39SywgonOl+nErk+S6npvJ/DMKiT1ZJNkoIp6c7QU2nZWJAdJn9PPfVDkgR36IjnJk4reKmvH56ZhFudQdFJJfPfIr7Qp5pmTnA5+dIaeOTDpvYpgYM+XAc4WRs0wsLdPQ9TUV1Z+YZJuTaS2+87ngE6qQjo61E42O5RHFRhOH4YiyXEBGLGsUYqpjrfD+T2/88qPOtLJpnVDNS8X1hCqpwDbDj3qHcQlVjCZ++X8f9a/STx/1h+Nsh0y8YI5IUjD9dTpsYfaRRP5+QCbhk9PIZAd4wuFEv5Im0ovcoCyN5wefFVQ0nRxXQU+Dns7qaa51HlVBn6+moC/7bVPQFIMQKOidK2jTi1xyxP/WxgvtSOjeY+1I6OKR6dB+x8HIdifefYUe3kQRUyXJSTfjijWppOf4B6pD84LM0m9m9oQUM2Kt/ko0OgS9Dnq9pXp9uJpeH5yDXn9Kej3APkYF/SJSX9sBNtOO2sAWhNE8UftUNdSo/A6VaTIS5oLnxZaC5swwcsiMYT6JQp6XSaFPhAvXlPOlzxtCfWttyZXvO7YZ++ZgT8CetNOeGMZqBsXoDsCigKdAPQVqdLyln1BvMJ6Yj/CONj+0iZep9JPX6y+qNKTQEN28YqQTnBX1sqAxE4nWu4os0ZFGISPRsbqbm2dZLfsp14S7rNIRqbabry+YqgMwVf1VTVV37UXCgzUOVGa/vE9k9w0ai/XenNbMPvULCm5ZkKOwZyA1NBu6J61xRDw/3jTwj5cf9S0di2wf1dsfB0+xa/2QFqE/qnV6m7RcYZvEPiulcUKOjUK+XSS7nUEfo6AgTzWmLJy9we6U8C0N3Uw6DjfYf2Fh054jLtWZgta2iEtVsInWmtiOI481T/gxQJYtVu276+u3QV6/DWr128y2LOzeCIHN96XYILHc/SEkMVsPOtBT9yp8r95KhD5PFcrDjALF1oR459H3c58s1Fn/gwOvmCNMcr4MYZLzqYE9nZEb5f4mptEUqV7AJ11S5VzWaxyaCrXnEe7L5EtPtS97vjAUVJxwXofyxC+2RTL7eVRGiIkyd4oSiDc916XKUzLaakOF2KasVJWXVY5pSNUspNrBZHZuigvSU6fifVZbJu9RKK//i/SiMNLut3CAuVqfOJ60l06o27eZ6bbMLKs5Vcfmbb67GXj52HpDe71QgcOLfnHvKi3rUZ2mvNG6X/bAw8NGFkzG6T9Ws2zv8DSW99wXNgq0pbsK72WlgYLiJsCl3ujlZ/8osZ22a9mfbStCTtHTljYey5t+79CdnZu848i8FTIpN0V2y+RNBcuNmbmn1bovVVcK67VAd7hiKkg2tlsidbTMgGQlwWG0Wagdy/Cm36Gw4MbFqr/wuND9hWSplSqDfd/qehYmxKIoDdwWK4SRp7/Bn9NKZzbZFuIK0t7WjHI85K2t5yWrWgPF1la1ROSINuuUvcZ0rlpYbDDHc/vEoi7RSdFjZpvoqQZflDns+i92GNruNPs7fhT4TiFIQAeA/oTi8XE2xskfvolCnxnQXA29ALnTwsMf3FuX7TsVG72P+P7u5K/j8vCOCu3r8L4K8Sswf9U97uW4r2TwSg4vlchUKi9y/cgk0+jlEu+KkS0uoRel/VrF6HKQ4lxpoIsuowLj61G+HOcrkb6A9aovS2yvylYCfjnk14B+ucETOfQ7/46BOy8vlf5AtU9Q5ReU+gZFUVD5COV+QomvoPYXWIKGQu1vNhCZRz1lfCqegG+KTnC+0iUEXuM/VPkIFX5Cta9Q6y9U+gwZv6FYaoXzUOpAKKPfLXMkst9PImDcucjkHBeD7/VORp2jsYKzUeJwZOqWczpKHY/q6H+JE1LliFQ6I5UOSdYpEeCg3fDWZOVZ5ZlkvROVnis4KCs5KcXZrXZWShyWSqelynEpd15q9HnRiSk2oejMKKdllU9Q6RfkfINeIU/tH1T4CKV+gmJVZ2ftMHbdjtJ5uFALVon/U+kDZQsqcwdbB/9vI/J2ckO1dPbhBXYc78vjeQqsBvknlU0Dl2LHLkUhsRmf4gJ8CvApwKcAn+LxfArpuZZ5FFLNDt6fYJYbvAnwJsCb2KE3Uf66r7TGmJ+F9admPMkVmouyvXf9TZdoVnQlnrKrIPd65erD2ZaewvlBOAol2cp9RXtwE2I/QCD9Rl5AlcnapxPQUvg32gT/ha7fCfsHXuRaR7ZrBpjaXdkV4HTLBWkTh+Dhj6+SSf/p+FiQ/FdTWhrb8b8Nx2/A6Wm5T4PSr0xCuZkqMOD0PXB62Vam9eBX15tFX9WbLpsSvNTCE6PXnjY+kcWCZ4ah59tg6OWWGDo4e0TYLAs7r0ab+49KA24+51jzQeCmP6PfYqnfB4EX/O8PyHawtTGGdjT+e49Mo7zMhwcqfU8NTNlwRAGGAHKbwTR3PuUBcSmA5/px5XRZ59nHkwclh1kfdDi5TSFjo7dtzHjYKKwrMioDw0INqDKAtYG1D3FfhwqzA3YuskmxecQP3Nk+tAuR3UYA+l08LrZYA940xqvSUsDSe9uMYcBmjEOL1pooUB3RxJKvkWWJPdMKuWQPvGPRDUWhnRUOzmLfT+/T+ee4P5l0876T+OqNiYTKCP8qjGjyxAzP41lNlarvOYjgGx+b1BwWaIMaDOoRClI8PVeiT8XcDfmvRvNypcEM/VJlkzBRCRvQbiW7DrZi12E5u8bXXBWIP4ndfpfquKLZnlFAdBgkvoqlqnjYD3dCuiuRr3r2PKyCahKHPeRwh109hf+7qhU7gA8hBRVn+hUZw6kO1OXvKdti62d3BfJwcEPIUYMVa5PDNQ54p7gmXhkeSt0UmvOTHRKPWvx5neTFv/+67KiHhB5UA6XfXSnV/CoSfydE99doPubUXuip+IEb+28VUugLddEVS2S1hrXccpbazXJVWzE8vmOTVGwrjc5C9MF3sYHKn126fKK8GxeFbjzElc82RUx651tGTIbdvUZMDmgrHbxxA2/ctDwyczsM+QW3LOxStw66YliGPjPuaAEWanF0S538lr1Cw2r48KDd32cqefDRnP8ahtrV9c+wVQ+iOBDFgT13uyDPwTaHS50ZW5Ln5cXeybMOLTdeyVPG0DdFy9bTYcNUCORXTX5fvOCWMh612Rb2yayC8dj/WSJjw9OvM+e3PvIqGyvy4JHs30nHa69ZxwORPTMi28tOtD03A3isHe9ADLY6tWd7WBu2BdbKX3JYi9bKdRXgFuCW4mib22iMTeKM8B02R+w4M9udVl0oxknL9OZztF3EbAPYiks9eN66Fr2sxV2vBZELzAUvrO74hdVDbWGrXhp45VGr4TkODuDVAQZgw7KjaHrw7kAz+696255MedZ/du8OPL3F6uIVVe1aqT5/ZgA99UZzPGcHHIajGUb+CDmOZ47GC4LD+z+98cuPuuDrE6pgAtsM46ilGqofP2apul7q8ED6Fzz3goX2IURTDK8D7DlsyWUfFpJhIRnATcQmz7Y+Unx46HsYHxf7WDMB+QD5doN8doAIPvIDz8RhODL9iFIZHSMrjLckrk19fxjzT8eAfpuh36vrD8B97QidGhsfqQJMCEz4rJiw392SCS96rUHCja+SgVjgkz1K5BmGAade4EWEjTbE/R4L/n5M+xzoD14fAcLb3yL4e+xgpukWz30h/FK9ED4cwjp4Q+vgg223enb3e7vKXk6sBvqFnaJN7RSNX6ceId8epe8ke+M/sUnCdV/Iju8v5FmPHAMVhT6BU/HECGhvxQhor9ihhcDDwMPAwxDxbM9tef1tsa3X6kvxgOqA6g6P6jSthusA6B4d6NihOAnUAc0BzQHNAc09ufXr4UULXkYBWgNaO/RjERNoo4qOtr6pUxGB37bYkSj6VfuejwgAHAAcANx+lqfjiQgvaHOrpV6YHnRhYTrt1eE2B0Zuuy49XDHAueE1y5sgbzzPVVkrcSnPBDB9lmA6tUnurO70E1XALz/qTngS4LlHGJZqVW/HdOcJlqbp8LrMpjsmbaK9E+MQakfpEBzDuUPAp8CnEGDcB0wpT3Tcmqb6QFNAU8+HpiaYmDMgqX2RlDnD5q0XEQCpdoNU0ycYAkbBVbTtuYq2dTfObhVQM873euOsYfTXuXK2ms72f7msTBBWFPCbUNOTS+K7ZjdgigbvoD3sO2YZD/zAOmyn18tWSxncH6s9t/tjwQA+DQOoDILs3QJePHMLuOoaBVhByQq+CU/e8U4DSwiWEG5S321EffX35rc9hnRgQES9qYg6nOnZrpD76se4B9j3TsRO1RYd6eR71tM5xD0/KyCIDke4QxgddiPs8Qj3ba+XHMDezjbvRgBs2hibyo69BE5q36GXgEmwaRMwabuXit5RvabdcL0GLxbxNQpD/WLRxcWTe7Fod8G54WWpVhKdCbG3XcfeAAJ3HjsTONjCE9EhfAZceMjhsyZvCgcsfLLRs0p+udyGXy4H1fxydgH8srv4F6BL8/ErYBUIYUEIC0JYwCpPjVWM7kUNrAzbCiubng8IsZYnBCxmYfNyDblof0XIJbbDNjQb1RuZm0OZ3nNBmVfae/qg9lvcxw0STReIBogGiKaOaKgKJMLa6v+kf7Tpu+tXfIXONnH47Rrv6vS2ooqKd3U4VZz3V7GgyRSbo+DW8r64ulrr5JID5FINXnLccaoFCL4j+iECYsXbEQ3B4+rvJBs1ka5+tzUXM5dkx/cy7+BU7PXINTkWWxDsRodiV9mUfZ6JraFQ+5t1M1Bv4y/zTQPfHAnGHc2QaznYik/Eqabg9HvULLDHfN+xTY7Rp1fL/8d2ozLeV3zbj/+2SQWgo/H/Mi3mWY98tE5aiYeHTvxJ1OPwb0QpjE968g4crA24DbgNAcRdMaD6tfxaCOwBBK4XvgSSA5LbkOTie0xO4+P+geBaSHDZsQF6A3oDegN6208Er1ez/NvvA7wBvAG8PQq8+YH3JzbJ6bX4F+CthfCWHZuN4Q0WuoHdgN2A3baKvNXC28XzgjeAM4CzncEZO8MjtCkCLU7fpf8FRFsN0aqHjR98nO6SDWxim4VNEEv9WpzD8TQsy/N8TkMKs5pYLdo9E9LUubBbAmZBuIAxgTGBMYEx9xIfPKvZyNkfAmICYgJiNoKYIQ5DtgXvRvwLcNnC+F92bIDNgM2AzYDN9hP/q4WzS4AzgDOAs0bgjN3LxODsd/EvwFkL4Sw7NgBnAGcAZwBnewmc9fs1b0B3gc2AzYDNmnl/1TS9yCWnV+JfYLMWsll2bIDNgM2AzYDN9hM4q4OzSwicAZwBnDW1qkmI7U7D05v4P4BnrVzXzAwO8BnwGfDZnq7V2umNWnK/tPkmrZ76Ji1jaRxbfJXWaifz7ekC18st7281jB6cCV3DsI9xUes6INt7ziArQHVEAcUO6d8cbGKSXYdAJ8imCHx84KCXgB1rEq0EVfKAeIB4gHiPfXMq1UX1hPci/jJTMUxjsNyzrviKHpozPEfxngd2VUNXJJOFKMNCwa14khqJpUTqV1Qdvnqtp79N8Nx3EPP+0k6gli0kkgxLNLe03fdZy+CIuw2U6psdtlwRAOTTKveAKpBJsTQ0A5sbzsL469QocUc+lzwTp3nIgEktgxNZ+MpRgEGJvOpzqiltxeOxGpH5T+bl1MjLylqnKjhgKMaOfCj2iDTaRiZ1iu9yB17r4a3tfwicm4VrKiqXSJRUuTxWS0OLHOf3ZCROv5ZLkQY9Mxz14V3mSdiuHY+Y6NyRUKNHt9EYj3zPGtnuxJNuYjjWdzzkWc2+6Yin9V1jwDNzJpWCtXtFho5lt5+kz578jgKbhaVPfuNlKCdTjZSdfi24KSsJ1fJW5ACme7hQhb8lrdWzuSUZseSKPpIyohC/F+pMhnalOEekGCPnqSO+FJmqTDknvkPSmJeI/9o6j9dCre2+Go14mQlopv/ZqfIrkfZEqtM6qIX6RRkKyL2hcsPKumPNDskBSXkdlHcZxpXIjG62CjSr2bLOy8s6b7oso1vRsG7TpZ1VlHbWeGnGrLxps4bL6pWX1Wu6rH55Wf2myxqWlzXMlvVCgcpLe2XMO+fzDpWoDh3njjHr9Gad/qyT+XnJvPTW5pX8vK+jlfWUdfFSkLh/VBZd7qRl/qHR6TTwIp8R/m60urrjarquEZHmDawqtOHy4jPaSotM8lefTGnP8OXQTrGAFeeMGYXEm7eJ7+O4W3wmccyzYrn4QBA/rnqzgL9it5Qwfvy1Jgh/ba432sH1q4jxuqr/Kh+ez0Cp4zyexm9GcmcYOWQ2Cgki2fnaoNbP91lVrzWifH/ijVqUFprkN1sqnXzTgL32nTGfhcBV+kyzpd9Eoc/WZKzSspdPNFvyL3Z1m5P8Zkt9jacBqmpu+kCz5X5wb938HXNysUn+6oY9lsaOJBuddKw6ce91kuZ0ihV4XMP/3DVmSHv5ielLJjhVikNkN1vm24i8nXCJLSt2+UTbZ7DooE5a4cbmKP+XLSE+iBUYm4tgvPYyEYu/Oi3pJI166DyWydL0zNd827zlq/Xxl2PkSyNnsuDq59Jik9GVPpzJH6TomBy90g35gxz8kWMzcuxEN2Lh+pS0gW3ILE6o8lLkH76Qf1gupdeXPyzFXR9Ycn2TumS672/PXQYQpJdVxBIZT4z4phf9r+vb6Y8/TQfxktrndNmNQ3FOr+lvJxPbtJGjiV/SXqNwNvZQYLENy95E+8/IxVqv2zP0Fw//D4RhmGe/kgEA +{{- end }} diff --git a/charts/kubezero-argocd/update.sh b/charts/kubezero-argocd/update.sh new file mode 100755 index 0000000..ccb8397 --- /dev/null +++ b/charts/kubezero-argocd/update.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# Create ZDT dashboard configmap +../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/grafana-dashboards.yaml diff --git a/charts/kubezero-argocd/values.yaml b/charts/kubezero-argocd/values.yaml index ecba336..9f7ec78 100644 --- a/charts/kubezero-argocd/values.yaml +++ b/charts/kubezero-argocd/values.yaml @@ -42,7 +42,7 @@ argo-cd: global: image: - tag: v1.8.7 + tag: v2.0.4 controller: args: @@ -55,8 +55,6 @@ argo-cd: enabled: false serviceMonitor: enabled: true - additionalLabels: - release: metrics resources: # limits: @@ -72,12 +70,13 @@ argo-cd: enabled: false serviceMonitor: enabled: true - additionalLabels: - release: metrics server: logFormat: json config: + ui.bannercontent: "KubeZero Release 2.20 incl. ArgoCD 2.0 -> Release notes" + ui.bannerurl: "https://blog.argoproj.io/argo-cd-v2-0-rc1-is-here-f7d21ff1aa64" + # argo-cd.server.config.url -- ArgoCD hostname to be exposed via Istio url: argocd.example.com @@ -117,8 +116,6 @@ argo-cd: enabled: false serviceMonitor: enabled: true - additionalLabels: - release: metrics extraArgs: - --insecure -- 2.40.1 From f3f2db3df8cd5c44e7120fb346faff1c1d096b83 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 1 Jul 2021 16:41:31 +0200 Subject: [PATCH 19/45] feat: aws-ebs and aws-efs csi driver upgrades, migrate to use service account tokens for AWS access --- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 8 +- charts/kubezero-aws-ebs-csi-driver/README.md | 21 +- .../charts/aws-ebs-csi-driver/Chart.yaml | 4 +- .../aws-ebs-csi-driver/templates/NOTES.txt | 36 ++++ .../aws-ebs-csi-driver/templates/_helpers.tpl | 15 +- .../templates/clusterrole-attacher.yaml | 24 +-- .../templates/clusterrole-csi-node.yaml | 11 + .../templates/clusterrole-resizer.yaml | 38 ++-- .../clusterrole-snapshot-controller.yaml | 51 +++-- .../templates/clusterrole-snapshotter.yaml | 32 ++- .../clusterrolebinding-csi-node.yaml | 15 ++ .../templates/clusterrolebinding-resizer.yaml | 3 - ...lusterrolebinding-snapshot-controller.yaml | 3 - .../clusterrolebinding-snapshotter.yaml | 3 - .../templates/controller.yaml | 195 ++++++++++-------- .../templates/csidriver.yaml | 2 +- .../aws-ebs-csi-driver/templates/node.yaml | 94 ++++----- .../poddisruptionbudget-controller.yaml | 17 ++ ...ddisruptionbudget-snapshot-controller.yaml | 14 ++ ...le-snapshot-controller-leaderelection.yaml | 9 +- ...ng-snapshot-controller-leaderelection.yaml | 3 - .../serviceaccount-csi-controller.yaml | 3 +- .../templates/serviceaccount-csi-node.yaml | 3 +- .../serviceaccount-snapshot-controller.yaml | 5 +- ...tefulset.yaml => snapshot-controller.yaml} | 39 ++-- .../templates/storageclass.yaml | 8 +- .../charts/aws-ebs-csi-driver/values.yaml | 163 +++++++++------ .../loglevel_leader.patch | 103 --------- charts/kubezero-aws-ebs-csi-driver/patch | 87 ++++++++ charts/kubezero-aws-ebs-csi-driver/update.sh | 4 +- .../kubezero-aws-ebs-csi-driver/values.yaml | 51 +++-- charts/kubezero-aws-ebs-csi-driver/zdt.patch | 75 +++++++ charts/kubezero-aws-efs-csi-driver/Chart.yaml | 6 +- charts/kubezero-aws-efs-csi-driver/README.md | 25 ++- .../affinity_resources.patch | 64 ------ .../charts/aws-efs-csi-driver/CHANGELOG.md | 106 ++++++++++ .../charts/aws-efs-csi-driver/Chart.yaml | 6 +- .../aws-efs-csi-driver/templates/_helpers.tpl | 15 +- .../templates/controller-deployment.yaml | 63 ++++-- .../templates/controller-serviceaccount.yaml | 7 +- .../templates/csidriver.yaml | 2 +- .../templates/node-daemonset.yaml | 51 ++--- .../templates/storageclass.yaml | 13 +- .../charts/aws-efs-csi-driver/values.yaml | 158 +++++++------- .../templates/storage-class.yaml | 13 -- charts/kubezero-aws-efs-csi-driver/update.sh | 4 +- .../kubezero-aws-efs-csi-driver/values.yaml | 54 +++-- charts/kubezero-aws-efs-csi-driver/zdt.patch | 23 +++ 48 files changed, 1041 insertions(+), 708 deletions(-) create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-snapshot-controller.yaml rename charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/{statefulset.yaml => snapshot-controller.yaml} (62%) delete mode 100644 charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch create mode 100644 charts/kubezero-aws-ebs-csi-driver/patch create mode 100644 charts/kubezero-aws-ebs-csi-driver/zdt.patch delete mode 100644 charts/kubezero-aws-efs-csi-driver/affinity_resources.patch create mode 100644 charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/CHANGELOG.md delete mode 100644 charts/kubezero-aws-efs-csi-driver/templates/storage-class.yaml create mode 100644 charts/kubezero-aws-efs-csi-driver/zdt.patch diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index 14f5d11..44b3e93 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.5.1 -appVersion: 0.10.0 +version: 0.6.0 +appVersion: 1.2.3 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png sources: @@ -18,8 +18,8 @@ maintainers: - name: Quarky9 dependencies: - name: aws-ebs-csi-driver - version: 0.10.0 - repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver + version: 1.2.3 + # repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: kubezero-lib version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ diff --git a/charts/kubezero-aws-ebs-csi-driver/README.md b/charts/kubezero-aws-ebs-csi-driver/README.md index 93b343c..5c389a2 100644 --- a/charts/kubezero-aws-ebs-csi-driver/README.md +++ b/charts/kubezero-aws-ebs-csi-driver/README.md @@ -1,6 +1,6 @@ # kubezero-aws-ebs-csi-driver -![Version: 0.5.1](https://img.shields.io/badge/Version-0.5.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.10.0](https://img.shields.io/badge/AppVersion-0.10.0-informational?style=flat-square) +![Version: 0.6.0](https://img.shields.io/badge/Version-0.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.3](https://img.shields.io/badge/AppVersion-1.2.3-informational?style=flat-square) KubeZero Umbrella Chart for aws-ebs-csi-driver @@ -23,7 +23,7 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| -| https://kubernetes-sigs.github.io/aws-ebs-csi-driver | aws-ebs-csi-driver | 0.10.0 | +| | aws-ebs-csi-driver | 1.2.3 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## IAM Role @@ -41,17 +41,16 @@ This class is by default also set as default storage class. | Key | Type | Default | Description | |-----|------|---------|-------------| -| aws-ebs-csi-driver.enableVolumeResizing | bool | `true` | | -| aws-ebs-csi-driver.enableVolumeScheduling | bool | `true` | | +| aws-ebs-csi-driver.controller.logLevel | int | `1` | | +| aws-ebs-csi-driver.controller.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| aws-ebs-csi-driver.controller.replicaCount | int | `1` | | +| aws-ebs-csi-driver.controller.resources.limits.memory | string | `"40Mi"` | | +| aws-ebs-csi-driver.controller.resources.requests.cpu | string | `"10m"` | | +| aws-ebs-csi-driver.controller.resources.requests.memory | string | `"24Mi"` | | +| aws-ebs-csi-driver.controller.tolerations[0].effect | string | `"NoSchedule"` | | +| aws-ebs-csi-driver.controller.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | aws-ebs-csi-driver.enableVolumeSnapshot | bool | `true` | | -| aws-ebs-csi-driver.extraVolumeTags | object | `{}` | Optional tags to be added to each EBS volume | -| aws-ebs-csi-driver.logLevel | int | `1` | | | aws-ebs-csi-driver.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | -| aws-ebs-csi-driver.podAnnotations | object | `{}` | iam.amazonaws.com/role: to assume | -| aws-ebs-csi-driver.replicaCount | int | `1` | | -| aws-ebs-csi-driver.resources.limits.memory | string | `"40Mi"` | | -| aws-ebs-csi-driver.resources.requests.cpu | string | `"10m"` | | -| aws-ebs-csi-driver.resources.requests.memory | string | `"24Mi"` | | | aws-ebs-csi-driver.storageClasses[0].allowVolumeExpansion | bool | `true` | | | aws-ebs-csi-driver.storageClasses[0].name | string | `"ebs-sc-gp2-xfs"` | | | aws-ebs-csi-driver.storageClasses[0].parameters."csi.storage.k8s.io/fstype" | string | `"xfs"` | | diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml index d246abe..b773b00 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: 0.10.0 +appVersion: 1.1.0 description: A Helm chart for AWS EBS CSI Driver home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver keywords: @@ -15,4 +15,4 @@ maintainers: name: aws-ebs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver -version: 0.10.0 +version: 1.2.3 diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt index 3717647..8f6d02e 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt @@ -1,3 +1,39 @@ To verify that aws-ebs-csi-driver has started, run: kubectl get pod -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" + + +WARNING: The following values have been deprecated in favor of moving them into the controller or node groups. They will be removed in a subsequent release. + +affinity: +extraCreateMetadata: +extraVolumeTags: +k8sTagClusterId: +nodeSelector: +podAnnotations: +priorityClassName: +region: +replicaCount: +resources: +tolerations: +topologySpreadConstraints: +volumeAttachLimit: + +are moving to + +controller: + affinity: + extraCreateMetadata: + extraVolumeTags: + k8sTagClusterId: + nodeSelector: + podAnnotations: + priorityClassName: + region: + replicaCount: + resources: + tolerations: + topologySpreadConstraints: + +node: + volumeAttachLimit: diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl index fdc77c4..ddf7b61 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl @@ -59,11 +59,24 @@ app.kubernetes.io/instance: {{ .Release.Name }} Convert the `--extra-volume-tags` command line arg from a map. */}} {{- define "aws-ebs-csi-driver.extra-volume-tags" -}} +{{- $evt := default .Values.extraVolumeTags .Values.controller.extraVolumeTags }} {{- $result := dict "pairs" (list) -}} -{{- range $key, $value := .Values.extraVolumeTags -}} +{{- range $key, $value := $evt -}} {{- $noop := printf "%s=%s" $key $value | append $result.pairs | set $result "pairs" -}} {{- end -}} {{- if gt (len $result.pairs) 0 -}} {{- printf "%s=%s" "- --extra-volume-tags" (join "," $result.pairs) -}} {{- end -}} {{- end -}} + +{{/* +Handle http proxy env vars +*/}} +{{- define "aws-ebs-csi-driver.http-proxy" -}} +- name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} +- name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} +- name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end -}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml index e0919ce..816fdf6 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml @@ -6,18 +6,18 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "nodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "csi.storage.k8s.io" ] + resources: [ "csinodeinfos" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments" ] + verbs: [ "get", "list", "watch", "update", "patch" ] - apiGroups: [ "storage.k8s.io" ] resources: [ "volumeattachments/status" ] verbs: [ "patch" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml new file mode 100644 index 0000000..3ca368e --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml @@ -0,0 +1,11 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-node-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml index 9d85b97..065f3ab 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeResizing }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -12,22 +11,21 @@ rules: # - apiGroups: [""] # resources: ["secrets"] # verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] -{{- end}} + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims/status" ] + verbs: [ "update", "patch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "storageclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "pods" ] + verbs: [ "get", "list", "watch" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml index ffdb1b7..b5b7978 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -7,29 +6,27 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - -{{- end }} + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims" ] + verbs: [ "get", "list", "watch", "update" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "storageclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents" ] + verbs: [ "create", "get", "list", "watch", "update", "delete" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshots" ] + verbs: [ "get", "list", "watch", "update" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshots/status" ] + verbs: [ "update" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml index 061b565..5fada8b 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -7,19 +6,18 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] -{{- end }} + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get", "list" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents" ] + verbs: [ "create", "get", "list", "watch", "update", "delete" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents/status" ] + verbs: [ "update" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml new file mode 100644 index 0000000..63d843e --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-node-getter-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.node.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: ebs-csi-node-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml index 6fe42d1..768e589 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeResizing }} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -14,5 +13,3 @@ roleRef: kind: ClusterRole name: ebs-external-resizer-role apiGroup: rbac.authorization.k8s.io - -{{- end}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml index b74484f..55a634f 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -14,5 +13,3 @@ roleRef: kind: ClusterRole name: ebs-snapshot-controller-role apiGroup: rbac.authorization.k8s.io - -{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml index cbc1169..e9f3624 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -14,5 +13,3 @@ roleRef: kind: ClusterRole name: ebs-external-snapshotter-role apiGroup: rbac.authorization.k8s.io - -{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml index 64c9233..48420c8 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -6,7 +6,7 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} spec: - replicas: {{ .Values.replicaCount }} + replicas: {{ default .Values.replicaCount .Values.controller.replicaCount }} selector: matchLabels: app: ebs-csi-controller @@ -16,40 +16,46 @@ spec: labels: app: ebs-csi-controller {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} - {{- if .Values.podAnnotations }} - annotations: {{ toYaml .Values.podAnnotations | nindent 8 }} + {{- if .Values.controller.podLabels }} + {{- toYaml .Values.controller.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.controller.podAnnotations }} + annotations: + {{- toYaml .Values.controller.podAnnotations | nindent 8 }} + {{- else if .Values.podAnnotations}} + annotations: + {{- toYaml .Values.podAnnotations | nindent 8 }} {{- end }} spec: nodeSelector: kubernetes.io/os: linux - {{- with .Values.nodeSelector }} -{{ toYaml . | indent 8 }} + {{- with default .Values.nodeSelector .Values.controller.nodeSelector }} + {{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ .Values.serviceAccount.controller.name }} - priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} - {{- with .Values.affinity }} - affinity: {{ toYaml . | nindent 8 }} + priorityClassName: {{ default .Values.priorityClassName .Values.controller.priorityClassName }} + {{- with default .Values.affinity .Values.controller.affinity }} + affinity: + {{- toYaml . | nindent 8 }} {{- end }} tolerations: - {{- if .Values.tolerateAllTaints }} - - operator: Exists - {{- else }} - key: CriticalAddonsOnly operator: Exists - operator: Exists effect: NoExecute tolerationSeconds: 300 + {{- with default .Values.tolerations .Values.controller.tolerations }} + {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} -{{ toYaml . | indent 8 }} - {{- end }} -{{- if .Values.topologySpreadConstraints }} -{{- $tscLabelSelector := dict "labelSelector" ( dict "matchLabels" ( dict "app" "ebs-csi-controller" ) ) }} + {{- if or .Values.controller.topologySpreadConstraints .Values.topologySpreadConstraints }} + {{- $tscLabelSelector := dict "labelSelector" ( dict "matchLabels" ( dict "app" "ebs-csi-controller" ) ) }} + {{- $constraints := list }} + {{- range default .Values.topologySpreadConstraints .Values.controller.topologySpreadConstraints }} + {{- $constraints = mustAppend $constraints (mergeOverwrite . $tscLabelSelector) }} + {{- end }} topologySpreadConstraints: - {{- range .Values.topologySpreadConstraints }} - - {{ mergeOverwrite . $tscLabelSelector | toJson }} - {{- end }} -{{- end }} + {{- $constraints | toYaml | nindent 8 }} + {{- end }} containers: - name: ebs-plugin image: {{ .Values.image.repository }}:{{ .Values.image.tag }} @@ -61,17 +67,24 @@ spec: # - {all,controller,node} # specify the driver mode {{- end }} - --endpoint=$(CSI_ENDPOINT) - {{- if .Values.extraVolumeTags }} + {{- if or .Values.controller.extraVolumeTags .Values.extraVolumeTags }} {{- include "aws-ebs-csi-driver.extra-volume-tags" . | nindent 12 }} {{- end }} - {{- if .Values.k8sTagClusterId }} - - --k8s-tag-cluster-id={{ .Values.k8sTagClusterId }} + {{- with default .Values.k8sTagClusterId .Values.controller.k8sTagClusterId }} + - --k8s-tag-cluster-id={{ . }} + {{- end }} + {{- with .Values.controller.httpEndpoint }} + - --http-endpoint={{ . }} {{- end }} - --logtostderr - - --v={{ .Values.logLevel }} + - --v={{ .Values.controller.logLevel }} env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -84,9 +97,9 @@ spec: name: aws-secret key: access_key optional: true - {{- if .Values.region }} + {{- with default .Values.region .Values.controller.region }} - name: AWS_REGION - value: {{ .Values.region }} + value: {{ . }} {{- end }} {{- if .Values.controller.extraVars }} {{- range $key, $val := .Values.controller.extraVars }} @@ -94,17 +107,18 @@ spec: value: "{{ $val }}" {{- end }} {{- end }} -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.controller.env.ebsPlugin }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: aws-token + mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" + readOnly: true ports: - name: healthz containerPort: 9808 @@ -125,111 +139,102 @@ spec: timeoutSeconds: 3 periodSeconds: 10 failureThreshold: 5 - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.ebsPlugin) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - name: csi-provisioner image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} args: - --csi-address=$(ADDRESS) - - --v={{ .Values.logLevel }} - {{- if .Values.enableVolumeScheduling }} + - --v={{ .Values.controller.logLevel }} - --feature-gates=Topology=true - {{- end}} - {{- if .Values.extraCreateMetadata }} + {{- if or .Values.controller.extraCreateMetadata .Values.extraCreateMetadata }} - --extra-create-metadata {{- end}} - - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + - --leader-election=true - --default-fstype=ext4 env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.controller.env.provisioner }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.provisioner) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - name: csi-attacher image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} args: - --csi-address=$(ADDRESS) - - --v={{ .Values.logLevel }} - - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + - --v={{ .Values.controller.logLevel }} + - --leader-election=true env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.controller.env.attacher }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.attacher) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - {{- if .Values.enableVolumeSnapshot }} - name: csi-snapshotter image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} args: - --csi-address=$(ADDRESS) - - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + - --leader-election=true env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.controller.env.snapshotter }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.snapshotter) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - {{- end }} - {{- if .Values.enableVolumeResizing }} - name: csi-resizer image: {{ printf "%s:%s" .Values.sidecars.resizerImage.repository .Values.sidecars.resizerImage.tag }} imagePullPolicy: Always args: - --csi-address=$(ADDRESS) - - --v={{ .Values.logLevel }} + - --v={{ .Values.controller.logLevel }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.controller.env.resizer }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.resizer) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - {{- end }} - name: liveness-probe image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} args: @@ -237,8 +242,9 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.liveness) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: @@ -249,3 +255,10 @@ spec: volumes: - name: socket-dir emptyDir: {} + - name: aws-token + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 86400 + audience: "sts.amazonaws.com" diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml index 1858e39..6598355 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml @@ -1,4 +1,4 @@ -apiVersion: storage.k8s.io/v1beta1 +apiVersion: {{ ternary "storage.k8s.io/v1" "storage.k8s.io/v1beta1" (semverCompare ">=1.18.0-0" .Capabilities.KubeVersion.Version) }} kind: CSIDriver metadata: name: ebs.csi.aws.com diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml index 5caeb33..c816adf 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml @@ -15,8 +15,12 @@ spec: labels: app: ebs-csi-node {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} - {{- if .Values.node.podAnnotations }} - annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- if .Values.node.podLabels }} + {{- toYaml .Values.node.podLabels | nindent 8 }} + {{- end }} + {{- with .Values.node.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} {{- end }} spec: affinity: @@ -31,11 +35,10 @@ spec: nodeSelector: kubernetes.io/os: linux {{- with .Values.node.nodeSelector }} -{{ toYaml . | indent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} - hostNetwork: true serviceAccountName: {{ .Values.serviceAccount.node.name }} - priorityClassName: {{ .Values.node.priorityClassName | default "system-cluster-critical" }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} tolerations: {{- if .Values.node.tolerateAllTaints }} - operator: Exists @@ -47,7 +50,7 @@ spec: tolerationSeconds: 300 {{- end }} {{- with .Values.node.tolerations }} -{{ toYaml . | indent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} containers: - name: ebs-plugin @@ -57,22 +60,24 @@ spec: args: - node - --endpoint=$(CSI_ENDPOINT) - {{- if .Values.volumeAttachLimit }} - - --volume-attach-limit={{ .Values.volumeAttachLimit }} + {{- with default .Values.volumeAttachLimit .Values.node.volumeAttachLimit }} + - --volume-attach-limit={{ . }} {{- end }} - --logtostderr - - --v={{ .Values.logLevel }} + - --v={{ .Values.node.logLevel }} env: - name: CSI_ENDPOINT value: unix:/csi/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.node.env.ebsPlugin }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet @@ -93,47 +98,35 @@ spec: timeoutSeconds: 3 periodSeconds: 10 failureThreshold: 5 - {{- if .Values.node.resources }} - {{- with .Values.node.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- else }} - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} + {{- with default .Values.resources (default .Values.node.resources .Values.node.containerResources.ebsPlugin) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - name: node-driver-registrar image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v={{ .Values.logLevel }} + - --v={{ .Values.node.logLevel }} env: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.node.env.nodeDriverRegistrar }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: plugin-dir mountPath: /csi - name: registration-dir mountPath: /registration - {{- if .Values.node.resources }} - {{- with .Values.node.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- else }} - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} + {{- with default .Values.resources (default .Values.node.resources .Values.node.containerResources.nodeDriverRegistrar) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - name: liveness-probe image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} @@ -142,14 +135,9 @@ spec: volumeMounts: - name: plugin-dir mountPath: /csi - {{- if .Values.node.resources }} - {{- with .Values.node.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- else }} - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} + {{- with default .Values.resources (default .Values.node.resources .Values.node.containerResources.liveness) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: @@ -160,15 +148,15 @@ spec: volumes: - name: kubelet-dir hostPath: - path: /var/lib/kubelet + path: {{ .Values.node.kubeletPath }} type: Directory - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + path: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} type: DirectoryOrCreate - name: registration-dir hostPath: - path: /var/lib/kubelet/plugins_registry/ + path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} type: Directory - name: device-dir hostPath: diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml new file mode 100644 index 0000000..373974a --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml @@ -0,0 +1,17 @@ +{{- $replicas := (default .Values.replicaCount .Values.controller.replicaCount) | int }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: ebs-csi-controller + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ebs-csi-controller + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + {{- if le $replicas 2 }} + maxUnavailable: 1 + {{- else }} + minAvailable: 2 +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-snapshot-controller.yaml new file mode 100644 index 0000000..d9383bc --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-snapshot-controller.yaml @@ -0,0 +1,14 @@ +{{- if .Values.enableVolumeSnapshot }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: ebs-snapshot-controller + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ebs-snapshot-controller + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + maxUnavailable: 1 +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml index 4d09e4c..2b55a16 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -7,8 +6,6 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} rules: - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - -{{- end }} + - apiGroups: [ "coordination.k8s.io" ] + resources: [ "leases" ] + verbs: [ "get", "watch", "list", "delete", "update", "create" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml index e8248bd..74095f3 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -14,5 +13,3 @@ roleRef: kind: Role name: ebs-snapshot-controller-leaderelection apiGroup: rbac.authorization.k8s.io - -{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml index 0490c32..b465c3b 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml @@ -6,7 +6,8 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} {{- with .Values.serviceAccount.controller.annotations }} - annotations: {{ toYaml . | nindent 4 }} + annotations: + {{- toYaml . | nindent 4 }} {{- end }} {{- if eq .Release.Name "kustomize" }} #Enable if EKS IAM for SA is used diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml index 2e93f72..4722b2a 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml @@ -6,6 +6,7 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} {{- with .Values.serviceAccount.node.annotations }} - annotations: {{ toYaml . | nindent 4 }} + annotations: + {{- toYaml . | nindent 4 }} {{- end }} {{- end -}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml index 19d27cb..9d2c681 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} {{- if .Values.serviceAccount.snapshot.create }} --- apiVersion: v1 @@ -8,7 +7,7 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} {{- with .Values.serviceAccount.snapshot.annotations }} - annotations: {{ toYaml . | nindent 4 }} + annotations: + {{- toYaml . | nindent 4 }} {{- end }} {{- end }} -{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml similarity index 62% rename from charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml index e87aa30..bccfd13 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml @@ -5,7 +5,7 @@ apiVersion: apps/v1 metadata: name: ebs-snapshot-controller labels: - {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} spec: serviceName: ebs-snapshot-controller replicas: 1 @@ -18,41 +18,46 @@ spec: labels: app: ebs-snapshot-controller {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.snapshotController.podLabels }} + {{- toYaml .Values.snapshotController.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.snapshotController.podAnnotations }} + annotations: {{ toYaml .Values.snapshotController.podAnnotations | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ .Values.serviceAccount.snapshot.name }} nodeSelector: kubernetes.io/os: linux {{- with .Values.nodeSelector }} -{{ toYaml . | indent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} {{- with .Values.affinity }} - affinity: {{ toYaml . | nindent 8 }} + affinity: + {{- toYaml . | nindent 8 }} {{- end }} tolerations: - {{- if .Values.tolerateAllTaints }} + - key: CriticalAddonsOnly + operator: Exists - operator: Exists - {{- end }} + effect: NoExecute + tolerationSeconds: 300 {{- with .Values.tolerations }} -{{ toYaml . | indent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} containers: - name: snapshot-controller image: {{ printf "%s:%s" .Values.snapshotController.repository .Values.snapshotController.tag }} {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} + {{- if .Values.proxy.http_proxy }} env: -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} args: - - --v={{ .Values.logLevel }} + - --v=2 - --leader-election=false {{- if .Values.imagePullSecrets }} imagePullSecrets: @@ -60,4 +65,4 @@ spec: - name: {{ . }} {{- end }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml index 3da90e3..847f5e2 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml @@ -4,11 +4,11 @@ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: {{ .name }} - {{- if .annotations }} - annotations: {{- .annotations | toYaml | trim | nindent 4 }} + {{- with .annotations }} + annotations: {{- . | toYaml | trim | nindent 4 }} {{- end }} - {{- if .labels }} - labels: {{- .labels | toYaml | trim | nindent 4 }} + {{- with .labels }} + labels: {{- . | toYaml | trim | nindent 4 }} {{- end }} provisioner: ebs.csi.aws.com {{ omit (dict "volumeBindingMode" "WaitForFirstConsumer" | merge .) "name" "annotations" "labels" | toYaml }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml index 0710d18..a33fda9 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml @@ -2,22 +2,18 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. -replicaCount: 2 - image: repository: k8s.gcr.io/provider-aws/aws-ebs-csi-driver - tag: "v0.10.0" + tag: "v1.1.0" pullPolicy: IfNotPresent -logLevel: 5 - sidecars: provisionerImage: repository: k8s.gcr.io/sig-storage/csi-provisioner - tag: "v2.0.2" + tag: "v2.1.1" attacherImage: repository: k8s.gcr.io/sig-storage/csi-attacher - tag: "v3.0.0" + tag: "v3.1.0" snapshotterImage: repository: k8s.gcr.io/sig-storage/csi-snapshotter tag: "v3.0.3" @@ -29,97 +25,128 @@ sidecars: tag: "v1.0.0" nodeDriverRegistrarImage: repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar - tag: "v2.0.1" + tag: "v2.1.0" snapshotController: repository: k8s.gcr.io/sig-storage/snapshot-controller tag: "v3.0.3" + podAnnotations: {} + podLabels: {} -proxy: {} -# http_proxy: -# no_proxy: +proxy: + http_proxy: + no_proxy: imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -podAnnotations: {} - -# True if enable volume scheduling for dynamic volume provisioning -enableVolumeScheduling: true - -# True if enable volume resizing -enableVolumeResizing: false +nameOverride: +fullnameOverride: # True if enable volume snapshot enableVolumeSnapshot: false -# The "maximum number of attachable volumes" per node -volumeAttachLimit: "" +# Moving to values under controller +affinity: {} +extraCreateMetadata: true +extraVolumeTags: {} +k8sTagClusterId: +nodeSelector: {} +podAnnotations: {} +priorityClassName: "system-cluster-critical" +region: +replicaCount: 2 +resources: {} +tolerations: [] +topologySpreadConstraints: [] -resources: - {} +controller: + affinity: {} + # True if enable volume scheduling for dynamic volume provisioning + env: + ebsPlugin: [] + provisioner: [] + attacher: [] + snapshotter: [] + resizer: [] + # If set, add pv/pvc metadata to plugin create requests as parameters. + extraCreateMetadata: true + # Will be removed in later version in favor of env.ebsPlugin + extraVars: {} + # Extra volume tags to attach to each dynamically provisioned volume. + # --- + # extraVolumeTags: + # key1: value1 + # key2: value2 + extraVolumeTags: {} + httpEndpoint: + # ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional). + k8sTagClusterId: + logLevel: 2 + nodeSelector: {} + podAnnotations: {} + podLabels: {} + priorityClassName: + # AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata + # service. + # --- + # region: us-east-1 + region: + replicaCount: + resources: {} + containerResources: + ebsPlugin: {} + provisioner: {} + attacher: {} + snapshotter: {} + resizer: {} + liveness: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # Note that you will need to set resource requests if you want the cluster autoscaler to + # scale your nodes when you increase/decrease the number of ebs-csi-controller replicas. # limits: # cpu: 100m # memory: 128Mi # requests: # cpu: 100m # memory: 128Mi + tolerations: [] + # TSCs without the label selector stanza + # + # Example: + # + # topologySpreadConstraints: + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: ScheduleAnyway + # - maxSkew: 1 + # topologyKey: kubernetes.io/hostname + # whenUnsatisfiable: ScheduleAnyway + topologySpreadConstraints: [] -priorityClassName: "" -nodeSelector: {} -tolerateAllTaints: false -tolerations: [] -affinity: {} - -# TSCs without the label selector stanza -# -# Example: -# -# topologySpreadConstraints: -# - maxSkew: 1 -# topologyKey: topology.kubernetes.io/zone -# whenUnsatisfiable: ScheduleAnyway -# - maxSkew: 1 -# topologyKey: kubernetes.io/hostname -# whenUnsatisfiable: ScheduleAnyway - -topologySpreadConstraints: [] - -# Extra volume tags to attach to each dynamically provisioned volume. -# --- -# extraVolumeTags: -# key1: value1 -# key2: value2 -extraVolumeTags: {} - -# If set, add pv/pvc metadata to plugin create requests as parameters. -extraCreateMetadata: false - -# ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional). -k8sTagClusterId: "" - -# AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata -# service. -# --- -# region: us-east-1 -region: "" - -# Additonal environment variables for the controller -controller: - extraVars: {} +# Moving to values under node +# The "maximum number of attachable volumes" per node +volumeAttachLimit: node: - priorityClassName: "" + env: + ebsPlugin: [] + nodeDriverRegistrar: [] + kubeletPath: /var/lib/kubelet + logLevel: 2 + priorityClassName: nodeSelector: {} podAnnotations: {} + podLabels: {} tolerateAllTaints: false tolerations: [] resources: {} + containerResources: + ebsPlugin: {} + nodeDriverRegistrar: {} + liveness: {} + volumeAttachLimit: serviceAccount: controller: diff --git a/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch b/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch deleted file mode 100644 index 349eea0..0000000 --- a/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch +++ /dev/null @@ -1,103 +0,0 @@ -diff -rtubN aws-ebs-csi-driver.orig/templates/controller.yaml aws-ebs-csi-driver/templates/controller.yaml ---- aws-ebs-csi-driver.orig/templates/controller.yaml 2021-03-05 03:10:41.000000000 +0100 -+++ aws-ebs-csi-driver/templates/controller.yaml 2021-03-05 10:29:31.878615411 +0100 -@@ -68,7 +68,7 @@ - - --k8s-tag-cluster-id={{ .Values.k8sTagClusterId }} - {{- end }} - - --logtostderr -- - --v=5 -+ - --v={{ .Values.logLevel }} - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock -@@ -126,14 +126,14 @@ - image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} - args: - - --csi-address=$(ADDRESS) -- - --v=5 -+ - --v={{ .Values.logLevel }} - {{- if .Values.enableVolumeScheduling }} - - --feature-gates=Topology=true - {{- end}} - {{- if .Values.extraCreateMetadata }} - - --extra-create-metadata - {{- end}} -- - --leader-election=true -+ - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} - - --default-fstype=ext4 - env: - - name: ADDRESS -@@ -156,8 +156,8 @@ - image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} - args: - - --csi-address=$(ADDRESS) -- - --v=5 -- - --leader-election=true -+ - --v={{ .Values.logLevel }} -+ - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock -@@ -180,7 +180,7 @@ - image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} - args: - - --csi-address=$(ADDRESS) -- - --leader-election=true -+ - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock -@@ -205,7 +205,7 @@ - imagePullPolicy: Always - args: - - --csi-address=$(ADDRESS) -- - --v=5 -+ - --v={{ .Values.logLevel }} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock -diff -rtubN aws-ebs-csi-driver.orig/templates/node.yaml aws-ebs-csi-driver/templates/node.yaml ---- aws-ebs-csi-driver.orig/templates/node.yaml 2021-03-05 03:10:41.000000000 +0100 -+++ aws-ebs-csi-driver/templates/node.yaml 2021-03-05 10:30:07.391950366 +0100 -@@ -61,7 +61,7 @@ - - --volume-attach-limit={{ .Values.volumeAttachLimit }} - {{- end }} - - --logtostderr -- - --v=5 -+ - --v={{ .Values.logLevel }} - env: - - name: CSI_ENDPOINT - value: unix:/csi/csi.sock -@@ -107,7 +107,7 @@ - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) -- - --v=5 -+ - --v={{ .Values.logLevel }} - env: - - name: ADDRESS - value: /csi/csi.sock -diff -rtubN aws-ebs-csi-driver.orig/templates/statefulset.yaml aws-ebs-csi-driver/templates/statefulset.yaml ---- aws-ebs-csi-driver.orig/templates/statefulset.yaml 2021-03-05 03:10:41.000000000 +0100 -+++ aws-ebs-csi-driver/templates/statefulset.yaml 2021-03-05 10:29:31.881948744 +0100 -@@ -49,7 +49,7 @@ - value: {{ .Values.proxy.no_proxy | quote }} - {{- end }} - args: -- - --v=5 -+ - --v={{ .Values.logLevel }} - - --leader-election=false - {{- if .Values.imagePullSecrets }} - imagePullSecrets: -diff -rtubN aws-ebs-csi-driver.orig/values.yaml aws-ebs-csi-driver/values.yaml ---- aws-ebs-csi-driver.orig/values.yaml 2021-03-05 03:10:41.000000000 +0100 -+++ aws-ebs-csi-driver/values.yaml 2021-03-05 10:29:31.881948744 +0100 -@@ -9,6 +9,8 @@ - tag: "v0.9.0" - pullPolicy: IfNotPresent - -+logLevel: 5 -+ - sidecars: - provisionerImage: - repository: k8s.gcr.io/sig-storage/csi-provisioner diff --git a/charts/kubezero-aws-ebs-csi-driver/patch b/charts/kubezero-aws-ebs-csi-driver/patch new file mode 100644 index 0000000..71fb008 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/patch @@ -0,0 +1,87 @@ +diff -rtubN charts/aws-ebs-csi-driver/templates/controller.yaml charts/aws-ebs-csi-driver.zdt/templates/controller.yaml +--- charts/aws-ebs-csi-driver/templates/controller.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/templates/controller.yaml 2021-06-24 16:31:37.042386198 +0200 +@@ -116,6 +116,9 @@ + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ ++ - name: aws-token ++ mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" ++ readOnly: true + ports: + - name: healthz + containerPort: 9808 +@@ -144,7 +147,7 @@ + image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.controller.logLevel }} + - --feature-gates=Topology=true + {{- if or .Values.controller.extraCreateMetadata .Values.extraCreateMetadata }} + - --extra-create-metadata +@@ -171,7 +174,7 @@ + image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.controller.logLevel }} + - --leader-election=true + env: + - name: ADDRESS +@@ -215,7 +218,7 @@ + imagePullPolicy: Always + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.controller.logLevel }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +@@ -252,3 +255,10 @@ + volumes: + - name: socket-dir + emptyDir: {} ++ - name: aws-token ++ projected: ++ sources: ++ - serviceAccountToken: ++ path: token ++ expirationSeconds: 86400 ++ audience: "sts.amazonaws.com" +diff -rtubN charts/aws-ebs-csi-driver/templates/node.yaml charts/aws-ebs-csi-driver.zdt/templates/node.yaml +--- charts/aws-ebs-csi-driver/templates/node.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/templates/node.yaml 2021-06-24 15:03:44.532351851 +0200 +@@ -107,7 +107,7 @@ + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) +- - --v=5 ++ - --v={{ .Values.node.logLevel }} + env: + - name: ADDRESS + value: /csi/csi.sock +diff -rtubN charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml charts/aws-ebs-csi-driver.zdt/templates/snapshot-controller.yaml +--- charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/templates/snapshot-controller.yaml 2021-06-24 15:04:17.999018733 +0200 +@@ -57,7 +57,7 @@ + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + args: +- - --v=5 ++ - --v=2 + - --leader-election=false + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +diff -rtubN charts/aws-ebs-csi-driver/values.yaml charts/aws-ebs-csi-driver.zdt/values.yaml +--- charts/aws-ebs-csi-driver/values.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/values.yaml 2021-06-22 17:08:52.501232371 +0200 +@@ -56,7 +56,7 @@ + replicaCount: 2 + resources: {} + tolerations: [] +-topologySpreadConstraints: [] ++topolk8sTagClusterIdogySpreadConstraints: [] + + controller: + affinity: {} diff --git a/charts/kubezero-aws-ebs-csi-driver/update.sh b/charts/kubezero-aws-ebs-csi-driver/update.sh index 1641beb..12e4d9d 100755 --- a/charts/kubezero-aws-ebs-csi-driver/update.sh +++ b/charts/kubezero-aws-ebs-csi-driver/update.sh @@ -1,8 +1,8 @@ #!/bin/bash -VERSION=0.10.0 +VERSION=1.2.3 rm -rf charts/aws-ebs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts -patch -d charts -i ../loglevel_leader.patch -p0 --no-backup-if-mismatch +patch -i zdt.patch -p0 --no-backup-if-mismatch diff --git a/charts/kubezero-aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/values.yaml index 8519805..b7340b8 100644 --- a/charts/kubezero-aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/values.yaml @@ -1,33 +1,33 @@ aws-ebs-csi-driver: - replicaCount: 1 - logLevel: 1 - - enableVolumeScheduling: true - enableVolumeResizing: true enableVolumeSnapshot: true - nodeSelector: - node-role.kubernetes.io/master: "" + controller: + replicaCount: 1 + logLevel: 1 - tolerations: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule - # aws-ebs-csi-driver.podAnnotations -- iam.amazonaws.com/role: to assume - podAnnotations: {} - # iam.amazonaws.com/role: '' + # k8sTagClusterId: - # aws-ebs-csi-driver.extraVolumeTags -- Optional tags to be added to each EBS volume - extraVolumeTags: {} - # Name: KubeZero-Cluster - - resources: - requests: - cpu: 10m - memory: 24Mi - limits: - # cpu: 50m - memory: 40Mi + #env: + # ebsPlugin: + #- name: AWS_ROLE_ARN + # value: "" + #- name: AWS_WEB_IDENTITY_TOKEN_FILE + # value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + #- name: AWS_STS_REGIONAL_ENDPOINTS + # value: regional + resources: + requests: + cpu: 10m + memory: 24Mi + limits: + # cpu: 50m + memory: 40Mi storageClasses: - name: ebs-sc-gp2-xfs @@ -46,3 +46,10 @@ aws-ebs-csi-driver: csi.storage.k8s.io/fstype: xfs type: gp3 encrypted: "true" + + # This will probably move to snapshot later on + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/charts/kubezero-aws-ebs-csi-driver/zdt.patch b/charts/kubezero-aws-ebs-csi-driver/zdt.patch new file mode 100644 index 0000000..ecddbcc --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/zdt.patch @@ -0,0 +1,75 @@ +diff -rtubN charts/aws-ebs-csi-driver/templates/controller.yaml charts/aws-ebs-csi-driver.zdt/templates/controller.yaml +--- charts/aws-ebs-csi-driver/templates/controller.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/templates/controller.yaml 2021-06-24 16:31:37.042386198 +0200 +@@ -116,6 +116,9 @@ + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ ++ - name: aws-token ++ mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" ++ readOnly: true + ports: + - name: healthz + containerPort: 9808 +@@ -144,7 +147,7 @@ + image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.controller.logLevel }} + - --feature-gates=Topology=true + {{- if or .Values.controller.extraCreateMetadata .Values.extraCreateMetadata }} + - --extra-create-metadata +@@ -171,7 +174,7 @@ + image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.controller.logLevel }} + - --leader-election=true + env: + - name: ADDRESS +@@ -215,7 +218,7 @@ + imagePullPolicy: Always + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.controller.logLevel }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +@@ -252,3 +255,10 @@ + volumes: + - name: socket-dir + emptyDir: {} ++ - name: aws-token ++ projected: ++ sources: ++ - serviceAccountToken: ++ path: token ++ expirationSeconds: 86400 ++ audience: "sts.amazonaws.com" +diff -rtubN charts/aws-ebs-csi-driver/templates/node.yaml charts/aws-ebs-csi-driver.zdt/templates/node.yaml +--- charts/aws-ebs-csi-driver/templates/node.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/templates/node.yaml 2021-06-24 15:03:44.532351851 +0200 +@@ -107,7 +107,7 @@ + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) +- - --v=5 ++ - --v={{ .Values.node.logLevel }} + env: + - name: ADDRESS + value: /csi/csi.sock +diff -rtubN charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml charts/aws-ebs-csi-driver.zdt/templates/snapshot-controller.yaml +--- charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/templates/snapshot-controller.yaml 2021-06-24 15:04:17.999018733 +0200 +@@ -57,7 +57,7 @@ + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + args: +- - --v=5 ++ - --v=2 + - --leader-election=false + {{- if .Values.imagePullSecrets }} + imagePullSecrets: diff --git a/charts/kubezero-aws-efs-csi-driver/Chart.yaml b/charts/kubezero-aws-efs-csi-driver/Chart.yaml index 8bd4a5d..2e00e92 100644 --- a/charts/kubezero-aws-efs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-efs-csi-driver/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: kubezero-aws-efs-csi-driver description: KubeZero Umbrella Chart for aws-efs-csi-driver -version: 0.3.5 -appVersion: 1.2.0 +version: 0.4.0 +appVersion: 1.3.1 kubeVersion: ">=1.18.0-0" home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -21,5 +21,5 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: aws-efs-csi-driver - version: 1.2.2 + version: 2.1.1 # repository: https://kubernetes-sigs.github.io/aws-efs-csi-driver/ diff --git a/charts/kubezero-aws-efs-csi-driver/README.md b/charts/kubezero-aws-efs-csi-driver/README.md index 6a17d54..01843b5 100644 --- a/charts/kubezero-aws-efs-csi-driver/README.md +++ b/charts/kubezero-aws-efs-csi-driver/README.md @@ -1,6 +1,6 @@ # kubezero-aws-efs-csi-driver -![Version: 0.3.5](https://img.shields.io/badge/Version-0.3.5-informational?style=flat-square) ![AppVersion: 1.2.0](https://img.shields.io/badge/AppVersion-1.2.0-informational?style=flat-square) +![Version: 0.4.0](https://img.shields.io/badge/Version-0.4.0-informational?style=flat-square) ![AppVersion: 1.3.1](https://img.shields.io/badge/AppVersion-1.3.1-informational?style=flat-square) KubeZero Umbrella Chart for aws-efs-csi-driver @@ -23,7 +23,7 @@ Kubernetes: `>=1.18.0-0` | Repository | Name | Version | |------------|------|---------| -| | aws-efs-csi-driver | 1.2.2 | +| | aws-efs-csi-driver | 2.1.1 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Storage Class @@ -37,11 +37,16 @@ Details also see: [Reserve PV](https://kubernetes.io/docs/concepts/storage/persi | Key | Type | Default | Description | |-----|------|---------|-------------| -| StorageClass.create | bool | `true` | | -| StorageClass.default | bool | `false` | | -| aws-efs-csi-driver.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"node.kubernetes.io/csi.efs.fs"` | | -| aws-efs-csi-driver.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator | string | `"Exists"` | | -| aws-efs-csi-driver.logLevel | int | `1` | | -| aws-efs-csi-driver.resources.limits.memory | string | `"128Mi"` | | -| aws-efs-csi-driver.resources.requests.cpu | string | `"20m"` | | -| aws-efs-csi-driver.resources.requests.memory | string | `"64Mi"` | | +| aws-efs-csi-driver.controller.create | bool | `true` | | +| aws-efs-csi-driver.controller.logLevel | int | `1` | | +| aws-efs-csi-driver.controller.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| aws-efs-csi-driver.controller.tolerations[0].effect | string | `"NoSchedule"` | | +| aws-efs-csi-driver.controller.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| aws-efs-csi-driver.node.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"node.kubernetes.io/csi.efs.fs"` | | +| aws-efs-csi-driver.node.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator | string | `"Exists"` | | +| aws-efs-csi-driver.node.logLevel | int | `1` | | +| aws-efs-csi-driver.node.resources.limits.memory | string | `"128Mi"` | | +| aws-efs-csi-driver.node.resources.requests.cpu | string | `"20m"` | | +| aws-efs-csi-driver.node.resources.requests.memory | string | `"64Mi"` | | +| aws-efs-csi-driver.replicaCount | int | `1` | | +| aws-efs-csi-driver.storageClasses[0].name | string | `"efs-sc"` | | diff --git a/charts/kubezero-aws-efs-csi-driver/affinity_resources.patch b/charts/kubezero-aws-efs-csi-driver/affinity_resources.patch deleted file mode 100644 index 60c546a..0000000 --- a/charts/kubezero-aws-efs-csi-driver/affinity_resources.patch +++ /dev/null @@ -1,64 +0,0 @@ -diff -tubrN charts/aws-efs-csi-driver/templates/node-daemonset.yaml charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml ---- charts/aws-efs-csi-driver/templates/node-daemonset.yaml 2021-03-23 14:34:03.000000000 +0100 -+++ charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml 2021-04-01 14:31:36.427375803 +0200 -@@ -40,15 +40,10 @@ - {{- with .Values.nodeSelector }} - {{- . | toYaml | nindent 8 }} - {{- end }} -+ {{- with .Values.affinity }} - affinity: -- nodeAffinity: -- requiredDuringSchedulingIgnoredDuringExecution: -- nodeSelectorTerms: -- - matchExpressions: -- - key: eks.amazonaws.com/compute-type -- operator: NotIn -- values: -- - fargate -+ {{- . | toYaml | nindent 8 }} -+ {{- end }} - hostNetwork: true - {{- if .Values.dnsPolicy }} - dnsPolicy: "{{ .Values.dnsPolicy }}" -@@ -99,6 +94,10 @@ - timeoutSeconds: 3 - periodSeconds: 2 - failureThreshold: 5 -+ {{- with .Values.resources }} -+ resources: -+ {{- . | toYaml | nindent 12 }} -+ {{- end }} - - name: csi-driver-registrar - image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} - args: -diff -tubrN charts/aws-efs-csi-driver/values.yaml charts/aws-efs-csi-driver.zdt/values.yaml ---- charts/aws-efs-csi-driver/values.yaml 2021-03-23 14:34:03.000000000 +0100 -+++ charts/aws-efs-csi-driver.zdt/values.yaml 2021-04-01 14:37:21.290724721 +0200 -@@ -11,8 +11,8 @@ - - sidecars: - livenessProbeImage: -- repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe -- tag: v2.1.0-eks-1-18-1 -+ repository: k8s.gcr.io/sig-storage/livenessprobe -+ tag: "v2.2.0" - nodeDriverRegistrarImage: - repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar - tag: v2.0.1-eks-1-18-1 -@@ -43,7 +43,15 @@ - - tolerations: [] - --affinity: {} -+affinity: -+ nodeAffinity: -+ requiredDuringSchedulingIgnoredDuringExecution: -+ nodeSelectorTerms: -+ - matchExpressions: -+ - key: eks.amazonaws.com/compute-type -+ operator: NotIn -+ values: -+ - fargate - - node: - podAnnotations: {} diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/CHANGELOG.md b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/CHANGELOG.md new file mode 100644 index 0000000..a842698 --- /dev/null +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/CHANGELOG.md @@ -0,0 +1,106 @@ +# Helm chart + +# v2.1.1 +* Update app/driver version to `v1.3.0` + +# v2.1.0 + +## New features +* Update app/driver version to `v1.3.0` + +## Bug fixes +* Put comments back in place inside the values file ([#475](https://github.com/kubernetes-sigs/aws-efs-csi-driver/pull/475), [@pierluigilenoci](https://github.com/pierluigilenoci)) + +# v2.0.1 + +## Bug fixes +* Helm chart: fix reclaimPolicy and volumeBindingMode ([#464](https://github.com/kubernetes-sigs/aws-efs-csi-driver/pull/464), [@devinsmith911](https://github.com/devinsmith911)) + + +# v2.0.0 + +## Breaking changes + +Multiple changes in values file at `sidecars`, `controller` and `node` + +--- +```yaml +sidecars: + xxxxxxxxx: + repository: + tag: +``` + +Moving to + +```yaml +sidecars: + xxxxxxxxx: + image: + repository: + tag: +``` + +--- +```yaml +podAnnotations: +resources: +nodeSelector: +tolerations: +affinity: +``` + +Moving to + +```yaml +controller: + podAnnotations: + resources: + nodeSelector: + tolerations: + affinity: +``` + +--- +```yaml +hostAliases: +dnsPolicy: +dnsConfig: +``` + +Moving to + +```yaml +node: + hostAliases: + dnsPolicy: + dnsConfig: +``` + +--- +```yaml +serviceAccount: + controller: +``` + +Moving to + +```yaml +controller: + serviceAccount: +``` + +## New features + +* Chart API `v2` (requires Helm 3) +* Set `resources` and `imagePullPolicy` fields independently for containers +* Set `logLevel`, `affinity`, `nodeSelector`, `podAnnotations` and `tolerations` fields independently +for Controller deployment and Node daemonset +* Set `reclaimPolicy` and `volumeBindingMode` fields in storage class + +## Fixes + +* Fixing Controller deployment using `podAnnotations` and `tolerations` values from Node daemonset +* Let the user define the whole `tolerations` array, default to `- operator: Exists` +* Default `logLevel` lowered from `5` to `2` +* Default `imagePullPolicy` everywhere set to `IfNotPresent` diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml index 4f1b11d..29c91a7 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ -apiVersion: v1 -appVersion: 1.2.0 +apiVersion: v2 +appVersion: 1.3.1 description: A Helm chart for AWS EFS CSI Driver home: https://github.com/kubernetes-sigs/aws-efs-csi-driver keywords: @@ -15,4 +15,4 @@ maintainers: name: aws-efs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-efs-csi-driver -version: 1.2.2 +version: 2.1.1 diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl index 2b4ed0b..3a74fef 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl @@ -49,8 +49,19 @@ Create the name of the service account to use */}} {{- define "aws-efs-csi-driver.serviceAccountName" -}} {{- if .Values.controller.create -}} - {{ default (include "aws-efs-csi-driver.fullname" .) .Values.serviceAccount.controller.name }} + {{ default (include "aws-efs-csi-driver.fullname" .) .Values.controller.serviceAccount.name }} {{- else -}} - {{ default "default" .Values.serviceAccount.controller.name }} + {{ default "default" .Values.controller.serviceAccount.name }} {{- end -}} {{- end -}} + +{{/* +Create a string out of the map for controller tags flag +*/}} +{{- define "aws-efs-csi-driver.tags" -}} +{{- $tags := list -}} +{{ range $key, $val := . }} +{{- $tags = print $key ":" $val | append $tags -}} +{{- end -}} +{{- join " " $tags -}} +{{- end -}} diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml index 122ffe2..67f3ada 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml @@ -19,10 +19,11 @@ spec: app: efs-csi-controller app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} - {{- if .Values.node.podAnnotations }} - annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- with .Values.controller.podAnnotations }} + annotations: {{- toYaml . | nindent 8 }} {{- end }} spec: + hostNetwork: true {{- if .Values.imagePullSecrets }} imagePullSecrets: {{- range .Values.imagePullSecrets }} @@ -31,37 +32,39 @@ spec: {{- end }} nodeSelector: kubernetes.io/os: linux - {{- with .Values.nodeSelector }} - {{- . | toYaml | nindent 8 }} + {{- with .Values.controller.nodeSelector }} + {{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ include "aws-efs-csi-driver.serviceAccountName" . }} priorityClassName: system-cluster-critical - tolerations: - - operator: Exists - {{- with .Values.node.tolerations }} - {{- . | toYaml | nindent 8 }} - {{- end }} + {{- with .Values.controller.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: efs-plugin securityContext: privileged: true image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: IfNotPresent + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - --endpoint=$(CSI_ENDPOINT) - --logtostderr - - --v=5 - # Uncomment below line to allow access point root directory to be deleted by controller. - #- --delete-access-point-root-dir + {{- if .Values.controller.tags }} + - --tags={{ include "aws-efs-csi-driver.tags" .Values.controller.tags }} + {{- end }} + - --v={{ .Values.controller.logLevel }} + - --delete-access-point-root-dir={{ hasKey .Values.controller "deleteAccessPointRootDir" | ternary .Values.controller.deleteAccessPointRootDir false }} env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: aws-token + mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" ports: - name: healthz - containerPort: 9808 + containerPort: 9909 protocol: TCP livenessProbe: httpGet: @@ -71,28 +74,52 @@ spec: timeoutSeconds: 3 periodSeconds: 10 failureThreshold: 5 + {{- with .Values.controller.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} - name: csi-provisioner - image: {{ printf "%s:%s" .Values.sidecars.csiProvisionerImage.repository .Values.sidecars.csiProvisionerImage.tag }} + image: {{ printf "%s:%s" .Values.sidecars.csiProvisioner.image.repository .Values.sidecars.csiProvisioner.image.tag }} + imagePullPolicy: {{ .Values.sidecars.csiProvisioner.image.pullPolicy }} args: - --csi-address=$(ADDRESS) - - --v=5 + - --v={{ .Values.controller.logLevel }} - --feature-gates=Topology=true - --leader-election env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock + {{- if .Values.controller.extraEnv }} +{{ toYaml .Values.extraEnv | indent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.sidecars.csiProvisioner.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} - name: liveness-probe - image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }} args: - --csi-address=/csi/csi.sock - - --health-port=9808 + - --health-port=9909 volumeMounts: - name: socket-dir mountPath: /csi + {{- with .Values.sidecars.livenessProbe.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} volumes: - name: socket-dir emptyDir: {} + - name: aws-token + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 86400 + audience: "sts.amazonaws.com" + {{- with .Values.controller.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml index 5a08044..63d45b6 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml @@ -1,11 +1,11 @@ -{{- if .Values.serviceAccount.controller.create }} +{{- if .Values.controller.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "aws-efs-csi-driver.serviceAccountName" . }} labels: app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} - {{- with .Values.serviceAccount.controller.annotations }} + {{- with .Values.controller.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} @@ -40,6 +40,9 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get", "watch", "list" ] --- diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/csidriver.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/csidriver.yaml index 33247d5..e6b4d41 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/csidriver.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/csidriver.yaml @@ -1,4 +1,4 @@ -apiVersion: storage.k8s.io/v1beta1 +apiVersion: {{ ternary "storage.k8s.io/v1" "storage.k8s.io/v1beta1" (semverCompare ">=1.18.0-0" .Capabilities.KubeVersion.Version) }} kind: CSIDriver metadata: name: efs.csi.aws.com diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/node-daemonset.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/node-daemonset.yaml index a8cd9a7..df10a2d 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/node-daemonset.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/node-daemonset.yaml @@ -21,9 +21,9 @@ spec: annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} {{- end }} spec: - {{- if .Values.hostAliases }} + {{- with .Values.node.hostAliases }} hostAliases: - {{- range $k, $v := .Values.hostAliases }} + {{- range $k, $v := . }} - ip: {{ $v.ip }} hostnames: - {{ $k }}.efs.{{ $v.region }}.amazonaws.com @@ -37,36 +37,32 @@ spec: {{- end }} nodeSelector: beta.kubernetes.io/os: linux - {{- with .Values.nodeSelector }} - {{- . | toYaml | nindent 8 }} + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with .Values.node.affinity }} affinity: {{- . | toYaml | nindent 8 }} {{- end }} hostNetwork: true - {{- if .Values.dnsPolicy }} - dnsPolicy: "{{ .Values.dnsPolicy }}" - {{- end }} - {{- with .Values.dnsConfig }} - dnsConfig: - {{- toYaml . | nindent 8 }} + dnsPolicy: {{ .Values.node.dnsPolicy }} + {{- with .Values.node.dnsConfig }} + dnsConfig: {{- toYaml . | nindent 8 }} {{- end }} priorityClassName: system-node-critical - tolerations: - - operator: Exists - {{- with .Values.node.tolerations }} - {{- . | toYaml | nindent 8 }} - {{- end }} + {{- with .Values.node.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: efs-plugin securityContext: privileged: true image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - --endpoint=$(CSI_ENDPOINT) - --logtostderr - - --v={{ .Values.logLevel }} + - --v={{ .Values.node.logLevel }} env: - name: CSI_ENDPOINT value: unix:/csi/csi.sock @@ -94,16 +90,16 @@ spec: timeoutSeconds: 3 periodSeconds: 2 failureThreshold: 5 - {{- with .Values.resources }} - resources: - {{- . | toYaml | nindent 12 }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} {{- end }} - name: csi-driver-registrar - image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} + imagePullPolicy: {{ .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v={{ .Values.logLevel }} + - --v={{ .Values.node.logLevel }} env: - name: ADDRESS value: /csi/csi.sock @@ -118,15 +114,22 @@ spec: mountPath: /csi - name: registration-dir mountPath: /registration + {{- with .Values.sidecars.nodeDriverRegistrar.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} - name: liveness-probe - image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }} args: - --csi-address=/csi/csi.sock - --health-port=9809 - - --v={{ .Values.logLevel }} + - --v={{ .Values.node.logLevel }} volumeMounts: - name: plugin-dir mountPath: /csi + {{- with .Values.sidecars.livenessProbe.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} volumes: - name: kubelet-dir hostPath: diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/storageclass.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/storageclass.yaml index 39f1fae..8ce1ec8 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/storageclass.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/storageclass.yaml @@ -3,13 +3,24 @@ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: {{ .name }} + {{- with .annotations }} + annotations: + {{ toYaml . | indent 4 }} + {{- end }} provisioner: efs.csi.aws.com {{- with .mountOptions }} -mountOptions: +mountOptions: {{ toYaml . }} {{- end }} {{- with .parameters }} parameters: {{ toYaml . | indent 2 }} {{- end }} +{{- with .reclaimPolicy }} +reclaimPolicy: {{ . }} {{- end }} +{{- with .volumeBindingMode }} +volumeBindingMode: {{ . }} +{{- end }} +--- +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml index b175d57..e85d3fb 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml @@ -2,98 +2,114 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. +nameOverride: "" +fullnameOverride: "" + replicaCount: 2 image: repository: amazon/aws-efs-csi-driver - tag: "v1.2.0" + tag: "v1.3.1" pullPolicy: IfNotPresent sidecars: - livenessProbeImage: - repository: k8s.gcr.io/sig-storage/livenessprobe - tag: "v2.2.0" - nodeDriverRegistrarImage: - repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar - tag: v2.0.1-eks-1-18-1 - csiProvisionerImage: - repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner - tag: v2.0.3-eks-1-18-1 + livenessProbe: + image: + repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe + tag: v2.2.0-eks-1-18-2 + pullPolicy: IfNotPresent + resources: {} + nodeDriverRegistrar: + image: + repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar + tag: v2.1.0-eks-1-18-2 + pullPolicy: IfNotPresent + resources: {} + csiProvisioner: + image: + repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner + tag: v2.1.1-eks-1-18-2 + pullPolicy: IfNotPresent + resources: {} imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" -podAnnotations: {} +## Controller deployment variables -resources: - {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: {} - -tolerations: [] - -affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: eks.amazonaws.com/compute-type - operator: NotIn - values: - - fargate - -node: +controller: + # Specifies whether a deployment should be created + create: true + # Number for the log level verbosity + logLevel: 2 + # Add additional tags to access points + tags: {} + # environment: prod + # region: us-east-1 + # Enable if you want the controller to also delete the + # path on efs when deleteing an access point + deleteAccessPointRootDir: false podAnnotations: {} + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + nodeSelector: {} tolerations: [] - -logLevel: 5 - -hostAliases: - {} - # for cross VPC EFS, you need to poison or overwrite the DNS for the efs volume as per - # https://docs.aws.amazon.com/efs/latest/ug/efs-different-vpc.html#wt6-efs-utils-step3 - # implementing the suggested solution found here: - # https://github.com/kubernetes-sigs/aws-efs-csi-driver/issues/240#issuecomment-676849346 - # EFS Vol ID, IP, Region - # "fs-01234567": - # ip: 10.10.2.2 - # region: us-east-2 - -dnsPolicy: "" -dnsConfig: - {} - # Example config which uses the AWS nameservers - # dnsPolicy: "None" - # dnsConfig: - # nameservers: - # - 169.254.169.253 - -serviceAccount: - controller: - # Specifies whether a service account should be created + affinity: {} + # Specifies whether a service account should be created + serviceAccount: create: true + name: efs-csi-controller-sa annotations: {} ## Enable if EKS IAM for SA is used # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/efs-csi-role - name: efs-csi-controller-sa -controller: - create: true +## Node daemonset variables + +node: + # Number for the log level verbosity + logLevel: 2 + hostAliases: {} + # For cross VPC EFS, you need to poison or overwrite the DNS for the efs volume as per + # https://docs.aws.amazon.com/efs/latest/ug/efs-different-vpc.html#wt6-efs-utils-step3 + # implementing the suggested solution found here: + # https://github.com/kubernetes-sigs/aws-efs-csi-driver/issues/240#issuecomment-676849346 + # EFS Vol ID, IP, Region + # "fs-01234567": + # ip: 10.10.2.2 + # region: us-east-2 + dnsPolicy: ClusterFirst + dnsConfig: {} + # Example config which uses the AWS nameservers + # dnsPolicy: "None" + # dnsConfig: + # nameservers: + # - 169.254.169.253 + podAnnotations: {} + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + nodeSelector: {} + tolerations: + - operator: Exists storageClasses: [] # Add StorageClass resources like: # - name: efs-sc +# annotations: +# # Use that annotation if you want this to your default storageclass +# storageclass.kubernetes.io/is-default-class: "true" # mountOptions: # - tls # parameters: @@ -103,3 +119,5 @@ storageClasses: [] # gidRangeStart: "1000" # gidRangeEnd: "2000" # basePath: "/dynamic_provisioning" +# reclaimPolicy: Delete +# volumeBindingMode: Immediate diff --git a/charts/kubezero-aws-efs-csi-driver/templates/storage-class.yaml b/charts/kubezero-aws-efs-csi-driver/templates/storage-class.yaml deleted file mode 100644 index 7559e4b..0000000 --- a/charts/kubezero-aws-efs-csi-driver/templates/storage-class.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.StorageClass.create }} -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: efs-sc - labels: -{{ include "kubezero-lib.labels" . | indent 4 }} - {{- if .Values.StorageClass.default }} - annotations: - storageclass.kubernetes.io/is-default-class: "true" - {{- end }} -provisioner: efs.csi.aws.com -{{- end }} diff --git a/charts/kubezero-aws-efs-csi-driver/update.sh b/charts/kubezero-aws-efs-csi-driver/update.sh index 3b19ef3..a0cc453 100755 --- a/charts/kubezero-aws-efs-csi-driver/update.sh +++ b/charts/kubezero-aws-efs-csi-driver/update.sh @@ -1,8 +1,8 @@ #!/bin/bash -VERSION=1.2.2 +VERSION=2.1.1 rm -rf charts/aws-efs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-efs-csi-driver/releases/download/helm-chart-aws-efs-csi-driver-${VERSION}/aws-efs-csi-driver-${VERSION}.tgz | tar xfz - -C charts -patch -i affinity_resources.patch -p0 --no-backup-if-mismatch +# patch -i zdt.patch -p0 --no-backup-if-mismatch diff --git a/charts/kubezero-aws-efs-csi-driver/values.yaml b/charts/kubezero-aws-efs-csi-driver/values.yaml index 5ad6088..288a630 100644 --- a/charts/kubezero-aws-efs-csi-driver/values.yaml +++ b/charts/kubezero-aws-efs-csi-driver/values.yaml @@ -1,24 +1,44 @@ aws-efs-csi-driver: - logLevel: 1 + replicaCount: 1 - resources: - requests: - cpu: 20m - memory: 64Mi - limits: - memory: 128Mi + controller: + create: true + logLevel: 1 - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node.kubernetes.io/csi.efs.fs - operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + nodeSelector: + node-role.kubernetes.io/master: "" -StorageClass: - create: true - default: false + #extraEnv: + #- name: AWS_ROLE_ARN + # value: "" + #- name: AWS_WEB_IDENTITY_TOKEN_FILE + # value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + #- name: AWS_STS_REGIONAL_ENDPOINTS + # value: regional + + node: + logLevel: 1 + + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + memory: 128Mi + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node.kubernetes.io/csi.efs.fs + operator: Exists + + storageClasses: + - name: efs-sc #PersistentVolumes: # - name: example-pv diff --git a/charts/kubezero-aws-efs-csi-driver/zdt.patch b/charts/kubezero-aws-efs-csi-driver/zdt.patch new file mode 100644 index 0000000..36637ef --- /dev/null +++ b/charts/kubezero-aws-efs-csi-driver/zdt.patch @@ -0,0 +1,23 @@ +Only in charts/aws-efs-csi-driver.zdt/: .values.yaml.swp +diff -rtub charts/aws-efs-csi-driver/templates/node-daemonset.yaml charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml +--- charts/aws-efs-csi-driver/templates/node-daemonset.yaml 2021-06-14 23:12:20.000000000 +0200 ++++ charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml 2021-06-25 12:29:05.279549688 +0200 +@@ -40,15 +40,10 @@ + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} ++ {{- with .Values.node.affinity }} + affinity: +- nodeAffinity: +- requiredDuringSchedulingIgnoredDuringExecution: +- nodeSelectorTerms: +- - matchExpressions: +- - key: eks.amazonaws.com/compute-type +- operator: NotIn +- values: +- - fargate ++ {{- . | toYaml | nindent 8 }} ++ {{- end }} + hostNetwork: true + dnsPolicy: {{ .Values.node.dnsPolicy }} + {{- with .Values.node.dnsConfig }} -- 2.40.1 From 3ee27d7da5dd768174f6b1718b6179e2e9315eef Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 1 Jul 2021 16:42:24 +0200 Subject: [PATCH 20/45] feat: Istio version bump, optional support for proxyprotocol for ingress, bugfixes --- charts/kubezero-istio-ingress/Chart.yaml | 8 +- charts/kubezero-istio-ingress/README.md | 29 +- .../charts/istio-ingress/Chart.yaml | 2 +- .../istio-ingress/templates/deployment.yaml | 18 +- .../charts/istio-ingress/values.yaml | 20 +- .../charts/istio-private-ingress/Chart.yaml | 2 +- .../templates/deployment.yaml | 18 +- .../charts/istio-private-ingress/values.yaml | 20 +- .../templates/bootstrap-config.yaml | 2 +- .../templates/envoyfilter-keepalive-nlb.yaml | 43 +- .../templates/envoyfilter-proxy-protocol.yaml | 44 + .../templates/ingress-certificate.yaml | 34 +- .../templates/ingress-gateway.yaml | 71 +- charts/kubezero-istio-ingress/values.yaml | 53 +- charts/kubezero-istio/Chart.yaml | 8 +- charts/kubezero-istio/README.md | 7 +- charts/kubezero-istio/charts/base/Chart.yaml | 2 +- .../charts/base/crds/crd-all.gen.yaml | 8733 ++++++++++------ .../charts/base/crds/crd-operator.yaml | 74 +- .../charts/base/files/gen-istio-cluster.yaml | 8819 +++++++++++------ .../charts/base/templates/clusterrole.yaml | 12 +- .../validatingwebhookconfiguration.yaml | 2 +- .../charts/istio-discovery/Chart.yaml | 2 +- .../charts/istio-discovery/NOTES.txt | 4 + .../files/gateway-injection-template.yaml | 1 + .../istio-discovery/files/gen-istio.yaml | 208 +- .../files/injection-template.yaml | 11 +- .../istio-discovery/templates/configmap.yaml | 9 +- .../istio-discovery/templates/deployment.yaml | 22 +- .../templates/mutatingwebhook.yaml | 25 +- .../templates/poddisruptionbudget.yaml | 2 +- .../templates/revision-tags.yaml | 113 + .../istio-discovery/templates/service.yaml | 2 +- ...metryv2_1.9.yaml => telemetryv2_1.10.yaml} | 98 +- .../charts/istio-discovery/values.yaml | 17 +- .../templates/grafana-dashboards.yaml | 8 +- charts/kubezero-istio/update.sh | 4 +- charts/kubezero-istio/values.yaml | 3 +- 38 files changed, 11926 insertions(+), 6624 deletions(-) create mode 100644 charts/kubezero-istio-ingress/templates/envoyfilter-proxy-protocol.yaml create mode 100644 charts/kubezero-istio/charts/istio-discovery/templates/revision-tags.yaml rename charts/kubezero-istio/charts/istio-discovery/templates/{telemetryv2_1.9.yaml => telemetryv2_1.10.yaml} (88%) diff --git a/charts/kubezero-istio-ingress/Chart.yaml b/charts/kubezero-istio-ingress/Chart.yaml index 645462c..d8ac819 100644 --- a/charts/kubezero-istio-ingress/Chart.yaml +++ b/charts/kubezero-istio-ingress/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-istio-ingress description: KubeZero Umbrella Chart for Istio based Ingress type: application -version: 0.5.6 -appVersion: 1.9.3 +version: 0.6.0 +appVersion: 1.10.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -16,9 +16,9 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: istio-ingress - version: 1.9.3 + version: 1.10.2 condition: istio-ingress.enabled - name: istio-private-ingress - version: 1.9.3 + version: 1.10.2 condition: istio-private-ingress.enabled kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-istio-ingress/README.md b/charts/kubezero-istio-ingress/README.md index 7caa87a..5ba6c9f 100644 --- a/charts/kubezero-istio-ingress/README.md +++ b/charts/kubezero-istio-ingress/README.md @@ -1,6 +1,6 @@ # kubezero-istio-ingress -![Version: 0.5.6](https://img.shields.io/badge/Version-0.5.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.9.3](https://img.shields.io/badge/AppVersion-1.9.3-informational?style=flat-square) +![Version: 0.6.0](https://img.shields.io/badge/Version-0.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.10.2](https://img.shields.io/badge/AppVersion-1.10.2-informational?style=flat-square) KubeZero Umbrella Chart for Istio based Ingress @@ -20,8 +20,8 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| -| | istio-ingress | 1.9.3 | -| | istio-private-ingress | 1.9.3 | +| | istio-ingress | 1.10.2 | +| | istio-private-ingress | 1.10.2 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Values @@ -30,10 +30,10 @@ Kubernetes: `>= 1.18.0` |-----|------|---------|-------------| | global.arch.amd64 | int | `2` | | | global.defaultPodDisruptionBudget.enabled | bool | `false` | | -| global.jwtPolicy | string | `"first-party-jwt"` | | | global.logAsJson | bool | `true` | | | global.priorityClassName | string | `"system-cluster-critical"` | | -| istio-ingress.dnsNames | list | `[]` | | +| istio-ingress.certificates[0].dnsNames | list | `[]` | | +| istio-ingress.certificates[0].name | string | `"ingress-cert"` | | | istio-ingress.enabled | bool | `false` | | | istio-ingress.gateways.istio-ingressgateway.autoscaleEnabled | bool | `false` | | | istio-ingress.gateways.istio-ingressgateway.configVolumes[0].configMapName | string | `"istio-gateway-bootstrap-config"` | | @@ -69,10 +69,16 @@ Kubernetes: `>= 1.18.0` | istio-ingress.gateways.istio-ingressgateway.rollingMaxUnavailable | int | `0` | | | istio-ingress.gateways.istio-ingressgateway.type | string | `"NodePort"` | | | istio-ingress.meshConfig.defaultConfig.proxyMetadata | string | `nil` | | +| istio-ingress.proxyProtocol | bool | `false` | | | istio-ingress.telemetry.enabled | bool | `false` | | -| istio-private-ingress.dnsNames | list | `[]` | | +| istio-private-ingress.certificates[0].dnsNames | list | `[]` | | +| istio-private-ingress.certificates[0].name | string | `"private-ingress-cert"` | | | istio-private-ingress.enabled | bool | `false` | | | istio-private-ingress.gateways.istio-ingressgateway.autoscaleEnabled | bool | `false` | | +| istio-private-ingress.gateways.istio-ingressgateway.configVolumes[0].configMapName | string | `"istio-gateway-bootstrap-config"` | | +| istio-private-ingress.gateways.istio-ingressgateway.configVolumes[0].mountPath | string | `"/etc/istio/custom-bootstrap"` | | +| istio-private-ingress.gateways.istio-ingressgateway.configVolumes[0].name | string | `"custom-bootstrap-volume"` | | +| istio-private-ingress.gateways.istio-ingressgateway.env.ISTIO_BOOTSTRAP_OVERRIDE | string | `"/etc/istio/custom-bootstrap/custom_bootstrap.json"` | | | istio-private-ingress.gateways.istio-ingressgateway.externalTrafficPolicy | string | `"Local"` | | | istio-private-ingress.gateways.istio-ingressgateway.labels.app | string | `"istio-private-ingressgateway"` | | | istio-private-ingress.gateways.istio-ingressgateway.labels.istio | string | `"private-ingressgateway"` | | @@ -97,16 +103,6 @@ Kubernetes: `>= 1.18.0` | istio-private-ingress.gateways.istio-ingressgateway.ports[2].port | int | `443` | | | istio-private-ingress.gateways.istio-ingressgateway.ports[2].protocol | string | `"TCP"` | | | istio-private-ingress.gateways.istio-ingressgateway.ports[2].targetPort | int | `8443` | | -| istio-private-ingress.gateways.istio-ingressgateway.ports[3].name | string | `"tcp-istiod"` | | -| istio-private-ingress.gateways.istio-ingressgateway.ports[3].nodePort | int | `31012` | | -| istio-private-ingress.gateways.istio-ingressgateway.ports[3].port | int | `15012` | | -| istio-private-ingress.gateways.istio-ingressgateway.ports[3].protocol | string | `"TCP"` | | -| istio-private-ingress.gateways.istio-ingressgateway.ports[3].targetPort | int | `15012` | | -| istio-private-ingress.gateways.istio-ingressgateway.ports[4].name | string | `"tls"` | | -| istio-private-ingress.gateways.istio-ingressgateway.ports[4].nodePort | int | `31044` | | -| istio-private-ingress.gateways.istio-ingressgateway.ports[4].port | int | `15443` | | -| istio-private-ingress.gateways.istio-ingressgateway.ports[4].protocol | string | `"TCP"` | | -| istio-private-ingress.gateways.istio-ingressgateway.ports[4].targetPort | int | `15443` | | | istio-private-ingress.gateways.istio-ingressgateway.replicaCount | int | `1` | | | istio-private-ingress.gateways.istio-ingressgateway.resources.limits.memory | string | `"512Mi"` | | | istio-private-ingress.gateways.istio-ingressgateway.resources.requests.cpu | string | `"50m"` | | @@ -115,6 +111,7 @@ Kubernetes: `>= 1.18.0` | istio-private-ingress.gateways.istio-ingressgateway.rollingMaxUnavailable | int | `0` | | | istio-private-ingress.gateways.istio-ingressgateway.type | string | `"NodePort"` | | | istio-private-ingress.meshConfig.defaultConfig.proxyMetadata | string | `nil` | | +| istio-private-ingress.proxyProtocol | bool | `false` | | | istio-private-ingress.telemetry.enabled | bool | `false` | | ## Resources diff --git a/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml b/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml index 75fb402..43ca564 100644 --- a/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: istio-ingress -version: 1.9.3 +version: 1.10.2 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio gateways keywords: diff --git a/charts/kubezero-istio-ingress/charts/istio-ingress/templates/deployment.yaml b/charts/kubezero-istio-ingress/charts/istio-ingress/templates/deployment.yaml index 0c10978..c7435d0 100644 --- a/charts/kubezero-istio-ingress/charts/istio-ingress/templates/deployment.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-ingress/templates/deployment.yaml @@ -1,4 +1,3 @@ - {{- $gateway := index .Values "gateways" "istio-ingressgateway" }} {{- if eq $gateway.injectionTemplate "" }} apiVersion: apps/v1 @@ -45,17 +44,14 @@ spec: istio.io/rev: {{ .Values.revision | default "default" }} install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }} operator.istio.io/component: "IngressGateways" - sidecar.istio.io/inject: "{{- ne $gateway.injectionTemplate "" }}" + sidecar.istio.io/inject: "false" annotations: {{- if .Values.meshConfig.enablePrometheusMerge }} prometheus.io/port: "15020" prometheus.io/scrape: "true" prometheus.io/path: "/stats/prometheus" {{- end }} - sidecar.istio.io/inject: "{{- ne $gateway.injectionTemplate "" }}" - {{- if ne $gateway.injectionTemplate "" }} - inject.istio.io/templates: "{{ $gateway.injectionTemplate }}" - {{- end}} + sidecar.istio.io/inject: "false" {{- if $gateway.podAnnotations }} {{ toYaml $gateway.podAnnotations | indent 8 }} {{ end }} @@ -219,13 +215,13 @@ spec: {{- if $.Values.global.meshID }} - name: ISTIO_META_MESH_ID value: "{{ $.Values.global.meshID }}" - {{- else if $.Values.global.trustDomain | default (index .Values.meshConfig "trustDomain") }} + {{- else if .Values.meshConfig.trustDomain }} - name: ISTIO_META_MESH_ID - value: "{{ $.Values.global.trustDomain | default (index .Values.meshConfig "trustDomain") }}" + value: "{{ .Values.meshConfig.trustDomain }}" {{- end }} - {{- if .Values.global.trustDomain | default (index .Values.meshConfig "trustDomain") }} + {{- if .Values.meshConfig.trustDomain }} - name: TRUST_DOMAIN - value: "{{ .Values.global.trustDomain | default (index .Values.meshConfig "trustDomain") }}" + value: "{{ .Values.meshConfig.trustDomain }}" {{- end }} {{- if not $gateway.runAsRoot }} - name: ISTIO_META_UNPRIVILEGED_POD @@ -233,7 +229,7 @@ spec: {{- end }} {{- range $key, $val := $gateway.env }} - name: {{ $key }} - value: {{ $val }} + value: "{{ $val }}" {{- end }} {{- range $key, $value := .Values.meshConfig.defaultConfig.proxyMetadata }} - name: {{ $key }} diff --git a/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml b/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml index 4aa40af..8a3ef0a 100644 --- a/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml @@ -24,17 +24,8 @@ gateways: targetPort: 8443 name: https protocol: TCP - - port: 15012 - targetPort: 15012 - name: tcp-istiod - protocol: TCP - # This is the port where sni routing happens - - port: 15443 - targetPort: 15443 - name: tls - protocol: TCP - # Scalability tunning + # Scalability tuning # replicaCount: 1 rollingMaxSurge: 100% rollingMaxUnavailable: 25% @@ -174,7 +165,7 @@ global: hub: docker.io/istio # Default tag for Istio images. - tag: 1.9.3 + tag: 1.10.2 # Specify image pull policy if default behavior isn't desired. # Default behavior: latest images will be Always else IfNotPresent. @@ -310,11 +301,14 @@ global: # Setting this port to a non-zero value enables STS server. servicePort: 0 - # Deprecated, use meshConfig.trustDomain - trustDomain: "" meshConfig: enablePrometheusMerge: true + + # The trust domain corresponds to the trust root of a system + # Refer to https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md#21-trust-domain + trustDomain: "cluster.local" + defaultConfig: proxyMetadata: {} tracing: diff --git a/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml b/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml index 39cecad..b0d3b2e 100644 --- a/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: istio-private-ingress -version: 1.9.3 +version: 1.10.2 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio gateways keywords: diff --git a/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/deployment.yaml b/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/deployment.yaml index 0c10978..c7435d0 100644 --- a/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/deployment.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/deployment.yaml @@ -1,4 +1,3 @@ - {{- $gateway := index .Values "gateways" "istio-ingressgateway" }} {{- if eq $gateway.injectionTemplate "" }} apiVersion: apps/v1 @@ -45,17 +44,14 @@ spec: istio.io/rev: {{ .Values.revision | default "default" }} install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }} operator.istio.io/component: "IngressGateways" - sidecar.istio.io/inject: "{{- ne $gateway.injectionTemplate "" }}" + sidecar.istio.io/inject: "false" annotations: {{- if .Values.meshConfig.enablePrometheusMerge }} prometheus.io/port: "15020" prometheus.io/scrape: "true" prometheus.io/path: "/stats/prometheus" {{- end }} - sidecar.istio.io/inject: "{{- ne $gateway.injectionTemplate "" }}" - {{- if ne $gateway.injectionTemplate "" }} - inject.istio.io/templates: "{{ $gateway.injectionTemplate }}" - {{- end}} + sidecar.istio.io/inject: "false" {{- if $gateway.podAnnotations }} {{ toYaml $gateway.podAnnotations | indent 8 }} {{ end }} @@ -219,13 +215,13 @@ spec: {{- if $.Values.global.meshID }} - name: ISTIO_META_MESH_ID value: "{{ $.Values.global.meshID }}" - {{- else if $.Values.global.trustDomain | default (index .Values.meshConfig "trustDomain") }} + {{- else if .Values.meshConfig.trustDomain }} - name: ISTIO_META_MESH_ID - value: "{{ $.Values.global.trustDomain | default (index .Values.meshConfig "trustDomain") }}" + value: "{{ .Values.meshConfig.trustDomain }}" {{- end }} - {{- if .Values.global.trustDomain | default (index .Values.meshConfig "trustDomain") }} + {{- if .Values.meshConfig.trustDomain }} - name: TRUST_DOMAIN - value: "{{ .Values.global.trustDomain | default (index .Values.meshConfig "trustDomain") }}" + value: "{{ .Values.meshConfig.trustDomain }}" {{- end }} {{- if not $gateway.runAsRoot }} - name: ISTIO_META_UNPRIVILEGED_POD @@ -233,7 +229,7 @@ spec: {{- end }} {{- range $key, $val := $gateway.env }} - name: {{ $key }} - value: {{ $val }} + value: "{{ $val }}" {{- end }} {{- range $key, $value := .Values.meshConfig.defaultConfig.proxyMetadata }} - name: {{ $key }} diff --git a/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml b/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml index 4aa40af..8a3ef0a 100644 --- a/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml @@ -24,17 +24,8 @@ gateways: targetPort: 8443 name: https protocol: TCP - - port: 15012 - targetPort: 15012 - name: tcp-istiod - protocol: TCP - # This is the port where sni routing happens - - port: 15443 - targetPort: 15443 - name: tls - protocol: TCP - # Scalability tunning + # Scalability tuning # replicaCount: 1 rollingMaxSurge: 100% rollingMaxUnavailable: 25% @@ -174,7 +165,7 @@ global: hub: docker.io/istio # Default tag for Istio images. - tag: 1.9.3 + tag: 1.10.2 # Specify image pull policy if default behavior isn't desired. # Default behavior: latest images will be Always else IfNotPresent. @@ -310,11 +301,14 @@ global: # Setting this port to a non-zero value enables STS server. servicePort: 0 - # Deprecated, use meshConfig.trustDomain - trustDomain: "" meshConfig: enablePrometheusMerge: true + + # The trust domain corresponds to the trust root of a system + # Refer to https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md#21-trust-domain + trustDomain: "cluster.local" + defaultConfig: proxyMetadata: {} tracing: diff --git a/charts/kubezero-istio-ingress/templates/bootstrap-config.yaml b/charts/kubezero-istio-ingress/templates/bootstrap-config.yaml index 7117673..aa6ec25 100644 --- a/charts/kubezero-istio-ingress/templates/bootstrap-config.yaml +++ b/charts/kubezero-istio-ingress/templates/bootstrap-config.yaml @@ -1,6 +1,6 @@ +{{- if or (index .Values "istio-ingress" "enabled") (index .Values "istio-private-ingress" "enabled") }} # https://www.envoyproxy.io/docs/envoy/v1.17.1/configuration/best_practices/edge#configuring-envoy-as-an-edge-proxy # https://github.com/istio/istio/issues/24715 -{{- if or (index .Values "istio-ingress" "enabled") (index .Values "istio-private-ingress" "enabled") }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/kubezero-istio-ingress/templates/envoyfilter-keepalive-nlb.yaml b/charts/kubezero-istio-ingress/templates/envoyfilter-keepalive-nlb.yaml index 402b38e..d72d34a 100644 --- a/charts/kubezero-istio-ingress/templates/envoyfilter-keepalive-nlb.yaml +++ b/charts/kubezero-istio-ingress/templates/envoyfilter-keepalive-nlb.yaml @@ -1,4 +1,4 @@ -{{- if or (index .Values "istio-ingress" "enabled") (index .Values "istio-private-ingress" "enabled") }} +{{- if index .Values "istio-ingress" "enabled" }} apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: @@ -7,6 +7,47 @@ metadata: labels: {{ include "kubezero-lib.labels" . | indent 4 }} spec: + workloadSelector: + labels: + istio: ingressgateway + configPatches: + - applyTo: LISTENER + patch: + operation: MERGE + value: + socket_options: + # SOL_SOCKET = 1 + # SO_KEEPALIVE = 9 + - level: 1 + name: 9 + int_value: 1 + state: STATE_LISTENING + # IPPROTO_TCP = 6 + # TCP_KEEPIDLE = 4 + - level: 6 + name: 4 + int_value: 120 + state: STATE_LISTENING + # TCP_KEEPINTVL = 5 + - level: 6 + name: 5 + int_value: 60 + state: STATE_LISTENING +{{- end }} + +{{- if index .Values "istio-private-ingress" "enabled" }} +--- +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: private-ingressgateway-listener-tcp-keepalive + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +spec: + workloadSelector: + labels: + istio: private-ingressgateway configPatches: - applyTo: LISTENER patch: diff --git a/charts/kubezero-istio-ingress/templates/envoyfilter-proxy-protocol.yaml b/charts/kubezero-istio-ingress/templates/envoyfilter-proxy-protocol.yaml new file mode 100644 index 0000000..e3d4fe4 --- /dev/null +++ b/charts/kubezero-istio-ingress/templates/envoyfilter-proxy-protocol.yaml @@ -0,0 +1,44 @@ +{{- if and (index .Values "istio-ingress" "enabled") (index .Values "istio-ingress" "proxyProtocol") }} +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: ingressgateway-proxy-protocol + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +spec: + workloadSelector: + labels: + istio: ingressgateway + configPatches: + - applyTo: LISTENER + patch: + operation: MERGE + value: + listener_filters: + - name: envoy.listener.proxy_protocol + - name: envoy.listener.tls_inspector +{{- end }} + +{{- if and (index .Values "istio-private-ingress" "enabled") (index .Values "istio-private-ingress" "proxyProtocol") }} +--- +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: private-ingressgateway-proxy-protocol + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +spec: + workloadSelector: + labels: + istio: private-ingressgateway + configPatches: + - applyTo: LISTENER + patch: + operation: MERGE + value: + listener_filters: + - name: envoy.listener.proxy_protocol + - name: envoy.listener.tls_inspector +{{- end }} diff --git a/charts/kubezero-istio-ingress/templates/ingress-certificate.yaml b/charts/kubezero-istio-ingress/templates/ingress-certificate.yaml index fbb2fee..53d05a6 100644 --- a/charts/kubezero-istio-ingress/templates/ingress-certificate.yaml +++ b/charts/kubezero-istio-ingress/templates/ingress-certificate.yaml @@ -1,35 +1,39 @@ -{{- if index .Values "istio-ingress" "dnsNames" }} +{{- range $cert := (index .Values "istio-ingress" "certificates") }} +{{- if $cert.dnsNames }} apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: ingress-cert - namespace: {{ .Release.Namespace }} + name: {{ $cert.name }} + namespace: {{ $.Release.Namespace }} labels: -{{ include "kubezero-lib.labels" . | indent 4 }} +{{ include "kubezero-lib.labels" $ | indent 4 }} spec: - secretName: ingress-cert + secretName: {{ $cert.name }} issuerRef: - name: letsencrypt-dns-prod + name: {{ default "letsencrypt-dns-prod" $cert.issuer }} kind: ClusterIssuer dnsNames: -{{ toYaml (index .Values "istio-ingress" "dnsNames") | indent 4 }} +{{ toYaml $cert.dnsNames | indent 4 }} +--- +{{- end }} {{- end }} -{{- if index .Values "istio-private-ingress" "dnsNames" }} ---- +{{- range $cert := (index .Values "istio-private-ingress" "certificates") }} +{{- if $cert.dnsNames }} apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: private-ingress-cert - namespace: {{ .Release.Namespace }} + name: {{ $cert.name }} + namespace: {{ $.Release.Namespace }} labels: -{{ include "kubezero-lib.labels" . | indent 4 }} +{{ include "kubezero-lib.labels" $ | indent 4 }} spec: secretName: private-ingress-cert issuerRef: - name: letsencrypt-dns-prod + name: {{ default "letsencrypt-dns-prod" $cert.issuer }} kind: ClusterIssuer dnsNames: -{{ toYaml (index .Values "istio-private-ingress" "dnsNames") | indent 4 }} +{{ toYaml $cert.dnsNames | indent 4 }} +--- +{{- end }} {{- end }} - diff --git a/charts/kubezero-istio-ingress/templates/ingress-gateway.yaml b/charts/kubezero-istio-ingress/templates/ingress-gateway.yaml index cd35cb2..64d9b34 100644 --- a/charts/kubezero-istio-ingress/templates/ingress-gateway.yaml +++ b/charts/kubezero-istio-ingress/templates/ingress-gateway.yaml @@ -1,6 +1,6 @@ +{{- if and (index .Values "istio-ingress" "enabled") (index .Values "istio-ingress" "certificates") }} # https://istio.io/latest/docs/tasks/traffic-management/ingress/secure-ingress/#configure-a-tls-ingress-gateway-for-multiple-hosts -{{- if and (index .Values "istio-ingress" "enabled") (index .Values "istio-ingress" "dnsNames") }} apiVersion: networking.istio.io/v1beta1 kind: Gateway metadata: @@ -17,23 +17,25 @@ spec: name: http protocol: HTTP2 hosts: - {{- toYaml (index .Values "istio-ingress" "dnsNames") | nindent 4 }} + {{- range $cert := (index .Values "istio-ingress" "certificates") }} + {{- toYaml $cert.dnsNames | nindent 4 }} + {{- end }} tls: httpsRedirect: true + {{- range $cert := (index .Values "istio-ingress" "certificates") }} - port: number: 443 name: https protocol: HTTPS hosts: - {{- toYaml (index .Values "istio-ingress" "dnsNames") | nindent 4 }} + {{- toYaml $cert.dnsNames | nindent 4 }} tls: mode: SIMPLE - privateKey: /etc/istio/ingressgateway-certs/tls.key - serverCertificate: /etc/istio/ingressgateway-certs/tls.crt - credentialName: ingress-cert + credentialName: {{ $cert.name }} + {{- end }} {{- end }} -{{- if and (index .Values "istio-private-ingress" "enabled") (index .Values "istio-private-ingress" "dnsNames") }} +{{- if and (index .Values "istio-private-ingress" "enabled") (index .Values "istio-private-ingress" "certificates") }} --- apiVersion: networking.istio.io/v1beta1 kind: Gateway @@ -51,53 +53,62 @@ spec: name: http protocol: HTTP2 hosts: - {{- toYaml (index .Values "istio-private-ingress" "dnsNames") | nindent 4 }} + {{- range $certs := (index .Values "istio-private-ingress" "certificates") }} + {{- toYaml $certs.dnsNames | nindent 4 }} + {{- end }} tls: httpsRedirect: true + # All SSL hosts one entry per ingress-certificate + {{- range $cert := (index .Values "istio-private-ingress" "certificates") }} - port: number: 443 name: https protocol: HTTPS hosts: - {{- toYaml (index .Values "istio-private-ingress" "dnsNames") | nindent 4 }} + {{- toYaml $cert.dnsNames | nindent 4 }} tls: mode: SIMPLE - privateKey: /etc/istio/ingressgateway-certs/tls.key - serverCertificate: /etc/istio/ingressgateway-certs/tls.crt - credentialName: private-ingress-cert - - port: - number: 5672 - name: amqp - protocol: TCP - hosts: - {{- toYaml (index .Values "istio-private-ingress" "dnsNames") | nindent 4 }} - - port: - number: 5671 - name: amqps - protocol: TCP - hosts: - {{- toYaml (index .Values "istio-private-ingress" "dnsNames") | nindent 4 }} + credentialName: {{ $cert.name }} - port: number: 24224 name: fluentd-forward protocol: TLS hosts: - {{- toYaml (index .Values "istio-private-ingress" "dnsNames") | nindent 4 }} + {{- toYaml $cert.dnsNames | nindent 4 }} tls: mode: SIMPLE - privateKey: /etc/istio/ingressgateway-certs/tls.key - serverCertificate: /etc/istio/ingressgateway-certs/tls.crt - credentialName: private-ingress-cert + credentialName: {{ $cert.name }} + {{- end }} + - port: + number: 5672 + name: amqp + protocol: TCP + hosts: + {{- range $certs := (index .Values "istio-private-ingress" "certificates") }} + {{- toYaml $certs.dnsNames | nindent 4 }} + {{- end }} + - port: + number: 5671 + name: amqps + protocol: TCP + hosts: + {{- range $certs := (index .Values "istio-private-ingress" "certificates") }} + {{- toYaml $certs.dnsNames | nindent 4 }} + {{- end }} - port: number: 6379 name: redis protocol: TCP hosts: - {{- toYaml (index .Values "istio-private-ingress" "dnsNames") | nindent 4 }} + {{- range $certs := (index .Values "istio-private-ingress" "certificates") }} + {{- toYaml $certs.dnsNames | nindent 4 }} + {{- end }} - port: number: 6380 name: redis-1 protocol: TCP hosts: - {{- toYaml (index .Values "istio-private-ingress" "dnsNames") | nindent 4 }} + {{- range $certs := (index .Values "istio-private-ingress" "certificates") }} + {{- toYaml $certs.dnsNames | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/kubezero-istio-ingress/values.yaml b/charts/kubezero-istio-ingress/values.yaml index 5583418..5e83d22 100644 --- a/charts/kubezero-istio-ingress/values.yaml +++ b/charts/kubezero-istio-ingress/values.yaml @@ -1,10 +1,9 @@ # Make sure these values match kuberzero-istio !!! global: #hub: docker.io/istio - #tag: 1.9.3 + #tag: 1.10.2 logAsJson: true - jwtPolicy: first-party-jwt priorityClassName: "system-cluster-critical" @@ -69,21 +68,13 @@ istio-ingress: targetPort: 8443 nodePort: 30443 protocol: TCP - ## multi-cluster - disabled on public LBs - #- name: tcp-istiod - # port: 15012 - # targetPort: 15012 - # nodePort: 30012 - # protocol: TCP - ## multi-cluster sni east-west - #- name: tls - # port: 15443 - # targetPort: 15443 - # nodePort: 30044 - # protocol: TCP - dnsNames: [] -# - '*.example.com' + certificates: + - name: ingress-cert + dnsNames: [] + # - '*.example.com' + + proxyProtocol: false meshConfig: defaultConfig: @@ -123,8 +114,16 @@ istio-private-ingress: values: istio-private-ingressgateway type: NodePort podAnnotations: - # sidecar.istio.io/bootstrapOverride: istio-gateway-bootstrap-config proxy.istio.io/config: '{ "terminationDrainDuration": "20s" }' + + # custom hardened bootstrap config + env: + ISTIO_BOOTSTRAP_OVERRIDE: /etc/istio/custom-bootstrap/custom_bootstrap.json + configVolumes: + - name: custom-bootstrap-volume + mountPath: /etc/istio/custom-bootstrap + configMapName: istio-gateway-bootstrap-config + nodeSelector: node.kubernetes.io/ingress.private: "31080_31443" #nodeSelector: "31080_31443_31671_31672_31224" @@ -143,18 +142,6 @@ istio-private-ingress: targetPort: 8443 nodePort: 31443 protocol: TCP - # multi-cluster - - name: tcp-istiod - port: 15012 - targetPort: 15012 - nodePort: 31012 - protocol: TCP - # multi-cluster sni east-west - - name: tls - port: 15443 - targetPort: 15443 - nodePort: 31044 - protocol: TCP #- name: fluentd-forward # port: 24224 # nodePort: 31224 @@ -168,8 +155,12 @@ istio-private-ingress: # port: 6379 # nodePort: 31379 - dnsNames: [] -# - '*.example.com' + certificates: + - name: private-ingress-cert + dnsNames: [] + #- '*.example.com' + + proxyProtocol: false meshConfig: defaultConfig: diff --git a/charts/kubezero-istio/Chart.yaml b/charts/kubezero-istio/Chart.yaml index 5c2a70a..deebed3 100644 --- a/charts/kubezero-istio/Chart.yaml +++ b/charts/kubezero-istio/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-istio description: KubeZero Umbrella Chart for Istio type: application -version: 0.5.6 -appVersion: 1.9.3 +version: 0.6.0 +appVersion: 1.10.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -16,7 +16,7 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: base - version: 1.9.3 + version: 1.10.2 - name: istio-discovery - version: 1.9.3 + version: 1.10.2 kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-istio/README.md b/charts/kubezero-istio/README.md index 0041be0..cc3a2be 100644 --- a/charts/kubezero-istio/README.md +++ b/charts/kubezero-istio/README.md @@ -1,6 +1,6 @@ # kubezero-istio -![Version: 0.5.6](https://img.shields.io/badge/Version-0.5.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.9.3](https://img.shields.io/badge/AppVersion-1.9.3-informational?style=flat-square) +![Version: 0.6.0](https://img.shields.io/badge/Version-0.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.10.2](https://img.shields.io/badge/AppVersion-1.10.2-informational?style=flat-square) KubeZero Umbrella Chart for Istio @@ -20,8 +20,8 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| -| | base | 1.9.3 | -| | istio-discovery | 1.9.3 | +| | base | 1.10.2 | +| | istio-discovery | 1.10.2 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Values @@ -29,7 +29,6 @@ Kubernetes: `>= 1.18.0` | Key | Type | Default | Description | |-----|------|---------|-------------| | global.defaultPodDisruptionBudget.enabled | bool | `false` | | -| global.jwtPolicy | string | `"first-party-jwt"` | | | global.logAsJson | bool | `true` | | | global.priorityClassName | string | `"system-cluster-critical"` | | | istio-discovery.meshConfig.accessLogEncoding | string | `"JSON"` | | diff --git a/charts/kubezero-istio/charts/base/Chart.yaml b/charts/kubezero-istio/charts/base/Chart.yaml index 1ed5b5c..eba13b9 100644 --- a/charts/kubezero-istio/charts/base/Chart.yaml +++ b/charts/kubezero-istio/charts/base/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: base -version: 1.9.3 +version: 1.10.2 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio cluster resources and CRDs keywords: diff --git a/charts/kubezero-istio/charts/base/crds/crd-all.gen.yaml b/charts/kubezero-istio/charts/base/crds/crd-all.gen.yaml index 0f64904..0387315 100644 --- a/charts/kubezero-istio/charts/base/crds/crd-all.gen.yaml +++ b/charts/kubezero-istio/charts/base/crds/crd-all.gen.yaml @@ -1,5 +1,5 @@ # DO NOT EDIT - Generated by Cue OpenAPI generator based on Istio APIs. -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -11,19 +11,6 @@ metadata: release: istio name: destinationrules.networking.istio.io spec: - additionalPrinterColumns: - - JSONPath: .spec.host - description: The name of a service from the service registry - name: Host - type: string - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date group: networking.istio.io names: categories: @@ -35,376 +22,126 @@ spec: shortNames: - dr singular: destinationrule - preserveUnknownFields: false scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting load balancing, outlier detection, - etc. See more details at: https://istio.io/docs/reference/config/networking/destination-rule.html' - properties: - exportTo: - description: A list of namespaces to which this destination rule is - exported. - items: + versions: + - additionalPrinterColumns: + - description: The name of a service from the service registry + jsonPath: .spec.host + name: Host + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting load balancing, outlier detection, + etc. See more details at: https://istio.io/docs/reference/config/networking/destination-rule.html' + properties: + exportTo: + description: A list of namespaces to which this destination rule is + exported. + items: + format: string + type: string + type: array + host: + description: The name of a service from the service registry. format: string type: string - type: array - host: - description: The name of a service from the service registry. - format: string - type: string - subsets: - items: - properties: - labels: - additionalProperties: + subsets: + items: + properties: + labels: + additionalProperties: + format: string + type: string + type: object + name: + description: Name of the subset. format: string type: string - type: object - name: - description: Name of the subset. - format: string - type: string - trafficPolicy: - description: Traffic policies that apply to this subset. - properties: - connectionPool: - properties: - http: - description: HTTP connection pool settings. - properties: - h2UpgradePolicy: - description: Specify if http1.1 connection should - be upgraded to http2 for the associated destination. - enum: - - DEFAULT - - DO_NOT_UPGRADE - - UPGRADE - type: string - http1MaxPendingRequests: - description: Maximum number of pending HTTP requests - to a destination. - format: int32 - type: integer - http2MaxRequests: - description: Maximum number of requests to a backend. - format: int32 - type: integer - idleTimeout: - description: The idle timeout for upstream connection - pool connections. - type: string - maxRequestsPerConnection: - description: Maximum number of requests per connection - to a backend. - format: int32 - type: integer - maxRetries: - format: int32 - type: integer - useClientProtocol: - description: If set to true, client protocol will - be preserved while initiating connection to backend. - type: boolean - type: object - tcp: - description: Settings common to both HTTP and TCP upstream - connections. - properties: - connectTimeout: - description: TCP connection timeout. - type: string - maxConnections: - description: Maximum number of HTTP1 /TCP connections - to a destination host. - format: int32 - type: integer - tcpKeepalive: - description: If set then set SO_KEEPALIVE on the socket - to enable TCP Keepalives. - properties: - interval: - description: The time duration between keep-alive - probes. - type: string - probes: - type: integer - time: - type: string - type: object - type: object - type: object - loadBalancer: - description: Settings controlling the load balancer algorithms. - oneOf: - - not: - anyOf: - - required: - - simple - - properties: - consistentHash: - oneOf: - - not: - anyOf: - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - required: - - consistentHash - - required: - - simple - - properties: - consistentHash: - oneOf: - - not: - anyOf: - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - required: - - consistentHash - properties: - consistentHash: - properties: - httpCookie: - description: Hash based on HTTP cookie. - properties: - name: - description: Name of the cookie. - format: string - type: string - path: - description: Path to set for the cookie. - format: string - type: string - ttl: - description: Lifetime of the cookie. - type: string - type: object - httpHeaderName: - description: Hash based on a specific HTTP header. - format: string - type: string - httpQueryParameterName: - description: Hash based on a specific HTTP query parameter. - format: string - type: string - minimumRingSize: - type: integer - useSourceIp: - description: Hash based on the source IP address. - type: boolean - type: object - localityLbSetting: - properties: - distribute: - description: 'Optional: only one of distribute or - failover can be set.' - items: - properties: - from: - description: Originating locality, '/' separated, - e.g. - format: string - type: string - to: - additionalProperties: - type: integer - description: Map of upstream localities to traffic - distribution weights. - type: object - type: object - type: array - enabled: - description: enable locality load balancing, this - is DestinationRule-level and will override mesh - wide settings in entirety. - nullable: true - type: boolean - failover: - description: 'Optional: only failover or distribute - can be set.' - items: - properties: - from: - description: Originating region. - format: string - type: string - to: - format: string - type: string - type: object - type: array - type: object - simple: - enum: - - ROUND_ROBIN - - LEAST_CONN - - RANDOM - - PASSTHROUGH - type: string - type: object - outlierDetection: - properties: - baseEjectionTime: - description: Minimum ejection duration. - type: string - consecutive5xxErrors: - description: Number of 5xx errors before a host is ejected - from the connection pool. - nullable: true - type: integer - consecutiveErrors: - format: int32 - type: integer - consecutiveGatewayErrors: - description: Number of gateway errors before a host is - ejected from the connection pool. - nullable: true - type: integer - interval: - description: Time interval between ejection sweep analysis. - type: string - maxEjectionPercent: - format: int32 - type: integer - minHealthPercent: - format: int32 - type: integer - type: object - portLevelSettings: - description: Traffic policies specific to individual ports. - items: + trafficPolicy: + description: Traffic policies that apply to this subset. + properties: + connectionPool: properties: - connectionPool: + http: + description: HTTP connection pool settings. properties: - http: - description: HTTP connection pool settings. + h2UpgradePolicy: + description: Specify if http1.1 connection should + be upgraded to http2 for the associated destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP requests + to a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream connection + pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per connection + to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol will + be preserved while initiating connection to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and TCP upstream + connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP connections + to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE on the + socket to enable TCP Keepalives. properties: - h2UpgradePolicy: - description: Specify if http1.1 connection should - be upgraded to http2 for the associated destination. - enum: - - DEFAULT - - DO_NOT_UPGRADE - - UPGRADE + interval: + description: The time duration between keep-alive + probes. type: string - http1MaxPendingRequests: - description: Maximum number of pending HTTP - requests to a destination. - format: int32 + probes: type: integer - http2MaxRequests: - description: Maximum number of requests to a - backend. - format: int32 - type: integer - idleTimeout: - description: The idle timeout for upstream connection - pool connections. + time: type: string - maxRequestsPerConnection: - description: Maximum number of requests per - connection to a backend. - format: int32 - type: integer - maxRetries: - format: int32 - type: integer - useClientProtocol: - description: If set to true, client protocol - will be preserved while initiating connection - to backend. - type: boolean - type: object - tcp: - description: Settings common to both HTTP and TCP - upstream connections. - properties: - connectTimeout: - description: TCP connection timeout. - type: string - maxConnections: - description: Maximum number of HTTP1 /TCP connections - to a destination host. - format: int32 - type: integer - tcpKeepalive: - description: If set then set SO_KEEPALIVE on - the socket to enable TCP Keepalives. - properties: - interval: - description: The time duration between keep-alive - probes. - type: string - probes: - type: integer - time: - type: string - type: object type: object type: object - loadBalancer: - description: Settings controlling the load balancer - algorithms. - oneOf: - - not: - anyOf: - - required: - - simple - - properties: - consistentHash: - oneOf: - - not: - anyOf: - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - required: - - consistentHash + type: object + loadBalancer: + description: Settings controlling the load balancer algorithms. + oneOf: + - not: + anyOf: - required: - simple - properties: @@ -430,277 +167,21 @@ spec: - httpQueryParameterName required: - consistentHash - properties: - consistentHash: - properties: - httpCookie: - description: Hash based on HTTP cookie. - properties: - name: - description: Name of the cookie. - format: string - type: string - path: - description: Path to set for the cookie. - format: string - type: string - ttl: - description: Lifetime of the cookie. - type: string - type: object - httpHeaderName: - description: Hash based on a specific HTTP header. - format: string - type: string - httpQueryParameterName: - description: Hash based on a specific HTTP query - parameter. - format: string - type: string - minimumRingSize: - type: integer - useSourceIp: - description: Hash based on the source IP address. - type: boolean - type: object - localityLbSetting: - properties: - distribute: - description: 'Optional: only one of distribute - or failover can be set.' - items: - properties: - from: - description: Originating locality, '/' - separated, e.g. - format: string - type: string - to: - additionalProperties: - type: integer - description: Map of upstream localities - to traffic distribution weights. - type: object - type: object - type: array - enabled: - description: enable locality load balancing, - this is DestinationRule-level and will override - mesh wide settings in entirety. - nullable: true - type: boolean - failover: - description: 'Optional: only failover or distribute - can be set.' - items: - properties: - from: - description: Originating region. - format: string - type: string - to: - format: string - type: string - type: object - type: array - type: object - simple: - enum: - - ROUND_ROBIN - - LEAST_CONN - - RANDOM - - PASSTHROUGH - type: string - type: object - outlierDetection: - properties: - baseEjectionTime: - description: Minimum ejection duration. - type: string - consecutive5xxErrors: - description: Number of 5xx errors before a host - is ejected from the connection pool. - nullable: true - type: integer - consecutiveErrors: - format: int32 - type: integer - consecutiveGatewayErrors: - description: Number of gateway errors before a host - is ejected from the connection pool. - nullable: true - type: integer - interval: - description: Time interval between ejection sweep - analysis. - type: string - maxEjectionPercent: - format: int32 - type: integer - minHealthPercent: - format: int32 - type: integer - type: object - port: - properties: - number: - type: integer - type: object - tls: - description: TLS related settings for connections to - the upstream service. - properties: - caCertificates: - format: string - type: string - clientCertificate: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - credentialName: - format: string - type: string - mode: - enum: - - DISABLE - - SIMPLE - - MUTUAL - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - sni: - description: SNI string to present to the server - during TLS handshake. - format: string - type: string - subjectAltNames: - items: - format: string - type: string - type: array - type: object - type: object - type: array - tls: - description: TLS related settings for connections to the upstream - service. - properties: - caCertificates: - format: string - type: string - clientCertificate: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - credentialName: - format: string - type: string - mode: - enum: - - DISABLE - - SIMPLE - - MUTUAL - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - sni: - description: SNI string to present to the server during - TLS handshake. - format: string - type: string - subjectAltNames: - items: - format: string - type: string - type: array - type: object - type: object - type: object - type: array - trafficPolicy: - properties: - connectionPool: - properties: - http: - description: HTTP connection pool settings. - properties: - h2UpgradePolicy: - description: Specify if http1.1 connection should be upgraded - to http2 for the associated destination. - enum: - - DEFAULT - - DO_NOT_UPGRADE - - UPGRADE - type: string - http1MaxPendingRequests: - description: Maximum number of pending HTTP requests to - a destination. - format: int32 - type: integer - http2MaxRequests: - description: Maximum number of requests to a backend. - format: int32 - type: integer - idleTimeout: - description: The idle timeout for upstream connection pool - connections. - type: string - maxRequestsPerConnection: - description: Maximum number of requests per connection to - a backend. - format: int32 - type: integer - maxRetries: - format: int32 - type: integer - useClientProtocol: - description: If set to true, client protocol will be preserved - while initiating connection to backend. - type: boolean - type: object - tcp: - description: Settings common to both HTTP and TCP upstream connections. - properties: - connectTimeout: - description: TCP connection timeout. - type: string - maxConnections: - description: Maximum number of HTTP1 /TCP connections to - a destination host. - format: int32 - type: integer - tcpKeepalive: - description: If set then set SO_KEEPALIVE on the socket - to enable TCP Keepalives. - properties: - interval: - description: The time duration between keep-alive probes. - type: string - probes: - type: integer - time: - type: string - type: object - type: object - type: object - loadBalancer: - description: Settings controlling the load balancer algorithms. - oneOf: - - not: - anyOf: - - required: - - simple - - properties: - consistentHash: - oneOf: - - not: - anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName - required: - httpHeaderName - required: @@ -709,231 +190,237 @@ spec: - useSourceIp - required: - httpQueryParameterName - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - required: - - consistentHash - - required: - - simple - - properties: - consistentHash: - oneOf: - - not: - anyOf: - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - required: - - consistentHash - properties: - consistentHash: - properties: - httpCookie: - description: Hash based on HTTP cookie. + required: + - consistentHash properties: - name: - description: Name of the cookie. - format: string - type: string - path: - description: Path to set for the cookie. - format: string - type: string - ttl: - description: Lifetime of the cookie. + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP query + parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute or + failover can be set.' + items: + properties: + from: + description: Originating locality, '/' separated, + e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities to + traffic distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, this + is DestinationRule-level and will override mesh + wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute + can be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH type: string type: object - httpHeaderName: - description: Hash based on a specific HTTP header. - format: string - type: string - httpQueryParameterName: - description: Hash based on a specific HTTP query parameter. - format: string - type: string - minimumRingSize: - type: integer - useSourceIp: - description: Hash based on the source IP address. - type: boolean - type: object - localityLbSetting: - properties: - distribute: - description: 'Optional: only one of distribute or failover - can be set.' + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host is ejected + from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a host + is ejected from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + portLevelSettings: + description: Traffic policies specific to individual ports. items: properties: - from: - description: Originating locality, '/' separated, - e.g. - format: string - type: string - to: - additionalProperties: - type: integer - description: Map of upstream localities to traffic - distribution weights. - type: object - type: object - type: array - enabled: - description: enable locality load balancing, this is DestinationRule-level - and will override mesh wide settings in entirety. - nullable: true - type: boolean - failover: - description: 'Optional: only failover or distribute can - be set.' - items: - properties: - from: - description: Originating region. - format: string - type: string - to: - format: string - type: string - type: object - type: array - type: object - simple: - enum: - - ROUND_ROBIN - - LEAST_CONN - - RANDOM - - PASSTHROUGH - type: string - type: object - outlierDetection: - properties: - baseEjectionTime: - description: Minimum ejection duration. - type: string - consecutive5xxErrors: - description: Number of 5xx errors before a host is ejected from - the connection pool. - nullable: true - type: integer - consecutiveErrors: - format: int32 - type: integer - consecutiveGatewayErrors: - description: Number of gateway errors before a host is ejected - from the connection pool. - nullable: true - type: integer - interval: - description: Time interval between ejection sweep analysis. - type: string - maxEjectionPercent: - format: int32 - type: integer - minHealthPercent: - format: int32 - type: integer - type: object - portLevelSettings: - description: Traffic policies specific to individual ports. - items: - properties: - connectionPool: - properties: - http: - description: HTTP connection pool settings. - properties: - h2UpgradePolicy: - description: Specify if http1.1 connection should - be upgraded to http2 for the associated destination. - enum: - - DEFAULT - - DO_NOT_UPGRADE - - UPGRADE - type: string - http1MaxPendingRequests: - description: Maximum number of pending HTTP requests - to a destination. - format: int32 - type: integer - http2MaxRequests: - description: Maximum number of requests to a backend. - format: int32 - type: integer - idleTimeout: - description: The idle timeout for upstream connection - pool connections. - type: string - maxRequestsPerConnection: - description: Maximum number of requests per connection - to a backend. - format: int32 - type: integer - maxRetries: - format: int32 - type: integer - useClientProtocol: - description: If set to true, client protocol will - be preserved while initiating connection to backend. - type: boolean - type: object - tcp: - description: Settings common to both HTTP and TCP upstream - connections. - properties: - connectTimeout: - description: TCP connection timeout. - type: string - maxConnections: - description: Maximum number of HTTP1 /TCP connections - to a destination host. - format: int32 - type: integer - tcpKeepalive: - description: If set then set SO_KEEPALIVE on the socket - to enable TCP Keepalives. + connectionPool: properties: - interval: - description: The time duration between keep-alive - probes. - type: string - probes: - type: integer - time: - type: string + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection + should be upgraded to http2 for the associated + destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP + requests to a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to + a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream + connection pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per + connection to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol + will be preserved while initiating connection + to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and + TCP upstream connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP + connections to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE + on the socket to enable TCP Keepalives. + properties: + interval: + description: The time duration between + keep-alive probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object type: object - type: object - type: object - loadBalancer: - description: Settings controlling the load balancer algorithms. - oneOf: - - not: - anyOf: - - required: - - simple - - properties: - consistentHash: - oneOf: - - not: - anyOf: + loadBalancer: + description: Settings controlling the load balancer + algorithms. + oneOf: + - not: + anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName - required: - httpHeaderName - required: @@ -942,16 +429,275 @@ spec: - useSourceIp - required: - httpQueryParameterName - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - required: - - consistentHash + required: + - consistentHash + properties: + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP + header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP + query parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute + or failover can be set.' + items: + properties: + from: + description: Originating locality, '/' + separated, e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities + to traffic distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, + this is DestinationRule-level and will override + mesh wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute + can be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string + type: object + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host + is ejected from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a + host is ejected from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep + analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + port: + properties: + number: + type: integer + type: object + tls: + description: TLS related settings for connections + to the upstream service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server + during TLS handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: array + tls: + description: TLS related settings for connections to the + upstream service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server during + TLS handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: object + type: array + trafficPolicy: + properties: + connectionPool: + properties: + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection should be upgraded + to http2 for the associated destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP requests to + a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream connection + pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per connection + to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol will be preserved + while initiating connection to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and TCP upstream + connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP connections + to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE on the socket + to enable TCP Keepalives. + properties: + interval: + description: The time duration between keep-alive + probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object + type: object + loadBalancer: + description: Settings controlling the load balancer algorithms. + oneOf: + - not: + anyOf: - required: - simple - properties: @@ -977,210 +723,1622 @@ spec: - httpQueryParameterName required: - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + properties: + consistentHash: properties: - consistentHash: + httpCookie: + description: Hash based on HTTP cookie. properties: - httpCookie: - description: Hash based on HTTP cookie. - properties: - name: - description: Name of the cookie. - format: string - type: string - path: - description: Path to set for the cookie. - format: string - type: string - ttl: - description: Lifetime of the cookie. - type: string - type: object - httpHeaderName: - description: Hash based on a specific HTTP header. + name: + description: Name of the cookie. format: string type: string - httpQueryParameterName: - description: Hash based on a specific HTTP query parameter. + path: + description: Path to set for the cookie. format: string type: string - minimumRingSize: - type: integer - useSourceIp: - description: Hash based on the source IP address. - type: boolean + ttl: + description: Lifetime of the cookie. + type: string type: object - localityLbSetting: - properties: - distribute: - description: 'Optional: only one of distribute or - failover can be set.' - items: - properties: - from: - description: Originating locality, '/' separated, - e.g. - format: string - type: string - to: - additionalProperties: - type: integer - description: Map of upstream localities to traffic - distribution weights. - type: object - type: object - type: array - enabled: - description: enable locality load balancing, this - is DestinationRule-level and will override mesh - wide settings in entirety. - nullable: true - type: boolean - failover: - description: 'Optional: only failover or distribute - can be set.' - items: - properties: - from: - description: Originating region. - format: string - type: string - to: - format: string - type: string - type: object - type: array - type: object - simple: - enum: - - ROUND_ROBIN - - LEAST_CONN - - RANDOM - - PASSTHROUGH + httpHeaderName: + description: Hash based on a specific HTTP header. + format: string type: string + httpQueryParameterName: + description: Hash based on a specific HTTP query parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean type: object - outlierDetection: + localityLbSetting: properties: - baseEjectionTime: - description: Minimum ejection duration. - type: string - consecutive5xxErrors: - description: Number of 5xx errors before a host is ejected - from the connection pool. - nullable: true - type: integer - consecutiveErrors: - format: int32 - type: integer - consecutiveGatewayErrors: - description: Number of gateway errors before a host is - ejected from the connection pool. - nullable: true - type: integer - interval: - description: Time interval between ejection sweep analysis. - type: string - maxEjectionPercent: - format: int32 - type: integer - minHealthPercent: - format: int32 - type: integer - type: object - port: - properties: - number: - type: integer - type: object - tls: - description: TLS related settings for connections to the upstream - service. - properties: - caCertificates: - format: string - type: string - clientCertificate: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - credentialName: - format: string - type: string - mode: - enum: - - DISABLE - - SIMPLE - - MUTUAL - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - sni: - description: SNI string to present to the server during - TLS handshake. - format: string - type: string - subjectAltNames: + distribute: + description: 'Optional: only one of distribute or failover + can be set.' items: - format: string - type: string + properties: + from: + description: Originating locality, '/' separated, + e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities to traffic + distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, this is DestinationRule-level + and will override mesh wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute can + be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object type: array type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string type: object - type: array - tls: - description: TLS related settings for connections to the upstream - service. - properties: - caCertificates: - format: string - type: string - clientCertificate: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - credentialName: - format: string - type: string - mode: - enum: - - DISABLE - - SIMPLE - - MUTUAL - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - sni: - description: SNI string to present to the server during TLS - handshake. - format: string - type: string - subjectAltNames: - items: + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host is ejected + from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a host is ejected + from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + portLevelSettings: + description: Traffic policies specific to individual ports. + items: + properties: + connectionPool: + properties: + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection should + be upgraded to http2 for the associated destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP requests + to a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream connection + pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per connection + to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol will + be preserved while initiating connection to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and TCP upstream + connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP connections + to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE on the + socket to enable TCP Keepalives. + properties: + interval: + description: The time duration between keep-alive + probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object + type: object + loadBalancer: + description: Settings controlling the load balancer algorithms. + oneOf: + - not: + anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + properties: + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP query + parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute or + failover can be set.' + items: + properties: + from: + description: Originating locality, '/' separated, + e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities to + traffic distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, this + is DestinationRule-level and will override mesh + wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute + can be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string + type: object + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host is ejected + from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a host + is ejected from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + port: + properties: + number: + type: integer + type: object + tls: + description: TLS related settings for connections to the + upstream service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server during + TLS handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: array + tls: + description: TLS related settings for connections to the upstream + service. + properties: + caCertificates: format: string type: string - type: array - type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1alpha3 + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server during TLS + handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true - - name: v1beta1 + subresources: + status: {} + - additionalPrinterColumns: + - description: The name of a service from the service registry + jsonPath: .spec.host + name: Host + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting load balancing, outlier detection, + etc. See more details at: https://istio.io/docs/reference/config/networking/destination-rule.html' + properties: + exportTo: + description: A list of namespaces to which this destination rule is + exported. + items: + format: string + type: string + type: array + host: + description: The name of a service from the service registry. + format: string + type: string + subsets: + items: + properties: + labels: + additionalProperties: + format: string + type: string + type: object + name: + description: Name of the subset. + format: string + type: string + trafficPolicy: + description: Traffic policies that apply to this subset. + properties: + connectionPool: + properties: + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection should + be upgraded to http2 for the associated destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP requests + to a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream connection + pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per connection + to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol will + be preserved while initiating connection to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and TCP upstream + connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP connections + to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE on the + socket to enable TCP Keepalives. + properties: + interval: + description: The time duration between keep-alive + probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object + type: object + loadBalancer: + description: Settings controlling the load balancer algorithms. + oneOf: + - not: + anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + properties: + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP query + parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute or + failover can be set.' + items: + properties: + from: + description: Originating locality, '/' separated, + e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities to + traffic distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, this + is DestinationRule-level and will override mesh + wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute + can be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string + type: object + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host is ejected + from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a host + is ejected from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + portLevelSettings: + description: Traffic policies specific to individual ports. + items: + properties: + connectionPool: + properties: + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection + should be upgraded to http2 for the associated + destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP + requests to a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to + a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream + connection pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per + connection to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol + will be preserved while initiating connection + to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and + TCP upstream connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP + connections to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE + on the socket to enable TCP Keepalives. + properties: + interval: + description: The time duration between + keep-alive probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object + type: object + loadBalancer: + description: Settings controlling the load balancer + algorithms. + oneOf: + - not: + anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + properties: + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP + header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP + query parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute + or failover can be set.' + items: + properties: + from: + description: Originating locality, '/' + separated, e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities + to traffic distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, + this is DestinationRule-level and will override + mesh wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute + can be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string + type: object + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host + is ejected from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a + host is ejected from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep + analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + port: + properties: + number: + type: integer + type: object + tls: + description: TLS related settings for connections + to the upstream service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server + during TLS handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: array + tls: + description: TLS related settings for connections to the + upstream service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server during + TLS handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: object + type: array + trafficPolicy: + properties: + connectionPool: + properties: + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection should be upgraded + to http2 for the associated destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP requests to + a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream connection + pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per connection + to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol will be preserved + while initiating connection to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and TCP upstream + connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP connections + to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE on the socket + to enable TCP Keepalives. + properties: + interval: + description: The time duration between keep-alive + probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object + type: object + loadBalancer: + description: Settings controlling the load balancer algorithms. + oneOf: + - not: + anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + properties: + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP query parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute or failover + can be set.' + items: + properties: + from: + description: Originating locality, '/' separated, + e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities to traffic + distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, this is DestinationRule-level + and will override mesh wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute can + be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string + type: object + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host is ejected + from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a host is ejected + from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + portLevelSettings: + description: Traffic policies specific to individual ports. + items: + properties: + connectionPool: + properties: + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection should + be upgraded to http2 for the associated destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP requests + to a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream connection + pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per connection + to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol will + be preserved while initiating connection to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and TCP upstream + connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP connections + to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE on the + socket to enable TCP Keepalives. + properties: + interval: + description: The time duration between keep-alive + probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object + type: object + loadBalancer: + description: Settings controlling the load balancer algorithms. + oneOf: + - not: + anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + properties: + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP query + parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute or + failover can be set.' + items: + properties: + from: + description: Originating locality, '/' separated, + e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities to + traffic distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, this + is DestinationRule-level and will override mesh + wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute + can be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string + type: object + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host is ejected + from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a host + is ejected from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + port: + properties: + number: + type: integer + type: object + tls: + description: TLS related settings for connections to the + upstream service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server during + TLS handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: array + tls: + description: TLS related settings for connections to the upstream + service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server during TLS + handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: false + subresources: + status: {} --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -1201,238 +2359,238 @@ spec: listKind: EnvoyFilterList plural: envoyfilters singular: envoyfilter - preserveUnknownFields: true scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Customizing Envoy configuration generated by Istio. See more - details at: https://istio.io/docs/reference/config/networking/envoy-filter.html' - properties: - configPatches: - description: One or more patches with match conditions. - items: - properties: - applyTo: - enum: - - INVALID - - LISTENER - - FILTER_CHAIN - - NETWORK_FILTER - - HTTP_FILTER - - ROUTE_CONFIGURATION - - VIRTUAL_HOST - - HTTP_ROUTE - - CLUSTER - - EXTENSION_CONFIG - type: string - match: - description: Match on listener/route configuration/cluster. - oneOf: - - not: - anyOf: - - required: - - listener - - required: - - routeConfiguration - - required: - - cluster - - required: - - listener - - required: - - routeConfiguration - - required: - - cluster - properties: - cluster: - description: Match on envoy cluster attributes. - properties: - name: - description: The exact name of the cluster to match. - format: string - type: string - portNumber: - description: The service port for which this cluster was - generated. - type: integer - service: - description: The fully qualified service name for this - cluster. - format: string - type: string - subset: - description: The subset associated with the service. - format: string - type: string - type: object - context: - description: The specific config generation context to match - on. - enum: - - ANY - - SIDECAR_INBOUND - - SIDECAR_OUTBOUND - - GATEWAY - type: string - listener: - description: Match on envoy listener attributes. - properties: - filterChain: - description: Match a specific filter chain in a listener. - properties: - applicationProtocols: - description: Applies only to sidecars. - format: string - type: string - destinationPort: - description: The destination_port value used by a - filter chain's match condition. - type: integer - filter: - description: The name of a specific filter to apply - the patch to. - properties: - name: - description: The filter name to match on. - format: string - type: string - subFilter: - properties: - name: - description: The filter name to match on. - format: string - type: string - type: object - type: object - name: - description: The name assigned to the filter chain. - format: string - type: string - sni: - description: The SNI value used by a filter chain's - match condition. - format: string - type: string - transportProtocol: - description: Applies only to `SIDECAR_INBOUND` context. - format: string - type: string - type: object - name: - description: Match a specific listener by its name. - format: string - type: string - portName: - format: string - type: string - portNumber: - type: integer - type: object - proxy: - description: Match on properties associated with a proxy. - properties: - metadata: - additionalProperties: - format: string - type: string - type: object - proxyVersion: - format: string - type: string - type: object - routeConfiguration: - description: Match on envoy HTTP route configuration attributes. - properties: - gateway: - format: string - type: string - name: - description: Route configuration name to match on. - format: string - type: string - portName: - description: Applicable only for GATEWAY context. - format: string - type: string - portNumber: - type: integer - vhost: - properties: - name: - format: string - type: string - route: - description: Match a specific route within the virtual - host. - properties: - action: - description: Match a route with specific action - type. - enum: - - ANY - - ROUTE - - REDIRECT - - DIRECT_RESPONSE - type: string - name: - format: string - type: string - type: object - type: object - type: object - type: object - patch: - description: The patch to apply along with the operation. - properties: - filterClass: - description: Determines the filter insertion order. - enum: - - UNSPECIFIED - - AUTHN - - AUTHZ - - STATS - type: string - operation: - description: Determines how the patch should be applied. - enum: - - INVALID - - MERGE - - ADD - - REMOVE - - INSERT_BEFORE - - INSERT_AFTER - - INSERT_FIRST - - REPLACE - type: string - value: - description: The JSON config of the object being patched. - type: object - type: object - type: object - type: array - workloadSelector: - properties: - labels: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object versions: - name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Customizing Envoy configuration generated by Istio. See + more details at: https://istio.io/docs/reference/config/networking/envoy-filter.html' + properties: + configPatches: + description: One or more patches with match conditions. + items: + properties: + applyTo: + enum: + - INVALID + - LISTENER + - FILTER_CHAIN + - NETWORK_FILTER + - HTTP_FILTER + - ROUTE_CONFIGURATION + - VIRTUAL_HOST + - HTTP_ROUTE + - CLUSTER + - EXTENSION_CONFIG + type: string + match: + description: Match on listener/route configuration/cluster. + oneOf: + - not: + anyOf: + - required: + - listener + - required: + - routeConfiguration + - required: + - cluster + - required: + - listener + - required: + - routeConfiguration + - required: + - cluster + properties: + cluster: + description: Match on envoy cluster attributes. + properties: + name: + description: The exact name of the cluster to match. + format: string + type: string + portNumber: + description: The service port for which this cluster + was generated. + type: integer + service: + description: The fully qualified service name for this + cluster. + format: string + type: string + subset: + description: The subset associated with the service. + format: string + type: string + type: object + context: + description: The specific config generation context to match + on. + enum: + - ANY + - SIDECAR_INBOUND + - SIDECAR_OUTBOUND + - GATEWAY + type: string + listener: + description: Match on envoy listener attributes. + properties: + filterChain: + description: Match a specific filter chain in a listener. + properties: + applicationProtocols: + description: Applies only to sidecars. + format: string + type: string + destinationPort: + description: The destination_port value used by + a filter chain's match condition. + type: integer + filter: + description: The name of a specific filter to apply + the patch to. + properties: + name: + description: The filter name to match on. + format: string + type: string + subFilter: + properties: + name: + description: The filter name to match on. + format: string + type: string + type: object + type: object + name: + description: The name assigned to the filter chain. + format: string + type: string + sni: + description: The SNI value used by a filter chain's + match condition. + format: string + type: string + transportProtocol: + description: Applies only to `SIDECAR_INBOUND` context. + format: string + type: string + type: object + name: + description: Match a specific listener by its name. + format: string + type: string + portName: + format: string + type: string + portNumber: + type: integer + type: object + proxy: + description: Match on properties associated with a proxy. + properties: + metadata: + additionalProperties: + format: string + type: string + type: object + proxyVersion: + format: string + type: string + type: object + routeConfiguration: + description: Match on envoy HTTP route configuration attributes. + properties: + gateway: + format: string + type: string + name: + description: Route configuration name to match on. + format: string + type: string + portName: + description: Applicable only for GATEWAY context. + format: string + type: string + portNumber: + type: integer + vhost: + properties: + name: + format: string + type: string + route: + description: Match a specific route within the virtual + host. + properties: + action: + description: Match a route with specific action + type. + enum: + - ANY + - ROUTE + - REDIRECT + - DIRECT_RESPONSE + type: string + name: + format: string + type: string + type: object + type: object + type: object + type: object + patch: + description: The patch to apply along with the operation. + properties: + filterClass: + description: Determines the filter insertion order. + enum: + - UNSPECIFIED + - AUTHN + - AUTHZ + - STATS + type: string + operation: + description: Determines how the patch should be applied. + enum: + - INVALID + - MERGE + - ADD + - REMOVE + - INSERT_BEFORE + - INSERT_AFTER + - INSERT_FIRST + - REPLACE + type: string + value: + description: The JSON config of the object being patched. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + type: array + workloadSelector: + properties: + labels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true + subresources: + status: {} --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -1455,146 +2613,273 @@ spec: shortNames: - gw singular: gateway - preserveUnknownFields: false scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting edge load balancer. See more details - at: https://istio.io/docs/reference/config/networking/gateway.html' - properties: - selector: - additionalProperties: - format: string - type: string - type: object - servers: - description: A list of server specifications. - items: - properties: - bind: - format: string - type: string - defaultEndpoint: - format: string - type: string - hosts: - description: One or more hosts exposed by this gateway. - items: - format: string - type: string - type: array - name: - description: An optional name of the server, when set must be - unique across all servers. - format: string - type: string - port: - properties: - name: - description: Label assigned to the port. - format: string - type: string - number: - description: A valid non-negative integer port number. - type: integer - protocol: - description: The protocol exposed on the port. - format: string - type: string - targetPort: - type: integer - type: object - tls: - description: Set of TLS related options that govern the server's - behavior. - properties: - caCertificates: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - cipherSuites: - description: 'Optional: If specified, only support the specified - cipher list.' - items: - format: string - type: string - type: array - credentialName: - format: string - type: string - httpsRedirect: - type: boolean - maxProtocolVersion: - description: 'Optional: Maximum TLS protocol version.' - enum: - - TLS_AUTO - - TLSV1_0 - - TLSV1_1 - - TLSV1_2 - - TLSV1_3 - type: string - minProtocolVersion: - description: 'Optional: Minimum TLS protocol version.' - enum: - - TLS_AUTO - - TLSV1_0 - - TLSV1_1 - - TLSV1_2 - - TLSV1_3 - type: string - mode: - enum: - - PASSTHROUGH - - SIMPLE - - MUTUAL - - AUTO_PASSTHROUGH - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. - format: string - type: string - serverCertificate: - description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. - format: string - type: string - subjectAltNames: - items: - format: string - type: string - type: array - verifyCertificateHash: - items: - format: string - type: string - type: array - verifyCertificateSpki: - items: - format: string - type: string - type: array - type: object - type: object - type: array - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object versions: - name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting edge load balancer. See more details + at: https://istio.io/docs/reference/config/networking/gateway.html' + properties: + selector: + additionalProperties: + format: string + type: string + type: object + servers: + description: A list of server specifications. + items: + properties: + bind: + format: string + type: string + defaultEndpoint: + format: string + type: string + hosts: + description: One or more hosts exposed by this gateway. + items: + format: string + type: string + type: array + name: + description: An optional name of the server, when set must be + unique across all servers. + format: string + type: string + port: + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + tls: + description: Set of TLS related options that govern the server's + behavior. + properties: + caCertificates: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + cipherSuites: + description: 'Optional: If specified, only support the specified + cipher list.' + items: + format: string + type: string + type: array + credentialName: + format: string + type: string + httpsRedirect: + type: boolean + maxProtocolVersion: + description: 'Optional: Maximum TLS protocol version.' + enum: + - TLS_AUTO + - TLSV1_0 + - TLSV1_1 + - TLSV1_2 + - TLSV1_3 + type: string + minProtocolVersion: + description: 'Optional: Minimum TLS protocol version.' + enum: + - TLS_AUTO + - TLSV1_0 + - TLSV1_1 + - TLSV1_2 + - TLSV1_3 + type: string + mode: + enum: + - PASSTHROUGH + - SIMPLE + - MUTUAL + - AUTO_PASSTHROUGH + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. + format: string + type: string + serverCertificate: + description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + verifyCertificateHash: + items: + format: string + type: string + type: array + verifyCertificateSpki: + items: + format: string + type: string + type: array + type: object + type: object + type: array + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true + subresources: + status: {} - name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting edge load balancer. See more details + at: https://istio.io/docs/reference/config/networking/gateway.html' + properties: + selector: + additionalProperties: + format: string + type: string + type: object + servers: + description: A list of server specifications. + items: + properties: + bind: + format: string + type: string + defaultEndpoint: + format: string + type: string + hosts: + description: One or more hosts exposed by this gateway. + items: + format: string + type: string + type: array + name: + description: An optional name of the server, when set must be + unique across all servers. + format: string + type: string + port: + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + tls: + description: Set of TLS related options that govern the server's + behavior. + properties: + caCertificates: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + cipherSuites: + description: 'Optional: If specified, only support the specified + cipher list.' + items: + format: string + type: string + type: array + credentialName: + format: string + type: string + httpsRedirect: + type: boolean + maxProtocolVersion: + description: 'Optional: Maximum TLS protocol version.' + enum: + - TLS_AUTO + - TLSV1_0 + - TLSV1_1 + - TLSV1_2 + - TLSV1_3 + type: string + minProtocolVersion: + description: 'Optional: Minimum TLS protocol version.' + enum: + - TLS_AUTO + - TLSV1_0 + - TLSV1_1 + - TLSV1_2 + - TLSV1_3 + type: string + mode: + enum: + - PASSTHROUGH + - SIMPLE + - MUTUAL + - AUTO_PASSTHROUGH + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. + format: string + type: string + serverCertificate: + description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + verifyCertificateHash: + items: + format: string + type: string + type: array + verifyCertificateSpki: + items: + format: string + type: string + type: array + type: object + type: object + type: array + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: false + subresources: + status: {} --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -1606,28 +2891,6 @@ metadata: release: istio name: serviceentries.networking.istio.io spec: - additionalPrinterColumns: - - JSONPath: .spec.hosts - description: The hosts associated with the ServiceEntry - name: Hosts - type: string - - JSONPath: .spec.location - description: Whether the service is external to the mesh or part of the mesh (MESH_EXTERNAL - or MESH_INTERNAL) - name: Location - type: string - - JSONPath: .spec.resolution - description: Service discovery mode for the hosts (NONE, STATIC, or DNS) - name: Resolution - type: string - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date group: networking.istio.io names: categories: @@ -1639,26 +2902,2457 @@ spec: shortNames: - se singular: serviceentry - preserveUnknownFields: false scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting service registry. See more details - at: https://istio.io/docs/reference/config/networking/service-entry.html' - properties: - addresses: - description: The virtual IP addresses associated with the service. - items: + versions: + - additionalPrinterColumns: + - description: The hosts associated with the ServiceEntry + jsonPath: .spec.hosts + name: Hosts + type: string + - description: Whether the service is external to the mesh or part of the mesh + (MESH_EXTERNAL or MESH_INTERNAL) + jsonPath: .spec.location + name: Location + type: string + - description: Service discovery mode for the hosts (NONE, STATIC, or DNS) + jsonPath: .spec.resolution + name: Resolution + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting service registry. See more details + at: https://istio.io/docs/reference/config/networking/service-entry.html' + properties: + addresses: + description: The virtual IP addresses associated with the service. + items: + format: string + type: string + type: array + endpoints: + description: One or more endpoints associated with the service. + items: + properties: + address: + format: string + type: string + labels: + additionalProperties: + format: string + type: string + description: One or more labels associated with the endpoint. + type: object + locality: + description: The locality associated with the endpoint. + format: string + type: string + network: + format: string + type: string + ports: + additionalProperties: + type: integer + description: Set of ports associated with the endpoint. + type: object + serviceAccount: + format: string + type: string + weight: + description: The load balancing weight associated with the endpoint. + type: integer + type: object + type: array + exportTo: + description: A list of namespaces to which this service is exported. + items: + format: string + type: string + type: array + hosts: + description: The hosts associated with the ServiceEntry. + items: + format: string + type: string + type: array + location: + enum: + - MESH_EXTERNAL + - MESH_INTERNAL + type: string + ports: + description: The ports associated with the external service. + items: + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + type: array + resolution: + description: Service discovery mode for the hosts. + enum: + - NONE + - STATIC + - DNS + type: string + subjectAltNames: + items: + format: string + type: string + type: array + workloadSelector: + description: Applicable only for MESH_INTERNAL services. + properties: + labels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: The hosts associated with the ServiceEntry + jsonPath: .spec.hosts + name: Hosts + type: string + - description: Whether the service is external to the mesh or part of the mesh + (MESH_EXTERNAL or MESH_INTERNAL) + jsonPath: .spec.location + name: Location + type: string + - description: Service discovery mode for the hosts (NONE, STATIC, or DNS) + jsonPath: .spec.resolution + name: Resolution + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting service registry. See more details + at: https://istio.io/docs/reference/config/networking/service-entry.html' + properties: + addresses: + description: The virtual IP addresses associated with the service. + items: + format: string + type: string + type: array + endpoints: + description: One or more endpoints associated with the service. + items: + properties: + address: + format: string + type: string + labels: + additionalProperties: + format: string + type: string + description: One or more labels associated with the endpoint. + type: object + locality: + description: The locality associated with the endpoint. + format: string + type: string + network: + format: string + type: string + ports: + additionalProperties: + type: integer + description: Set of ports associated with the endpoint. + type: object + serviceAccount: + format: string + type: string + weight: + description: The load balancing weight associated with the endpoint. + type: integer + type: object + type: array + exportTo: + description: A list of namespaces to which this service is exported. + items: + format: string + type: string + type: array + hosts: + description: The hosts associated with the ServiceEntry. + items: + format: string + type: string + type: array + location: + enum: + - MESH_EXTERNAL + - MESH_INTERNAL + type: string + ports: + description: The ports associated with the external service. + items: + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + type: array + resolution: + description: Service discovery mode for the hosts. + enum: + - NONE + - STATIC + - DNS + type: string + subjectAltNames: + items: + format: string + type: string + type: array + workloadSelector: + description: Applicable only for MESH_INTERNAL services. + properties: + labels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + "helm.sh/resource-policy": keep + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + name: sidecars.networking.istio.io +spec: + group: networking.istio.io + names: + categories: + - istio-io + - networking-istio-io + kind: Sidecar + listKind: SidecarList + plural: sidecars + singular: sidecar + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting network reachability of a sidecar. + See more details at: https://istio.io/docs/reference/config/networking/sidecar.html' + properties: + egress: + items: + properties: + bind: + format: string + type: string + captureMode: + enum: + - DEFAULT + - IPTABLES + - NONE + type: string + hosts: + items: + format: string + type: string + type: array + port: + description: The port associated with the listener. + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + type: object + type: array + ingress: + items: + properties: + bind: + description: The IP to which the listener should be bound. + format: string + type: string + captureMode: + enum: + - DEFAULT + - IPTABLES + - NONE + type: string + defaultEndpoint: + format: string + type: string + port: + description: The port associated with the listener. + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + type: object + type: array + outboundTrafficPolicy: + description: Configuration for the outbound traffic policy. + properties: + egressProxy: + properties: + host: + description: The name of a service from the service registry. + format: string + type: string + port: + description: Specifies the port on the host that is being + addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + mode: + enum: + - REGISTRY_ONLY + - ALLOW_ANY + type: string + type: object + workloadSelector: + properties: + labels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting network reachability of a sidecar. + See more details at: https://istio.io/docs/reference/config/networking/sidecar.html' + properties: + egress: + items: + properties: + bind: + format: string + type: string + captureMode: + enum: + - DEFAULT + - IPTABLES + - NONE + type: string + hosts: + items: + format: string + type: string + type: array + port: + description: The port associated with the listener. + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + type: object + type: array + ingress: + items: + properties: + bind: + description: The IP to which the listener should be bound. + format: string + type: string + captureMode: + enum: + - DEFAULT + - IPTABLES + - NONE + type: string + defaultEndpoint: + format: string + type: string + port: + description: The port associated with the listener. + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + type: object + type: array + outboundTrafficPolicy: + description: Configuration for the outbound traffic policy. + properties: + egressProxy: + properties: + host: + description: The name of a service from the service registry. + format: string + type: string + port: + description: Specifies the port on the host that is being + addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + mode: + enum: + - REGISTRY_ONLY + - ALLOW_ANY + type: string + type: object + workloadSelector: + properties: + labels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + "helm.sh/resource-policy": keep + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + name: virtualservices.networking.istio.io +spec: + group: networking.istio.io + names: + categories: + - istio-io + - networking-istio-io + kind: VirtualService + listKind: VirtualServiceList + plural: virtualservices + shortNames: + - vs + singular: virtualservice + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The names of gateways and sidecars that should apply these routes + jsonPath: .spec.gateways + name: Gateways + type: string + - description: The destination hosts to which traffic is being sent + jsonPath: .spec.hosts + name: Hosts + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting label/content routing, sni routing, + etc. See more details at: https://istio.io/docs/reference/config/networking/virtual-service.html' + properties: + exportTo: + description: A list of namespaces to which this virtual service is + exported. + items: + format: string + type: string + type: array + gateways: + description: The names of gateways and sidecars that should apply + these routes. + items: + format: string + type: string + type: array + hosts: + description: The destination hosts to which traffic is being sent. + items: + format: string + type: string + type: array + http: + description: An ordered list of route rules for HTTP traffic. + items: + properties: + corsPolicy: + description: Cross-Origin Resource Sharing policy (CORS). + properties: + allowCredentials: + nullable: true + type: boolean + allowHeaders: + items: + format: string + type: string + type: array + allowMethods: + description: List of HTTP methods allowed to access the + resource. + items: + format: string + type: string + type: array + allowOrigin: + description: The list of origins that are allowed to perform + CORS requests. + items: + format: string + type: string + type: array + allowOrigins: + description: String patterns that match allowed origins. + items: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + type: array + exposeHeaders: + items: + format: string + type: string + type: array + maxAge: + type: string + type: object + delegate: + properties: + name: + description: Name specifies the name of the delegate VirtualService. + format: string + type: string + namespace: + description: Namespace specifies the namespace where the + delegate VirtualService resides. + format: string + type: string + type: object + fault: + description: Fault injection policy to apply on HTTP traffic + at the client side. + properties: + abort: + oneOf: + - not: + anyOf: + - required: + - httpStatus + - required: + - grpcStatus + - required: + - http2Error + - required: + - httpStatus + - required: + - grpcStatus + - required: + - http2Error + properties: + grpcStatus: + format: string + type: string + http2Error: + format: string + type: string + httpStatus: + description: HTTP status code to use to abort the Http + request. + format: int32 + type: integer + percentage: + description: Percentage of requests to be aborted with + the error code provided. + properties: + value: + format: double + type: number + type: object + type: object + delay: + oneOf: + - not: + anyOf: + - required: + - fixedDelay + - required: + - exponentialDelay + - required: + - fixedDelay + - required: + - exponentialDelay + properties: + exponentialDelay: + type: string + fixedDelay: + description: Add a fixed delay before forwarding the + request. + type: string + percent: + description: Percentage of requests on which the delay + will be injected (0-100). + format: int32 + type: integer + percentage: + description: Percentage of requests on which the delay + will be injected. + properties: + value: + format: double + type: number + type: object + type: object + type: object + headers: + properties: + request: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + response: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + match: + items: + properties: + authority: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + gateways: + description: Names of gateways where the rule should be + applied. + items: + format: string + type: string + type: array + headers: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + type: object + ignoreUriCase: + description: Flag to specify whether the URI matching + should be case-insensitive. + type: boolean + method: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + name: + description: The name assigned to a match. + format: string + type: string + port: + description: Specifies the ports on the host that is being + addressed. + type: integer + queryParams: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + description: Query parameters for matching. + type: object + scheme: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + sourceLabels: + additionalProperties: + format: string + type: string + type: object + sourceNamespace: + description: Source namespace constraining the applicability + of a rule to workloads in that namespace. + format: string + type: string + uri: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + withoutHeaders: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + description: withoutHeader has the same syntax with the + header, but has opposite meaning. + type: object + type: object + type: array + mirror: + properties: + host: + description: The name of a service from the service registry. + format: string + type: string + port: + description: Specifies the port on the host that is being + addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + mirror_percent: + description: Percentage of the traffic to be mirrored by the + `mirror` field. + nullable: true + type: integer + mirrorPercent: + description: Percentage of the traffic to be mirrored by the + `mirror` field. + nullable: true + type: integer + mirrorPercentage: + description: Percentage of the traffic to be mirrored by the + `mirror` field. + properties: + value: + format: double + type: number + type: object + name: + description: The name assigned to the route for debugging purposes. + format: string + type: string + redirect: + description: A HTTP rule can either redirect or forward (default) + traffic. + properties: + authority: + format: string + type: string + redirectCode: + type: integer + uri: + format: string + type: string + type: object + retries: + description: Retry policy for HTTP requests. + properties: + attempts: + description: Number of retries to be allowed for a given + request. + format: int32 + type: integer + perTryTimeout: + description: Timeout per attempt for a given request, including + the initial call and any retries. + type: string + retryOn: + description: Specifies the conditions under which retry + takes place. + format: string + type: string + retryRemoteLocalities: + description: Flag to specify whether the retries should + retry to other localities. + nullable: true + type: boolean + type: object + rewrite: + description: Rewrite HTTP URIs and Authority headers. + properties: + authority: + description: rewrite the Authority/Host header with this + value. + format: string + type: string + uri: + format: string + type: string + type: object + route: + description: A HTTP rule can either redirect or forward (default) + traffic. + items: + properties: + destination: + properties: + host: + description: The name of a service from the service + registry. + format: string + type: string + port: + description: Specifies the port on the host that is + being addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + headers: + properties: + request: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + response: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + weight: + format: int32 + type: integer + type: object + type: array + timeout: + description: Timeout for HTTP requests, default is disabled. + type: string + type: object + type: array + tcp: + description: An ordered list of route rules for opaque TCP traffic. + items: + properties: + match: + items: + properties: + destinationSubnets: + description: IPv4 or IPv6 ip addresses of destination + with optional subnet. + items: + format: string + type: string + type: array + gateways: + description: Names of gateways where the rule should be + applied. + items: + format: string + type: string + type: array + port: + description: Specifies the port on the host that is being + addressed. + type: integer + sourceLabels: + additionalProperties: + format: string + type: string + type: object + sourceNamespace: + description: Source namespace constraining the applicability + of a rule to workloads in that namespace. + format: string + type: string + sourceSubnet: + description: IPv4 or IPv6 ip address of source with optional + subnet. + format: string + type: string + type: object + type: array + route: + description: The destination to which the connection should + be forwarded to. + items: + properties: + destination: + properties: + host: + description: The name of a service from the service + registry. + format: string + type: string + port: + description: Specifies the port on the host that is + being addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + weight: + format: int32 + type: integer + type: object + type: array + type: object + type: array + tls: + items: + properties: + match: + items: + properties: + destinationSubnets: + description: IPv4 or IPv6 ip addresses of destination + with optional subnet. + items: + format: string + type: string + type: array + gateways: + description: Names of gateways where the rule should be + applied. + items: + format: string + type: string + type: array + port: + description: Specifies the port on the host that is being + addressed. + type: integer + sniHosts: + description: SNI (server name indicator) to match on. + items: + format: string + type: string + type: array + sourceLabels: + additionalProperties: + format: string + type: string + type: object + sourceNamespace: + description: Source namespace constraining the applicability + of a rule to workloads in that namespace. + format: string + type: string + type: object + type: array + route: + description: The destination to which the connection should + be forwarded to. + items: + properties: + destination: + properties: + host: + description: The name of a service from the service + registry. + format: string + type: string + port: + description: Specifies the port on the host that is + being addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + weight: + format: int32 + type: integer + type: object + type: array + type: object + type: array + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: The names of gateways and sidecars that should apply these routes + jsonPath: .spec.gateways + name: Gateways + type: string + - description: The destination hosts to which traffic is being sent + jsonPath: .spec.hosts + name: Hosts + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting label/content routing, sni routing, + etc. See more details at: https://istio.io/docs/reference/config/networking/virtual-service.html' + properties: + exportTo: + description: A list of namespaces to which this virtual service is + exported. + items: + format: string + type: string + type: array + gateways: + description: The names of gateways and sidecars that should apply + these routes. + items: + format: string + type: string + type: array + hosts: + description: The destination hosts to which traffic is being sent. + items: + format: string + type: string + type: array + http: + description: An ordered list of route rules for HTTP traffic. + items: + properties: + corsPolicy: + description: Cross-Origin Resource Sharing policy (CORS). + properties: + allowCredentials: + nullable: true + type: boolean + allowHeaders: + items: + format: string + type: string + type: array + allowMethods: + description: List of HTTP methods allowed to access the + resource. + items: + format: string + type: string + type: array + allowOrigin: + description: The list of origins that are allowed to perform + CORS requests. + items: + format: string + type: string + type: array + allowOrigins: + description: String patterns that match allowed origins. + items: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + type: array + exposeHeaders: + items: + format: string + type: string + type: array + maxAge: + type: string + type: object + delegate: + properties: + name: + description: Name specifies the name of the delegate VirtualService. + format: string + type: string + namespace: + description: Namespace specifies the namespace where the + delegate VirtualService resides. + format: string + type: string + type: object + fault: + description: Fault injection policy to apply on HTTP traffic + at the client side. + properties: + abort: + oneOf: + - not: + anyOf: + - required: + - httpStatus + - required: + - grpcStatus + - required: + - http2Error + - required: + - httpStatus + - required: + - grpcStatus + - required: + - http2Error + properties: + grpcStatus: + format: string + type: string + http2Error: + format: string + type: string + httpStatus: + description: HTTP status code to use to abort the Http + request. + format: int32 + type: integer + percentage: + description: Percentage of requests to be aborted with + the error code provided. + properties: + value: + format: double + type: number + type: object + type: object + delay: + oneOf: + - not: + anyOf: + - required: + - fixedDelay + - required: + - exponentialDelay + - required: + - fixedDelay + - required: + - exponentialDelay + properties: + exponentialDelay: + type: string + fixedDelay: + description: Add a fixed delay before forwarding the + request. + type: string + percent: + description: Percentage of requests on which the delay + will be injected (0-100). + format: int32 + type: integer + percentage: + description: Percentage of requests on which the delay + will be injected. + properties: + value: + format: double + type: number + type: object + type: object + type: object + headers: + properties: + request: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + response: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + match: + items: + properties: + authority: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + gateways: + description: Names of gateways where the rule should be + applied. + items: + format: string + type: string + type: array + headers: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + type: object + ignoreUriCase: + description: Flag to specify whether the URI matching + should be case-insensitive. + type: boolean + method: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + name: + description: The name assigned to a match. + format: string + type: string + port: + description: Specifies the ports on the host that is being + addressed. + type: integer + queryParams: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + description: Query parameters for matching. + type: object + scheme: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + sourceLabels: + additionalProperties: + format: string + type: string + type: object + sourceNamespace: + description: Source namespace constraining the applicability + of a rule to workloads in that namespace. + format: string + type: string + uri: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + withoutHeaders: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + description: withoutHeader has the same syntax with the + header, but has opposite meaning. + type: object + type: object + type: array + mirror: + properties: + host: + description: The name of a service from the service registry. + format: string + type: string + port: + description: Specifies the port on the host that is being + addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + mirror_percent: + description: Percentage of the traffic to be mirrored by the + `mirror` field. + nullable: true + type: integer + mirrorPercent: + description: Percentage of the traffic to be mirrored by the + `mirror` field. + nullable: true + type: integer + mirrorPercentage: + description: Percentage of the traffic to be mirrored by the + `mirror` field. + properties: + value: + format: double + type: number + type: object + name: + description: The name assigned to the route for debugging purposes. + format: string + type: string + redirect: + description: A HTTP rule can either redirect or forward (default) + traffic. + properties: + authority: + format: string + type: string + redirectCode: + type: integer + uri: + format: string + type: string + type: object + retries: + description: Retry policy for HTTP requests. + properties: + attempts: + description: Number of retries to be allowed for a given + request. + format: int32 + type: integer + perTryTimeout: + description: Timeout per attempt for a given request, including + the initial call and any retries. + type: string + retryOn: + description: Specifies the conditions under which retry + takes place. + format: string + type: string + retryRemoteLocalities: + description: Flag to specify whether the retries should + retry to other localities. + nullable: true + type: boolean + type: object + rewrite: + description: Rewrite HTTP URIs and Authority headers. + properties: + authority: + description: rewrite the Authority/Host header with this + value. + format: string + type: string + uri: + format: string + type: string + type: object + route: + description: A HTTP rule can either redirect or forward (default) + traffic. + items: + properties: + destination: + properties: + host: + description: The name of a service from the service + registry. + format: string + type: string + port: + description: Specifies the port on the host that is + being addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + headers: + properties: + request: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + response: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + weight: + format: int32 + type: integer + type: object + type: array + timeout: + description: Timeout for HTTP requests, default is disabled. + type: string + type: object + type: array + tcp: + description: An ordered list of route rules for opaque TCP traffic. + items: + properties: + match: + items: + properties: + destinationSubnets: + description: IPv4 or IPv6 ip addresses of destination + with optional subnet. + items: + format: string + type: string + type: array + gateways: + description: Names of gateways where the rule should be + applied. + items: + format: string + type: string + type: array + port: + description: Specifies the port on the host that is being + addressed. + type: integer + sourceLabels: + additionalProperties: + format: string + type: string + type: object + sourceNamespace: + description: Source namespace constraining the applicability + of a rule to workloads in that namespace. + format: string + type: string + sourceSubnet: + description: IPv4 or IPv6 ip address of source with optional + subnet. + format: string + type: string + type: object + type: array + route: + description: The destination to which the connection should + be forwarded to. + items: + properties: + destination: + properties: + host: + description: The name of a service from the service + registry. + format: string + type: string + port: + description: Specifies the port on the host that is + being addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + weight: + format: int32 + type: integer + type: object + type: array + type: object + type: array + tls: + items: + properties: + match: + items: + properties: + destinationSubnets: + description: IPv4 or IPv6 ip addresses of destination + with optional subnet. + items: + format: string + type: string + type: array + gateways: + description: Names of gateways where the rule should be + applied. + items: + format: string + type: string + type: array + port: + description: Specifies the port on the host that is being + addressed. + type: integer + sniHosts: + description: SNI (server name indicator) to match on. + items: + format: string + type: string + type: array + sourceLabels: + additionalProperties: + format: string + type: string + type: object + sourceNamespace: + description: Source namespace constraining the applicability + of a rule to workloads in that namespace. + format: string + type: string + type: object + type: array + route: + description: The destination to which the connection should + be forwarded to. + items: + properties: + destination: + properties: + host: + description: The name of a service from the service + registry. + format: string + type: string + port: + description: Specifies the port on the host that is + being addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + weight: + format: int32 + type: integer + type: object + type: array + type: object + type: array + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + "helm.sh/resource-policy": keep + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + name: workloadentries.networking.istio.io +spec: + group: networking.istio.io + names: + categories: + - istio-io + - networking-istio-io + kind: WorkloadEntry + listKind: WorkloadEntryList + plural: workloadentries + shortNames: + - we + singular: workloadentry + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Address associated with the network endpoint. + jsonPath: .spec.address + name: Address + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting VMs onboarded into the mesh. See + more details at: https://istio.io/docs/reference/config/networking/workload-entry.html' + properties: + address: format: string type: string - type: array - endpoints: - description: One or more endpoints associated with the service. - items: + labels: + additionalProperties: + format: string + type: string + description: One or more labels associated with the endpoint. + type: object + locality: + description: The locality associated with the endpoint. + format: string + type: string + network: + format: string + type: string + ports: + additionalProperties: + type: integer + description: Set of ports associated with the endpoint. + type: object + serviceAccount: + format: string + type: string + weight: + description: The load balancing weight associated with the endpoint. + type: integer + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Address associated with the network endpoint. + jsonPath: .spec.address + name: Address + type: string + name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting VMs onboarded into the mesh. See + more details at: https://istio.io/docs/reference/config/networking/workload-entry.html' + properties: + address: + format: string + type: string + labels: + additionalProperties: + format: string + type: string + description: One or more labels associated with the endpoint. + type: object + locality: + description: The locality associated with the endpoint. + format: string + type: string + network: + format: string + type: string + ports: + additionalProperties: + type: integer + description: Set of ports associated with the endpoint. + type: object + serviceAccount: + format: string + type: string + weight: + description: The load balancing weight associated with the endpoint. + type: integer + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + name: workloadgroups.networking.istio.io +spec: + group: networking.istio.io + names: + categories: + - istio-io + - networking-istio-io + kind: WorkloadGroup + listKind: WorkloadGroupList + plural: workloadgroups + shortNames: + - wg + singular: workloadgroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Describes a collection of workload instances. See more details + at: https://istio.io/docs/reference/config/networking/workload-group.html' + properties: + metadata: + description: Metadata that will be used for all corresponding `WorkloadEntries`. + properties: + annotations: + additionalProperties: + format: string + type: string + type: object + labels: + additionalProperties: + format: string + type: string + type: object + type: object + probe: + description: '`ReadinessProbe` describes the configuration the user + must provide for healthchecking on their workload.' + oneOf: + - not: + anyOf: + - required: + - httpGet + - required: + - tcpSocket + - required: + - exec + - required: + - httpGet + - required: + - tcpSocket + - required: + - exec + properties: + exec: + description: Health is determined by how the command that is executed + exited. + properties: + command: + description: Command to run. + items: + format: string + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be + considered failed after having succeeded. + format: int32 + type: integer + httpGet: + properties: + host: + description: Host name to connect to, defaults to the pod + IP. + format: string + type: string + httpHeaders: + description: Headers the proxy will pass on to make the request. + items: + properties: + name: + format: string + type: string + value: + format: string + type: string + type: object + type: array + path: + description: Path to access on the HTTP server. + format: string + type: string + port: + description: Port on which the endpoint lives. + type: integer + scheme: + format: string + type: string + type: object + initialDelaySeconds: + description: Number of seconds after the container has started + before readiness probes are initiated. + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be + considered successful after having failed. + format: int32 + type: integer + tcpSocket: + description: Health is determined by if the proxy is able to connect. + properties: + host: + format: string + type: string + port: + type: integer + type: object + timeoutSeconds: + description: Number of seconds after which the probe times out. + format: int32 + type: integer + type: object + template: + description: Template to be used for the generation of `WorkloadEntry` + resources that belong to this `WorkloadGroup`. properties: address: format: string @@ -1688,1328 +5382,18 @@ spec: description: The load balancing weight associated with the endpoint. type: integer type: object - type: array - exportTo: - description: A list of namespaces to which this service is exported. - items: - format: string - type: string - type: array - hosts: - description: The hosts associated with the ServiceEntry. - items: - format: string - type: string - type: array - location: - enum: - - MESH_EXTERNAL - - MESH_INTERNAL - type: string - ports: - description: The ports associated with the external service. - items: - properties: - name: - description: Label assigned to the port. - format: string - type: string - number: - description: A valid non-negative integer port number. - type: integer - protocol: - description: The protocol exposed on the port. - format: string - type: string - targetPort: - type: integer - type: object - type: array - resolution: - description: Service discovery mode for the hosts. - enum: - - NONE - - STATIC - - DNS - type: string - subjectAltNames: - items: - format: string - type: string - type: array - workloadSelector: - description: Applicable only for MESH_INTERNAL services. - properties: - labels: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1alpha3 + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true - - name: v1beta1 - served: true - storage: false + subresources: + status: {} --- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - "helm.sh/resource-policy": keep - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: sidecars.networking.istio.io -spec: - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: Sidecar - listKind: SidecarList - plural: sidecars - singular: sidecar - preserveUnknownFields: false - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting network reachability of a sidecar. - See more details at: https://istio.io/docs/reference/config/networking/sidecar.html' - properties: - egress: - items: - properties: - bind: - format: string - type: string - captureMode: - enum: - - DEFAULT - - IPTABLES - - NONE - type: string - hosts: - items: - format: string - type: string - type: array - port: - description: The port associated with the listener. - properties: - name: - description: Label assigned to the port. - format: string - type: string - number: - description: A valid non-negative integer port number. - type: integer - protocol: - description: The protocol exposed on the port. - format: string - type: string - targetPort: - type: integer - type: object - type: object - type: array - ingress: - items: - properties: - bind: - description: The IP to which the listener should be bound. - format: string - type: string - captureMode: - enum: - - DEFAULT - - IPTABLES - - NONE - type: string - defaultEndpoint: - format: string - type: string - port: - description: The port associated with the listener. - properties: - name: - description: Label assigned to the port. - format: string - type: string - number: - description: A valid non-negative integer port number. - type: integer - protocol: - description: The protocol exposed on the port. - format: string - type: string - targetPort: - type: integer - type: object - type: object - type: array - outboundTrafficPolicy: - description: Configuration for the outbound traffic policy. - properties: - egressProxy: - properties: - host: - description: The name of a service from the service registry. - format: string - type: string - port: - description: Specifies the port on the host that is being addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - mode: - enum: - - REGISTRY_ONLY - - ALLOW_ANY - type: string - type: object - workloadSelector: - properties: - labels: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1alpha3 - served: true - storage: true - - name: v1beta1 - served: true - storage: false - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - "helm.sh/resource-policy": keep - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: virtualservices.networking.istio.io -spec: - additionalPrinterColumns: - - JSONPath: .spec.gateways - description: The names of gateways and sidecars that should apply these routes - name: Gateways - type: string - - JSONPath: .spec.hosts - description: The destination hosts to which traffic is being sent - name: Hosts - type: string - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: VirtualService - listKind: VirtualServiceList - plural: virtualservices - shortNames: - - vs - singular: virtualservice - preserveUnknownFields: false - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting label/content routing, sni routing, - etc. See more details at: https://istio.io/docs/reference/config/networking/virtual-service.html' - properties: - exportTo: - description: A list of namespaces to which this virtual service is exported. - items: - format: string - type: string - type: array - gateways: - description: The names of gateways and sidecars that should apply these - routes. - items: - format: string - type: string - type: array - hosts: - description: The destination hosts to which traffic is being sent. - items: - format: string - type: string - type: array - http: - description: An ordered list of route rules for HTTP traffic. - items: - properties: - corsPolicy: - description: Cross-Origin Resource Sharing policy (CORS). - properties: - allowCredentials: - nullable: true - type: boolean - allowHeaders: - items: - format: string - type: string - type: array - allowMethods: - description: List of HTTP methods allowed to access the resource. - items: - format: string - type: string - type: array - allowOrigin: - description: The list of origins that are allowed to perform - CORS requests. - items: - format: string - type: string - type: array - allowOrigins: - description: String patterns that match allowed origins. - items: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - type: array - exposeHeaders: - items: - format: string - type: string - type: array - maxAge: - type: string - type: object - delegate: - properties: - name: - description: Name specifies the name of the delegate VirtualService. - format: string - type: string - namespace: - description: Namespace specifies the namespace where the delegate - VirtualService resides. - format: string - type: string - type: object - fault: - description: Fault injection policy to apply on HTTP traffic at - the client side. - properties: - abort: - oneOf: - - not: - anyOf: - - required: - - httpStatus - - required: - - grpcStatus - - required: - - http2Error - - required: - - httpStatus - - required: - - grpcStatus - - required: - - http2Error - properties: - grpcStatus: - format: string - type: string - http2Error: - format: string - type: string - httpStatus: - description: HTTP status code to use to abort the Http - request. - format: int32 - type: integer - percentage: - description: Percentage of requests to be aborted with - the error code provided. - properties: - value: - format: double - type: number - type: object - type: object - delay: - oneOf: - - not: - anyOf: - - required: - - fixedDelay - - required: - - exponentialDelay - - required: - - fixedDelay - - required: - - exponentialDelay - properties: - exponentialDelay: - type: string - fixedDelay: - description: Add a fixed delay before forwarding the request. - type: string - percent: - description: Percentage of requests on which the delay - will be injected (0-100). - format: int32 - type: integer - percentage: - description: Percentage of requests on which the delay - will be injected. - properties: - value: - format: double - type: number - type: object - type: object - type: object - headers: - properties: - request: - properties: - add: - additionalProperties: - format: string - type: string - type: object - remove: - items: - format: string - type: string - type: array - set: - additionalProperties: - format: string - type: string - type: object - type: object - response: - properties: - add: - additionalProperties: - format: string - type: string - type: object - remove: - items: - format: string - type: string - type: array - set: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - match: - items: - properties: - authority: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - gateways: - description: Names of gateways where the rule should be - applied. - items: - format: string - type: string - type: array - headers: - additionalProperties: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - type: object - ignoreUriCase: - description: Flag to specify whether the URI matching should - be case-insensitive. - type: boolean - method: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - name: - description: The name assigned to a match. - format: string - type: string - port: - description: Specifies the ports on the host that is being - addressed. - type: integer - queryParams: - additionalProperties: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - description: Query parameters for matching. - type: object - scheme: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - sourceLabels: - additionalProperties: - format: string - type: string - type: object - sourceNamespace: - description: Source namespace constraining the applicability - of a rule to workloads in that namespace. - format: string - type: string - uri: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - withoutHeaders: - additionalProperties: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - description: withoutHeader has the same syntax with the - header, but has opposite meaning. - type: object - type: object - type: array - mirror: - properties: - host: - description: The name of a service from the service registry. - format: string - type: string - port: - description: Specifies the port on the host that is being - addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - mirror_percent: - description: Percentage of the traffic to be mirrored by the `mirror` - field. - nullable: true - type: integer - mirrorPercent: - description: Percentage of the traffic to be mirrored by the `mirror` - field. - nullable: true - type: integer - mirrorPercentage: - description: Percentage of the traffic to be mirrored by the `mirror` - field. - properties: - value: - format: double - type: number - type: object - name: - description: The name assigned to the route for debugging purposes. - format: string - type: string - redirect: - description: A HTTP rule can either redirect or forward (default) - traffic. - properties: - authority: - format: string - type: string - redirectCode: - type: integer - uri: - format: string - type: string - type: object - retries: - description: Retry policy for HTTP requests. - properties: - attempts: - description: Number of retries to be allowed for a given request. - format: int32 - type: integer - perTryTimeout: - description: Timeout per attempt for a given request, including - the initial call and any retries. - type: string - retryOn: - description: Specifies the conditions under which retry takes - place. - format: string - type: string - retryRemoteLocalities: - description: Flag to specify whether the retries should retry - to other localities. - nullable: true - type: boolean - type: object - rewrite: - description: Rewrite HTTP URIs and Authority headers. - properties: - authority: - description: rewrite the Authority/Host header with this value. - format: string - type: string - uri: - format: string - type: string - type: object - route: - description: A HTTP rule can either redirect or forward (default) - traffic. - items: - properties: - destination: - properties: - host: - description: The name of a service from the service - registry. - format: string - type: string - port: - description: Specifies the port on the host that is - being addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - headers: - properties: - request: - properties: - add: - additionalProperties: - format: string - type: string - type: object - remove: - items: - format: string - type: string - type: array - set: - additionalProperties: - format: string - type: string - type: object - type: object - response: - properties: - add: - additionalProperties: - format: string - type: string - type: object - remove: - items: - format: string - type: string - type: array - set: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - weight: - format: int32 - type: integer - type: object - type: array - timeout: - description: Timeout for HTTP requests, default is disabled. - type: string - type: object - type: array - tcp: - description: An ordered list of route rules for opaque TCP traffic. - items: - properties: - match: - items: - properties: - destinationSubnets: - description: IPv4 or IPv6 ip addresses of destination with - optional subnet. - items: - format: string - type: string - type: array - gateways: - description: Names of gateways where the rule should be - applied. - items: - format: string - type: string - type: array - port: - description: Specifies the port on the host that is being - addressed. - type: integer - sourceLabels: - additionalProperties: - format: string - type: string - type: object - sourceNamespace: - description: Source namespace constraining the applicability - of a rule to workloads in that namespace. - format: string - type: string - sourceSubnet: - description: IPv4 or IPv6 ip address of source with optional - subnet. - format: string - type: string - type: object - type: array - route: - description: The destination to which the connection should be - forwarded to. - items: - properties: - destination: - properties: - host: - description: The name of a service from the service - registry. - format: string - type: string - port: - description: Specifies the port on the host that is - being addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - weight: - format: int32 - type: integer - type: object - type: array - type: object - type: array - tls: - items: - properties: - match: - items: - properties: - destinationSubnets: - description: IPv4 or IPv6 ip addresses of destination with - optional subnet. - items: - format: string - type: string - type: array - gateways: - description: Names of gateways where the rule should be - applied. - items: - format: string - type: string - type: array - port: - description: Specifies the port on the host that is being - addressed. - type: integer - sniHosts: - description: SNI (server name indicator) to match on. - items: - format: string - type: string - type: array - sourceLabels: - additionalProperties: - format: string - type: string - type: object - sourceNamespace: - description: Source namespace constraining the applicability - of a rule to workloads in that namespace. - format: string - type: string - type: object - type: array - route: - description: The destination to which the connection should be - forwarded to. - items: - properties: - destination: - properties: - host: - description: The name of a service from the service - registry. - format: string - type: string - port: - description: Specifies the port on the host that is - being addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - weight: - format: int32 - type: integer - type: object - type: array - type: object - type: array - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1alpha3 - served: true - storage: true - - name: v1beta1 - served: true - storage: false - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - "helm.sh/resource-policy": keep - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: workloadentries.networking.istio.io -spec: - additionalPrinterColumns: - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date - - JSONPath: .spec.address - description: Address associated with the network endpoint. - name: Address - type: string - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: WorkloadEntry - listKind: WorkloadEntryList - plural: workloadentries - shortNames: - - we - singular: workloadentry - preserveUnknownFields: false - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting VMs onboarded into the mesh. See more - details at: https://istio.io/docs/reference/config/networking/workload-entry.html' - properties: - address: - format: string - type: string - labels: - additionalProperties: - format: string - type: string - description: One or more labels associated with the endpoint. - type: object - locality: - description: The locality associated with the endpoint. - format: string - type: string - network: - format: string - type: string - ports: - additionalProperties: - type: integer - description: Set of ports associated with the endpoint. - type: object - serviceAccount: - format: string - type: string - weight: - description: The load balancing weight associated with the endpoint. - type: integer - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1alpha3 - served: true - storage: true - - name: v1beta1 - served: true - storage: false - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: workloadgroups.networking.istio.io -spec: - additionalPrinterColumns: - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: WorkloadGroup - listKind: WorkloadGroupList - plural: workloadgroups - shortNames: - - wg - singular: workloadgroup - preserveUnknownFields: false - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Describes a collection of workload instances. See more details - at: https://istio.io/docs/reference/config/networking/workload-group.html' - properties: - metadata: - description: Metadata that will be used for all corresponding `WorkloadEntries`. - properties: - annotations: - additionalProperties: - format: string - type: string - type: object - labels: - additionalProperties: - format: string - type: string - type: object - type: object - probe: - description: '`ReadinessProbe` describes the configuration the user - must provide for healthchecking on their workload.' - oneOf: - - not: - anyOf: - - required: - - httpGet - - required: - - tcpSocket - - required: - - exec - - required: - - httpGet - - required: - - tcpSocket - - required: - - exec - properties: - exec: - description: Health is determined by how the command that is executed - exited. - properties: - command: - description: Command to run. - items: - format: string - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for the probe to be considered - failed after having succeeded. - format: int32 - type: integer - httpGet: - properties: - host: - description: Host name to connect to, defaults to the pod IP. - format: string - type: string - httpHeaders: - description: Headers the proxy will pass on to make the request. - items: - properties: - name: - format: string - type: string - value: - format: string - type: string - type: object - type: array - path: - description: Path to access on the HTTP server. - format: string - type: string - port: - description: Port on which the endpoint lives. - type: integer - scheme: - format: string - type: string - type: object - initialDelaySeconds: - description: Number of seconds after the container has started before - readiness probes are initiated. - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform the probe. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for the probe to be considered - successful after having failed. - format: int32 - type: integer - tcpSocket: - description: Health is determined by if the proxy is able to connect. - properties: - host: - format: string - type: string - port: - type: integer - type: object - timeoutSeconds: - description: Number of seconds after which the probe times out. - format: int32 - type: integer - type: object - template: - description: Template to be used for the generation of `WorkloadEntry` - resources that belong to this `WorkloadGroup`. - properties: - address: - format: string - type: string - labels: - additionalProperties: - format: string - type: string - description: One or more labels associated with the endpoint. - type: object - locality: - description: The locality associated with the endpoint. - format: string - type: string - network: - format: string - type: string - ports: - additionalProperties: - type: integer - description: Set of ports associated with the endpoint. - type: object - serviceAccount: - format: string - type: string - weight: - description: The load balancing weight associated with the endpoint. - type: integer - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1alpha3 - served: true - storage: true - ---- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -3031,217 +5415,216 @@ spec: listKind: AuthorizationPolicyList plural: authorizationpolicies singular: authorizationpolicy - preserveUnknownFields: false scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration for access control on workloads. See more details - at: https://istio.io/docs/reference/config/security/authorization-policy.html' - oneOf: - - not: - anyOf: - - required: - - provider - - required: - - provider - properties: - action: - description: Optional. - enum: - - ALLOW - - DENY - - AUDIT - - CUSTOM - type: string - provider: - description: Specifies detailed configuration of the CUSTOM action. - properties: - name: - description: Specifies the name of the extension provider. - format: string - type: string - type: object - rules: - description: Optional. - items: - properties: - from: - description: Optional. - items: - properties: - source: - description: Source specifies the source of a request. - properties: - ipBlocks: - description: Optional. - items: - format: string - type: string - type: array - namespaces: - description: Optional. - items: - format: string - type: string - type: array - notIpBlocks: - description: Optional. - items: - format: string - type: string - type: array - notNamespaces: - description: Optional. - items: - format: string - type: string - type: array - notPrincipals: - description: Optional. - items: - format: string - type: string - type: array - notRemoteIpBlocks: - description: Optional. - items: - format: string - type: string - type: array - notRequestPrincipals: - description: Optional. - items: - format: string - type: string - type: array - principals: - description: Optional. - items: - format: string - type: string - type: array - remoteIpBlocks: - description: Optional. - items: - format: string - type: string - type: array - requestPrincipals: - description: Optional. - items: - format: string - type: string - type: array - type: object - type: object - type: array - to: - description: Optional. - items: - properties: - operation: - description: Operation specifies the operation of a request. - properties: - hosts: - description: Optional. - items: - format: string - type: string - type: array - methods: - description: Optional. - items: - format: string - type: string - type: array - notHosts: - description: Optional. - items: - format: string - type: string - type: array - notMethods: - description: Optional. - items: - format: string - type: string - type: array - notPaths: - description: Optional. - items: - format: string - type: string - type: array - notPorts: - description: Optional. - items: - format: string - type: string - type: array - paths: - description: Optional. - items: - format: string - type: string - type: array - ports: - description: Optional. - items: - format: string - type: string - type: array - type: object - type: object - type: array - when: - description: Optional. - items: - properties: - key: - description: The name of an Istio attribute. - format: string - type: string - notValues: - description: Optional. - items: - format: string - type: string - type: array - values: - description: Optional. - items: - format: string - type: string - type: array - type: object - type: array - type: object - type: array - selector: - description: Optional. - properties: - matchLabels: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object versions: - name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration for access control on workloads. See more + details at: https://istio.io/docs/reference/config/security/authorization-policy.html' + oneOf: + - not: + anyOf: + - required: + - provider + - required: + - provider + properties: + action: + description: Optional. + enum: + - ALLOW + - DENY + - AUDIT + - CUSTOM + type: string + provider: + description: Specifies detailed configuration of the CUSTOM action. + properties: + name: + description: Specifies the name of the extension provider. + format: string + type: string + type: object + rules: + description: Optional. + items: + properties: + from: + description: Optional. + items: + properties: + source: + description: Source specifies the source of a request. + properties: + ipBlocks: + description: Optional. + items: + format: string + type: string + type: array + namespaces: + description: Optional. + items: + format: string + type: string + type: array + notIpBlocks: + description: Optional. + items: + format: string + type: string + type: array + notNamespaces: + description: Optional. + items: + format: string + type: string + type: array + notPrincipals: + description: Optional. + items: + format: string + type: string + type: array + notRemoteIpBlocks: + description: Optional. + items: + format: string + type: string + type: array + notRequestPrincipals: + description: Optional. + items: + format: string + type: string + type: array + principals: + description: Optional. + items: + format: string + type: string + type: array + remoteIpBlocks: + description: Optional. + items: + format: string + type: string + type: array + requestPrincipals: + description: Optional. + items: + format: string + type: string + type: array + type: object + type: object + type: array + to: + description: Optional. + items: + properties: + operation: + description: Operation specifies the operation of a request. + properties: + hosts: + description: Optional. + items: + format: string + type: string + type: array + methods: + description: Optional. + items: + format: string + type: string + type: array + notHosts: + description: Optional. + items: + format: string + type: string + type: array + notMethods: + description: Optional. + items: + format: string + type: string + type: array + notPaths: + description: Optional. + items: + format: string + type: string + type: array + notPorts: + description: Optional. + items: + format: string + type: string + type: array + paths: + description: Optional. + items: + format: string + type: string + type: array + ports: + description: Optional. + items: + format: string + type: string + type: array + type: object + type: object + type: array + when: + description: Optional. + items: + properties: + key: + description: The name of an Istio attribute. + format: string + type: string + notValues: + description: Optional. + items: + format: string + type: string + type: array + values: + description: Optional. + items: + format: string + type: string + type: array + type: object + type: array + type: object + type: array + selector: + description: Optional. + properties: + matchLabels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true + subresources: + status: {} --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -3254,19 +5637,6 @@ metadata: release: istio name: peerauthentications.security.istio.io spec: - additionalPrinterColumns: - - JSONPath: .spec.mtls.mode - description: Defines the mTLS mode used for peer authentication. - name: Mode - type: string - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date group: security.istio.io names: categories: @@ -3278,31 +5648,31 @@ spec: shortNames: - pa singular: peerauthentication - preserveUnknownFields: false scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: PeerAuthentication defines how traffic will be tunneled (or - not) to the sidecar. - properties: - mtls: - description: Mutual TLS settings for workload. - properties: - mode: - description: Defines the mTLS mode used for peer authentication. - enum: - - UNSET - - DISABLE - - PERMISSIVE - - STRICT - type: string - type: object - portLevelMtls: - additionalProperties: + versions: + - additionalPrinterColumns: + - description: Defines the mTLS mode used for peer authentication. + jsonPath: .spec.mtls.mode + name: Mode + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: PeerAuthentication defines how traffic will be tunneled (or + not) to the sidecar. + properties: + mtls: + description: Mutual TLS settings for workload. properties: mode: description: Defines the mTLS mode used for peer authentication. @@ -3313,30 +5683,42 @@ spec: - STRICT type: string type: object - description: Port specific mutual TLS settings. - type: object - selector: - description: The selector determines the workloads to apply the ChannelAuthentication - on. - properties: - matchLabels: - additionalProperties: - format: string - type: string + portLevelMtls: + additionalProperties: + properties: + mode: + description: Defines the mTLS mode used for peer authentication. + enum: + - UNSET + - DISABLE + - PERMISSIVE + - STRICT + type: string type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1beta1 + description: Port specific mutual TLS settings. + type: object + selector: + description: The selector determines the workloads to apply the ChannelAuthentication + on. + properties: + matchLabels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true + subresources: + status: {} --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -3360,90 +5742,233 @@ spec: shortNames: - ra singular: requestauthentication - preserveUnknownFields: false scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: RequestAuthentication defines what request authentication methods - are supported by a workload. - properties: - jwtRules: - description: Define the list of JWTs that can be validated at the selected - workloads' proxy. - items: - properties: - audiences: - items: - format: string - type: string - type: array - forwardOriginalToken: - description: If set to true, the orginal token will be kept for - the ustream request. - type: boolean - fromHeaders: - description: List of header locations from which JWT is expected. - items: - properties: - name: - description: The HTTP header name. - format: string - type: string - prefix: - description: The prefix that should be stripped before decoding - the token. - format: string - type: string - type: object - type: array - fromParams: - description: List of query parameters from which JWT is expected. - items: - format: string - type: string - type: array - issuer: - description: Identifies the issuer that issued the JWT. - format: string - type: string - jwks: - description: JSON Web Key Set of public keys to validate signature - of the JWT. - format: string - type: string - jwks_uri: - format: string - type: string - jwksUri: - format: string - type: string - outputPayloadToHeader: - format: string - type: string - type: object - type: array - selector: - description: The selector determines the workloads to apply the RequestAuthentication - on. - properties: - matchLabels: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object versions: - name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: RequestAuthentication defines what request authentication + methods are supported by a workload. + properties: + jwtRules: + description: Define the list of JWTs that can be validated at the + selected workloads' proxy. + items: + properties: + audiences: + items: + format: string + type: string + type: array + forwardOriginalToken: + description: If set to true, the orginal token will be kept + for the ustream request. + type: boolean + fromHeaders: + description: List of header locations from which JWT is expected. + items: + properties: + name: + description: The HTTP header name. + format: string + type: string + prefix: + description: The prefix that should be stripped before + decoding the token. + format: string + type: string + type: object + type: array + fromParams: + description: List of query parameters from which JWT is expected. + items: + format: string + type: string + type: array + issuer: + description: Identifies the issuer that issued the JWT. + format: string + type: string + jwks: + description: JSON Web Key Set of public keys to validate signature + of the JWT. + format: string + type: string + jwks_uri: + format: string + type: string + jwksUri: + format: string + type: string + outputPayloadToHeader: + format: string + type: string + type: object + type: array + selector: + description: The selector determines the workloads to apply the RequestAuthentication + on. + properties: + matchLabels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + "helm.sh/resource-policy": keep + labels: + app: istio-pilot + chart: istio + heritage: Tiller + istio: telemetry + release: istio + name: telemetries.telemetry.istio.io +spec: + group: telemetry.istio.io + names: + categories: + - istio-io + - telemetry-istio-io + kind: Telemetry + listKind: TelemetryList + plural: telemetries + shortNames: + - telemetry + singular: telemetry + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + spec: + description: Telemetry defines how the telemetry is generated for workloads + within a mesh. + properties: + selector: + description: Optional. + properties: + matchLabels: + additionalProperties: + format: string + type: string + type: object + type: object + tracing: + description: Optional. + items: + properties: + customTags: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - literal + - required: + - environment + - required: + - header + - required: + - literal + - required: + - environment + - required: + - header + properties: + environment: + description: Environment adds the value of an environment + variable to each span. + properties: + defaultValue: + description: Optional. + format: string + type: string + name: + description: Name of the environment variable from + which to extract the tag value. + format: string + type: string + type: object + header: + description: RequestHeader adds the value of an header + from the request to each span. + properties: + defaultValue: + description: Optional. + format: string + type: string + name: + description: Name of the header from which to extract + the tag value. + format: string + type: string + type: object + literal: + description: Literal adds the same, hard-coded value to + each span. + properties: + value: + description: The tag value to use. + format: string + type: string + type: object + type: object + description: Optional. + type: object + disableSpanReporting: + description: Controls span reporting. + nullable: true + type: boolean + providers: + description: Optional. + items: + properties: + name: + description: Required. + format: string + type: string + type: object + type: array + randomSamplingPercentage: + nullable: true + type: number + type: object + type: array + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} --- diff --git a/charts/kubezero-istio/charts/base/crds/crd-operator.yaml b/charts/kubezero-istio/charts/base/crds/crd-operator.yaml index d0be4c3..2a80f41 100644 --- a/charts/kubezero-istio/charts/base/crds/crd-operator.yaml +++ b/charts/kubezero-istio/charts/base/crds/crd-operator.yaml @@ -1,66 +1,48 @@ # SYNC WITH manifests/charts/istio-operator/templates -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: istiooperators.install.istio.io labels: release: istio spec: - additionalPrinterColumns: - - JSONPath: .spec.revision - description: Istio control plane revision - name: Revision - type: string - - JSONPath: .status.status - description: IOP current state - type: string - name: Status - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date + conversion: + strategy: None group: install.istio.io names: kind: IstioOperator + listKind: IstioOperatorList plural: istiooperators singular: istiooperator shortNames: - iop - io scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. - More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. - More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - spec: - description: 'Specification of the desired state of the istio control plane resource. - More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' - type: object - status: - description: 'Status describes each of istio control plane component status at the current time. - 0 means NONE, 1 means UPDATING, 2 means HEALTHY, 3 means ERROR, 4 means RECONCILING. - More info: https://github.com/istio/api/blob/master/operator/v1alpha1/istio.operator.v1alpha1.pb.html & - https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' - type: object versions: - - name: v1alpha1 + - additionalPrinterColumns: + - description: Istio control plane revision + jsonPath: .spec.revision + name: Revision + type: string + - description: IOP current state + jsonPath: .status.status + name: Status + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + subresources: + status: {} + name: v1alpha1 + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true served: true storage: true --- diff --git a/charts/kubezero-istio/charts/base/files/gen-istio-cluster.yaml b/charts/kubezero-istio/charts/base/files/gen-istio-cluster.yaml index 365ca20..aec8e17 100644 --- a/charts/kubezero-istio/charts/base/files/gen-istio-cluster.yaml +++ b/charts/kubezero-istio/charts/base/files/gen-istio-cluster.yaml @@ -1,7 +1,7 @@ --- # Source: crds/crd-all.gen.yaml # DO NOT EDIT - Generated by Cue OpenAPI generator based on Istio APIs. -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -13,19 +13,6 @@ metadata: release: istio name: destinationrules.networking.istio.io spec: - additionalPrinterColumns: - - JSONPath: .spec.host - description: The name of a service from the service registry - name: Host - type: string - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date group: networking.istio.io names: categories: @@ -37,376 +24,126 @@ spec: shortNames: - dr singular: destinationrule - preserveUnknownFields: false scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting load balancing, outlier detection, - etc. See more details at: https://istio.io/docs/reference/config/networking/destination-rule.html' - properties: - exportTo: - description: A list of namespaces to which this destination rule is - exported. - items: + versions: + - additionalPrinterColumns: + - description: The name of a service from the service registry + jsonPath: .spec.host + name: Host + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting load balancing, outlier detection, + etc. See more details at: https://istio.io/docs/reference/config/networking/destination-rule.html' + properties: + exportTo: + description: A list of namespaces to which this destination rule is + exported. + items: + format: string + type: string + type: array + host: + description: The name of a service from the service registry. format: string type: string - type: array - host: - description: The name of a service from the service registry. - format: string - type: string - subsets: - items: - properties: - labels: - additionalProperties: + subsets: + items: + properties: + labels: + additionalProperties: + format: string + type: string + type: object + name: + description: Name of the subset. format: string type: string - type: object - name: - description: Name of the subset. - format: string - type: string - trafficPolicy: - description: Traffic policies that apply to this subset. - properties: - connectionPool: - properties: - http: - description: HTTP connection pool settings. - properties: - h2UpgradePolicy: - description: Specify if http1.1 connection should - be upgraded to http2 for the associated destination. - enum: - - DEFAULT - - DO_NOT_UPGRADE - - UPGRADE - type: string - http1MaxPendingRequests: - description: Maximum number of pending HTTP requests - to a destination. - format: int32 - type: integer - http2MaxRequests: - description: Maximum number of requests to a backend. - format: int32 - type: integer - idleTimeout: - description: The idle timeout for upstream connection - pool connections. - type: string - maxRequestsPerConnection: - description: Maximum number of requests per connection - to a backend. - format: int32 - type: integer - maxRetries: - format: int32 - type: integer - useClientProtocol: - description: If set to true, client protocol will - be preserved while initiating connection to backend. - type: boolean - type: object - tcp: - description: Settings common to both HTTP and TCP upstream - connections. - properties: - connectTimeout: - description: TCP connection timeout. - type: string - maxConnections: - description: Maximum number of HTTP1 /TCP connections - to a destination host. - format: int32 - type: integer - tcpKeepalive: - description: If set then set SO_KEEPALIVE on the socket - to enable TCP Keepalives. - properties: - interval: - description: The time duration between keep-alive - probes. - type: string - probes: - type: integer - time: - type: string - type: object - type: object - type: object - loadBalancer: - description: Settings controlling the load balancer algorithms. - oneOf: - - not: - anyOf: - - required: - - simple - - properties: - consistentHash: - oneOf: - - not: - anyOf: - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - required: - - consistentHash - - required: - - simple - - properties: - consistentHash: - oneOf: - - not: - anyOf: - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - required: - - consistentHash - properties: - consistentHash: - properties: - httpCookie: - description: Hash based on HTTP cookie. - properties: - name: - description: Name of the cookie. - format: string - type: string - path: - description: Path to set for the cookie. - format: string - type: string - ttl: - description: Lifetime of the cookie. - type: string - type: object - httpHeaderName: - description: Hash based on a specific HTTP header. - format: string - type: string - httpQueryParameterName: - description: Hash based on a specific HTTP query parameter. - format: string - type: string - minimumRingSize: - type: integer - useSourceIp: - description: Hash based on the source IP address. - type: boolean - type: object - localityLbSetting: - properties: - distribute: - description: 'Optional: only one of distribute or - failover can be set.' - items: - properties: - from: - description: Originating locality, '/' separated, - e.g. - format: string - type: string - to: - additionalProperties: - type: integer - description: Map of upstream localities to traffic - distribution weights. - type: object - type: object - type: array - enabled: - description: enable locality load balancing, this - is DestinationRule-level and will override mesh - wide settings in entirety. - nullable: true - type: boolean - failover: - description: 'Optional: only failover or distribute - can be set.' - items: - properties: - from: - description: Originating region. - format: string - type: string - to: - format: string - type: string - type: object - type: array - type: object - simple: - enum: - - ROUND_ROBIN - - LEAST_CONN - - RANDOM - - PASSTHROUGH - type: string - type: object - outlierDetection: - properties: - baseEjectionTime: - description: Minimum ejection duration. - type: string - consecutive5xxErrors: - description: Number of 5xx errors before a host is ejected - from the connection pool. - nullable: true - type: integer - consecutiveErrors: - format: int32 - type: integer - consecutiveGatewayErrors: - description: Number of gateway errors before a host is - ejected from the connection pool. - nullable: true - type: integer - interval: - description: Time interval between ejection sweep analysis. - type: string - maxEjectionPercent: - format: int32 - type: integer - minHealthPercent: - format: int32 - type: integer - type: object - portLevelSettings: - description: Traffic policies specific to individual ports. - items: + trafficPolicy: + description: Traffic policies that apply to this subset. + properties: + connectionPool: properties: - connectionPool: + http: + description: HTTP connection pool settings. properties: - http: - description: HTTP connection pool settings. + h2UpgradePolicy: + description: Specify if http1.1 connection should + be upgraded to http2 for the associated destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP requests + to a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream connection + pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per connection + to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol will + be preserved while initiating connection to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and TCP upstream + connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP connections + to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE on the + socket to enable TCP Keepalives. properties: - h2UpgradePolicy: - description: Specify if http1.1 connection should - be upgraded to http2 for the associated destination. - enum: - - DEFAULT - - DO_NOT_UPGRADE - - UPGRADE + interval: + description: The time duration between keep-alive + probes. type: string - http1MaxPendingRequests: - description: Maximum number of pending HTTP - requests to a destination. - format: int32 + probes: type: integer - http2MaxRequests: - description: Maximum number of requests to a - backend. - format: int32 - type: integer - idleTimeout: - description: The idle timeout for upstream connection - pool connections. + time: type: string - maxRequestsPerConnection: - description: Maximum number of requests per - connection to a backend. - format: int32 - type: integer - maxRetries: - format: int32 - type: integer - useClientProtocol: - description: If set to true, client protocol - will be preserved while initiating connection - to backend. - type: boolean - type: object - tcp: - description: Settings common to both HTTP and TCP - upstream connections. - properties: - connectTimeout: - description: TCP connection timeout. - type: string - maxConnections: - description: Maximum number of HTTP1 /TCP connections - to a destination host. - format: int32 - type: integer - tcpKeepalive: - description: If set then set SO_KEEPALIVE on - the socket to enable TCP Keepalives. - properties: - interval: - description: The time duration between keep-alive - probes. - type: string - probes: - type: integer - time: - type: string - type: object type: object type: object - loadBalancer: - description: Settings controlling the load balancer - algorithms. - oneOf: - - not: - anyOf: - - required: - - simple - - properties: - consistentHash: - oneOf: - - not: - anyOf: - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - required: - - consistentHash + type: object + loadBalancer: + description: Settings controlling the load balancer algorithms. + oneOf: + - not: + anyOf: - required: - simple - properties: @@ -432,277 +169,21 @@ spec: - httpQueryParameterName required: - consistentHash - properties: - consistentHash: - properties: - httpCookie: - description: Hash based on HTTP cookie. - properties: - name: - description: Name of the cookie. - format: string - type: string - path: - description: Path to set for the cookie. - format: string - type: string - ttl: - description: Lifetime of the cookie. - type: string - type: object - httpHeaderName: - description: Hash based on a specific HTTP header. - format: string - type: string - httpQueryParameterName: - description: Hash based on a specific HTTP query - parameter. - format: string - type: string - minimumRingSize: - type: integer - useSourceIp: - description: Hash based on the source IP address. - type: boolean - type: object - localityLbSetting: - properties: - distribute: - description: 'Optional: only one of distribute - or failover can be set.' - items: - properties: - from: - description: Originating locality, '/' - separated, e.g. - format: string - type: string - to: - additionalProperties: - type: integer - description: Map of upstream localities - to traffic distribution weights. - type: object - type: object - type: array - enabled: - description: enable locality load balancing, - this is DestinationRule-level and will override - mesh wide settings in entirety. - nullable: true - type: boolean - failover: - description: 'Optional: only failover or distribute - can be set.' - items: - properties: - from: - description: Originating region. - format: string - type: string - to: - format: string - type: string - type: object - type: array - type: object - simple: - enum: - - ROUND_ROBIN - - LEAST_CONN - - RANDOM - - PASSTHROUGH - type: string - type: object - outlierDetection: - properties: - baseEjectionTime: - description: Minimum ejection duration. - type: string - consecutive5xxErrors: - description: Number of 5xx errors before a host - is ejected from the connection pool. - nullable: true - type: integer - consecutiveErrors: - format: int32 - type: integer - consecutiveGatewayErrors: - description: Number of gateway errors before a host - is ejected from the connection pool. - nullable: true - type: integer - interval: - description: Time interval between ejection sweep - analysis. - type: string - maxEjectionPercent: - format: int32 - type: integer - minHealthPercent: - format: int32 - type: integer - type: object - port: - properties: - number: - type: integer - type: object - tls: - description: TLS related settings for connections to - the upstream service. - properties: - caCertificates: - format: string - type: string - clientCertificate: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - credentialName: - format: string - type: string - mode: - enum: - - DISABLE - - SIMPLE - - MUTUAL - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - sni: - description: SNI string to present to the server - during TLS handshake. - format: string - type: string - subjectAltNames: - items: - format: string - type: string - type: array - type: object - type: object - type: array - tls: - description: TLS related settings for connections to the upstream - service. - properties: - caCertificates: - format: string - type: string - clientCertificate: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - credentialName: - format: string - type: string - mode: - enum: - - DISABLE - - SIMPLE - - MUTUAL - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - sni: - description: SNI string to present to the server during - TLS handshake. - format: string - type: string - subjectAltNames: - items: - format: string - type: string - type: array - type: object - type: object - type: object - type: array - trafficPolicy: - properties: - connectionPool: - properties: - http: - description: HTTP connection pool settings. - properties: - h2UpgradePolicy: - description: Specify if http1.1 connection should be upgraded - to http2 for the associated destination. - enum: - - DEFAULT - - DO_NOT_UPGRADE - - UPGRADE - type: string - http1MaxPendingRequests: - description: Maximum number of pending HTTP requests to - a destination. - format: int32 - type: integer - http2MaxRequests: - description: Maximum number of requests to a backend. - format: int32 - type: integer - idleTimeout: - description: The idle timeout for upstream connection pool - connections. - type: string - maxRequestsPerConnection: - description: Maximum number of requests per connection to - a backend. - format: int32 - type: integer - maxRetries: - format: int32 - type: integer - useClientProtocol: - description: If set to true, client protocol will be preserved - while initiating connection to backend. - type: boolean - type: object - tcp: - description: Settings common to both HTTP and TCP upstream connections. - properties: - connectTimeout: - description: TCP connection timeout. - type: string - maxConnections: - description: Maximum number of HTTP1 /TCP connections to - a destination host. - format: int32 - type: integer - tcpKeepalive: - description: If set then set SO_KEEPALIVE on the socket - to enable TCP Keepalives. - properties: - interval: - description: The time duration between keep-alive probes. - type: string - probes: - type: integer - time: - type: string - type: object - type: object - type: object - loadBalancer: - description: Settings controlling the load balancer algorithms. - oneOf: - - not: - anyOf: - - required: - - simple - - properties: - consistentHash: - oneOf: - - not: - anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName - required: - httpHeaderName - required: @@ -711,231 +192,237 @@ spec: - useSourceIp - required: - httpQueryParameterName - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - required: - - consistentHash - - required: - - simple - - properties: - consistentHash: - oneOf: - - not: - anyOf: - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - required: - - consistentHash - properties: - consistentHash: - properties: - httpCookie: - description: Hash based on HTTP cookie. + required: + - consistentHash properties: - name: - description: Name of the cookie. - format: string - type: string - path: - description: Path to set for the cookie. - format: string - type: string - ttl: - description: Lifetime of the cookie. + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP query + parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute or + failover can be set.' + items: + properties: + from: + description: Originating locality, '/' separated, + e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities to + traffic distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, this + is DestinationRule-level and will override mesh + wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute + can be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH type: string type: object - httpHeaderName: - description: Hash based on a specific HTTP header. - format: string - type: string - httpQueryParameterName: - description: Hash based on a specific HTTP query parameter. - format: string - type: string - minimumRingSize: - type: integer - useSourceIp: - description: Hash based on the source IP address. - type: boolean - type: object - localityLbSetting: - properties: - distribute: - description: 'Optional: only one of distribute or failover - can be set.' + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host is ejected + from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a host + is ejected from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + portLevelSettings: + description: Traffic policies specific to individual ports. items: properties: - from: - description: Originating locality, '/' separated, - e.g. - format: string - type: string - to: - additionalProperties: - type: integer - description: Map of upstream localities to traffic - distribution weights. - type: object - type: object - type: array - enabled: - description: enable locality load balancing, this is DestinationRule-level - and will override mesh wide settings in entirety. - nullable: true - type: boolean - failover: - description: 'Optional: only failover or distribute can - be set.' - items: - properties: - from: - description: Originating region. - format: string - type: string - to: - format: string - type: string - type: object - type: array - type: object - simple: - enum: - - ROUND_ROBIN - - LEAST_CONN - - RANDOM - - PASSTHROUGH - type: string - type: object - outlierDetection: - properties: - baseEjectionTime: - description: Minimum ejection duration. - type: string - consecutive5xxErrors: - description: Number of 5xx errors before a host is ejected from - the connection pool. - nullable: true - type: integer - consecutiveErrors: - format: int32 - type: integer - consecutiveGatewayErrors: - description: Number of gateway errors before a host is ejected - from the connection pool. - nullable: true - type: integer - interval: - description: Time interval between ejection sweep analysis. - type: string - maxEjectionPercent: - format: int32 - type: integer - minHealthPercent: - format: int32 - type: integer - type: object - portLevelSettings: - description: Traffic policies specific to individual ports. - items: - properties: - connectionPool: - properties: - http: - description: HTTP connection pool settings. - properties: - h2UpgradePolicy: - description: Specify if http1.1 connection should - be upgraded to http2 for the associated destination. - enum: - - DEFAULT - - DO_NOT_UPGRADE - - UPGRADE - type: string - http1MaxPendingRequests: - description: Maximum number of pending HTTP requests - to a destination. - format: int32 - type: integer - http2MaxRequests: - description: Maximum number of requests to a backend. - format: int32 - type: integer - idleTimeout: - description: The idle timeout for upstream connection - pool connections. - type: string - maxRequestsPerConnection: - description: Maximum number of requests per connection - to a backend. - format: int32 - type: integer - maxRetries: - format: int32 - type: integer - useClientProtocol: - description: If set to true, client protocol will - be preserved while initiating connection to backend. - type: boolean - type: object - tcp: - description: Settings common to both HTTP and TCP upstream - connections. - properties: - connectTimeout: - description: TCP connection timeout. - type: string - maxConnections: - description: Maximum number of HTTP1 /TCP connections - to a destination host. - format: int32 - type: integer - tcpKeepalive: - description: If set then set SO_KEEPALIVE on the socket - to enable TCP Keepalives. + connectionPool: properties: - interval: - description: The time duration between keep-alive - probes. - type: string - probes: - type: integer - time: - type: string + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection + should be upgraded to http2 for the associated + destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP + requests to a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to + a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream + connection pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per + connection to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol + will be preserved while initiating connection + to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and + TCP upstream connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP + connections to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE + on the socket to enable TCP Keepalives. + properties: + interval: + description: The time duration between + keep-alive probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object type: object - type: object - type: object - loadBalancer: - description: Settings controlling the load balancer algorithms. - oneOf: - - not: - anyOf: - - required: - - simple - - properties: - consistentHash: - oneOf: - - not: - anyOf: + loadBalancer: + description: Settings controlling the load balancer + algorithms. + oneOf: + - not: + anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName - required: - httpHeaderName - required: @@ -944,16 +431,275 @@ spec: - useSourceIp - required: - httpQueryParameterName - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - - required: - - httpQueryParameterName - required: - - consistentHash + required: + - consistentHash + properties: + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP + header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP + query parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute + or failover can be set.' + items: + properties: + from: + description: Originating locality, '/' + separated, e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities + to traffic distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, + this is DestinationRule-level and will override + mesh wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute + can be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string + type: object + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host + is ejected from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a + host is ejected from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep + analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + port: + properties: + number: + type: integer + type: object + tls: + description: TLS related settings for connections + to the upstream service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server + during TLS handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: array + tls: + description: TLS related settings for connections to the + upstream service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server during + TLS handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: object + type: array + trafficPolicy: + properties: + connectionPool: + properties: + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection should be upgraded + to http2 for the associated destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP requests to + a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream connection + pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per connection + to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol will be preserved + while initiating connection to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and TCP upstream + connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP connections + to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE on the socket + to enable TCP Keepalives. + properties: + interval: + description: The time duration between keep-alive + probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object + type: object + loadBalancer: + description: Settings controlling the load balancer algorithms. + oneOf: + - not: + anyOf: - required: - simple - properties: @@ -979,210 +725,1622 @@ spec: - httpQueryParameterName required: - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + properties: + consistentHash: properties: - consistentHash: + httpCookie: + description: Hash based on HTTP cookie. properties: - httpCookie: - description: Hash based on HTTP cookie. - properties: - name: - description: Name of the cookie. - format: string - type: string - path: - description: Path to set for the cookie. - format: string - type: string - ttl: - description: Lifetime of the cookie. - type: string - type: object - httpHeaderName: - description: Hash based on a specific HTTP header. + name: + description: Name of the cookie. format: string type: string - httpQueryParameterName: - description: Hash based on a specific HTTP query parameter. + path: + description: Path to set for the cookie. format: string type: string - minimumRingSize: - type: integer - useSourceIp: - description: Hash based on the source IP address. - type: boolean + ttl: + description: Lifetime of the cookie. + type: string type: object - localityLbSetting: - properties: - distribute: - description: 'Optional: only one of distribute or - failover can be set.' - items: - properties: - from: - description: Originating locality, '/' separated, - e.g. - format: string - type: string - to: - additionalProperties: - type: integer - description: Map of upstream localities to traffic - distribution weights. - type: object - type: object - type: array - enabled: - description: enable locality load balancing, this - is DestinationRule-level and will override mesh - wide settings in entirety. - nullable: true - type: boolean - failover: - description: 'Optional: only failover or distribute - can be set.' - items: - properties: - from: - description: Originating region. - format: string - type: string - to: - format: string - type: string - type: object - type: array - type: object - simple: - enum: - - ROUND_ROBIN - - LEAST_CONN - - RANDOM - - PASSTHROUGH + httpHeaderName: + description: Hash based on a specific HTTP header. + format: string type: string + httpQueryParameterName: + description: Hash based on a specific HTTP query parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean type: object - outlierDetection: + localityLbSetting: properties: - baseEjectionTime: - description: Minimum ejection duration. - type: string - consecutive5xxErrors: - description: Number of 5xx errors before a host is ejected - from the connection pool. - nullable: true - type: integer - consecutiveErrors: - format: int32 - type: integer - consecutiveGatewayErrors: - description: Number of gateway errors before a host is - ejected from the connection pool. - nullable: true - type: integer - interval: - description: Time interval between ejection sweep analysis. - type: string - maxEjectionPercent: - format: int32 - type: integer - minHealthPercent: - format: int32 - type: integer - type: object - port: - properties: - number: - type: integer - type: object - tls: - description: TLS related settings for connections to the upstream - service. - properties: - caCertificates: - format: string - type: string - clientCertificate: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - credentialName: - format: string - type: string - mode: - enum: - - DISABLE - - SIMPLE - - MUTUAL - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - sni: - description: SNI string to present to the server during - TLS handshake. - format: string - type: string - subjectAltNames: + distribute: + description: 'Optional: only one of distribute or failover + can be set.' items: - format: string - type: string + properties: + from: + description: Originating locality, '/' separated, + e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities to traffic + distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, this is DestinationRule-level + and will override mesh wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute can + be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object type: array type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string type: object - type: array - tls: - description: TLS related settings for connections to the upstream - service. - properties: - caCertificates: - format: string - type: string - clientCertificate: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - credentialName: - format: string - type: string - mode: - enum: - - DISABLE - - SIMPLE - - MUTUAL - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - sni: - description: SNI string to present to the server during TLS - handshake. - format: string - type: string - subjectAltNames: - items: + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host is ejected + from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a host is ejected + from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + portLevelSettings: + description: Traffic policies specific to individual ports. + items: + properties: + connectionPool: + properties: + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection should + be upgraded to http2 for the associated destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP requests + to a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream connection + pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per connection + to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol will + be preserved while initiating connection to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and TCP upstream + connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP connections + to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE on the + socket to enable TCP Keepalives. + properties: + interval: + description: The time duration between keep-alive + probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object + type: object + loadBalancer: + description: Settings controlling the load balancer algorithms. + oneOf: + - not: + anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + properties: + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP query + parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute or + failover can be set.' + items: + properties: + from: + description: Originating locality, '/' separated, + e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities to + traffic distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, this + is DestinationRule-level and will override mesh + wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute + can be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string + type: object + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host is ejected + from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a host + is ejected from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + port: + properties: + number: + type: integer + type: object + tls: + description: TLS related settings for connections to the + upstream service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server during + TLS handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: array + tls: + description: TLS related settings for connections to the upstream + service. + properties: + caCertificates: format: string type: string - type: array - type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1alpha3 + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server during TLS + handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true - - name: v1beta1 + subresources: + status: {} + - additionalPrinterColumns: + - description: The name of a service from the service registry + jsonPath: .spec.host + name: Host + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting load balancing, outlier detection, + etc. See more details at: https://istio.io/docs/reference/config/networking/destination-rule.html' + properties: + exportTo: + description: A list of namespaces to which this destination rule is + exported. + items: + format: string + type: string + type: array + host: + description: The name of a service from the service registry. + format: string + type: string + subsets: + items: + properties: + labels: + additionalProperties: + format: string + type: string + type: object + name: + description: Name of the subset. + format: string + type: string + trafficPolicy: + description: Traffic policies that apply to this subset. + properties: + connectionPool: + properties: + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection should + be upgraded to http2 for the associated destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP requests + to a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream connection + pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per connection + to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol will + be preserved while initiating connection to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and TCP upstream + connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP connections + to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE on the + socket to enable TCP Keepalives. + properties: + interval: + description: The time duration between keep-alive + probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object + type: object + loadBalancer: + description: Settings controlling the load balancer algorithms. + oneOf: + - not: + anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + properties: + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP query + parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute or + failover can be set.' + items: + properties: + from: + description: Originating locality, '/' separated, + e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities to + traffic distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, this + is DestinationRule-level and will override mesh + wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute + can be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string + type: object + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host is ejected + from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a host + is ejected from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + portLevelSettings: + description: Traffic policies specific to individual ports. + items: + properties: + connectionPool: + properties: + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection + should be upgraded to http2 for the associated + destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP + requests to a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to + a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream + connection pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per + connection to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol + will be preserved while initiating connection + to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and + TCP upstream connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP + connections to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE + on the socket to enable TCP Keepalives. + properties: + interval: + description: The time duration between + keep-alive probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object + type: object + loadBalancer: + description: Settings controlling the load balancer + algorithms. + oneOf: + - not: + anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + properties: + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP + header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP + query parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute + or failover can be set.' + items: + properties: + from: + description: Originating locality, '/' + separated, e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities + to traffic distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, + this is DestinationRule-level and will override + mesh wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute + can be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string + type: object + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host + is ejected from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a + host is ejected from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep + analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + port: + properties: + number: + type: integer + type: object + tls: + description: TLS related settings for connections + to the upstream service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server + during TLS handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: array + tls: + description: TLS related settings for connections to the + upstream service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server during + TLS handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: object + type: array + trafficPolicy: + properties: + connectionPool: + properties: + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection should be upgraded + to http2 for the associated destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP requests to + a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream connection + pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per connection + to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol will be preserved + while initiating connection to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and TCP upstream + connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP connections + to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE on the socket + to enable TCP Keepalives. + properties: + interval: + description: The time duration between keep-alive + probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object + type: object + loadBalancer: + description: Settings controlling the load balancer algorithms. + oneOf: + - not: + anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + properties: + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP query parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute or failover + can be set.' + items: + properties: + from: + description: Originating locality, '/' separated, + e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities to traffic + distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, this is DestinationRule-level + and will override mesh wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute can + be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string + type: object + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host is ejected + from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a host is ejected + from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + portLevelSettings: + description: Traffic policies specific to individual ports. + items: + properties: + connectionPool: + properties: + http: + description: HTTP connection pool settings. + properties: + h2UpgradePolicy: + description: Specify if http1.1 connection should + be upgraded to http2 for the associated destination. + enum: + - DEFAULT + - DO_NOT_UPGRADE + - UPGRADE + type: string + http1MaxPendingRequests: + description: Maximum number of pending HTTP requests + to a destination. + format: int32 + type: integer + http2MaxRequests: + description: Maximum number of requests to a backend. + format: int32 + type: integer + idleTimeout: + description: The idle timeout for upstream connection + pool connections. + type: string + maxRequestsPerConnection: + description: Maximum number of requests per connection + to a backend. + format: int32 + type: integer + maxRetries: + format: int32 + type: integer + useClientProtocol: + description: If set to true, client protocol will + be preserved while initiating connection to backend. + type: boolean + type: object + tcp: + description: Settings common to both HTTP and TCP upstream + connections. + properties: + connectTimeout: + description: TCP connection timeout. + type: string + maxConnections: + description: Maximum number of HTTP1 /TCP connections + to a destination host. + format: int32 + type: integer + tcpKeepalive: + description: If set then set SO_KEEPALIVE on the + socket to enable TCP Keepalives. + properties: + interval: + description: The time duration between keep-alive + probes. + type: string + probes: + type: integer + time: + type: string + type: object + type: object + type: object + loadBalancer: + description: Settings controlling the load balancer algorithms. + oneOf: + - not: + anyOf: + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + - required: + - simple + - properties: + consistentHash: + oneOf: + - not: + anyOf: + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + - required: + - httpHeaderName + - required: + - httpCookie + - required: + - useSourceIp + - required: + - httpQueryParameterName + required: + - consistentHash + properties: + consistentHash: + properties: + httpCookie: + description: Hash based on HTTP cookie. + properties: + name: + description: Name of the cookie. + format: string + type: string + path: + description: Path to set for the cookie. + format: string + type: string + ttl: + description: Lifetime of the cookie. + type: string + type: object + httpHeaderName: + description: Hash based on a specific HTTP header. + format: string + type: string + httpQueryParameterName: + description: Hash based on a specific HTTP query + parameter. + format: string + type: string + minimumRingSize: + type: integer + useSourceIp: + description: Hash based on the source IP address. + type: boolean + type: object + localityLbSetting: + properties: + distribute: + description: 'Optional: only one of distribute or + failover can be set.' + items: + properties: + from: + description: Originating locality, '/' separated, + e.g. + format: string + type: string + to: + additionalProperties: + type: integer + description: Map of upstream localities to + traffic distribution weights. + type: object + type: object + type: array + enabled: + description: enable locality load balancing, this + is DestinationRule-level and will override mesh + wide settings in entirety. + nullable: true + type: boolean + failover: + description: 'Optional: only failover or distribute + can be set.' + items: + properties: + from: + description: Originating region. + format: string + type: string + to: + format: string + type: string + type: object + type: array + type: object + simple: + enum: + - ROUND_ROBIN + - LEAST_CONN + - RANDOM + - PASSTHROUGH + type: string + type: object + outlierDetection: + properties: + baseEjectionTime: + description: Minimum ejection duration. + type: string + consecutive5xxErrors: + description: Number of 5xx errors before a host is ejected + from the connection pool. + nullable: true + type: integer + consecutiveErrors: + format: int32 + type: integer + consecutiveGatewayErrors: + description: Number of gateway errors before a host + is ejected from the connection pool. + nullable: true + type: integer + interval: + description: Time interval between ejection sweep analysis. + type: string + maxEjectionPercent: + format: int32 + type: integer + minHealthPercent: + format: int32 + type: integer + type: object + port: + properties: + number: + type: integer + type: object + tls: + description: TLS related settings for connections to the + upstream service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server during + TLS handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: array + tls: + description: TLS related settings for connections to the upstream + service. + properties: + caCertificates: + format: string + type: string + clientCertificate: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + credentialName: + format: string + type: string + mode: + enum: + - DISABLE + - SIMPLE + - MUTUAL + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + sni: + description: SNI string to present to the server during TLS + handshake. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: false + subresources: + status: {} --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -1203,238 +2361,238 @@ spec: listKind: EnvoyFilterList plural: envoyfilters singular: envoyfilter - preserveUnknownFields: true scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Customizing Envoy configuration generated by Istio. See more - details at: https://istio.io/docs/reference/config/networking/envoy-filter.html' - properties: - configPatches: - description: One or more patches with match conditions. - items: - properties: - applyTo: - enum: - - INVALID - - LISTENER - - FILTER_CHAIN - - NETWORK_FILTER - - HTTP_FILTER - - ROUTE_CONFIGURATION - - VIRTUAL_HOST - - HTTP_ROUTE - - CLUSTER - - EXTENSION_CONFIG - type: string - match: - description: Match on listener/route configuration/cluster. - oneOf: - - not: - anyOf: - - required: - - listener - - required: - - routeConfiguration - - required: - - cluster - - required: - - listener - - required: - - routeConfiguration - - required: - - cluster - properties: - cluster: - description: Match on envoy cluster attributes. - properties: - name: - description: The exact name of the cluster to match. - format: string - type: string - portNumber: - description: The service port for which this cluster was - generated. - type: integer - service: - description: The fully qualified service name for this - cluster. - format: string - type: string - subset: - description: The subset associated with the service. - format: string - type: string - type: object - context: - description: The specific config generation context to match - on. - enum: - - ANY - - SIDECAR_INBOUND - - SIDECAR_OUTBOUND - - GATEWAY - type: string - listener: - description: Match on envoy listener attributes. - properties: - filterChain: - description: Match a specific filter chain in a listener. - properties: - applicationProtocols: - description: Applies only to sidecars. - format: string - type: string - destinationPort: - description: The destination_port value used by a - filter chain's match condition. - type: integer - filter: - description: The name of a specific filter to apply - the patch to. - properties: - name: - description: The filter name to match on. - format: string - type: string - subFilter: - properties: - name: - description: The filter name to match on. - format: string - type: string - type: object - type: object - name: - description: The name assigned to the filter chain. - format: string - type: string - sni: - description: The SNI value used by a filter chain's - match condition. - format: string - type: string - transportProtocol: - description: Applies only to `SIDECAR_INBOUND` context. - format: string - type: string - type: object - name: - description: Match a specific listener by its name. - format: string - type: string - portName: - format: string - type: string - portNumber: - type: integer - type: object - proxy: - description: Match on properties associated with a proxy. - properties: - metadata: - additionalProperties: - format: string - type: string - type: object - proxyVersion: - format: string - type: string - type: object - routeConfiguration: - description: Match on envoy HTTP route configuration attributes. - properties: - gateway: - format: string - type: string - name: - description: Route configuration name to match on. - format: string - type: string - portName: - description: Applicable only for GATEWAY context. - format: string - type: string - portNumber: - type: integer - vhost: - properties: - name: - format: string - type: string - route: - description: Match a specific route within the virtual - host. - properties: - action: - description: Match a route with specific action - type. - enum: - - ANY - - ROUTE - - REDIRECT - - DIRECT_RESPONSE - type: string - name: - format: string - type: string - type: object - type: object - type: object - type: object - patch: - description: The patch to apply along with the operation. - properties: - filterClass: - description: Determines the filter insertion order. - enum: - - UNSPECIFIED - - AUTHN - - AUTHZ - - STATS - type: string - operation: - description: Determines how the patch should be applied. - enum: - - INVALID - - MERGE - - ADD - - REMOVE - - INSERT_BEFORE - - INSERT_AFTER - - INSERT_FIRST - - REPLACE - type: string - value: - description: The JSON config of the object being patched. - type: object - type: object - type: object - type: array - workloadSelector: - properties: - labels: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object versions: - name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Customizing Envoy configuration generated by Istio. See + more details at: https://istio.io/docs/reference/config/networking/envoy-filter.html' + properties: + configPatches: + description: One or more patches with match conditions. + items: + properties: + applyTo: + enum: + - INVALID + - LISTENER + - FILTER_CHAIN + - NETWORK_FILTER + - HTTP_FILTER + - ROUTE_CONFIGURATION + - VIRTUAL_HOST + - HTTP_ROUTE + - CLUSTER + - EXTENSION_CONFIG + type: string + match: + description: Match on listener/route configuration/cluster. + oneOf: + - not: + anyOf: + - required: + - listener + - required: + - routeConfiguration + - required: + - cluster + - required: + - listener + - required: + - routeConfiguration + - required: + - cluster + properties: + cluster: + description: Match on envoy cluster attributes. + properties: + name: + description: The exact name of the cluster to match. + format: string + type: string + portNumber: + description: The service port for which this cluster + was generated. + type: integer + service: + description: The fully qualified service name for this + cluster. + format: string + type: string + subset: + description: The subset associated with the service. + format: string + type: string + type: object + context: + description: The specific config generation context to match + on. + enum: + - ANY + - SIDECAR_INBOUND + - SIDECAR_OUTBOUND + - GATEWAY + type: string + listener: + description: Match on envoy listener attributes. + properties: + filterChain: + description: Match a specific filter chain in a listener. + properties: + applicationProtocols: + description: Applies only to sidecars. + format: string + type: string + destinationPort: + description: The destination_port value used by + a filter chain's match condition. + type: integer + filter: + description: The name of a specific filter to apply + the patch to. + properties: + name: + description: The filter name to match on. + format: string + type: string + subFilter: + properties: + name: + description: The filter name to match on. + format: string + type: string + type: object + type: object + name: + description: The name assigned to the filter chain. + format: string + type: string + sni: + description: The SNI value used by a filter chain's + match condition. + format: string + type: string + transportProtocol: + description: Applies only to `SIDECAR_INBOUND` context. + format: string + type: string + type: object + name: + description: Match a specific listener by its name. + format: string + type: string + portName: + format: string + type: string + portNumber: + type: integer + type: object + proxy: + description: Match on properties associated with a proxy. + properties: + metadata: + additionalProperties: + format: string + type: string + type: object + proxyVersion: + format: string + type: string + type: object + routeConfiguration: + description: Match on envoy HTTP route configuration attributes. + properties: + gateway: + format: string + type: string + name: + description: Route configuration name to match on. + format: string + type: string + portName: + description: Applicable only for GATEWAY context. + format: string + type: string + portNumber: + type: integer + vhost: + properties: + name: + format: string + type: string + route: + description: Match a specific route within the virtual + host. + properties: + action: + description: Match a route with specific action + type. + enum: + - ANY + - ROUTE + - REDIRECT + - DIRECT_RESPONSE + type: string + name: + format: string + type: string + type: object + type: object + type: object + type: object + patch: + description: The patch to apply along with the operation. + properties: + filterClass: + description: Determines the filter insertion order. + enum: + - UNSPECIFIED + - AUTHN + - AUTHZ + - STATS + type: string + operation: + description: Determines how the patch should be applied. + enum: + - INVALID + - MERGE + - ADD + - REMOVE + - INSERT_BEFORE + - INSERT_AFTER + - INSERT_FIRST + - REPLACE + type: string + value: + description: The JSON config of the object being patched. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: object + type: array + workloadSelector: + properties: + labels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true + subresources: + status: {} --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -1457,146 +2615,273 @@ spec: shortNames: - gw singular: gateway - preserveUnknownFields: false scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting edge load balancer. See more details - at: https://istio.io/docs/reference/config/networking/gateway.html' - properties: - selector: - additionalProperties: - format: string - type: string - type: object - servers: - description: A list of server specifications. - items: - properties: - bind: - format: string - type: string - defaultEndpoint: - format: string - type: string - hosts: - description: One or more hosts exposed by this gateway. - items: - format: string - type: string - type: array - name: - description: An optional name of the server, when set must be - unique across all servers. - format: string - type: string - port: - properties: - name: - description: Label assigned to the port. - format: string - type: string - number: - description: A valid non-negative integer port number. - type: integer - protocol: - description: The protocol exposed on the port. - format: string - type: string - targetPort: - type: integer - type: object - tls: - description: Set of TLS related options that govern the server's - behavior. - properties: - caCertificates: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - cipherSuites: - description: 'Optional: If specified, only support the specified - cipher list.' - items: - format: string - type: string - type: array - credentialName: - format: string - type: string - httpsRedirect: - type: boolean - maxProtocolVersion: - description: 'Optional: Maximum TLS protocol version.' - enum: - - TLS_AUTO - - TLSV1_0 - - TLSV1_1 - - TLSV1_2 - - TLSV1_3 - type: string - minProtocolVersion: - description: 'Optional: Minimum TLS protocol version.' - enum: - - TLS_AUTO - - TLSV1_0 - - TLSV1_1 - - TLSV1_2 - - TLSV1_3 - type: string - mode: - enum: - - PASSTHROUGH - - SIMPLE - - MUTUAL - - AUTO_PASSTHROUGH - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. - format: string - type: string - serverCertificate: - description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. - format: string - type: string - subjectAltNames: - items: - format: string - type: string - type: array - verifyCertificateHash: - items: - format: string - type: string - type: array - verifyCertificateSpki: - items: - format: string - type: string - type: array - type: object - type: object - type: array - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object versions: - name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting edge load balancer. See more details + at: https://istio.io/docs/reference/config/networking/gateway.html' + properties: + selector: + additionalProperties: + format: string + type: string + type: object + servers: + description: A list of server specifications. + items: + properties: + bind: + format: string + type: string + defaultEndpoint: + format: string + type: string + hosts: + description: One or more hosts exposed by this gateway. + items: + format: string + type: string + type: array + name: + description: An optional name of the server, when set must be + unique across all servers. + format: string + type: string + port: + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + tls: + description: Set of TLS related options that govern the server's + behavior. + properties: + caCertificates: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + cipherSuites: + description: 'Optional: If specified, only support the specified + cipher list.' + items: + format: string + type: string + type: array + credentialName: + format: string + type: string + httpsRedirect: + type: boolean + maxProtocolVersion: + description: 'Optional: Maximum TLS protocol version.' + enum: + - TLS_AUTO + - TLSV1_0 + - TLSV1_1 + - TLSV1_2 + - TLSV1_3 + type: string + minProtocolVersion: + description: 'Optional: Minimum TLS protocol version.' + enum: + - TLS_AUTO + - TLSV1_0 + - TLSV1_1 + - TLSV1_2 + - TLSV1_3 + type: string + mode: + enum: + - PASSTHROUGH + - SIMPLE + - MUTUAL + - AUTO_PASSTHROUGH + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. + format: string + type: string + serverCertificate: + description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + verifyCertificateHash: + items: + format: string + type: string + type: array + verifyCertificateSpki: + items: + format: string + type: string + type: array + type: object + type: object + type: array + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true + subresources: + status: {} - name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting edge load balancer. See more details + at: https://istio.io/docs/reference/config/networking/gateway.html' + properties: + selector: + additionalProperties: + format: string + type: string + type: object + servers: + description: A list of server specifications. + items: + properties: + bind: + format: string + type: string + defaultEndpoint: + format: string + type: string + hosts: + description: One or more hosts exposed by this gateway. + items: + format: string + type: string + type: array + name: + description: An optional name of the server, when set must be + unique across all servers. + format: string + type: string + port: + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + tls: + description: Set of TLS related options that govern the server's + behavior. + properties: + caCertificates: + description: REQUIRED if mode is `MUTUAL`. + format: string + type: string + cipherSuites: + description: 'Optional: If specified, only support the specified + cipher list.' + items: + format: string + type: string + type: array + credentialName: + format: string + type: string + httpsRedirect: + type: boolean + maxProtocolVersion: + description: 'Optional: Maximum TLS protocol version.' + enum: + - TLS_AUTO + - TLSV1_0 + - TLSV1_1 + - TLSV1_2 + - TLSV1_3 + type: string + minProtocolVersion: + description: 'Optional: Minimum TLS protocol version.' + enum: + - TLS_AUTO + - TLSV1_0 + - TLSV1_1 + - TLSV1_2 + - TLSV1_3 + type: string + mode: + enum: + - PASSTHROUGH + - SIMPLE + - MUTUAL + - AUTO_PASSTHROUGH + - ISTIO_MUTUAL + type: string + privateKey: + description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. + format: string + type: string + serverCertificate: + description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. + format: string + type: string + subjectAltNames: + items: + format: string + type: string + type: array + verifyCertificateHash: + items: + format: string + type: string + type: array + verifyCertificateSpki: + items: + format: string + type: string + type: array + type: object + type: object + type: array + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: false + subresources: + status: {} --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -1608,28 +2893,6 @@ metadata: release: istio name: serviceentries.networking.istio.io spec: - additionalPrinterColumns: - - JSONPath: .spec.hosts - description: The hosts associated with the ServiceEntry - name: Hosts - type: string - - JSONPath: .spec.location - description: Whether the service is external to the mesh or part of the mesh (MESH_EXTERNAL - or MESH_INTERNAL) - name: Location - type: string - - JSONPath: .spec.resolution - description: Service discovery mode for the hosts (NONE, STATIC, or DNS) - name: Resolution - type: string - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date group: networking.istio.io names: categories: @@ -1641,26 +2904,2457 @@ spec: shortNames: - se singular: serviceentry - preserveUnknownFields: false scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting service registry. See more details - at: https://istio.io/docs/reference/config/networking/service-entry.html' - properties: - addresses: - description: The virtual IP addresses associated with the service. - items: + versions: + - additionalPrinterColumns: + - description: The hosts associated with the ServiceEntry + jsonPath: .spec.hosts + name: Hosts + type: string + - description: Whether the service is external to the mesh or part of the mesh + (MESH_EXTERNAL or MESH_INTERNAL) + jsonPath: .spec.location + name: Location + type: string + - description: Service discovery mode for the hosts (NONE, STATIC, or DNS) + jsonPath: .spec.resolution + name: Resolution + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting service registry. See more details + at: https://istio.io/docs/reference/config/networking/service-entry.html' + properties: + addresses: + description: The virtual IP addresses associated with the service. + items: + format: string + type: string + type: array + endpoints: + description: One or more endpoints associated with the service. + items: + properties: + address: + format: string + type: string + labels: + additionalProperties: + format: string + type: string + description: One or more labels associated with the endpoint. + type: object + locality: + description: The locality associated with the endpoint. + format: string + type: string + network: + format: string + type: string + ports: + additionalProperties: + type: integer + description: Set of ports associated with the endpoint. + type: object + serviceAccount: + format: string + type: string + weight: + description: The load balancing weight associated with the endpoint. + type: integer + type: object + type: array + exportTo: + description: A list of namespaces to which this service is exported. + items: + format: string + type: string + type: array + hosts: + description: The hosts associated with the ServiceEntry. + items: + format: string + type: string + type: array + location: + enum: + - MESH_EXTERNAL + - MESH_INTERNAL + type: string + ports: + description: The ports associated with the external service. + items: + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + type: array + resolution: + description: Service discovery mode for the hosts. + enum: + - NONE + - STATIC + - DNS + type: string + subjectAltNames: + items: + format: string + type: string + type: array + workloadSelector: + description: Applicable only for MESH_INTERNAL services. + properties: + labels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: The hosts associated with the ServiceEntry + jsonPath: .spec.hosts + name: Hosts + type: string + - description: Whether the service is external to the mesh or part of the mesh + (MESH_EXTERNAL or MESH_INTERNAL) + jsonPath: .spec.location + name: Location + type: string + - description: Service discovery mode for the hosts (NONE, STATIC, or DNS) + jsonPath: .spec.resolution + name: Resolution + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting service registry. See more details + at: https://istio.io/docs/reference/config/networking/service-entry.html' + properties: + addresses: + description: The virtual IP addresses associated with the service. + items: + format: string + type: string + type: array + endpoints: + description: One or more endpoints associated with the service. + items: + properties: + address: + format: string + type: string + labels: + additionalProperties: + format: string + type: string + description: One or more labels associated with the endpoint. + type: object + locality: + description: The locality associated with the endpoint. + format: string + type: string + network: + format: string + type: string + ports: + additionalProperties: + type: integer + description: Set of ports associated with the endpoint. + type: object + serviceAccount: + format: string + type: string + weight: + description: The load balancing weight associated with the endpoint. + type: integer + type: object + type: array + exportTo: + description: A list of namespaces to which this service is exported. + items: + format: string + type: string + type: array + hosts: + description: The hosts associated with the ServiceEntry. + items: + format: string + type: string + type: array + location: + enum: + - MESH_EXTERNAL + - MESH_INTERNAL + type: string + ports: + description: The ports associated with the external service. + items: + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + type: array + resolution: + description: Service discovery mode for the hosts. + enum: + - NONE + - STATIC + - DNS + type: string + subjectAltNames: + items: + format: string + type: string + type: array + workloadSelector: + description: Applicable only for MESH_INTERNAL services. + properties: + labels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + "helm.sh/resource-policy": keep + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + name: sidecars.networking.istio.io +spec: + group: networking.istio.io + names: + categories: + - istio-io + - networking-istio-io + kind: Sidecar + listKind: SidecarList + plural: sidecars + singular: sidecar + scope: Namespaced + versions: + - name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting network reachability of a sidecar. + See more details at: https://istio.io/docs/reference/config/networking/sidecar.html' + properties: + egress: + items: + properties: + bind: + format: string + type: string + captureMode: + enum: + - DEFAULT + - IPTABLES + - NONE + type: string + hosts: + items: + format: string + type: string + type: array + port: + description: The port associated with the listener. + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + type: object + type: array + ingress: + items: + properties: + bind: + description: The IP to which the listener should be bound. + format: string + type: string + captureMode: + enum: + - DEFAULT + - IPTABLES + - NONE + type: string + defaultEndpoint: + format: string + type: string + port: + description: The port associated with the listener. + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + type: object + type: array + outboundTrafficPolicy: + description: Configuration for the outbound traffic policy. + properties: + egressProxy: + properties: + host: + description: The name of a service from the service registry. + format: string + type: string + port: + description: Specifies the port on the host that is being + addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + mode: + enum: + - REGISTRY_ONLY + - ALLOW_ANY + type: string + type: object + workloadSelector: + properties: + labels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting network reachability of a sidecar. + See more details at: https://istio.io/docs/reference/config/networking/sidecar.html' + properties: + egress: + items: + properties: + bind: + format: string + type: string + captureMode: + enum: + - DEFAULT + - IPTABLES + - NONE + type: string + hosts: + items: + format: string + type: string + type: array + port: + description: The port associated with the listener. + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + type: object + type: array + ingress: + items: + properties: + bind: + description: The IP to which the listener should be bound. + format: string + type: string + captureMode: + enum: + - DEFAULT + - IPTABLES + - NONE + type: string + defaultEndpoint: + format: string + type: string + port: + description: The port associated with the listener. + properties: + name: + description: Label assigned to the port. + format: string + type: string + number: + description: A valid non-negative integer port number. + type: integer + protocol: + description: The protocol exposed on the port. + format: string + type: string + targetPort: + type: integer + type: object + type: object + type: array + outboundTrafficPolicy: + description: Configuration for the outbound traffic policy. + properties: + egressProxy: + properties: + host: + description: The name of a service from the service registry. + format: string + type: string + port: + description: Specifies the port on the host that is being + addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + mode: + enum: + - REGISTRY_ONLY + - ALLOW_ANY + type: string + type: object + workloadSelector: + properties: + labels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + "helm.sh/resource-policy": keep + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + name: virtualservices.networking.istio.io +spec: + group: networking.istio.io + names: + categories: + - istio-io + - networking-istio-io + kind: VirtualService + listKind: VirtualServiceList + plural: virtualservices + shortNames: + - vs + singular: virtualservice + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The names of gateways and sidecars that should apply these routes + jsonPath: .spec.gateways + name: Gateways + type: string + - description: The destination hosts to which traffic is being sent + jsonPath: .spec.hosts + name: Hosts + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting label/content routing, sni routing, + etc. See more details at: https://istio.io/docs/reference/config/networking/virtual-service.html' + properties: + exportTo: + description: A list of namespaces to which this virtual service is + exported. + items: + format: string + type: string + type: array + gateways: + description: The names of gateways and sidecars that should apply + these routes. + items: + format: string + type: string + type: array + hosts: + description: The destination hosts to which traffic is being sent. + items: + format: string + type: string + type: array + http: + description: An ordered list of route rules for HTTP traffic. + items: + properties: + corsPolicy: + description: Cross-Origin Resource Sharing policy (CORS). + properties: + allowCredentials: + nullable: true + type: boolean + allowHeaders: + items: + format: string + type: string + type: array + allowMethods: + description: List of HTTP methods allowed to access the + resource. + items: + format: string + type: string + type: array + allowOrigin: + description: The list of origins that are allowed to perform + CORS requests. + items: + format: string + type: string + type: array + allowOrigins: + description: String patterns that match allowed origins. + items: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + type: array + exposeHeaders: + items: + format: string + type: string + type: array + maxAge: + type: string + type: object + delegate: + properties: + name: + description: Name specifies the name of the delegate VirtualService. + format: string + type: string + namespace: + description: Namespace specifies the namespace where the + delegate VirtualService resides. + format: string + type: string + type: object + fault: + description: Fault injection policy to apply on HTTP traffic + at the client side. + properties: + abort: + oneOf: + - not: + anyOf: + - required: + - httpStatus + - required: + - grpcStatus + - required: + - http2Error + - required: + - httpStatus + - required: + - grpcStatus + - required: + - http2Error + properties: + grpcStatus: + format: string + type: string + http2Error: + format: string + type: string + httpStatus: + description: HTTP status code to use to abort the Http + request. + format: int32 + type: integer + percentage: + description: Percentage of requests to be aborted with + the error code provided. + properties: + value: + format: double + type: number + type: object + type: object + delay: + oneOf: + - not: + anyOf: + - required: + - fixedDelay + - required: + - exponentialDelay + - required: + - fixedDelay + - required: + - exponentialDelay + properties: + exponentialDelay: + type: string + fixedDelay: + description: Add a fixed delay before forwarding the + request. + type: string + percent: + description: Percentage of requests on which the delay + will be injected (0-100). + format: int32 + type: integer + percentage: + description: Percentage of requests on which the delay + will be injected. + properties: + value: + format: double + type: number + type: object + type: object + type: object + headers: + properties: + request: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + response: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + match: + items: + properties: + authority: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + gateways: + description: Names of gateways where the rule should be + applied. + items: + format: string + type: string + type: array + headers: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + type: object + ignoreUriCase: + description: Flag to specify whether the URI matching + should be case-insensitive. + type: boolean + method: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + name: + description: The name assigned to a match. + format: string + type: string + port: + description: Specifies the ports on the host that is being + addressed. + type: integer + queryParams: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + description: Query parameters for matching. + type: object + scheme: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + sourceLabels: + additionalProperties: + format: string + type: string + type: object + sourceNamespace: + description: Source namespace constraining the applicability + of a rule to workloads in that namespace. + format: string + type: string + uri: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + withoutHeaders: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + description: withoutHeader has the same syntax with the + header, but has opposite meaning. + type: object + type: object + type: array + mirror: + properties: + host: + description: The name of a service from the service registry. + format: string + type: string + port: + description: Specifies the port on the host that is being + addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + mirror_percent: + description: Percentage of the traffic to be mirrored by the + `mirror` field. + nullable: true + type: integer + mirrorPercent: + description: Percentage of the traffic to be mirrored by the + `mirror` field. + nullable: true + type: integer + mirrorPercentage: + description: Percentage of the traffic to be mirrored by the + `mirror` field. + properties: + value: + format: double + type: number + type: object + name: + description: The name assigned to the route for debugging purposes. + format: string + type: string + redirect: + description: A HTTP rule can either redirect or forward (default) + traffic. + properties: + authority: + format: string + type: string + redirectCode: + type: integer + uri: + format: string + type: string + type: object + retries: + description: Retry policy for HTTP requests. + properties: + attempts: + description: Number of retries to be allowed for a given + request. + format: int32 + type: integer + perTryTimeout: + description: Timeout per attempt for a given request, including + the initial call and any retries. + type: string + retryOn: + description: Specifies the conditions under which retry + takes place. + format: string + type: string + retryRemoteLocalities: + description: Flag to specify whether the retries should + retry to other localities. + nullable: true + type: boolean + type: object + rewrite: + description: Rewrite HTTP URIs and Authority headers. + properties: + authority: + description: rewrite the Authority/Host header with this + value. + format: string + type: string + uri: + format: string + type: string + type: object + route: + description: A HTTP rule can either redirect or forward (default) + traffic. + items: + properties: + destination: + properties: + host: + description: The name of a service from the service + registry. + format: string + type: string + port: + description: Specifies the port on the host that is + being addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + headers: + properties: + request: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + response: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + weight: + format: int32 + type: integer + type: object + type: array + timeout: + description: Timeout for HTTP requests, default is disabled. + type: string + type: object + type: array + tcp: + description: An ordered list of route rules for opaque TCP traffic. + items: + properties: + match: + items: + properties: + destinationSubnets: + description: IPv4 or IPv6 ip addresses of destination + with optional subnet. + items: + format: string + type: string + type: array + gateways: + description: Names of gateways where the rule should be + applied. + items: + format: string + type: string + type: array + port: + description: Specifies the port on the host that is being + addressed. + type: integer + sourceLabels: + additionalProperties: + format: string + type: string + type: object + sourceNamespace: + description: Source namespace constraining the applicability + of a rule to workloads in that namespace. + format: string + type: string + sourceSubnet: + description: IPv4 or IPv6 ip address of source with optional + subnet. + format: string + type: string + type: object + type: array + route: + description: The destination to which the connection should + be forwarded to. + items: + properties: + destination: + properties: + host: + description: The name of a service from the service + registry. + format: string + type: string + port: + description: Specifies the port on the host that is + being addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + weight: + format: int32 + type: integer + type: object + type: array + type: object + type: array + tls: + items: + properties: + match: + items: + properties: + destinationSubnets: + description: IPv4 or IPv6 ip addresses of destination + with optional subnet. + items: + format: string + type: string + type: array + gateways: + description: Names of gateways where the rule should be + applied. + items: + format: string + type: string + type: array + port: + description: Specifies the port on the host that is being + addressed. + type: integer + sniHosts: + description: SNI (server name indicator) to match on. + items: + format: string + type: string + type: array + sourceLabels: + additionalProperties: + format: string + type: string + type: object + sourceNamespace: + description: Source namespace constraining the applicability + of a rule to workloads in that namespace. + format: string + type: string + type: object + type: array + route: + description: The destination to which the connection should + be forwarded to. + items: + properties: + destination: + properties: + host: + description: The name of a service from the service + registry. + format: string + type: string + port: + description: Specifies the port on the host that is + being addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + weight: + format: int32 + type: integer + type: object + type: array + type: object + type: array + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: The names of gateways and sidecars that should apply these routes + jsonPath: .spec.gateways + name: Gateways + type: string + - description: The destination hosts to which traffic is being sent + jsonPath: .spec.hosts + name: Hosts + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting label/content routing, sni routing, + etc. See more details at: https://istio.io/docs/reference/config/networking/virtual-service.html' + properties: + exportTo: + description: A list of namespaces to which this virtual service is + exported. + items: + format: string + type: string + type: array + gateways: + description: The names of gateways and sidecars that should apply + these routes. + items: + format: string + type: string + type: array + hosts: + description: The destination hosts to which traffic is being sent. + items: + format: string + type: string + type: array + http: + description: An ordered list of route rules for HTTP traffic. + items: + properties: + corsPolicy: + description: Cross-Origin Resource Sharing policy (CORS). + properties: + allowCredentials: + nullable: true + type: boolean + allowHeaders: + items: + format: string + type: string + type: array + allowMethods: + description: List of HTTP methods allowed to access the + resource. + items: + format: string + type: string + type: array + allowOrigin: + description: The list of origins that are allowed to perform + CORS requests. + items: + format: string + type: string + type: array + allowOrigins: + description: String patterns that match allowed origins. + items: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + type: array + exposeHeaders: + items: + format: string + type: string + type: array + maxAge: + type: string + type: object + delegate: + properties: + name: + description: Name specifies the name of the delegate VirtualService. + format: string + type: string + namespace: + description: Namespace specifies the namespace where the + delegate VirtualService resides. + format: string + type: string + type: object + fault: + description: Fault injection policy to apply on HTTP traffic + at the client side. + properties: + abort: + oneOf: + - not: + anyOf: + - required: + - httpStatus + - required: + - grpcStatus + - required: + - http2Error + - required: + - httpStatus + - required: + - grpcStatus + - required: + - http2Error + properties: + grpcStatus: + format: string + type: string + http2Error: + format: string + type: string + httpStatus: + description: HTTP status code to use to abort the Http + request. + format: int32 + type: integer + percentage: + description: Percentage of requests to be aborted with + the error code provided. + properties: + value: + format: double + type: number + type: object + type: object + delay: + oneOf: + - not: + anyOf: + - required: + - fixedDelay + - required: + - exponentialDelay + - required: + - fixedDelay + - required: + - exponentialDelay + properties: + exponentialDelay: + type: string + fixedDelay: + description: Add a fixed delay before forwarding the + request. + type: string + percent: + description: Percentage of requests on which the delay + will be injected (0-100). + format: int32 + type: integer + percentage: + description: Percentage of requests on which the delay + will be injected. + properties: + value: + format: double + type: number + type: object + type: object + type: object + headers: + properties: + request: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + response: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + match: + items: + properties: + authority: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + gateways: + description: Names of gateways where the rule should be + applied. + items: + format: string + type: string + type: array + headers: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + type: object + ignoreUriCase: + description: Flag to specify whether the URI matching + should be case-insensitive. + type: boolean + method: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + name: + description: The name assigned to a match. + format: string + type: string + port: + description: Specifies the ports on the host that is being + addressed. + type: integer + queryParams: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + description: Query parameters for matching. + type: object + scheme: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + sourceLabels: + additionalProperties: + format: string + type: string + type: object + sourceNamespace: + description: Source namespace constraining the applicability + of a rule to workloads in that namespace. + format: string + type: string + uri: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + withoutHeaders: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - exact + - required: + - prefix + - required: + - regex + - required: + - exact + - required: + - prefix + - required: + - regex + properties: + exact: + format: string + type: string + prefix: + format: string + type: string + regex: + description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). + format: string + type: string + type: object + description: withoutHeader has the same syntax with the + header, but has opposite meaning. + type: object + type: object + type: array + mirror: + properties: + host: + description: The name of a service from the service registry. + format: string + type: string + port: + description: Specifies the port on the host that is being + addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + mirror_percent: + description: Percentage of the traffic to be mirrored by the + `mirror` field. + nullable: true + type: integer + mirrorPercent: + description: Percentage of the traffic to be mirrored by the + `mirror` field. + nullable: true + type: integer + mirrorPercentage: + description: Percentage of the traffic to be mirrored by the + `mirror` field. + properties: + value: + format: double + type: number + type: object + name: + description: The name assigned to the route for debugging purposes. + format: string + type: string + redirect: + description: A HTTP rule can either redirect or forward (default) + traffic. + properties: + authority: + format: string + type: string + redirectCode: + type: integer + uri: + format: string + type: string + type: object + retries: + description: Retry policy for HTTP requests. + properties: + attempts: + description: Number of retries to be allowed for a given + request. + format: int32 + type: integer + perTryTimeout: + description: Timeout per attempt for a given request, including + the initial call and any retries. + type: string + retryOn: + description: Specifies the conditions under which retry + takes place. + format: string + type: string + retryRemoteLocalities: + description: Flag to specify whether the retries should + retry to other localities. + nullable: true + type: boolean + type: object + rewrite: + description: Rewrite HTTP URIs and Authority headers. + properties: + authority: + description: rewrite the Authority/Host header with this + value. + format: string + type: string + uri: + format: string + type: string + type: object + route: + description: A HTTP rule can either redirect or forward (default) + traffic. + items: + properties: + destination: + properties: + host: + description: The name of a service from the service + registry. + format: string + type: string + port: + description: Specifies the port on the host that is + being addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + headers: + properties: + request: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + response: + properties: + add: + additionalProperties: + format: string + type: string + type: object + remove: + items: + format: string + type: string + type: array + set: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + weight: + format: int32 + type: integer + type: object + type: array + timeout: + description: Timeout for HTTP requests, default is disabled. + type: string + type: object + type: array + tcp: + description: An ordered list of route rules for opaque TCP traffic. + items: + properties: + match: + items: + properties: + destinationSubnets: + description: IPv4 or IPv6 ip addresses of destination + with optional subnet. + items: + format: string + type: string + type: array + gateways: + description: Names of gateways where the rule should be + applied. + items: + format: string + type: string + type: array + port: + description: Specifies the port on the host that is being + addressed. + type: integer + sourceLabels: + additionalProperties: + format: string + type: string + type: object + sourceNamespace: + description: Source namespace constraining the applicability + of a rule to workloads in that namespace. + format: string + type: string + sourceSubnet: + description: IPv4 or IPv6 ip address of source with optional + subnet. + format: string + type: string + type: object + type: array + route: + description: The destination to which the connection should + be forwarded to. + items: + properties: + destination: + properties: + host: + description: The name of a service from the service + registry. + format: string + type: string + port: + description: Specifies the port on the host that is + being addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + weight: + format: int32 + type: integer + type: object + type: array + type: object + type: array + tls: + items: + properties: + match: + items: + properties: + destinationSubnets: + description: IPv4 or IPv6 ip addresses of destination + with optional subnet. + items: + format: string + type: string + type: array + gateways: + description: Names of gateways where the rule should be + applied. + items: + format: string + type: string + type: array + port: + description: Specifies the port on the host that is being + addressed. + type: integer + sniHosts: + description: SNI (server name indicator) to match on. + items: + format: string + type: string + type: array + sourceLabels: + additionalProperties: + format: string + type: string + type: object + sourceNamespace: + description: Source namespace constraining the applicability + of a rule to workloads in that namespace. + format: string + type: string + type: object + type: array + route: + description: The destination to which the connection should + be forwarded to. + items: + properties: + destination: + properties: + host: + description: The name of a service from the service + registry. + format: string + type: string + port: + description: Specifies the port on the host that is + being addressed. + properties: + number: + type: integer + type: object + subset: + description: The name of a subset within the service. + format: string + type: string + type: object + weight: + format: int32 + type: integer + type: object + type: array + type: object + type: array + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + "helm.sh/resource-policy": keep + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + name: workloadentries.networking.istio.io +spec: + group: networking.istio.io + names: + categories: + - istio-io + - networking-istio-io + kind: WorkloadEntry + listKind: WorkloadEntryList + plural: workloadentries + shortNames: + - we + singular: workloadentry + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Address associated with the network endpoint. + jsonPath: .spec.address + name: Address + type: string + name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting VMs onboarded into the mesh. See + more details at: https://istio.io/docs/reference/config/networking/workload-entry.html' + properties: + address: format: string type: string - type: array - endpoints: - description: One or more endpoints associated with the service. - items: + labels: + additionalProperties: + format: string + type: string + description: One or more labels associated with the endpoint. + type: object + locality: + description: The locality associated with the endpoint. + format: string + type: string + network: + format: string + type: string + ports: + additionalProperties: + type: integer + description: Set of ports associated with the endpoint. + type: object + serviceAccount: + format: string + type: string + weight: + description: The load balancing weight associated with the endpoint. + type: integer + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Address associated with the network endpoint. + jsonPath: .spec.address + name: Address + type: string + name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration affecting VMs onboarded into the mesh. See + more details at: https://istio.io/docs/reference/config/networking/workload-entry.html' + properties: + address: + format: string + type: string + labels: + additionalProperties: + format: string + type: string + description: One or more labels associated with the endpoint. + type: object + locality: + description: The locality associated with the endpoint. + format: string + type: string + network: + format: string + type: string + ports: + additionalProperties: + type: integer + description: Set of ports associated with the endpoint. + type: object + serviceAccount: + format: string + type: string + weight: + description: The load balancing weight associated with the endpoint. + type: integer + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: false + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + name: workloadgroups.networking.istio.io +spec: + group: networking.istio.io + names: + categories: + - istio-io + - networking-istio-io + kind: WorkloadGroup + listKind: WorkloadGroupList + plural: workloadgroups + shortNames: + - wg + singular: workloadgroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha3 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Describes a collection of workload instances. See more details + at: https://istio.io/docs/reference/config/networking/workload-group.html' + properties: + metadata: + description: Metadata that will be used for all corresponding `WorkloadEntries`. + properties: + annotations: + additionalProperties: + format: string + type: string + type: object + labels: + additionalProperties: + format: string + type: string + type: object + type: object + probe: + description: '`ReadinessProbe` describes the configuration the user + must provide for healthchecking on their workload.' + oneOf: + - not: + anyOf: + - required: + - httpGet + - required: + - tcpSocket + - required: + - exec + - required: + - httpGet + - required: + - tcpSocket + - required: + - exec + properties: + exec: + description: Health is determined by how the command that is executed + exited. + properties: + command: + description: Command to run. + items: + format: string + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be + considered failed after having succeeded. + format: int32 + type: integer + httpGet: + properties: + host: + description: Host name to connect to, defaults to the pod + IP. + format: string + type: string + httpHeaders: + description: Headers the proxy will pass on to make the request. + items: + properties: + name: + format: string + type: string + value: + format: string + type: string + type: object + type: array + path: + description: Path to access on the HTTP server. + format: string + type: string + port: + description: Port on which the endpoint lives. + type: integer + scheme: + format: string + type: string + type: object + initialDelaySeconds: + description: Number of seconds after the container has started + before readiness probes are initiated. + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be + considered successful after having failed. + format: int32 + type: integer + tcpSocket: + description: Health is determined by if the proxy is able to connect. + properties: + host: + format: string + type: string + port: + type: integer + type: object + timeoutSeconds: + description: Number of seconds after which the probe times out. + format: int32 + type: integer + type: object + template: + description: Template to be used for the generation of `WorkloadEntry` + resources that belong to this `WorkloadGroup`. properties: address: format: string @@ -1690,1328 +5384,18 @@ spec: description: The load balancing weight associated with the endpoint. type: integer type: object - type: array - exportTo: - description: A list of namespaces to which this service is exported. - items: - format: string - type: string - type: array - hosts: - description: The hosts associated with the ServiceEntry. - items: - format: string - type: string - type: array - location: - enum: - - MESH_EXTERNAL - - MESH_INTERNAL - type: string - ports: - description: The ports associated with the external service. - items: - properties: - name: - description: Label assigned to the port. - format: string - type: string - number: - description: A valid non-negative integer port number. - type: integer - protocol: - description: The protocol exposed on the port. - format: string - type: string - targetPort: - type: integer - type: object - type: array - resolution: - description: Service discovery mode for the hosts. - enum: - - NONE - - STATIC - - DNS - type: string - subjectAltNames: - items: - format: string - type: string - type: array - workloadSelector: - description: Applicable only for MESH_INTERNAL services. - properties: - labels: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1alpha3 + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true - - name: v1beta1 - served: true - storage: false + subresources: + status: {} --- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - "helm.sh/resource-policy": keep - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: sidecars.networking.istio.io -spec: - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: Sidecar - listKind: SidecarList - plural: sidecars - singular: sidecar - preserveUnknownFields: false - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting network reachability of a sidecar. - See more details at: https://istio.io/docs/reference/config/networking/sidecar.html' - properties: - egress: - items: - properties: - bind: - format: string - type: string - captureMode: - enum: - - DEFAULT - - IPTABLES - - NONE - type: string - hosts: - items: - format: string - type: string - type: array - port: - description: The port associated with the listener. - properties: - name: - description: Label assigned to the port. - format: string - type: string - number: - description: A valid non-negative integer port number. - type: integer - protocol: - description: The protocol exposed on the port. - format: string - type: string - targetPort: - type: integer - type: object - type: object - type: array - ingress: - items: - properties: - bind: - description: The IP to which the listener should be bound. - format: string - type: string - captureMode: - enum: - - DEFAULT - - IPTABLES - - NONE - type: string - defaultEndpoint: - format: string - type: string - port: - description: The port associated with the listener. - properties: - name: - description: Label assigned to the port. - format: string - type: string - number: - description: A valid non-negative integer port number. - type: integer - protocol: - description: The protocol exposed on the port. - format: string - type: string - targetPort: - type: integer - type: object - type: object - type: array - outboundTrafficPolicy: - description: Configuration for the outbound traffic policy. - properties: - egressProxy: - properties: - host: - description: The name of a service from the service registry. - format: string - type: string - port: - description: Specifies the port on the host that is being addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - mode: - enum: - - REGISTRY_ONLY - - ALLOW_ANY - type: string - type: object - workloadSelector: - properties: - labels: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1alpha3 - served: true - storage: true - - name: v1beta1 - served: true - storage: false - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - "helm.sh/resource-policy": keep - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: virtualservices.networking.istio.io -spec: - additionalPrinterColumns: - - JSONPath: .spec.gateways - description: The names of gateways and sidecars that should apply these routes - name: Gateways - type: string - - JSONPath: .spec.hosts - description: The destination hosts to which traffic is being sent - name: Hosts - type: string - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: VirtualService - listKind: VirtualServiceList - plural: virtualservices - shortNames: - - vs - singular: virtualservice - preserveUnknownFields: false - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting label/content routing, sni routing, - etc. See more details at: https://istio.io/docs/reference/config/networking/virtual-service.html' - properties: - exportTo: - description: A list of namespaces to which this virtual service is exported. - items: - format: string - type: string - type: array - gateways: - description: The names of gateways and sidecars that should apply these - routes. - items: - format: string - type: string - type: array - hosts: - description: The destination hosts to which traffic is being sent. - items: - format: string - type: string - type: array - http: - description: An ordered list of route rules for HTTP traffic. - items: - properties: - corsPolicy: - description: Cross-Origin Resource Sharing policy (CORS). - properties: - allowCredentials: - nullable: true - type: boolean - allowHeaders: - items: - format: string - type: string - type: array - allowMethods: - description: List of HTTP methods allowed to access the resource. - items: - format: string - type: string - type: array - allowOrigin: - description: The list of origins that are allowed to perform - CORS requests. - items: - format: string - type: string - type: array - allowOrigins: - description: String patterns that match allowed origins. - items: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - type: array - exposeHeaders: - items: - format: string - type: string - type: array - maxAge: - type: string - type: object - delegate: - properties: - name: - description: Name specifies the name of the delegate VirtualService. - format: string - type: string - namespace: - description: Namespace specifies the namespace where the delegate - VirtualService resides. - format: string - type: string - type: object - fault: - description: Fault injection policy to apply on HTTP traffic at - the client side. - properties: - abort: - oneOf: - - not: - anyOf: - - required: - - httpStatus - - required: - - grpcStatus - - required: - - http2Error - - required: - - httpStatus - - required: - - grpcStatus - - required: - - http2Error - properties: - grpcStatus: - format: string - type: string - http2Error: - format: string - type: string - httpStatus: - description: HTTP status code to use to abort the Http - request. - format: int32 - type: integer - percentage: - description: Percentage of requests to be aborted with - the error code provided. - properties: - value: - format: double - type: number - type: object - type: object - delay: - oneOf: - - not: - anyOf: - - required: - - fixedDelay - - required: - - exponentialDelay - - required: - - fixedDelay - - required: - - exponentialDelay - properties: - exponentialDelay: - type: string - fixedDelay: - description: Add a fixed delay before forwarding the request. - type: string - percent: - description: Percentage of requests on which the delay - will be injected (0-100). - format: int32 - type: integer - percentage: - description: Percentage of requests on which the delay - will be injected. - properties: - value: - format: double - type: number - type: object - type: object - type: object - headers: - properties: - request: - properties: - add: - additionalProperties: - format: string - type: string - type: object - remove: - items: - format: string - type: string - type: array - set: - additionalProperties: - format: string - type: string - type: object - type: object - response: - properties: - add: - additionalProperties: - format: string - type: string - type: object - remove: - items: - format: string - type: string - type: array - set: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - match: - items: - properties: - authority: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - gateways: - description: Names of gateways where the rule should be - applied. - items: - format: string - type: string - type: array - headers: - additionalProperties: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - type: object - ignoreUriCase: - description: Flag to specify whether the URI matching should - be case-insensitive. - type: boolean - method: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - name: - description: The name assigned to a match. - format: string - type: string - port: - description: Specifies the ports on the host that is being - addressed. - type: integer - queryParams: - additionalProperties: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - description: Query parameters for matching. - type: object - scheme: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - sourceLabels: - additionalProperties: - format: string - type: string - type: object - sourceNamespace: - description: Source namespace constraining the applicability - of a rule to workloads in that namespace. - format: string - type: string - uri: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - withoutHeaders: - additionalProperties: - oneOf: - - not: - anyOf: - - required: - - exact - - required: - - prefix - - required: - - regex - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - description: RE2 style regex-based match (https://github.com/google/re2/wiki/Syntax). - format: string - type: string - type: object - description: withoutHeader has the same syntax with the - header, but has opposite meaning. - type: object - type: object - type: array - mirror: - properties: - host: - description: The name of a service from the service registry. - format: string - type: string - port: - description: Specifies the port on the host that is being - addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - mirror_percent: - description: Percentage of the traffic to be mirrored by the `mirror` - field. - nullable: true - type: integer - mirrorPercent: - description: Percentage of the traffic to be mirrored by the `mirror` - field. - nullable: true - type: integer - mirrorPercentage: - description: Percentage of the traffic to be mirrored by the `mirror` - field. - properties: - value: - format: double - type: number - type: object - name: - description: The name assigned to the route for debugging purposes. - format: string - type: string - redirect: - description: A HTTP rule can either redirect or forward (default) - traffic. - properties: - authority: - format: string - type: string - redirectCode: - type: integer - uri: - format: string - type: string - type: object - retries: - description: Retry policy for HTTP requests. - properties: - attempts: - description: Number of retries to be allowed for a given request. - format: int32 - type: integer - perTryTimeout: - description: Timeout per attempt for a given request, including - the initial call and any retries. - type: string - retryOn: - description: Specifies the conditions under which retry takes - place. - format: string - type: string - retryRemoteLocalities: - description: Flag to specify whether the retries should retry - to other localities. - nullable: true - type: boolean - type: object - rewrite: - description: Rewrite HTTP URIs and Authority headers. - properties: - authority: - description: rewrite the Authority/Host header with this value. - format: string - type: string - uri: - format: string - type: string - type: object - route: - description: A HTTP rule can either redirect or forward (default) - traffic. - items: - properties: - destination: - properties: - host: - description: The name of a service from the service - registry. - format: string - type: string - port: - description: Specifies the port on the host that is - being addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - headers: - properties: - request: - properties: - add: - additionalProperties: - format: string - type: string - type: object - remove: - items: - format: string - type: string - type: array - set: - additionalProperties: - format: string - type: string - type: object - type: object - response: - properties: - add: - additionalProperties: - format: string - type: string - type: object - remove: - items: - format: string - type: string - type: array - set: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - weight: - format: int32 - type: integer - type: object - type: array - timeout: - description: Timeout for HTTP requests, default is disabled. - type: string - type: object - type: array - tcp: - description: An ordered list of route rules for opaque TCP traffic. - items: - properties: - match: - items: - properties: - destinationSubnets: - description: IPv4 or IPv6 ip addresses of destination with - optional subnet. - items: - format: string - type: string - type: array - gateways: - description: Names of gateways where the rule should be - applied. - items: - format: string - type: string - type: array - port: - description: Specifies the port on the host that is being - addressed. - type: integer - sourceLabels: - additionalProperties: - format: string - type: string - type: object - sourceNamespace: - description: Source namespace constraining the applicability - of a rule to workloads in that namespace. - format: string - type: string - sourceSubnet: - description: IPv4 or IPv6 ip address of source with optional - subnet. - format: string - type: string - type: object - type: array - route: - description: The destination to which the connection should be - forwarded to. - items: - properties: - destination: - properties: - host: - description: The name of a service from the service - registry. - format: string - type: string - port: - description: Specifies the port on the host that is - being addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - weight: - format: int32 - type: integer - type: object - type: array - type: object - type: array - tls: - items: - properties: - match: - items: - properties: - destinationSubnets: - description: IPv4 or IPv6 ip addresses of destination with - optional subnet. - items: - format: string - type: string - type: array - gateways: - description: Names of gateways where the rule should be - applied. - items: - format: string - type: string - type: array - port: - description: Specifies the port on the host that is being - addressed. - type: integer - sniHosts: - description: SNI (server name indicator) to match on. - items: - format: string - type: string - type: array - sourceLabels: - additionalProperties: - format: string - type: string - type: object - sourceNamespace: - description: Source namespace constraining the applicability - of a rule to workloads in that namespace. - format: string - type: string - type: object - type: array - route: - description: The destination to which the connection should be - forwarded to. - items: - properties: - destination: - properties: - host: - description: The name of a service from the service - registry. - format: string - type: string - port: - description: Specifies the port on the host that is - being addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - weight: - format: int32 - type: integer - type: object - type: array - type: object - type: array - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1alpha3 - served: true - storage: true - - name: v1beta1 - served: true - storage: false - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - "helm.sh/resource-policy": keep - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: workloadentries.networking.istio.io -spec: - additionalPrinterColumns: - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date - - JSONPath: .spec.address - description: Address associated with the network endpoint. - name: Address - type: string - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: WorkloadEntry - listKind: WorkloadEntryList - plural: workloadentries - shortNames: - - we - singular: workloadentry - preserveUnknownFields: false - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting VMs onboarded into the mesh. See more - details at: https://istio.io/docs/reference/config/networking/workload-entry.html' - properties: - address: - format: string - type: string - labels: - additionalProperties: - format: string - type: string - description: One or more labels associated with the endpoint. - type: object - locality: - description: The locality associated with the endpoint. - format: string - type: string - network: - format: string - type: string - ports: - additionalProperties: - type: integer - description: Set of ports associated with the endpoint. - type: object - serviceAccount: - format: string - type: string - weight: - description: The load balancing weight associated with the endpoint. - type: integer - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1alpha3 - served: true - storage: true - - name: v1beta1 - served: true - storage: false - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: workloadgroups.networking.istio.io -spec: - additionalPrinterColumns: - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: WorkloadGroup - listKind: WorkloadGroupList - plural: workloadgroups - shortNames: - - wg - singular: workloadgroup - preserveUnknownFields: false - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Describes a collection of workload instances. See more details - at: https://istio.io/docs/reference/config/networking/workload-group.html' - properties: - metadata: - description: Metadata that will be used for all corresponding `WorkloadEntries`. - properties: - annotations: - additionalProperties: - format: string - type: string - type: object - labels: - additionalProperties: - format: string - type: string - type: object - type: object - probe: - description: '`ReadinessProbe` describes the configuration the user - must provide for healthchecking on their workload.' - oneOf: - - not: - anyOf: - - required: - - httpGet - - required: - - tcpSocket - - required: - - exec - - required: - - httpGet - - required: - - tcpSocket - - required: - - exec - properties: - exec: - description: Health is determined by how the command that is executed - exited. - properties: - command: - description: Command to run. - items: - format: string - type: string - type: array - type: object - failureThreshold: - description: Minimum consecutive failures for the probe to be considered - failed after having succeeded. - format: int32 - type: integer - httpGet: - properties: - host: - description: Host name to connect to, defaults to the pod IP. - format: string - type: string - httpHeaders: - description: Headers the proxy will pass on to make the request. - items: - properties: - name: - format: string - type: string - value: - format: string - type: string - type: object - type: array - path: - description: Path to access on the HTTP server. - format: string - type: string - port: - description: Port on which the endpoint lives. - type: integer - scheme: - format: string - type: string - type: object - initialDelaySeconds: - description: Number of seconds after the container has started before - readiness probes are initiated. - format: int32 - type: integer - periodSeconds: - description: How often (in seconds) to perform the probe. - format: int32 - type: integer - successThreshold: - description: Minimum consecutive successes for the probe to be considered - successful after having failed. - format: int32 - type: integer - tcpSocket: - description: Health is determined by if the proxy is able to connect. - properties: - host: - format: string - type: string - port: - type: integer - type: object - timeoutSeconds: - description: Number of seconds after which the probe times out. - format: int32 - type: integer - type: object - template: - description: Template to be used for the generation of `WorkloadEntry` - resources that belong to this `WorkloadGroup`. - properties: - address: - format: string - type: string - labels: - additionalProperties: - format: string - type: string - description: One or more labels associated with the endpoint. - type: object - locality: - description: The locality associated with the endpoint. - format: string - type: string - network: - format: string - type: string - ports: - additionalProperties: - type: integer - description: Set of ports associated with the endpoint. - type: object - serviceAccount: - format: string - type: string - weight: - description: The load balancing weight associated with the endpoint. - type: integer - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1alpha3 - served: true - storage: true - ---- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -3033,217 +5417,216 @@ spec: listKind: AuthorizationPolicyList plural: authorizationpolicies singular: authorizationpolicy - preserveUnknownFields: false scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration for access control on workloads. See more details - at: https://istio.io/docs/reference/config/security/authorization-policy.html' - oneOf: - - not: - anyOf: - - required: - - provider - - required: - - provider - properties: - action: - description: Optional. - enum: - - ALLOW - - DENY - - AUDIT - - CUSTOM - type: string - provider: - description: Specifies detailed configuration of the CUSTOM action. - properties: - name: - description: Specifies the name of the extension provider. - format: string - type: string - type: object - rules: - description: Optional. - items: - properties: - from: - description: Optional. - items: - properties: - source: - description: Source specifies the source of a request. - properties: - ipBlocks: - description: Optional. - items: - format: string - type: string - type: array - namespaces: - description: Optional. - items: - format: string - type: string - type: array - notIpBlocks: - description: Optional. - items: - format: string - type: string - type: array - notNamespaces: - description: Optional. - items: - format: string - type: string - type: array - notPrincipals: - description: Optional. - items: - format: string - type: string - type: array - notRemoteIpBlocks: - description: Optional. - items: - format: string - type: string - type: array - notRequestPrincipals: - description: Optional. - items: - format: string - type: string - type: array - principals: - description: Optional. - items: - format: string - type: string - type: array - remoteIpBlocks: - description: Optional. - items: - format: string - type: string - type: array - requestPrincipals: - description: Optional. - items: - format: string - type: string - type: array - type: object - type: object - type: array - to: - description: Optional. - items: - properties: - operation: - description: Operation specifies the operation of a request. - properties: - hosts: - description: Optional. - items: - format: string - type: string - type: array - methods: - description: Optional. - items: - format: string - type: string - type: array - notHosts: - description: Optional. - items: - format: string - type: string - type: array - notMethods: - description: Optional. - items: - format: string - type: string - type: array - notPaths: - description: Optional. - items: - format: string - type: string - type: array - notPorts: - description: Optional. - items: - format: string - type: string - type: array - paths: - description: Optional. - items: - format: string - type: string - type: array - ports: - description: Optional. - items: - format: string - type: string - type: array - type: object - type: object - type: array - when: - description: Optional. - items: - properties: - key: - description: The name of an Istio attribute. - format: string - type: string - notValues: - description: Optional. - items: - format: string - type: string - type: array - values: - description: Optional. - items: - format: string - type: string - type: array - type: object - type: array - type: object - type: array - selector: - description: Optional. - properties: - matchLabels: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object versions: - name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: 'Configuration for access control on workloads. See more + details at: https://istio.io/docs/reference/config/security/authorization-policy.html' + oneOf: + - not: + anyOf: + - required: + - provider + - required: + - provider + properties: + action: + description: Optional. + enum: + - ALLOW + - DENY + - AUDIT + - CUSTOM + type: string + provider: + description: Specifies detailed configuration of the CUSTOM action. + properties: + name: + description: Specifies the name of the extension provider. + format: string + type: string + type: object + rules: + description: Optional. + items: + properties: + from: + description: Optional. + items: + properties: + source: + description: Source specifies the source of a request. + properties: + ipBlocks: + description: Optional. + items: + format: string + type: string + type: array + namespaces: + description: Optional. + items: + format: string + type: string + type: array + notIpBlocks: + description: Optional. + items: + format: string + type: string + type: array + notNamespaces: + description: Optional. + items: + format: string + type: string + type: array + notPrincipals: + description: Optional. + items: + format: string + type: string + type: array + notRemoteIpBlocks: + description: Optional. + items: + format: string + type: string + type: array + notRequestPrincipals: + description: Optional. + items: + format: string + type: string + type: array + principals: + description: Optional. + items: + format: string + type: string + type: array + remoteIpBlocks: + description: Optional. + items: + format: string + type: string + type: array + requestPrincipals: + description: Optional. + items: + format: string + type: string + type: array + type: object + type: object + type: array + to: + description: Optional. + items: + properties: + operation: + description: Operation specifies the operation of a request. + properties: + hosts: + description: Optional. + items: + format: string + type: string + type: array + methods: + description: Optional. + items: + format: string + type: string + type: array + notHosts: + description: Optional. + items: + format: string + type: string + type: array + notMethods: + description: Optional. + items: + format: string + type: string + type: array + notPaths: + description: Optional. + items: + format: string + type: string + type: array + notPorts: + description: Optional. + items: + format: string + type: string + type: array + paths: + description: Optional. + items: + format: string + type: string + type: array + ports: + description: Optional. + items: + format: string + type: string + type: array + type: object + type: object + type: array + when: + description: Optional. + items: + properties: + key: + description: The name of an Istio attribute. + format: string + type: string + notValues: + description: Optional. + items: + format: string + type: string + type: array + values: + description: Optional. + items: + format: string + type: string + type: array + type: object + type: array + type: object + type: array + selector: + description: Optional. + properties: + matchLabels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true + subresources: + status: {} --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -3256,19 +5639,6 @@ metadata: release: istio name: peerauthentications.security.istio.io spec: - additionalPrinterColumns: - - JSONPath: .spec.mtls.mode - description: Defines the mTLS mode used for peer authentication. - name: Mode - type: string - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date group: security.istio.io names: categories: @@ -3280,31 +5650,31 @@ spec: shortNames: - pa singular: peerauthentication - preserveUnknownFields: false scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: PeerAuthentication defines how traffic will be tunneled (or - not) to the sidecar. - properties: - mtls: - description: Mutual TLS settings for workload. - properties: - mode: - description: Defines the mTLS mode used for peer authentication. - enum: - - UNSET - - DISABLE - - PERMISSIVE - - STRICT - type: string - type: object - portLevelMtls: - additionalProperties: + versions: + - additionalPrinterColumns: + - description: Defines the mTLS mode used for peer authentication. + jsonPath: .spec.mtls.mode + name: Mode + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: PeerAuthentication defines how traffic will be tunneled (or + not) to the sidecar. + properties: + mtls: + description: Mutual TLS settings for workload. properties: mode: description: Defines the mTLS mode used for peer authentication. @@ -3315,30 +5685,42 @@ spec: - STRICT type: string type: object - description: Port specific mutual TLS settings. - type: object - selector: - description: The selector determines the workloads to apply the ChannelAuthentication - on. - properties: - matchLabels: - additionalProperties: - format: string - type: string + portLevelMtls: + additionalProperties: + properties: + mode: + description: Defines the mTLS mode used for peer authentication. + enum: + - UNSET + - DISABLE + - PERMISSIVE + - STRICT + type: string type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - versions: - - name: v1beta1 + description: Port specific mutual TLS settings. + type: object + selector: + description: The selector determines the workloads to apply the ChannelAuthentication + on. + properties: + matchLabels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true + subresources: + status: {} --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -3362,159 +5744,284 @@ spec: shortNames: - ra singular: requestauthentication - preserveUnknownFields: false scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: RequestAuthentication defines what request authentication methods - are supported by a workload. - properties: - jwtRules: - description: Define the list of JWTs that can be validated at the selected - workloads' proxy. - items: - properties: - audiences: - items: - format: string - type: string - type: array - forwardOriginalToken: - description: If set to true, the orginal token will be kept for - the ustream request. - type: boolean - fromHeaders: - description: List of header locations from which JWT is expected. - items: - properties: - name: - description: The HTTP header name. - format: string - type: string - prefix: - description: The prefix that should be stripped before decoding - the token. - format: string - type: string - type: object - type: array - fromParams: - description: List of query parameters from which JWT is expected. - items: - format: string - type: string - type: array - issuer: - description: Identifies the issuer that issued the JWT. - format: string - type: string - jwks: - description: JSON Web Key Set of public keys to validate signature - of the JWT. - format: string - type: string - jwks_uri: - format: string - type: string - jwksUri: - format: string - type: string - outputPayloadToHeader: - format: string - type: string - type: object - type: array - selector: - description: The selector determines the workloads to apply the RequestAuthentication - on. - properties: - matchLabels: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - status: - type: object - x-kubernetes-preserve-unknown-fields: true - type: object versions: - name: v1beta1 + schema: + openAPIV3Schema: + properties: + spec: + description: RequestAuthentication defines what request authentication + methods are supported by a workload. + properties: + jwtRules: + description: Define the list of JWTs that can be validated at the + selected workloads' proxy. + items: + properties: + audiences: + items: + format: string + type: string + type: array + forwardOriginalToken: + description: If set to true, the orginal token will be kept + for the ustream request. + type: boolean + fromHeaders: + description: List of header locations from which JWT is expected. + items: + properties: + name: + description: The HTTP header name. + format: string + type: string + prefix: + description: The prefix that should be stripped before + decoding the token. + format: string + type: string + type: object + type: array + fromParams: + description: List of query parameters from which JWT is expected. + items: + format: string + type: string + type: array + issuer: + description: Identifies the issuer that issued the JWT. + format: string + type: string + jwks: + description: JSON Web Key Set of public keys to validate signature + of the JWT. + format: string + type: string + jwks_uri: + format: string + type: string + jwksUri: + format: string + type: string + outputPayloadToHeader: + format: string + type: string + type: object + type: array + selector: + description: The selector determines the workloads to apply the RequestAuthentication + on. + properties: + matchLabels: + additionalProperties: + format: string + type: string + type: object + type: object + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object served: true storage: true + subresources: + status: {} + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + "helm.sh/resource-policy": keep + labels: + app: istio-pilot + chart: istio + heritage: Tiller + istio: telemetry + release: istio + name: telemetries.telemetry.istio.io +spec: + group: telemetry.istio.io + names: + categories: + - istio-io + - telemetry-istio-io + kind: Telemetry + listKind: TelemetryList + plural: telemetries + shortNames: + - telemetry + singular: telemetry + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + spec: + description: Telemetry defines how the telemetry is generated for workloads + within a mesh. + properties: + selector: + description: Optional. + properties: + matchLabels: + additionalProperties: + format: string + type: string + type: object + type: object + tracing: + description: Optional. + items: + properties: + customTags: + additionalProperties: + oneOf: + - not: + anyOf: + - required: + - literal + - required: + - environment + - required: + - header + - required: + - literal + - required: + - environment + - required: + - header + properties: + environment: + description: Environment adds the value of an environment + variable to each span. + properties: + defaultValue: + description: Optional. + format: string + type: string + name: + description: Name of the environment variable from + which to extract the tag value. + format: string + type: string + type: object + header: + description: RequestHeader adds the value of an header + from the request to each span. + properties: + defaultValue: + description: Optional. + format: string + type: string + name: + description: Name of the header from which to extract + the tag value. + format: string + type: string + type: object + literal: + description: Literal adds the same, hard-coded value to + each span. + properties: + value: + description: The tag value to use. + format: string + type: string + type: object + type: object + description: Optional. + type: object + disableSpanReporting: + description: Controls span reporting. + nullable: true + type: boolean + providers: + description: Optional. + items: + properties: + name: + description: Required. + format: string + type: string + type: object + type: array + randomSamplingPercentage: + nullable: true + type: number + type: object + type: array + type: object + status: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} --- --- # Source: crds/crd-operator.yaml # SYNC WITH manifests/charts/istio-operator/templates -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: istiooperators.install.istio.io labels: release: istio spec: - additionalPrinterColumns: - - JSONPath: .spec.revision - description: Istio control plane revision - name: Revision - type: string - - JSONPath: .status.status - description: IOP current state - type: string - name: Status - - JSONPath: .metadata.creationTimestamp - description: 'CreationTimestamp is a timestamp representing the server time when - this object was created. It is not guaranteed to be set in happens-before order - across separate operations. Clients may not set this value. It is represented - in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for - lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - name: Age - type: date + conversion: + strategy: None group: install.istio.io names: kind: IstioOperator + listKind: IstioOperatorList plural: istiooperators singular: istiooperator shortNames: - iop - io scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. - More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. - More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - spec: - description: 'Specification of the desired state of the istio control plane resource. - More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' - type: object - status: - description: 'Status describes each of istio control plane component status at the current time. - 0 means NONE, 1 means UPDATING, 2 means HEALTHY, 3 means ERROR, 4 means RECONCILING. - More info: https://github.com/istio/api/blob/master/operator/v1alpha1/istio.operator.v1alpha1.pb.html & - https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' - type: object versions: - - name: v1alpha1 + - additionalPrinterColumns: + - description: Istio control plane revision + jsonPath: .spec.revision + name: Revision + type: string + - description: IOP current state + jsonPath: .status.status + name: Status + type: string + - description: 'CreationTimestamp is a timestamp representing the server time + when this object was created. It is not guaranteed to be set in happens-before + order across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + jsonPath: .metadata.creationTimestamp + name: Age + type: date + subresources: + status: {} + name: v1alpha1 + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true served: true storage: true --- @@ -3562,7 +6069,7 @@ rules: # istio configuration # removing CRD permissions can break older versions of Istio running alongside this control plane (https://github.com/istio/istio/issues/29382) # please proceed with caution - - apiGroups: ["config.istio.io", "security.istio.io", "networking.istio.io", "authentication.istio.io", "rbac.istio.io"] + - apiGroups: ["config.istio.io", "security.istio.io", "networking.istio.io", "authentication.istio.io", "rbac.istio.io", "telemetry.istio.io"] verbs: ["get", "watch", "list"] resources: ["*"] - apiGroups: ["networking.istio.io"] @@ -3626,11 +6133,19 @@ rules: - apiGroups: ["networking.x-k8s.io"] resources: ["*"] verbs: ["get", "watch", "list"] + - apiGroups: ["networking.x-k8s.io"] + resources: ["*"] # TODO: should be on just */status but wildcard is not supported + verbs: ["update"] # Needed for multicluster secret reading, possibly ingress certs in the future - apiGroups: [""] resources: ["secrets"] verbs: ["get", "watch", "list"] + + # Used for MCS serviceexport management + - apiGroups: ["multicluster.x-k8s.io"] + resources: ["serviceexports"] + verbs: ["get", "watch", "list", "create", "delete"] --- # Source: base/templates/clusterrole.yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -3746,7 +6261,7 @@ subjects: namespace: istio-system --- # Source: base/templates/validatingwebhookconfiguration.yaml -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: istiod-istio-system diff --git a/charts/kubezero-istio/charts/base/templates/clusterrole.yaml b/charts/kubezero-istio/charts/base/templates/clusterrole.yaml index e4176d5..014970c 100644 --- a/charts/kubezero-istio/charts/base/templates/clusterrole.yaml +++ b/charts/kubezero-istio/charts/base/templates/clusterrole.yaml @@ -19,11 +19,11 @@ rules: # istio configuration # removing CRD permissions can break older versions of Istio running alongside this control plane (https://github.com/istio/istio/issues/29382) # please proceed with caution - - apiGroups: ["config.istio.io", "security.istio.io", "networking.istio.io", "authentication.istio.io", "rbac.istio.io"] + - apiGroups: ["config.istio.io", "security.istio.io", "networking.istio.io", "authentication.istio.io", "rbac.istio.io", "telemetry.istio.io"] verbs: ["get", "watch", "list"] resources: ["*"] {{- if .Values.global.istiod.enableAnalysis }} - - apiGroups: ["config.istio.io", "security.istio.io", "networking.istio.io", "authentication.istio.io", "rbac.istio.io"] + - apiGroups: ["config.istio.io", "security.istio.io", "networking.istio.io", "authentication.istio.io", "rbac.istio.io", "telemetry.istio.io"] verbs: ["update"] # TODO: should be on just */status but wildcard is not supported resources: ["*"] @@ -97,12 +97,20 @@ rules: - apiGroups: ["networking.x-k8s.io"] resources: ["*"] verbs: ["get", "watch", "list"] + - apiGroups: ["networking.x-k8s.io"] + resources: ["*"] # TODO: should be on just */status but wildcard is not supported + verbs: ["update"] # Needed for multicluster secret reading, possibly ingress certs in the future - apiGroups: [""] resources: ["secrets"] verbs: ["get", "watch", "list"] + # Used for MCS serviceexport management + - apiGroups: ["multicluster.x-k8s.io"] + resources: ["serviceexports"] + verbs: ["get", "watch", "list", "create", "delete"] + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/charts/kubezero-istio/charts/base/templates/validatingwebhookconfiguration.yaml b/charts/kubezero-istio/charts/base/templates/validatingwebhookconfiguration.yaml index 80124a9..bd5d1cf 100644 --- a/charts/kubezero-istio/charts/base/templates/validatingwebhookconfiguration.yaml +++ b/charts/kubezero-istio/charts/base/templates/validatingwebhookconfiguration.yaml @@ -1,5 +1,5 @@ {{- if .Values.global.configValidation }} -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: istiod-{{ .Values.global.istioNamespace }} diff --git a/charts/kubezero-istio/charts/istio-discovery/Chart.yaml b/charts/kubezero-istio/charts/istio-discovery/Chart.yaml index 06bd7e2..6afbc9b 100644 --- a/charts/kubezero-istio/charts/istio-discovery/Chart.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: istio-discovery -version: 1.9.3 +version: 1.10.2 tillerVersion: ">=2.7.2" description: Helm chart for istio control plane keywords: diff --git a/charts/kubezero-istio/charts/istio-discovery/NOTES.txt b/charts/kubezero-istio/charts/istio-discovery/NOTES.txt index 997f4ac..620f3e1 100644 --- a/charts/kubezero-istio/charts/istio-discovery/NOTES.txt +++ b/charts/kubezero-istio/charts/istio-discovery/NOTES.txt @@ -3,3 +3,7 @@ Minimal control plane for Istio. Pilot and mesh config are included. MCP and injector should optionally be installed in the same namespace. Alternatively remote address of an MCP server can be set. + +Thank you for installing Istio 1.10. Please take a few minutes to tell us about your install/upgrade experience! + https://forms.gle/KjkrDnMPByq7akrYA" + diff --git a/charts/kubezero-istio/charts/istio-discovery/files/gateway-injection-template.yaml b/charts/kubezero-istio/charts/istio-discovery/files/gateway-injection-template.yaml index ab3e652..865d2c1 100644 --- a/charts/kubezero-istio/charts/istio-discovery/files/gateway-injection-template.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/files/gateway-injection-template.yaml @@ -8,6 +8,7 @@ metadata: annotations: { {{- if eq (len $containers) 1 }} kubectl.kubernetes.io/default-logs-container: "{{ index $containers 0 }}", + kubectl.kubernetes.io/default-container: "{{ index $containers 0 }}", {{ end }} } spec: diff --git a/charts/kubezero-istio/charts/istio-discovery/files/gen-istio.yaml b/charts/kubezero-istio/charts/istio-discovery/files/gen-istio.yaml index 71d892c..817cce5 100644 --- a/charts/kubezero-istio/charts/istio-discovery/files/gen-istio.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/files/gen-istio.yaml @@ -167,7 +167,6 @@ data: "address": "" } }, - "trustDomain": "", "useMCP": false }, "revision": "", @@ -183,7 +182,7 @@ data: }, "rewriteAppHTTPProbe": true, "templates": {}, - "useLegacySelectors": true + "useLegacySelectors": false } } @@ -215,6 +214,7 @@ data: annotations: { {{- if eq (len $containers) 1 }} kubectl.kubernetes.io/default-logs-container: "{{ index $containers 0 }}", + kubectl.kubernetes.io/default-container: "{{ index $containers 0 }}", {{ end }} {{- if .Values.istio_cni.enabled }} {{- if not .Values.istio_cni.chained }} @@ -286,7 +286,7 @@ data: - "--run-validation" - "--skip-rule-apply" {{ end -}} - imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" + {{with .Values.global.imagePullPolicy }}imagePullPolicy: "{{.}}"{{end}} {{- if .ProxyConfig.ProxyMetadata }} env: {{- range $key, $value := .ProxyConfig.ProxyMetadata }} @@ -355,7 +355,7 @@ data: {{- else }} image: "{{ .Values.global.hub }}/{{ .Values.global.proxy_init.image }}:{{ .Values.global.tag }}" {{- end }} - imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" + {{with .Values.global.imagePullPolicy }}imagePullPolicy: "{{.}}"{{end}} resources: {} securityContext: allowPrivilegeEscalation: true @@ -417,6 +417,10 @@ data: - wait {{- end }} env: + {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} + - name: REWRITE_PROBE_LEGACY_LOCALHOST_DESTINATION + value: "true" + {{- end }} - name: JWT_POLICY value: {{ .Values.global.jwtPolicy }} - name: PILOT_CERT_PROVIDER @@ -519,7 +523,7 @@ data: - name: {{ $key }} value: "{{ $value }}" {{- end }} - imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" + {{with .Values.global.imagePullPolicy }}imagePullPolicy: "{{.}}"{{end}} {{ if ne (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) `0` }} readinessProbe: httpGet: @@ -706,6 +710,7 @@ data: annotations: { {{- if eq (len $containers) 1 }} kubectl.kubernetes.io/default-logs-container: "{{ index $containers 0 }}", + kubectl.kubernetes.io/default-container: "{{ index $containers 0 }}", {{ end }} } spec: @@ -1063,8 +1068,6 @@ spec: value: "false" - name: CLUSTER_ID value: "Kubernetes" - - name: EXTERNAL_ISTIOD - value: "false" resources: requests: cpu: 500m @@ -1077,8 +1080,6 @@ spec: drop: - ALL volumeMounts: - - name: config-volume - mountPath: /etc/istio/config - name: istio-token mountPath: /var/run/secrets/tokens readOnly: true @@ -1090,9 +1091,6 @@ spec: - name: istio-kubeconfig mountPath: /var/run/secrets/remote readOnly: true - - name: inject - mountPath: /var/lib/istio/inject - readOnly: true volumes: # Technically not needed on this pod - but it helps debugging/testing SDS # Should be removed after everything works. @@ -1115,13 +1113,6 @@ spec: secret: secretName: istio-kubeconfig optional: true - # Optional - image should have - - name: inject - configMap: - name: istio-sidecar-injector - - name: config-volume - configMap: - name: istio --- # Source: istio-discovery/templates/autoscale.yaml apiVersion: autoscaling/v2beta1 @@ -1148,12 +1139,17 @@ spec: name: cpu targetAverageUtilization: 80 --- -# Source: istio-discovery/templates/telemetryv2_1.8.yaml +# Source: istio-discovery/templates/revision-tags.yaml +# Adapted from istio-discovery/templates/mutatingwebhook.yaml +# Removed paths for legacy and default selectors since a revision tag +# is inherently created from a specific revision +--- +# Source: istio-discovery/templates/telemetryv2_1.10.yaml # Note: metadata exchange filter is wasm enabled only in sidecars. apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: metadata-exchange-1.8 + name: metadata-exchange-1.10 namespace: istio-system labels: istio.io/rev: default @@ -1165,7 +1161,7 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -1192,7 +1188,7 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -1219,7 +1215,7 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -1243,11 +1239,11 @@ spec: local: inline_string: envoy.wasm.metadata_exchange --- -# Source: istio-discovery/templates/telemetryv2_1.8.yaml +# Source: istio-discovery/templates/telemetryv2_1.10.yaml apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: tcp-metadata-exchange-1.8 + name: tcp-metadata-exchange-1.10 namespace: istio-system labels: istio.io/rev: default @@ -1257,7 +1253,7 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.10.*' listener: {} patch: operation: INSERT_BEFORE @@ -1272,7 +1268,7 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.10.*' cluster: {} patch: operation: MERGE @@ -1288,7 +1284,7 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.10.*' cluster: {} patch: operation: MERGE @@ -1301,12 +1297,12 @@ spec: value: protocol: istio-peer-exchange --- -# Source: istio-discovery/templates/telemetryv2_1.8.yaml +# Source: istio-discovery/templates/telemetryv2_1.10.yaml # Note: http stats filter is wasm enabled only in sidecars. apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: stats-filter-1.8 + name: stats-filter-1.10 namespace: istio-system labels: istio.io/rev: default @@ -1316,7 +1312,7 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -1337,6 +1333,8 @@ spec: "@type": "type.googleapis.com/google.protobuf.StringValue" value: | { + "debug": "false", + "stat_prefix": "istio" } vm_config: vm_id: stats_outbound @@ -1348,7 +1346,7 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -1369,6 +1367,16 @@ spec: "@type": "type.googleapis.com/google.protobuf.StringValue" value: | { + "debug": "false", + "stat_prefix": "istio", + "metrics": [ + { + "dimensions": { + "destination_cluster": "node.metadata['CLUSTER_ID']", + "source_cluster": "downstream_peer.cluster_id" + } + } + ] } vm_config: vm_id: stats_inbound @@ -1380,7 +1388,7 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -1401,6 +1409,8 @@ spec: "@type": "type.googleapis.com/google.protobuf.StringValue" value: | { + "debug": "false", + "stat_prefix": "istio", "disable_host_header_fallback": true } vm_config: @@ -1410,12 +1420,12 @@ spec: local: inline_string: envoy.wasm.stats --- -# Source: istio-discovery/templates/telemetryv2_1.8.yaml +# Source: istio-discovery/templates/telemetryv2_1.10.yaml # Note: tcp stats filter is wasm enabled only in sidecars. apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: tcp-stats-filter-1.8 + name: tcp-stats-filter-1.10 namespace: istio-system labels: istio.io/rev: default @@ -1425,7 +1435,7 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -1444,6 +1454,16 @@ spec: "@type": "type.googleapis.com/google.protobuf.StringValue" value: | { + "debug": "false", + "stat_prefix": "istio", + "metrics": [ + { + "dimensions": { + "destination_cluster": "node.metadata['CLUSTER_ID']", + "source_cluster": "downstream_peer.cluster_id" + } + } + ] } vm_config: vm_id: tcp_stats_inbound @@ -1455,7 +1475,7 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -1474,6 +1494,8 @@ spec: "@type": "type.googleapis.com/google.protobuf.StringValue" value: | { + "debug": "false", + "stat_prefix": "istio" } vm_config: vm_id: tcp_stats_outbound @@ -1485,7 +1507,7 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -1504,6 +1526,8 @@ spec: "@type": "type.googleapis.com/google.protobuf.StringValue" value: | { + "debug": "false", + "stat_prefix": "istio" } vm_config: vm_id: tcp_stats_outbound @@ -1937,7 +1961,7 @@ spec: inline_string: "envoy.wasm.stats" --- # Source: istio-discovery/templates/mutatingwebhook.yaml -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: name: istio-sidecar-injector @@ -1948,12 +1972,13 @@ metadata: app: sidecar-injector release: istio webhooks: -- name: sidecar-injector.istio.io +- name: rev.namespace.sidecar-injector.istio.io clientConfig: service: name: istiod namespace: istio-system path: "/inject" + port: 443 caBundle: "" sideEffects: None rules: @@ -1964,11 +1989,106 @@ webhooks: failurePolicy: Fail admissionReviewVersions: ["v1beta1", "v1"] namespaceSelector: - matchLabels: - istio-injection: enabled + matchExpressions: + - key: istio.io/rev + operator: In + values: + - "default" + - key: istio-injection + operator: DoesNotExist objectSelector: matchExpressions: - - key: "sidecar.istio.io/inject" + - key: sidecar.istio.io/inject operator: NotIn values: - "false" +- name: rev.object.sidecar-injector.istio.io + clientConfig: + service: + name: istiod + namespace: istio-system + path: "/inject" + port: 443 + caBundle: "" + sideEffects: None + rules: + - operations: [ "CREATE" ] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + failurePolicy: Fail + admissionReviewVersions: ["v1beta1", "v1"] + namespaceSelector: + matchExpressions: + - key: istio.io/rev + operator: DoesNotExist + - key: istio-injection + operator: DoesNotExist + objectSelector: + matchExpressions: + - key: sidecar.istio.io/inject + operator: NotIn + values: + - "false" + - key: istio.io/rev + operator: In + values: + - "default" +- name: namespace.sidecar-injector.istio.io + clientConfig: + service: + name: istiod + namespace: istio-system + path: "/inject" + port: 443 + caBundle: "" + sideEffects: None + rules: + - operations: [ "CREATE" ] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + failurePolicy: Fail + admissionReviewVersions: ["v1beta1", "v1"] + namespaceSelector: + matchExpressions: + - key: istio-injection + operator: In + values: + - enabled + objectSelector: + matchExpressions: + - key: sidecar.istio.io/inject + operator: NotIn + values: + - "false" +- name: object.sidecar-injector.istio.io + clientConfig: + service: + name: istiod + namespace: istio-system + path: "/inject" + port: 443 + caBundle: "" + sideEffects: None + rules: + - operations: [ "CREATE" ] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + failurePolicy: Fail + admissionReviewVersions: ["v1beta1", "v1"] + namespaceSelector: + matchExpressions: + - key: istio-injection + operator: DoesNotExist + - key: istio.io/rev + operator: DoesNotExist + objectSelector: + matchExpressions: + - key: sidecar.istio.io/inject + operator: In + values: + - "true" + - key: istio.io/rev + operator: DoesNotExist diff --git a/charts/kubezero-istio/charts/istio-discovery/files/injection-template.yaml b/charts/kubezero-istio/charts/istio-discovery/files/injection-template.yaml index 8b1f156..39a6424 100644 --- a/charts/kubezero-istio/charts/istio-discovery/files/injection-template.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/files/injection-template.yaml @@ -9,6 +9,7 @@ metadata: annotations: { {{- if eq (len $containers) 1 }} kubectl.kubernetes.io/default-logs-container: "{{ index $containers 0 }}", + kubectl.kubernetes.io/default-container: "{{ index $containers 0 }}", {{ end }} {{- if .Values.istio_cni.enabled }} {{- if not .Values.istio_cni.chained }} @@ -80,7 +81,7 @@ spec: - "--run-validation" - "--skip-rule-apply" {{ end -}} - imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" + {{with .Values.global.imagePullPolicy }}imagePullPolicy: "{{.}}"{{end}} {{- if .ProxyConfig.ProxyMetadata }} env: {{- range $key, $value := .ProxyConfig.ProxyMetadata }} @@ -149,7 +150,7 @@ spec: {{- else }} image: "{{ .Values.global.hub }}/{{ .Values.global.proxy_init.image }}:{{ .Values.global.tag }}" {{- end }} - imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" + {{with .Values.global.imagePullPolicy }}imagePullPolicy: "{{.}}"{{end}} resources: {} securityContext: allowPrivilegeEscalation: true @@ -211,6 +212,10 @@ spec: - wait {{- end }} env: + {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} + - name: REWRITE_PROBE_LEGACY_LOCALHOST_DESTINATION + value: "true" + {{- end }} - name: JWT_POLICY value: {{ .Values.global.jwtPolicy }} - name: PILOT_CERT_PROVIDER @@ -313,7 +318,7 @@ spec: - name: {{ $key }} value: "{{ $value }}" {{- end }} - imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" + {{with .Values.global.imagePullPolicy }}imagePullPolicy: "{{.}}"{{end}} {{ if ne (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) `0` }} readinessProbe: httpGet: diff --git a/charts/kubezero-istio/charts/istio-discovery/templates/configmap.yaml b/charts/kubezero-istio/charts/istio-discovery/templates/configmap.yaml index 3a6bb68..f63fff1 100644 --- a/charts/kubezero-istio/charts/istio-discovery/templates/configmap.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/templates/configmap.yaml @@ -1,8 +1,7 @@ - {{- define "mesh" }} # The trust domain corresponds to the trust root of a system. # Refer to https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md#21-trust-domain - trustDomain: {{ .Values.global.trustDomain | default "cluster.local" | quote }} + trustDomain: "cluster.local" # The namespace to treat as the administrative root namespace for Istio configuration. # When processing a leaf namespace Istio will search for declarations in that namespace first @@ -13,8 +12,6 @@ defaultConfig: {{- if .Values.global.meshID }} meshId: {{ .Values.global.meshID }} - {{- else if .Values.global.trustDomain }} - meshId: {{ .Values.global.trustDomain }} {{- end }} tracing: {{- if eq .Values.global.proxy.tracer "lightstep" }} @@ -50,8 +47,8 @@ maxNumberOfMessageEvents: {{ $.Values.global.tracer.stackdriver.maxNumberOfMessageEvents | default "200" }} {{- end }} {{- else if eq .Values.global.proxy.tracer "openCensusAgent" }} - {{- /* Fill in openCensusAgent configuration from meshConfig so it isn't overwritten below */ -}} - {{ toYaml $.Values.meshConfig.defaultConfig.tracing }} + {{/* Fill in openCensusAgent configuration from meshConfig so it isn't overwritten below */}} +{{ toYaml $.Values.meshConfig.defaultConfig.tracing | indent 8 }} {{- end }} {{- if .Values.global.remotePilotAddress }} {{- if .Values.pilot.enabled }} diff --git a/charts/kubezero-istio/charts/istio-discovery/templates/deployment.yaml b/charts/kubezero-istio/charts/istio-discovery/templates/deployment.yaml index c7a42c0..9c226dc 100644 --- a/charts/kubezero-istio/charts/istio-discovery/templates/deployment.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/templates/deployment.yaml @@ -25,7 +25,7 @@ spec: maxUnavailable: {{ .Values.pilot.rollingMaxUnavailable }} selector: matchLabels: - {{- if ne .Values.revision ""}} + {{- if ne .Values.revision "" }} app: istiod istio.io/rev: {{ .Values.revision | default "default" }} {{- else }} @@ -39,10 +39,10 @@ spec: install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }} sidecar.istio.io/inject: "false" operator.istio.io/component: "Pilot" - {{- if eq .Values.revision ""}} - istio: pilot - {{- else }} + {{- if ne .Values.revision "" }} istio: istiod + {{- else }} + istio: pilot {{- end }} annotations: {{- if .Values.meshConfig.enablePrometheusMerge }} @@ -153,8 +153,6 @@ spec: value: "{{ .Values.global.istiod.enableAnalysis }}" - name: CLUSTER_ID value: "{{ $.Values.global.multiCluster.clusterName | default `Kubernetes` }}" - - name: EXTERNAL_ISTIOD - value: "{{ $.Values.global.externalIstiod | default "false" }}" {{- if not .Values.telemetry.v2.enabled }} - name: PILOT_ENDPOINT_TELEMETRY_LABEL value: "false" @@ -173,8 +171,6 @@ spec: drop: - ALL volumeMounts: - - name: config-volume - mountPath: /etc/istio/config {{- if eq .Values.global.jwtPolicy "third-party-jwt" }} - name: istio-token mountPath: /var/run/secrets/tokens @@ -188,9 +184,6 @@ spec: - name: istio-kubeconfig mountPath: /var/run/secrets/remote readOnly: true - - name: inject - mountPath: /var/lib/istio/inject - readOnly: true {{- if .Values.pilot.jwksResolverExtraRootCA }} - name: extracacerts mountPath: /cacerts @@ -219,13 +212,6 @@ spec: secret: secretName: istio-kubeconfig optional: true - # Optional - image should have - - name: inject - configMap: - name: istio-sidecar-injector{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} - - name: config-volume - configMap: - name: istio{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.pilot.jwksResolverExtraRootCA }} - name: extracacerts configMap: diff --git a/charts/kubezero-istio/charts/istio-discovery/templates/mutatingwebhook.yaml b/charts/kubezero-istio/charts/istio-discovery/templates/mutatingwebhook.yaml index f9fd67b..b50f0b7 100644 --- a/charts/kubezero-istio/charts/istio-discovery/templates/mutatingwebhook.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/templates/mutatingwebhook.yaml @@ -11,6 +11,7 @@ a unique prefix to each. */}} name: istiod{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} namespace: {{ .Release.Namespace }} path: "/inject" + port: 443 {{- end }} caBundle: "" sideEffects: None @@ -24,7 +25,7 @@ a unique prefix to each. */}} {{- end }} {{- /* Installed for each revision - not installed for cluster resources ( cluster roles, bindings, crds) */}} {{- if not .Values.global.operatorManageWebhooks }} -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: {{- if eq .Release.Namespace "istio-system"}} @@ -41,7 +42,7 @@ metadata: webhooks: {{- if .Values.sidecarInjectorWebhook.useLegacySelectors}} {{- /* Setup the "legacy" selectors. These are for backwards compatibility, will be removed in the future. */}} -{{- include "core" (mergeOverwrite (deepCopy .) (dict "Prefix" "")) }} +{{- include "core" . }} namespaceSelector: {{- if .Values.sidecarInjectorWebhook.enableNamespacesByDefault }} matchExpressions: @@ -92,18 +93,21 @@ webhooks: {{- end }} {{- else }} -{{- /* Set up the selectors. First section is for revision, rest is for "default" revision */}} -{{- if .Values.revision }} + {{- /* Set up the selectors. First section is for revision, rest is for "default" revision */}} {{- /* Case 1: namespace selector matches, and object doesn't disable */}} {{- /* Note: if both revision and legacy selector, we give precedence to the legacy one */}} -{{- include "core" (mergeOverwrite (deepCopy .) (dict "Prefix" "namespace.") ) }} +{{- include "core" (mergeOverwrite (deepCopy .) (dict "Prefix" "rev.namespace.") ) }} namespaceSelector: matchExpressions: - key: istio.io/rev operator: In values: + {{- if (eq .Values.revision "") }} + - "default" + {{- else }} - "{{ .Values.revision }}" + {{- end }} - key: istio-injection operator: DoesNotExist objectSelector: @@ -114,7 +118,7 @@ webhooks: - "false" {{- /* Case 2: No namespace selector, but object selects our revision (and doesn't disable) */}} -{{- include "core" (mergeOverwrite (deepCopy .) (dict "Prefix" "object.") ) }} +{{- include "core" (mergeOverwrite (deepCopy .) (dict "Prefix" "rev.object.") ) }} namespaceSelector: matchExpressions: - key: istio.io/rev @@ -130,10 +134,15 @@ webhooks: - key: istio.io/rev operator: In values: + {{- if (eq .Values.revision "") }} + - "default" + {{- else }} - "{{ .Values.revision }}" + {{- end }} -{{- else }} -{{- /* "default" revision */}} + +{{- /* Webhooks for default revision */}} +{{- if (eq .Values.revision "") }} {{- /* Case 1: Namespace selector enabled, and object selector is not injected */}} {{- include "core" (mergeOverwrite (deepCopy .) (dict "Prefix" "namespace.") ) }} diff --git a/charts/kubezero-istio/charts/istio-discovery/templates/poddisruptionbudget.yaml b/charts/kubezero-istio/charts/istio-discovery/templates/poddisruptionbudget.yaml index fef641a..40b2e60 100644 --- a/charts/kubezero-istio/charts/istio-discovery/templates/poddisruptionbudget.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/templates/poddisruptionbudget.yaml @@ -16,7 +16,7 @@ spec: selector: matchLabels: app: istiod - {{- if ne .Values.revision ""}} + {{- if ne .Values.revision "" }} istio.io/rev: {{ .Values.revision }} {{- else }} istio: pilot diff --git a/charts/kubezero-istio/charts/istio-discovery/templates/revision-tags.yaml b/charts/kubezero-istio/charts/istio-discovery/templates/revision-tags.yaml new file mode 100644 index 0000000..3df335d --- /dev/null +++ b/charts/kubezero-istio/charts/istio-discovery/templates/revision-tags.yaml @@ -0,0 +1,113 @@ +# Adapted from istio-discovery/templates/mutatingwebhook.yaml +# Removed paths for legacy and default selectors since a revision tag +# is inherently created from a specific revision +{{- define "core" }} +- name: {{.Prefix}}sidecar-injector.istio.io + clientConfig: + {{- if .Values.istiodRemote.injectionURL }} + url: {{ .Values.istiodRemote.injectionURL }} + {{- else }} + service: + name: istiod{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + namespace: {{ .Release.Namespace }} + path: "/inject" + {{- end }} + caBundle: "" + sideEffects: None + rules: + - operations: [ "CREATE" ] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + failurePolicy: Fail + admissionReviewVersions: ["v1beta1", "v1"] +{{- end }} + +{{- range $tagName := $.Values.revisionTags }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: +{{- if eq $.Release.Namespace "istio-system"}} + name: istio-revision-tag-{{ $tagName }} +{{- else }} + name: istio-revision-tag-{{ $tagName }}-{{ $.Release.Namespace }} +{{- end }} + labels: + istio.io/tag: {{ $tagName }} + istio.io/rev: {{ $.Values.revision | default "default" }} + install.operator.istio.io/owning-resource: {{ $.Values.ownerName | default "unknown" }} + operator.istio.io/component: "Pilot" + app: sidecar-injector + release: {{ $.Release.Name }} +webhooks: +{{- include "core" (mergeOverwrite (deepCopy $) (dict "Prefix" "rev.namespace.") ) }} + namespaceSelector: + matchExpressions: + - key: istio.io/rev + operator: In + values: + - "{{ $tagName }}" + - key: istio-injection + operator: DoesNotExist + objectSelector: + matchExpressions: + - key: sidecar.istio.io/inject + operator: NotIn + values: + - "false" +{{- include "core" (mergeOverwrite (deepCopy $) (dict "Prefix" "rev.object.") ) }} + namespaceSelector: + matchExpressions: + - key: istio.io/rev + operator: DoesNotExist + - key: istio-injection + operator: DoesNotExist + objectSelector: + matchExpressions: + - key: sidecar.istio.io/inject + operator: NotIn + values: + - "false" + - key: istio.io/rev + operator: In + values: + - "{{ $tagName }}" + +{{- /* When the tag is "default" we want to create webhooks for the default revision */}} +{{- /* These webhooks should be kept in sync with istio-discovery/templates/mutatingwebhook.yaml */}} +{{- if (eq $tagName "default") }} + +{{- /* Case 1: Namespace selector enabled, and object selector is not injected */}} +{{- include "core" (mergeOverwrite (deepCopy $) (dict "Prefix" "namespace.") ) }} + namespaceSelector: + matchExpressions: + - key: istio-injection + operator: In + values: + - enabled + objectSelector: + matchExpressions: + - key: sidecar.istio.io/inject + operator: NotIn + values: + - "false" + +{{- /* Case 2: no namespace label, but object selector is enabled (and revision label is not, which has priority) */}} +{{- include "core" (mergeOverwrite (deepCopy $) (dict "Prefix" "object.") ) }} + namespaceSelector: + matchExpressions: + - key: istio-injection + operator: DoesNotExist + - key: istio.io/rev + operator: DoesNotExist + objectSelector: + matchExpressions: + - key: sidecar.istio.io/inject + operator: In + values: + - "true" + - key: istio.io/rev + operator: DoesNotExist + +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-istio/charts/istio-discovery/templates/service.yaml b/charts/kubezero-istio/charts/istio-discovery/templates/service.yaml index 1fe1b5a..1d4d9fe 100644 --- a/charts/kubezero-istio/charts/istio-discovery/templates/service.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/templates/service.yaml @@ -27,7 +27,7 @@ spec: protocol: TCP selector: app: istiod - {{- if ne .Values.revision ""}} + {{- if ne .Values.revision "" }} istio.io/rev: {{ .Values.revision }} {{- else }} # Label used by the 'default' service. For versioned deployments we match with app and version. diff --git a/charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.9.yaml b/charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.10.yaml similarity index 88% rename from charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.9.yaml rename to charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.10.yaml index b1db1b9..3e4e597 100644 --- a/charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.9.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.10.yaml @@ -3,7 +3,7 @@ apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: metadata-exchange-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: metadata-exchange-1.10{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -19,7 +19,7 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -54,7 +54,7 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -89,7 +89,7 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -124,7 +124,7 @@ spec: apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: tcp-metadata-exchange-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: tcp-metadata-exchange-1.10{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -138,7 +138,7 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: {} patch: operation: INSERT_BEFORE @@ -153,7 +153,7 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' cluster: {} patch: operation: MERGE @@ -169,7 +169,7 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' cluster: {} patch: operation: MERGE @@ -187,7 +187,7 @@ spec: apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: stats-filter-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: stats-filter-1.10{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -201,7 +201,7 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -224,15 +224,7 @@ spec: {{- if not .Values.telemetry.v2.prometheus.configOverride.outboundSidecar }} { "debug": "false", - "stat_prefix": "istio", - "metrics": [ - { - "dimensions": { - "source_cluster": "node.metadata['CLUSTER_ID']", - "destination_cluster": "upstream_peer.cluster_id" - } - } - ] + "stat_prefix": "istio" } {{- else }} {{ toJson .Values.telemetry.v2.prometheus.configOverride.outboundSidecar | indent 18 }} @@ -255,7 +247,7 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -309,7 +301,7 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -333,15 +325,7 @@ spec: { "debug": "false", "stat_prefix": "istio", - "disable_host_header_fallback": true, - "metrics": [ - { - "dimensions": { - "source_cluster": "node.metadata['CLUSTER_ID']", - "destination_cluster": "upstream_peer.cluster_id" - } - } - ] + "disable_host_header_fallback": true } {{- else }} {{ toJson .Values.telemetry.v2.prometheus.configOverride.gateway | indent 18 }} @@ -365,7 +349,7 @@ spec: apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: tcp-stats-filter-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: tcp-stats-filter-1.10{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -379,7 +363,7 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -431,7 +415,7 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -452,15 +436,7 @@ spec: {{- if not .Values.telemetry.v2.prometheus.configOverride.outboundSidecar }} { "debug": "false", - "stat_prefix": "istio", - "metrics": [ - { - "dimensions": { - "source_cluster": "node.metadata['CLUSTER_ID']", - "destination_cluster": "upstream_peer.cluster_id" - } - } - ] + "stat_prefix": "istio" } {{- else }} {{ toJson .Values.telemetry.v2.prometheus.configOverride.outboundSidecar | indent 18 }} @@ -483,7 +459,7 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -504,15 +480,7 @@ spec: {{- if not .Values.telemetry.v2.prometheus.configOverride.gateway }} { "debug": "false", - "stat_prefix": "istio", - "metrics": [ - { - "dimensions": { - "source_cluster": "node.metadata['CLUSTER_ID']", - "destination_cluster": "upstream_peer.cluster_id" - } - } - ] + "stat_prefix": "istio" } {{- else }} {{ toJson .Values.telemetry.v2.prometheus.configOverride.gateway | indent 18 }} @@ -537,7 +505,7 @@ spec: apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: stackdriver-filter-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: stackdriver-filter-1.10{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -552,7 +520,7 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -573,7 +541,7 @@ spec: "@type": "type.googleapis.com/google.protobuf.StringValue" value: | {{- if not .Values.telemetry.v2.stackdriver.configOverride }} - {"enable_mesh_edges_reporting": {{ .Values.telemetry.v2.stackdriver.topology }}, "access_logging": "{{ .Values.telemetry.v2.stackdriver.outboundAccessLogging }}", "meshEdgesReportingDuration": "600s"} + {"access_logging": "{{ .Values.telemetry.v2.stackdriver.outboundAccessLogging }}"} {{- else }} {{ toJson .Values.telemetry.v2.stackdriver.configOverride | indent 18 }} {{- end }} @@ -587,7 +555,7 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -608,7 +576,7 @@ spec: "@type": "type.googleapis.com/google.protobuf.StringValue" value: | {{- if not .Values.telemetry.v2.stackdriver.configOverride }} - {"enable_mesh_edges_reporting": {{ .Values.telemetry.v2.stackdriver.topology }}, "disable_server_access_logging": {{ not .Values.telemetry.v2.stackdriver.logging }}, "access_logging": "{{ .Values.telemetry.v2.stackdriver.inboundAccessLogging }}", "meshEdgesReportingDuration": "600s", "disable_host_header_fallback": true} + {"disable_server_access_logging": {{ not .Values.telemetry.v2.stackdriver.logging }}, "access_logging": "{{ .Values.telemetry.v2.stackdriver.inboundAccessLogging }}", "disable_host_header_fallback": true} {{- else }} {{ toJson .Values.telemetry.v2.stackdriver.configOverride | indent 18 }} {{- end }} @@ -621,7 +589,7 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -642,7 +610,7 @@ spec: "@type": "type.googleapis.com/google.protobuf.StringValue" value: | {{- if not .Values.telemetry.v2.stackdriver.configOverride }} - {"enable_mesh_edges_reporting": {{ .Values.telemetry.v2.stackdriver.topology }}, "access_logging": "{{ .Values.telemetry.v2.stackdriver.outboundAccessLogging }}", "meshEdgesReportingDuration": "600s", "disable_host_header_fallback": true} + {"access_logging": "{{ .Values.telemetry.v2.stackdriver.outboundAccessLogging }}", "disable_host_header_fallback": true} {{- else }} {{ toJson .Values.telemetry.v2.stackdriver.configOverride | indent 18 }} {{- end }} @@ -655,7 +623,7 @@ spec: apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: tcp-stackdriver-filter-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: tcp-stackdriver-filter-1.10{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -670,7 +638,7 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -703,7 +671,7 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -735,7 +703,7 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.9.*' + proxyVersion: '^1\.10.*' listener: filterChain: filter: @@ -768,7 +736,7 @@ spec: apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: stackdriver-sampling-accesslog-filter-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: stackdriver-sampling-accesslog-filter-1.10{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -782,7 +750,7 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '1\.9.*' + proxyVersion: '1\.10.*' listener: filterChain: filter: diff --git a/charts/kubezero-istio/charts/istio-discovery/values.yaml b/charts/kubezero-istio/charts/istio-discovery/values.yaml index 1290bee..fc595d4 100644 --- a/charts/kubezero-istio/charts/istio-discovery/values.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/values.yaml @@ -68,7 +68,7 @@ sidecarInjectorWebhook: # If enabled, the legacy webhook selection logic will be used. This relies on filtering of webhook # requests in Istiod, rather than at the webhook selection level. # This is option is intended for migration purposes only and will be removed in Istio 1.10. - useLegacySelectors: true + useLegacySelectors: false # You can use the field called alwaysInjectSelector and neverInjectSelector which will always inject the sidecar or # always skip the injection on pods that match that label selector, regardless of the global policy. # See https://istio.io/docs/setup/kubernetes/additional-setup/sidecar-injection/#more-control-adding-exceptions @@ -157,15 +157,13 @@ telemetry: enabled: false logging: false monitoring: false - topology: false + topology: false # deprecated. setting this to true will have no effect, as this option is no longer supported. disableOutbound: false # configOverride parts give you the ability to override the low level configuration params passed to envoy filter. configOverride: {} # e.g. - # enable_mesh_edges_reporting: true # disable_server_access_logging: false - # meshEdgesReportingDuration: 500s # disable_host_header_fallback: true # Access Log Policy Filter Settings. This enables filtering of access logs from stackdriver. accessLogPolicy: @@ -176,6 +174,9 @@ telemetry: # Revision is set as 'version' label and part of the resource names when installing multiple control planes. revision: "" +# Revision tags are aliases to Istio control plane revisions +revisionTags: [] + # For Helm compatibility. ownerName: "" @@ -197,6 +198,10 @@ meshConfig: rootNamespace: + # The trust domain corresponds to the trust root of a system + # Refer to https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md#21-trust-domain + trustDomain: "cluster.local" + # TODO: the intent is to eventually have this enabled by default when security is used. # It is not clear if user should normally need to configure - the metadata is typically # used as an escape and to control testing and rollout, but it is not intended as a long-term @@ -232,7 +237,7 @@ global: # Dev builds from prow are on gcr.io hub: docker.io/istio # Default tag for Istio images. - tag: 1.9.3 + tag: 1.10.2 # Specify image pull policy if default behavior isn't desired. # Default behavior: latest images will be Always else IfNotPresent. @@ -505,8 +510,6 @@ global: # Use the Mesh Control Protocol (MCP) for configuring Istiod. Requires an MCP source. useMCP: false - # Deprecated, use meshConfig.trustDomain - trustDomain: "" base: # For istioctl usage to disable istio config crds in base enableIstioConfigCRDs: true diff --git a/charts/kubezero-istio/templates/grafana-dashboards.yaml b/charts/kubezero-istio/templates/grafana-dashboards.yaml index edb9810..e0e0de3 100644 --- a/charts/kubezero-istio/templates/grafana-dashboards.yaml +++ b/charts/kubezero-istio/templates/grafana-dashboards.yaml @@ -11,11 +11,11 @@ metadata: k8s-sidecar-target-directory: Istio binaryData: istio-control-plane.json.gz: - H4sIAAAAAAAC/+1dW2/bOBZ+768ghMUiWSSBnVvbAeYhTdJOMOlMELedRaeFQEuMzY0sakgqsZvN/vY9JGXr7qvsKBm9JDYpk+fGcz4eXvTwCiHLtqkfhFJYP6E/4TtCD/ov1Ph4QKDUOuvYV9e/fzz/9Mv55461M672cJd4qv6KswGRfRKKuNIlwuE0kJT56pG4Qo4C3aiLJRYs5A6J6wIv7FH/wlX1QUGjpv63iKxEt/qBR/j7fcewxMlfIeWkgKlx/z2Ob7CP48apW1g8FsKHbMUd4SLi7njvcO8gImKnuLsA+yCsfGdBv7CrZHGio+l9FImUlgrTz4uxqMujvdZeawneJO56JN/bp3RxnreJErHvM4mVASktmk4tjwo50WlMCtR0Q+rJC9VSeycuTYgEuvhHgYTgIeJron5CkockUd6nbkEpdZh/yjzGVYu818VbrR20327Dn6OjHdTeTjY9ZvskZgb9E514hEuRfC5Woeh3GeauFdU96v/fX0XSz46rCwGf0CnzJWceuvJAE+hs3AaKpIvae2/BPvXviUtlxO0N9oRmzOr5ROpR9/r48MiUKBP8xJgnaTAWKQjfv9UjyihIq71ghDnM83AgiJvsQ9f0OHWvWKxOI+WMyu7h+/5homAIBa3E95H6PhZP2uqOW7GvmFD3PfY9VHrGo5HAYyPioi9GQiLvnzi7L7Z67FEstAFoRmIKuliXpFlWCr0kfk9qNlupclL0+Cx7tW6o5yVFVi7Vo0Wl2i6W6tFx7PNJj/huuit818vyoawg5Jz4sqBmgIdFpdQvKBV9dp8fghLGklfw9B32wlioOWbAfnVtsjVdeE9dmTLDjKmboRx63hWjvvzItFvQBYnYREBLvsQ9klNqoH7FsUtDkdSKKc/bAIjNJZxo/3LjMRn3IQinRPwOwxpUTjL0iQA7pMjUhMTOba4XIUkQEPcS2M/VScx7JIEI0q5WeZFhoMkT4WCLKhdkK/frPjhsEDAfxPDzNyugQPs363EbdUdoCwSTdIzKjhkfYKlDBR0Q2/CWfgTEQzgo9T12pPa37VS1Mcb3k3YeHhB0gx4f061wcmMgxYk1KX6MPiV8Qx/gQp95bs5nDMh7CJFQqhSeKr8mvSg8ZX7Q6dMbmf9F5H2ulGSKXM/E3z4kRwDm2pVmxoBgXGaGsDZ/e+zAqO/SO+qGMFJyIyEBgpJIY4iHNONHuqFza0whyYsar9EoUOwWRLzM08XjeDJe//yeI3GEh2SKBcbWA01zmdb4GJmmiVAVrPcOC5KzJOOSco8bn5QrTjATG9RObelMOsMiyx/ltQ4xrlfkjnX5JbmbUJ1CKdlIuQkgcFwCBPbnAwLXxIRX9Fkoz/0SUcDrjEiPZ0j0dQkIaDBAgwGKMADMLR0ihH1HuYRgYw/IgPGR3R1JIh5wEAAU0BDBBSywBAIAyossqAgd7E9FB18Mfeijpq8EIlyky5WAVMMz3XxWGIAlQDu+rFoaC3F8HRExneVfVmS5xxSToCYp7D7BgS1GoiJ+Cyfgi8tBUYWAqjlg4aK8Ys9jTg251XSV8Hu2HL9VsrogSydTuHlftflSPxSkWpWu7Lt+UTql/i5QViKG8wrFoINFpXJYkN2OImA6vx9W5NdhEKch7vGxh75n/Jb6PSBfRkxPHvn5f98slwqHQdAdfbN2UMBcVWbksbv3r//qT7t6xgtfn9JSzsZ0oq0J/dslMny3YRlGUuJsuC4pLpgw6IDQHcznENVpTVIHmUjepAyWShlo22xSBjVMGbycafXxfNPqJrXeTKvLU+scSxIHJ9sJQjtUiSoIflDqClurKxFlFwYqf7YH37fXnplfBJUsMB0z8hnPuZV00nLJwFbNax2QWURxOdhYDZctazoL47Mljacqic4P32Yi3afEdKdXnxtA16wBbZDOBs8tjufa+/MButcNoGsA3fRsyY2IQnE2R7L+NNOymGXtGaQpMqlr2qjmu0xAd7cNrlhbosiynhhVgA3SAfZEUTsxBz7zyaIMtJK7LpoU0tNBjjfzQY7Dp4Yc2eIGc9QEc/SY3WOchVJJdtOreL+Fgy7hiN2gDxMaSmNmjefnBdQ34bTqafqTh9O/8SR9E/s024cl2wrfzLdR0+yZvgpFH134RlfqCEx1GzZTQad6VJA5t9MBjQsk+wSpPK3ykXr+goJQd7AEmHiT0cibWQo5Kts5u/+yMhhpVa0GJxAW6AfhLAErSoJ6/eFGyuArXrLSxmwPXWEbg35QwxOwh+OKja05nXohcMXn2NCxsypbZINsnftuZCgr7+6bzZi3QcYuAZgSn/AKtmnOZoxvkLFrQI+lyPe8Sq5EKVdjkguQVja5RRxOSq3rXZXk+hWQqw6D6wOtonR/aJ0OfV1l4vv0eUQuPr6oiURSXSbGT0pWmGOwYK07u6yW1WzseoZZuQz+PsWBDGHsI4zuMHh4OYoxOMApxteCwd/Mi8GrXURU+y3OB4EcFV9x8FUB2lxNs/L4oraSxTEYMLjNyX+II7MJwW3EONrCXQESmvMHu+n7Lqre/zMDWGmiiItOzzrqDowb2hPVwP2YebKotEjtpXU+U1pny0qLLyotXntpXc+U1smy0vIWlZZXe2ldzpTWgjOIzATinlMJcBf4gUlV4f7OtU/n/lAkoE+GhHlmHYtxqaOm5lXT5cMXA0gq4rYqlV9E1KHzNFzKHi5dXQ7G4uvGv7FyZCxfXbJ0DbRXM8kvmDTbakMMGcp62oJOy58aCqfbw6/PffRrVmcM/g+1Sjlk9NEsXTY7jJsdxhtY2NMBDCnDRVQiiW8JlDO9wocwckz4CANXrf9BOUZ6h+M60g7t47nX/g6b3cvV5RD2X1AOoQ9xloEjH9h/hdiX1CNbrb2jHZSJ0tqGFVgBysF+HBOtbePfTYTW18F5ZP2hOjhqodWnbIWMv603429bq0++ivmuO+NvV8//lDBee8733s6RzXlSOKrkhSYAukGkKyLSBo02aHT5zedZ0Jg631a0YWz/oGT3+VGzWFUl0CzYY/a3vFZQx1g1TfKoI23qd1nou7YX7RLa4NVcs/KgmjA0a/vSAtArwzoLZZp3uy9lYKtzeXY0jmzpBLWRyO8RvbFI0JaiGCmKUUQxAooruAFipqigm+clKaBvbkGdrlNQSmXPS1KK4u2ag+DTSANNQrZJyDZbveqOgXOXPpSC4Obah2bzVWHgBdVH+SJ1QbYKknfUyR/GXHuGaHz9dSciYHXsFXP2ZEzNYObdkjtz8rtrVB4vYO76D60w3492aY3PeaCHB+i59KUm5zUBNidnHfSR+RS4pH6vQTcNumnOyi55Vvag5D1cx4fznZU99+/YaE3HZJ9kOd0lElNPIAxzQIkMe2r1Bbwtor5ebR+An9vEsdmD/RIA2GoAYAMAp5+DI8pwbcecS7XDQEhO8MB2huO7OKMaFW0AfgAQ2e3xwHn6/X3/Nrt6fbPTUVR5pLZcJo7p0L6Bof9cRIPeA7HqXFFFZ1h9B6QhxlJSMFtlfwGkQhdggNImAXP6mzp4rNzutel4HsT9lGjUEHtm4kYDRlcDo2s+UlkRFH0iKps0W+XHIksxVrvBWA3GKguW5UgC4hu9I1MwxPa6r0JTKOFEUzEXjqrxZWgzOGkibJPuafZzbeB0gaA/9LVhajxy8heYoBQI+y58EQGMyPVcIpY7SVAaqvfXtimsqC7aFdZE+Bcb4UGVWzMSBnxoLg9/DtmU62iUoneKYvQRDHXlo/jpgxkvV1gnYJXJl3UvveY3h03JNYtpKWlobz/Tck6WsJw5DKf2EpluHnV5kWOCboE6EMwbCL0ahH7XJKlqmKQqXy5Ngd/UGKluIfWwVXbO9Xi+ldQ/SLfP2K2o5QJqSmg5jK8KPigoaBB1aw0nQg7z69QQ9l3id4xfz1Ifyf5tMzWYb2owYx7AAhlFnoT4lIVcTqYUud7/lmeXJ+c2e9jzyMgGDVFXb4ywAyzAOa12BemXSXMCbXVCZ9qrHU+WuP8kT7VaEq2OarQVrVpu13xRz1y5E3JDdcxAA52a7GOTfawOS2SSjtZGQUZuy/1yKOOgVTnKaJBEgyTGMVmYVwDa1I+uf7OFCfwmLbJaaL4YN7oePJGn/cbE/6ppfyawYvwa6QnpDZ5o8MTLwBOvombVoFPjSXF90DLpDEs4fTLAXwgXJtCb1w2CoxyZceFibl4gCv6xFyveulDHYqxJ05IMAg9wuN+bkGypU8UJS3koCrEppQji6WMvOYvUHQy1ulxyg0MvozBtsMnaIv2PVyOSQ5P6jhe65MQrirzFZmANoANa8Hg0wqyCFdxUdI3dElJJd/VeXXU0G9wZkX0SJlODpvrLmLtkTazLdqq0R4bZR8UtDT5zrzPynaLXNEY+J0F2ynyUC3g0OqaawUi7N8b/Wj67322PnRX4yqjMSv0soOCuePzjiHp7HFKSHsU6SmTa2q3El4Pkl/Yg/nyU+NxOfjloJWsSLnU/8bntGo6/j3lQqxIJfc3sJdnwcbLhZC/7h8kvbvz5tZukd0xLSnw/mMYrVpezexG9zSgOXXoo6ptXOfPQlYd9gs4A7ncZ5qZtK9R42DrY3f14+eXr12hE38Wjvv3q8f87VdPrCKQAAA== + H4sIAAAAAAAC/+1dW1PjOBZ+71+h8sMWbAGVcOvuqZoHGugeaukZCrp7tvpSLsUWiRbHcksyJMOyv32PJDu+5+qAYfwCieRI56ZzPh1dfP8KIcu2qR+EUli/oG/wHaF7/RdqfDwkUGqdXNkXl398PP302+nnK2srrvZwj3iq/oKzIZEDEoqk0iXC4TSQlPnqkaRCjgPdqIslFizkDknqAi/sU//MVfVBSaOm/veIrFS3+oEH+Ptjy7DEyc+QclLCVNx/n+Nr7OOkceqWFsdC+JCvuCVcRNwd7uzv7EVEbJV3F2AfhFXsLBiUdpUuTnU0vY8ykdJKYfpFMZZ1ebDT2ekswZvEPY8Ue/uULS7yNlEi9n0msTIgpUXTqeVRISc6TUiBml5IPXmmWupuJaUpkZSzCs8QX9P0C5I8JKnyAXVLSqnD/GPmMa4a5P0e3uhsod1uF/4cHGyh7ma66Zjro4QX9A905BEuMyQkGhSDHsPctaK6B/3/x6tI+PlhdSbgEzpmvuTMQxceKAKdxG2gSLiou/MWzFP/nrhURtxeY09oxqy+T6QedK8P9w9MibLAT4x5kgaxREH2/o0eUEY/WuslA8xhnocDQdx0H7qmz6l7wRJtGinnNHYH33f3UwUjKOikvo/V91g8WaM77CSuYkLdj8T1UOkZh0YCj42Ji74YCYmie+LsrtzosUex0AagGUko6GFdkmVZKfSc+H2p2exkyknZ4zPM1bqmnpeWWLVQDxYVardcqAeHiccnfeK72a7wbT/PhjKCkHPiy5KaIR6VlVK/pFQM2F1xBEoYSl7J07fYCxOZFpgB89W16dZ04R11ZcYKc5ZuRnLoeReM+vIj015BF6QiEwGd+RL3SUGngfoVxy4NRVorprxoAiA2l3Ci3cu1x2TShyCcEvEHjGpQOcnRJwLskDJLExI7N4VehCRBQNxzYL9QJzHvkxQeyDpa5URGgSZPhMMNqjyQrZyve++wYcB8EMOv362AAu3frYdN1BujDRBM2i8qO2Z8iKUOFHRIbMNb9hEQD+Gg1PfYkdrddjPVxhjfT9q5v0fQDXp4yLbCybUBFEfWpPgh+pRyDQMACwPmuQWXMSTvYRhCqVJ4pvyS9KPglPvB1YBey+IvIudzoSRT5nkm7vY+PQIw1540NwYE4zI3hLX527H/or5Lb6kbwkgpjIQUBErjjBEe0Zwf6YXOjTGFNC9qvEajQLFbEvByT5eP48l4/fajQOIYj8gUC0ysB5rmMqvxGJdmiVAVrP8OC1KwJOOSCo8bn1QoTjGTGNRWY+lMO8Myyx8XtQ4hrl/mjnX5ObmdUJ0BKflA+Rg44LACB+zOhwMuiQm26LNQnvsFgoDXOYkezhDo6woM0EKAFgKUQQCYWDpECPuWcgmxxh6SIeNjuzeWRNzjIAAkoBGCC1BgCQAAlJdZUBk42J0KDr4Y+tBHTV8FQjjLlisBqYZnevm8MABKgHZ8Wbc0FuL4MiJiOsu/rchynykmQU1S2AOCA1uMRU38lk6/F5eDogoBVXOgwkV5xZ7HnAZyq+mq4PdkOX7rZHVBlo6mcPO+bvOlfihIvSpd2Xf9pnRK/W2grEIMpzWKQQeLWuWwILtXioDp/H5YkV+HQZyGuMdjD33H+A31+0C+jJiePPLr/75bLhUOg6A7/m5toYC5qszIY3vnn//Vn7b1hBe+PqWlnMR0oo0J/ZsVMnz3yDKMpMTZaF1SXDBfcAVCdzCfQ1THDckc5CJ5mzFYKmOgbbPNGDQwY/BiZtWH882q28R6O6uuTqxzLEkSm2wnCO1Qpakg9kGpK2ytrlSQXRinfOsOf2yuPS+/CChZYDZm5BNPuZV0snLJoVbNaxOAWURxNdZYDZYtazoLw7Mljacuic6P3mYC3aeEdMcXn1s8164APSKdLZxbGM51d+fDc69bPNfiuem5kmsRReJ8hmT9SaZlIcva80dTZNLUpFHDt5iA7m5aWLG2NJFlPTGoABukQ+yJsnYSDnzmk0UZ6KS3XLQJpCdDHG/mQxz7T4048sUt5GgI5Ogzu884C6WS7GMv4f0eDnuEI3aNPkxoqAyZDZ6dl1DfRtO6J+lPHk3/xlP0x9ij2d2v2FP4Zr5Nmma/9EUoBujMN7pSp1/q26yZCTq1g4LciZ0rULhA8AxSSVrlIvXsBQWhbn8JLPEmp5A3s/RxULVpdvdl5S+ymloNTSAs0F+EsxSqqIjpzUcbGXuveb1KG7M9coVtDPpejU6AHo4rHm3B6dgLgSs+x2aOrVXZIo/I1qnvRoay8s6+2Yx5j8jYOeBS4hNewxbN2YzxR2TsEsBjJfA9rZMrUclVTHIJ0MqntojDSaV1vauTXL8GctUpcH2UVVTuDW3Sea+LXHyfPo0oxMcXNY9Iq8vE+EnJClMMFqx1V5fVsdpNXc8vJ5eD38c4kCEMfYTRLQYHL8cJBAc0xfhaIPibeSF4vSuIaq/F6TCQ4/K7Db4qPFuoaZcdX9Q2siQEAwS3OfkPcWQ+HbiJGEcbuCdAQnP+YDt70UXde39m4CpNFHHR8cmVuvzimvZFPWg/YZ4sKi3SeGmdzpTWybLS4otKizdeWpczpXW0rLS8RaXlNV5a5zOlteAEIjd/uONUAtoFfmBOVbq3c+2zuT8VCeiTIWGeScdiXOqoqXnVdPnwxQCSmritS+VnEXXoNAuX8udKV5eDsfim8W+sHBnLV7crXQLt9czxS+bMttoNQ0aymbagk/LHhsLp9vCv5z76NaszBv+HRmUccvpoFy7b3cXt7uL1L+vp+IWU3SIqkcQ3BMqZXt9DGDkmeoSBq1b/oBwjvbtxHVmH7uHcK3/77c7l+lIIuy8ohTCAMMvAjw/tnyH2JfXIRmfnYAvlgrS2YYVVgHKwH8cEa9u4dxOg9T1wHll/pA4OOmj1GVsp42+bzfjbzupzr3K+m87429XTPxWMN57znbdzJHOeFI0qeaEJfm4B6YqAtAWjLRhdeuN5HjNmjraV7Rbb3avYeX7QLlXViTNLNpj9Le8T1CFWzZI86kib+j0W+q7tRVuEHvFOrllZUE0YmrV3aQHklWOdhTLLuz2QMrDVkTw7Gke2dILGSOSPiN5EJGhDUYwUxSiiGAHFNdz9MFNU0M3zkhTQN7egjtcpKKWy5yUpRfFmwzHwcaSBNh3bpmPbfV4Nh8CF6x4qMXB74UO786o07oLqo2yRuhhbxchb6hTPYa49PxRfe30VEbA69Eo4ezKmZjDzbsltOcWtNSqLFzB3/QdWmO9HW7TiMx7o/h56rnyXyWlDcM3RyRX6yHwKXFK/34KbFty0x2SXPCa7V/H6rcP9+Y7Jnvq3bLymE7JPsZbuEompJxCGGaBEhju19ALOFlFfL7UPwc09xonZvd0K/Ndp8V+L/6YfgSPKcG3HHEm1w0BITvDQdkbxHZxRjQo2gD4Ah2z3eeA8/d6+f5sdvb7Z5SjqPE1bLRPHdGhfw9B/LqJB74FYdaaopuOrvgPSELGUFMpWuV/AqNAFGKC0ScCcwWOdOVZu99J0PA/gfkowaog9MXGjxaKrYdE1n6asCYk+EZVtkq3uE5GVEKvbQqwWYlXFymogAeGN3pIpEGJz3XegKZBwpKmYC0Y1+Ba0GZy0AbZN9rR7udZ/sEDQv/R9YWo4cvITLFAKhH0XvogABuR6bg8rHCKojNS7a9sQVlYX7QhrA/yLDfCgyo0Z6QI+MneGP4dcymU0StE7RTH6CIa68iH87JmMlyusI7DK9Au6l17wm8Om5JrFtJQ0tLefaTlHS1jOHIbTeIlMN4+mvL0xRbdAVxDMWwS9GoJ+16aoGpiiql4rzWDfzBipbxV1v1N1xPVwvmXUP0lvwNiNaOTqaUZoBYyvCj4oKGgQdWcNp0H2i4vUEPZd4l8Zv56nPpL923ZqMN/UYMY8gAUyijwp8SkLOZ9MKQq9/y2PLU+ObPax55GxDRqirt4VYQdYgHNa7e7RL5PmBNq4Cp1pL3Q8WuLmkyLVakG0PqrRRrRmudnwJT1z2U7IDdUJAy10apOPbfKxPiyRSzpajwoyCvvtl0MZe53aUUaLJFokEcdkYd78Z1M/uvjNFibwm7TIaqH5LG50PXiiSPu1if910/5MYEX88ugJ6S2eaPHEy8ATr6Jm1aBT40lxvdcx6QxLOAMyxF8IFybQm9cMgqMcm3HhYm7eGwr+sZ8o3jpTZ2KsSdOSDAMPcLjfn5BsqRPFKUu5LwuxGaUI4ukzLwWL1B2MtLpcco1DL6cwbbDp2jL9x6sR6aFJfccLXXLklUXecjOwhtABLXk8GmFpSJWqTaJr4paQSrqr1+mqY9nFdd+4+kvMXbom0WU3U9ono/yj4oYGn7l3NfadstczRj4nRXbGfJQLeDA6pprBSLvXxv9aPrvb7sbOCnxlVGZlfhZQcFc8+XFEvR2HlLRHsQ5SmbZuJ/VlL/2lO0w+H6Q+d9Nf9jrpmpRL3U197rqG4x8xD2pVIqWvmb2kGz5MN5zuZXc//cVNPr920/TGtGTE9xfTeMXqcXYnotcYJaFLD0V95ypnHrrwsE/QCcD9HsPctG2FGg9be9vbH8+/fP0ajejbZNR3Xz38H2KxXAX6owAA istio-mesh.json.gz: - H4sIAAAAAAAC/+1de1PbuBb/v59C1/sYugNJnBBK2NeU0m27U7pMYXvn3tLJKLaSaPFrJTnAMtzPfo9kO5ZjJYRAW5qqMw2JjqzHef70sHT1CCGn36dRkgru7KH38BuhK/UJlAiHBFKdg+P+0ds/Dp+fvHz+57GzWZADPCCBpB+xOCRiTFJeEn3CPUYTQeNIZikJ4jJRhfpYYB6nzCMlLQnSEY1e+ZKeGArN6G/yZmnVqgzX8PlhM+sSI3+nlBFDp4r6RwwPcYTLwqlvTC6Y8GKWMCGM573baWw3OnkjNs3VJTgCZtUrS8bGqvRkraLFdZhYSucyM6qz0VRlt9FqtFbom8CDgNRrO6km1/s2FSKOolhgqUBSilmlTkC5mMq0bApQBikNxCtZkrtZpmosgSq+NXAIMpFINWoPCZYSLX1MfUMq9eLoWRzETJbIRgO80dpEbdeFj253E7mP9aKLbj8tO4O+R08DwgTX85Ui5ONBjJnv5LRr9ffDo5z7s3b1isM3dEj4GB0Uj6Kcqcht9EAt1WPEpyLv5BAHXPXHGUVEKGN7stPpZSlS807iOBA0gfSWSlTyjNIgUL8CGp0ps8qkpGRvMDNgkiCRlJXzk08nv5xGCMkviIvLgPx86iQxp7IXewgPeBykgvyIBrEQcQj1njrqAXgEozEjQ8g/FiLhe80mlT1u0PjUQQKzERFA6w8CHJ1BSlH4EGrf4vQfsoc6reTiRyTIhdjyiRcznFUaxRFU6Ek57iEajQmjAmr9iYYjxJlnqLAZYEG4aEKOLG1rEKQkiEfxVhQPsHc2YnEa+VtpNGQgdb/BJyOtSWNCR2Oxh7rQHqgIKcn91MQZZ5q359GPiGUlwjetv25XlZ+xL1MPyhGODKwcUTFOBw0vDrMOZZ8mxv4SJyRCCXBgGLNQthqJMRYI/MoEjATKR2lEJQ2d40skYuBsFBFPbN4kxqYfe7wJuT2SCN4U4GWH1NsKwdWOSAga1IxBnSeUnDfGIgyMjcsyy1ZtQkd9dMs6wQrOY3a2BY9u4VSMm/Jjfm2ceClTtaGQeizmsnke4Y282gHLmf+GEB+NSZD8in6PaQQcI4saBWIIgYnismmsNhPlNJOsv6GrTva39KwjRv2juHSdmUeDnx3N7ZzD7/a2lnBRWH3++1L+LlzRtOxMl1V0AG2e8fxup0QIFV+hksJY+VRHsrd8UCl8talTlZa53d1EOLVmCCrUUzq6YDjiCWaZ69E999TFSldgjmce9sbkhIYkToXm8nJ/Bp5if2rluh8t6e8weAQziWtRC+Vxo70N8aK7nf1vNXqVyJHl6DyBiNLubaLtlsyya8jTBYr7pA1ZuqqUJ48LTpVcvzkMOtJ4sZJpnGhIYITT0YxkQnxRdNRt6coS0qgg6Ml8HJ/PMkXKAwwBSIH/WuJIvijHIWZnRLFQSrSmB0ur+s4Nmt6pFa00ut0qf0NUYxMczKqHQdPjpIpdSl04zG1gopiltUGF34IKSox14l8peIDh5ZScilgnx4yC0uMCGIzh9z9gQjioKAzxU4/8YWiZUv/Aq6ppDvO5eBOLN9BdR6N82NSfHVIS+Lxiixm+k12cynZKudalDPZY6VRNCCFOEhqNTjIDdk3p1XZXulVAMNUUGZqUA6i3UhZtaqCxMHAzoxsKa5eF1ewRTOgAbPAIwoIaeGl25Ei9UoSCKXkoJb5TyXMia55RQwAMYkgvqj4xT/wNtOE496fd1ncaHQJS7RmVNvcR1f1DnCxg+xCGFfI52b4qg0TWcOdN8+kMIZ4+sIB30r2fgb3NxgsaBFVY3gFI7rq78LHbk57R3a04z6Gsp4buZcF6MVkpbelje53K87lbM3okhbWhmDScHf2quL6AbeQiyaqWUWaDp+EGBchKNhRMUMNawKC8L2A0EVwxksQMHBLghMydnzrX793ww+PHsr+tVmUsonmv37AnVA/dChlkno29Z+TCBZGjge0FQpm6aj7T3yJEvwgAIwfobdYB9E7yhtRnBTjYcwAZsNDGiNKgdFXc1VVRURerYixb7/y8pBYW9nujIqqMxWQEnowsqFgAKhIChEhI7HpncLH7sdCFHqTuAV7sLAkvXAsvLLyw8MLCi08HLySwuBWu2ETgLRMwa9L3QHH+9b9Tp9v4YYo3UBPdusj80aqQyoghIGb2OQG/wz8PiulB6Ov15P9Gdz6iOU49j3CO3kLX0UYUR1vdi4sps/hji3HsxMnaTZy47SWhTdtCGwttLLSx0ObTQht6F2zzM2CbbQ3bPFiAYgAl2xcX3CIOizjWD3HsLok4OhZxWMRhEYdFHF8Y4uh+oYijaxHHfSOOb9q9nre9c1uI8Y2/vY07+I6QQm6M+0q3f+yYIYXrWkxhMYXFFOuJKWa9zkMBFaAiGwkNYtE/2+V9bzjqkwkYOL+SIRagxDvKRIqD42z3qYQUig4U7PuAJB6jLbSxeiE+CYggqpyYIVlOmqAfUOujgpOVcEhIfstUrWIJMv14TIeiTsiBS953lHd+nUCMlzK1A9YCma8DyOwsC2S2LZCxQMYCGQtkHhCQOYAISyNl72/TYFUks6CU9YcyWueR7L3FMhbLrM3WkrlgpmvBjAUzFsxYMPOAwMwLLMg5vlwRxBieXn/wknfaYhaLWdZmc8pczLJjMYvFLBazWMzygDDLv2PgQoz955FgqyKXuWWsP34puo5k36mde7E4Zv03xDyxMMbCGAtjLIx5QDAm38dxFxQzr4j1BzF5zy2GsRjm69gL07NnulkIYyGMhTAPCcIcEcKepmIMadRTZr8ikFlc0PrDmXr/0VEcUM9CGwtt1n9rTM8eKGexjcU2Fts8JGyTn/x6L/DmxrLWH+EYWWBBjgU5X8temp49Ws6CHAtyLMh5SCBHhmNp48riVTBedSnqhpLWH+BUGPD1ARulpTPBaynIUHYa/Nd3S9wH1XZveyFUr34hFPUVDPtjQhjLbs2rIioVsp8svBkqwSOSN72iEYwkBAtz6gFl4ITz6DrR7pPyWFw7MUr5hpcE+4TVKDETNUwACV39KkHCvXmeRd1ftUCfcEAx1zeLVRVIByB6L+eBUQ19LjiXcFmMWuabfz5hHVyA9oGJFd7lP/Bv6/Bw6+AAvXy5F4Z7fMbL+MSjIVaQsl0hSE2oe/yccKIc8768Am1+nul1heVWvPIaxVr2P5m6s7Q5zdL0B/l9fuf541tT2q8TzLYkvID45pGfv73q9z0SBP3OHsPn199LavFQSWwrYrXmBAvwskpN/fJlzf65UR0gzstL5Ur21jvOcURFZix1YsUZv6+KrXCbYLE+iar1qss59pSdMOHcCLumSm2V2SBm6Q6dzyqWfIqAW/EYxKMGxeibp7cXUZSGA4ggRhHJ81iXF9BRt4VeQ9ci79LKaL6M9u9XRreSUM9KaAkJPfucEupZCd0soYPPJyH9gpT5InIkdHK+biE9XySkaqsbve5sR5DbaLXmN3sZseqXtC0vYDummTOmqR5XXB+ufJsNGO53tDJvPPL9wqHMrUYrfSjqnv3JLaFtvl3Zatwn1bj8pvMZhctTS41yl9cobpLjMsrEBaPRyA5fP1pIMtr91H183IFtfXZ52fn5QC5b9v+KabRx1xvugNJutcoL7gaXaMPElE20mFVVeq7wjzfR6VzXKpty6jSyP6ZM8ylltafO/JUAuQxSJY4Nc7b54gAXOBIm13LDukFARiTyS628ujIy6vq6MYdS9gVdz3gT45rE5tKqsTEGnYhHDIf9v1PoHQ3IRqshDc6sNH0/ZVnbQhoElBMvjnzeH6TeGRELbjZUOhMQs4bcpDfqTkW31WqpFZ6VW/ypGmsV+l4Uev8eFbr3xSl0zyr0w1NodDeNfnafGt374jS6ZzV63TT6YDWNvvcrl1dFpHe5rHn1StdJ/eqjxgehmM8XjqHm7sTJN9y8PDk5ar54e/QMFRN6+m4bhiMu+Vnn5nSIlyV/7u0slf2jS21n6bRW3M/itnqrb2ixW1fs1pWPPRdsd67coNP7l4JwdCy34VmtXm2TRLXVd1t/2r/V9olMeG+JR+iEWLf06XZQ3E5MVjAPcneYDecPadX2E0Tq1VZt7ULbGi+02WV2u8x+b3s2PsaqrfCSPssBXn8g8d7Hm6GyC7U3zU3d6yKsQdIcxmFWyg9Uyvt3mlw8eXb0EaYVVaB6VoScq3JSa4BZ7eV1R/rT1yQaCTVN2KqkE1P2m6cnqZqpc2+elezddlJye3fe2ctlbZlMq5UBS0bRU36imFZ/kVZeb19PLV5FrFNCfGFKpZEhldHRWBwbVVd/LVd/y1xauSG38QX1a32ulfDZeVCZeE79TLiLpmVrr3JXXoYu9sDiUX3SN5FPMezTlOsTrFl6XYGApT5hasbWGQax9n5q9hpuMbk80z7lL0x6Cqbv1eYSIZUkCfFfZ+9eV2lLR2DpjDM/PEhp4Gfu1YvDJI6AF5sI2HHvLxYbPNK0RnBDoNVXst7l4s7iF5Lff1jqheS3ZJQfwTDzwOI3lV9JxqFnRdu5ZN47wric2y9zT9HhlW4VmBG/bhf5nH5r1iT6hU+kkU8n1E9xUD+Yocijzq0oG3CBL+iMY8oWnHkNxzthbhmiOktRHLQwk9ts21Mbfv+h1sRLfLFoyaHUrQxjVtVGQgjT0COIR/uYk5qeZS6slj3zYbXk+vkB81HNA2jnzAkeNTO4rEtdRQhTaJDpr8lk2upHeqHyUxUr7U+alux2p5WZvMO9MQlxofTFIly2tqQGI5idZTnBokvJZ6bjTIsWJEwCACTRaNpk8OBcaKpyZYpaFalwEmQHcxhAUvEKv0+GOA3mnBEypRpPRsmxmW6bNPKC1CdPzUdXGPXACaECashenGVigBuV42v0AZvzd0rYpXqBAzwbEWOS6q44IxenADnVo2cKWbqV1BG5mM3Kz2gCg9bjy8gzHQuUOx2t2RX1kT7gOpMxDctDQsoDSeLzLbfwVsWZI6DhlccSCv6KlQ/nre8XgUZ3KU5Xg5luS/vR0X+4Yfm9q3139R+dlk7RfGpb++76WY8/FH2QYVGT14216AXv6AXrtbS39R/a4TNPfL29RVsq7PsnVhDBGbD4nOej+tkodgj8RAeVaQsnVcDTebF7/pr9/uq/uSFPpsbefXT9f8ToGnNJtAAA + H4sIAAAAAAAC/+1de2/bthb/v5+CV9uKdEhsy47TOFs3NE3Xdmi6oMl6cW9TGLREy1z0Gkk5yYLcz34PKcmiLPkRJ2lTlwXq2DwUyfPkjw+RV48Qsvp9GsaJ4NYe+gi/EbpSn0AJcUAg1To47h+9/+Pw5cnrl38eW5s52ccD4kv6EYsCIkYk4QXRJdxhNBY0CmWWgiAuY1WoiwXmUcIcUtBiP/Fo+MaV9Lim0JT+LmuWVq3KcA2fnzZTlhj5O6GM1DCV1+8xPMQhLgqnbm1yLoRX04QxYTzjbqex3ehkjdisry7GIQirWlk8qq1KT9Yqml9HnUjpTGGGVTHWVdlttBqtFXgTeOCTam0n5eQqbxMl4jCMBJYGJLWYVmr5lIuJToumAGWQUF+8kSXZm0WqJpJ6ViEPCVWb9pBgCdHSR9StSaVOFL6I/IjJApk3wButTdS2bfjodjeR/UQvOuf6ecELeoye+4SJUhMKDfLRIMLMtTLatfr76VEm/Gm3esPhGzokfIQO8kdRJlNkN3pgleox4lKRMTnEPlf8WF5IhPK1pzudXpoiDe8kinxBY0hvqUSlzjDxffXLp+GZ8qpUSUr1NV4GQhIklKqyfnbp+JfTECH5BXFx6ZNnp1YccSq52EN4wCM/EeQnNIiEiAKo99RSD8AjGI0YGUL+kRAx32s2qeS4QaNTCwnMPCKA1h/4ODyDlLzwIdS+xek/ZA91WvHFT0iQC7HlEidiOK00jEKo0JF63EM0HBFGBdT6Mw08xJlTU2HTx4Jw0YQcadrWwE+IH3nRVhgNsHPmsSgJ3a0kHDLQutvgY09r0ohQbyT2UBfaAxUhpbmfmziVTPPmMvoJsbRE+Kbxa3dV+an4UvOgHOGwRpQeFaNk0HCiIGUo/awT7C9RTEIUgwSGEQtkq5EYYYEgrIzBSaB8lIRU0tA5vkQiAsmGIXHE5iI1Nt3I4U3I7ZBY8KaAIDukzlYAkdYjAVhQMwJzHlNy3hiJwK9tXJpZtmoTGHXRDesELziP2NkWPLqFEzFqyo/ZtXHiJEzVhgLqsIjL5jmEN7JqBywT/jtCXDQifvwr+j2iIUiMzGsUqCEAIYrLZm21qSonmWT9Dd100r9FYPUYdY+iInKmEQ1+drSwcw6/29tawkXu9dnvS/k7D0WTslNbVp0DWPNU4Lc7BUAoxQqVFEQqplpSvMWDyuDLTZ2YtMxt78bCqjRDUKGe0sEFwyGPMUtDjx65JyFWhoL67szBzoic0IBEidBCXhbPIFLsT7xcj6MF/QOGiFBP4lqnhbJ+o70N/UV3O/3favRKPUeao/MUepR2bxNtt2SW3Zo8XaDYT9uQpatKefokl1Qh9YW9oCV9FyuVRrGW7OHEm1JMgC9yPu2WbisBDXOCnsxH0fm0TKQ6wA+A5LtvJYrk83IcYnZGlASlQitmsLSl7yww9E6laGXQ7VbxGzo1Nsb+tHXUGHoUl5FLYQqHmQuMlbC0NqjeN6eCDWOd+FcCAWB4OSEnItLJEaNg8zjHBSP4/Q94EPZL9kLcxCF/1LRMWb/vlK00A/lcvIvEO2DX0iifNvVnh5T4Li+5YoruJIsT3U4o17qWwR1LTFWUEOA4pqF3kvqvXZdebneJrRyBqabInkn5f7WVsui6BtYWBlHGW1BYuyis4o7gQgfgkUfQK6hhl+ZHlrQrRciFkvWkxLVKeU5kzVNmCHhBDOlFOSRmib+BNRxn4bTb+kGjQ39UeUalzXxEsX+I4zliH0KMkc/J9pUFJNKGW++az6cI0eSBObKT0f0M/G26u6C+X0blHUDktr0LH7s9GRjt3VLsHMp6KuBeFqwXk5bSliG21yk9n4W12oikoDYUkwTTY1/Vrc8RG7mI06plJ7PBk2CDAmIlGwolqEEtQFDeFzCY8K8YiSMGAQlgQhrcT63rj3bw6ckTyW+rVRqKaNHrN+wIxaFdIoPO05H3lF64IHIwsD1HKZNQzaf4zXvoVz5AZB+9TxlAH6RsSHVOgIM/+5ABC22EKB1KN8Vd3RQVdb4pRrL11rMlrTD334WGqDLmUxF47BlMMRtTxASyhUIi11tji937Ahd6H3UH6GJnSXRhG3Rh0IVBFwZdfD50IXHFjWDFJoJoGYNbk74DhvOv/51a3caPE7iBmujGRWaPlpVU9BgCusw+JxB3+JcBMT3o+Xo9+b/RnQ1ojhPHIZyj98A62gijcKt7cTERFn9iII6ZNlm3aRO7vSSyaRtkY5CNQTYG2XxeZENvA22eAbTZ1qDNg8UnNZhk++KCG8BhAMfaAY7dJQFHxwAOAzgM4DCA4ysDHN2vFHB0DeC4a8DxXbvXc7Z3boowvnO3t3EH3w5RyD1x3+jWj516RGHbBlIYSGEgxXpCiumo81AwBZjIRkz9SPTPdnnfGXp9MgYH51eyhwUk8YEykWD/ON14KhGFogMFuy4AiSdoC22sXohLfCKIKidiSJaTxOhH1LpXbLISDAnIb6mplTxBph+P6FBUCRluyXhHGfPrhGGchKnNrwbHfBM4ZmdZHLNtcIzBMQbHGBzzgHDMAXSwNFT+/j7xVwUyc0pZfySjMY8k9wbKGCizLttKZmKZrsEyBssYLGOwzAPCMq+wIOf4ckUMU/P0+mOXjGkDWQxkWZeNKTMhy46BLAayGMhiIMsDgiz/jkAKEXZfhoKtClxmlrH+8CVnHUneqZl5MTBm7TfDPDUoxqAYg2IMinlAKCbbw3EbEDOriPXHMBnnBsIYCPNN7IPpmaPcDIIxCMYgmIeEYI4IYc8TiOKhoI5y+xVxzPyC1h/NVPlHR5FPHYNsDLJZ+20xPXOOnIE2BtoYaPOQoE123uudoJuFZa0/wKkVgcE4BuN8I/toeuZEOYNxDMYxGOchYRzZG0sfVx6v+uJV16EWlLT++KYkgG8P1ygrneq8lkEMBc8Qvn5Y4gqotn3TO6B61TugqKtA2B9jwlh6UV4ZT6ke++ncy6Bi7JGs6SWDYCQmWNSnHlAGMTjrXMfaFVIOiyoHRanQ8Jpgl7AKJWKiAgkgoatfHki4MyuwqCur5pgT9inm+j6xsv3o+EPnchYU1bDnnNMIl0WoRb7ZpxJWsQXYInhYHlz+A/+2Dg+3Dg7Q69d7QbDHp4KMSxwaYIUo2yWCtIRqwM8IJyou78tbz2bnmdxQWOzCK25OrGT/k6lbSpuTLE13kF3hd549vjWh/TrGbEuiC+jeHPLs+6t+3yG+3+/sMXx+/VhS84cKYlsRyzXHWECQVWbqFm9p9s9rzQG6eXmPXCHeKuMch1SkzlIllmLxx7La8qgJHuuSsFyvupFjT/kJE9ZC1DUxamPMNWqW4dD6omrJJgi4UU+NetSYGH33/OYqCpNgAD1IrYrkMazLK+io20JvgbXQuTQ6mq2j/bvV0Y001DMaWkJDL76khnpGQ4s1dPDlNKTfijJbRZaETta3raSX85RUbnWj151mBNmNVmt2s5dRq34z2/IKNmOaGWOa8inF1eHK9+mA4W5HK7PGI4/nDmVuNFrpQ1F3HE9uCG2zrcrG4j6rxWWXm08ZXJZaWJS9vEXxOj0uY0xcMBp6Zvh6b11Srd9Pwsf9Dmyrk8vLTs/7ctWy/1dEw43bXmsHlHarVdxqN7hEG3VC2UTzRVWmZwb/ZBOdzgytsimnViP9U5dpNqWo9tSavRAgV0HKxFHNnG22NsAFDkVdaFmwbOATj4RuYZVXV7WCur5uzKAUvKDrqWhSuySxubRpbIzAJiKP4aD/dwLcUZ9stBrS4eqNpu8mLG1bQH2fcuJEocv7g8Q5I2LOdYbKZnxSbyGL7EZdpGi3Wi21wLNyiz9XY41B34lB79+hQfe+OoPuGYN+eAaNbmfRL+7SontfnUX3jEWvm0UfrGbRd37P8qqI9DY3NK9e6TqZX3XU+CAM8+XcMdTMjTjZfpvXJydHzVfvj16gfEJP32zDcMilPKvSnAzx0uQvvJultHt0qd0sndaK21nsVm/1/Sxm54rZuXLfU8Fm48oCm96/FISjY7kJz1j1anskyq2+3fLT/o12T6TKe08cQsfEhKXPt4HiZmoyinmQm8NMd/6QFm0/Q0+92qKtWWdb43U2s8puVtnvbMvGfSzaCifuswzg9QcS793fBJVZp100NXWna7A1muYwDjNafqBa3r/V3OLJi6N7mFVUHdWLvMu5Kia1BphVXl23ZDx9S0JPqGnCVimd1GVfODtJ1USdvXhSsnfTOcnt3VmnLhe1pSotVwYS8cLn/ETJrPoWrbzTvpqav4dYpQT4oi6VhjWpjHojcVxrufo7ufor5tLJa3LXvp1+rU+1Ej49DSoTz6mb6nberGzlPe7Sm9D5DljsVed8Y/kUwy5NuD6/mqZX7QdE6hKmJmytoR9pL6em7+Dmc8tT7VPhos5MwfOdylQipJI4Ju7b9MXrMm3pDljG4jQMDxLqu2l0daIgjkKQxSYCcdz5W8U1AWlSI0QhsOorWe9y3c78t5E/flrqbeT3xMvOX5h6YP5rym+k4NCLvO1cCu8DYVxO7Re5J+DwSvcKzIhb9YtsSr817RL9PCTS0KVj6ibYr57KkOdRh1YUDbjAF3QqMKXLzbwC460g8wxRnqTIT1mYyl3v2xMf/vip0sRLfDFvxaGwrRRils1GIoi6kYcfefuYk4qdpSGskj2NYZXk6uEBs0HNA2jn1PEdFTe4rGpd9RB1XYNMf0vGk1Y/0guVn6pY6X/StSTbnVbq8hZ3RiTAudHna3Dp0pIai2B2luYEjy40n7qONSlakCD2AY+E3qTJEMG50Ezlqq7XKmmFEz89laMGI+Xv77tkiBN/xgEhE2rtsSgZNNN9k4aOn7jkef25FbV2YAVQAa3Jnh9kooEP/TCZyQkx+njN+jsh7FK9vlEFKTk5PwLIKp87k+vSLqV65GI6Kz+jMYxZjy9Dp+5MoCzoaM0umY+MAdepjmlQnBBSnEYSnW/ZebTKDxwBCy89FlOIV6x4OGt9P+9o9JBidTWcZre0Hx39hx0U37vad1v/0WnpFC2mtrXvtpty/CnnQXaLmr4W1qIXvKMXrNfS3tZ/aCfPPHX19uZtKYnvn0hBBGvAonOeDeqne7FDkCc6KM1aWIkCntar3fO37Pc3/80ceTxx9u6j6/8DB3BWLzm0AAA= istio-service.json.gz: - H4sIAAAAAAAC/+1dbVPbSBL+zq8Y6/auIEWIZTBYW9lUgSHZ7LIJh7ls1S05l5AGW4csOXohEIr77TczepdGRvKLLNv9IQQ0I81bd8/zdLc0T1sICf2+ZoxdxxZ+Rn+RvxF6Yj9JiSGPMLkqnPb6F5ef/zi7+vXsXz1hNyjW5Rus0/ILyxxhZ4hdOypUsa1Y2tjRTINWiQqcxzF7qCo7sm26loKjsrHuDjTjo0rLx5yHeuWf/G7FmmUVnsnPr7vekCz8zdUszBlU0P7Akm9lQ44erqncy8EkfEgX3GPL9kd3uHewt+93Ypff3Fg2yGRlGxsPuU3FL8camtwGb0q13Mk0stPIa7K919xrTjE2R77Rcba1q+Tl7NjCRZQNw3RkKkB0Fb1GBV2znXBNo66QkhtX052P9EnibnQ1NiWkiZ84M0QqYYN16mfkWC6OXR9qKueqpphG19RNiz7RGtzI281d1BJF8qPd3kXiTvzRwbCPo8Ggf6BjHVuOHa8XLaE9vDFlSxX8smf2/9ctf/bTevXRJr+hHrbuNQWj0+Bu5M8rEvckIpnsTqxqjj/OW1m32ZCEgYEdpm9Hh/uH3hUqfFemqTvamFxvsouagy3Zb1NsS+22JLYk8Ug6YqW6ZtwxTfMWjokDR/MUU9flsY3V1JySJjX1woxW2Zv81Ep+J3+3DmIXHoLe+X8/0r+DWUsKo9g8jGxIsndJKWK9NBxsUCET3qraPVJ02bZ/uY5W5vUQyyq2kIMfnNcKqYqta+HdtfHWJo9+1zu7/PKxe/Yz+sn2FuXtG3adlL8hj3sXW3TS2K2GdbVrGrfaIDF634Teyq7u2JkS2kvXdswRLXlOFD3vJp9hEkEgs+sZwq9bORW5KxCuwn7qmemV4K1GuCJibptsZTpS4lpSksLLI5MpojB0RnpyAs1x0kAsZRnTPcwdsrd/fYls3tGeSIxroo6jOUxHhdRlSzZIDyxvTEmTFLMfdDRR+7HWkzIuK0N8pY2w6dKnGa6u7yZ1gJi3E1m5G1ima6hxi5Gs80XWXZxfnFQzdp0ZzNYBMZTtA+9fc0/aSU+nV2v/iJjTlrSLDpq0WienXpuUikctUq3Nnna0E1+BpCQV2Q9qpJe3pjWSmRCbYzvZxYHsDnC2cyP5IVgUsZlWyBERPr8wXWQPze+8haSyNSRAamjq6jkFfPZLtf6QrTvMlp5K6XQm54Bjcg4LWpyDyRZHbCWvUcW/l3WeGuRYowk2h8j8H74luGcTneof21yDGkSZ5XSF/xL50W4fwyquY6armJZGjECwGwtD8vcPYuxkPaMcWHUV/Dmnt8wM6EpWRX1sbzufTOcTmRIhVfp1N/0cpi12xmZ5AI9OQygzE3WC2q7EwHMXciSPx5oxuPKMnphXlh1bZgoCfMa6iRyTbQc5o0jtZpkB5D6cmO5BgYe3kg/PtWFExU+J3bowiezaWT0XqByzwmAyyV5oYMUhwCtT74r2iCP6Y9N2brWH7D7kF7wnEtfTfrDHt5t/T9WxMP9edn3irWyq/iAo8eWlu7WYoWXDyE6q4w1M+PTmmFNohjcWnHO69d4Re8CxuLearif5wD7hAqLYIT8IvCE7ktjJbFy3tO3sPu4ZHRx/nPe0Ft3jpP3Mc3yzPdHSMtRPHumODA6ukK0BdgpMN34Ye12iiGDbdkfbGqEEeFujDITxbWw7dt8hNEd/svDYtIhhJYDrp2/kj2thl+x1jmYws9X3MdUv/yPF/u/XwvNf7dHXnR06Yc2muJNdtWgzdAh0oQ/RsJ2tFlj097LisFkUM1WIHHpOBo5s2A4ep3aRSYIRbns2Z24DKNfVqclGl94koS90LVKIIwBwNrFdOqkkJw2FZyTiqtNJqw6rUUx1TDpC4ZeSWhPYqVKKw24KPDaKazEAW3uAyoGUsyDULN6dFaCqWNFGMgNjmcmpH3gdY4syLNfQnLmC2E4VKDYNXBYAYw+ngLEHAGMBxgKMBRgLMHYaGLsAALtLto4xsVG4rxCJbZCy9t6rENeiN2hxoLkisDwVJpYI9pEk+m+vPRkf91xFwbaNLskUoW3DNF63Hx5QMKv2zgYiZvl+8DJalnVNtrsBnn1Kqs2NbHHBDvOEn2Nj4LBISzNThvNuWy0PqsYslJi5+MGSVc3zpjfnj+iSXsYXId1QU1Vs9Dwl5c26Fyw5SuI6PMCM+6T6ScRhYBzbV+lgX7zK/SCnJKBo/FIaET0bjZ3HCeX/xpaZU0y2/LwSzcgpsbTBkKgwJxKb3jnSOIwa1ZyH5sK55zRyxjYv3EILvmuqM8yIVg7YzoCZzHYfkCR5wOerRWJHY9oClWuXdqGdLeMLF1lwFVuY7Yu3upmyo97W8TmhbmlQoeA8S0LssnLHbZU6WsZYPffQSLa89Ja+PSTbqkm4yaj/zZUNR9PxdnOPUvmcjbevul5Uuz8iBkGzMUGYqt2/cZU77Ey5HYtsr795RNs6Zps+gbXNHWRaaPreLaBjMwMGPk0qACc8q/U+bO2i3RSKY47dWURBqrUoSKsmCn56DMe+TikJ0iRJOJmrJEi1lgRp4yVBmiAJ3al4SGrTouN571HvjNeAll3ige9v4tzYG2q3Dv9Ovsf/1F+1FIMJs6ye0qBGtjJpUkEZWWZO5Jshmn7AiTRD1e411ZUnJKPEEiFj2YYegpUfNA4A9kSN53OOZcDQ+UnTe9+DxLkrH76FEG0CuH+UH4p4xyIp5shukMWa7RwtNAcnso25QuuBWe5tHprlFvE9HS/bs9gYhlQCajIOjjs0Vx0f+VLF6EoeT6Fl5/g+HNkWr6EaRpH+1pIk5eBw2qjR39SDA3lfXsM8phPIY+K5CzrlI0AdiABBBAgiQBABggjQXCJAjjImLEvB2j1W+zePDp4Ql0FFuVVtKJUw97hOTiznqntBaI83jeiETiOEbSALH7LwNywLvzMZvEpHAF4BvAJ4BfAK4LWKLPwYXt3wXHz6rjS2IBcfcvEhFx9y8Yvl4r8EZjsAZgHMApgFMAtgdrG5+CVh7Bwz8qcD0Cuel+9jZcjLh7z8dc7L78yYly9JkJcPefmQl195Xn7pTbna7PwZuwc5+jUVC2k1xWLD8/UXLhUSSMVa5O6nIgSQuw+5+5C7D7n7kLsPufuFc/dfiBil/esQMoKQEYSMIGQEIaMZkvdtYpAgcX+mxP0efWkZkvZj93yNDq4JJuoDNrAV2+jC2bGIFnAPSqng5Asx7+SLg6pOvuiefzz7dIX+/Hz5+/nn49Pehh980ZqMAA/a9Tz4oswqrty5FxCQfTEg25xTQPaQozuZ4Gue7rRnDMi22sUCsnOItnKw0zTR1FUOlyLZRj9ocBnCpvMMmxZK8fdJlR9acC3NeeyPTV1THonJH7mOK+t9R7cLJktl4LJnsPrfTUI7TFn1brCU73q2rE95Jpu7oJZhR6lWNEyRumEX5T5hFyUStyp5DyGNu5+e0v1Dz8972atRr0k5IjbmKdl5enX72lU7+yr5qYrN0dV5b0eY5j2I1lQRtRnlqJEWpJcpFF2+LPOaJEtolYRp3gxvjpI2MR5bUKzqGJL7aCgm2TwHQVDOJiQV9dj0oGNDJZe9eUBdui+VDNRx9+ONi9RlvVOOnOLRcwzmpV+kXnAYTOBkKkA0r7poHhCvyjJhCxMvbirszMzrcJnMawFprpDHCoRs6iMjUHF6hqZ9mWV5wHq6Iy3mOyfLG/yK04elUtK5Sk0DVGlOrB10aSpdWnvS/dL7fxEZhxzZ5ebI5n3cYR68VCBwUBSAYAPBLvxdFFuxtHGQIrix7LtTMOopirOS71qdD1UNP4+/hgr8Hfj7Us+HqiG3FaeKmFV5dtVaTVqNmEyqp4zH5Pb++RldtJvL9Qos/tCuzdJPCfRzrfRTKqmfJ9XoZxv0c0r9bIN+rpV+tsvpZ7ca/ZRAP6fUTwn0c630Uyqnn6eV6GdF/LMBBHQzZm21GegEdTxbJ7rZAL4J6lh/wjlBHd+vE7tsAL0Edaw/v5ygjh/WiUw2gE2COtafTk5Qx1/X6W2r8BOIkOi1kR9DhOQuSO7a4LenePlbnYXkb3Vqnr8FCVqQoLXABK3STl7vY28QeoHUomkkSwLJWnnJWn5WTGnfG4gW5HPMIFkSSNYaGK0lpyJUibTALbm+MfQqcRXI0foGf6sEUSBH6xu1rBIygRxBuG0x4Tb6EX0ItdUl1KZi5SZzIgJE3CDiBhG3KiNu4uEiQm6HEHKDkBuE3Io4gvwvpoGTEWJu8/YNgWhB0G1R/iKQLYi6LciFBKIFYbdFhd1mly3wT0LcDQQJAm/1AVIgSBB5A0GC0Ft9Qm/huWEQcoOQG4TcIOQGR4TFY27cr5QfzRhy6zQXekQYHPIFAbEZAmL803IdZUyAq4K1e6z6wHWVDriaBrauwzm59Tx6eU7C1NgsaapIYNb1MKcTKmfo0hc7dEtaQmHi4VX3AnVDQQP2s1z2cwJnI8+Z+GTHAbwHeE/Jo5FnJz4tID5AfFaP+NhkXhM4tQTB4UFagKnry2tKyEoDhAU4zRw4TY+IHHJMYDPAZoDNVMZmtlLNRVrZ1SkyR3/6hiha/VD8LTKMrdgCBL0TFFPX5bGd0UMurBdS8JHB+dZB7EIqhMFQfGDWwrX3sLnYDMG5MJYNrCdlPMHYyJbmeDBceEt0Him6bNtkz6K868aULfX1EMsEEiIHPzivKSrF1rXw7tp4S+yx8a53dvnlY/cM/fn58vfzz8envbdv2HVS/oY87l09qdhEYrXPIVbxlZgYUNrPbdN7Q0sqwhACUzh0RiliYLIDiDkdr3IZ0z3MHXIRnhJqWuqyJRukBz5DzJCuQPnoaATwSszulWjWIRp7OKNPQlpsMHYBLzeusssCyTb6Qc+/BtfF4l0X/ic+yjstXqacMaIaq5y6NcFHVdthfJRXIcVJSVWPk7ZDTsq7K0mNeWmHYaalQkRwWWSV10mfsU7uP6mEiB16Sg4D1dP1UVjWpgra5oobelHe0AoL3ALyB+YvjevqW0l/lMam2bGn0RyF/A4dGyoK82i7dLcr6XHh7vIb53JJi4qHZhJX5+iVMcErA8m1QOdqF2Semc+Jy+RztPjflN9AhBoi1HOneS+A7nyUPNeE3AQAbJCy9t6rukFxehpdPWarXtOyNlRkqRR4AZLVAEVc9HSBJi5CE9ee/vdcRcG2jS6JAKNtwzRetx8eQi+IvZPnFoC0i+WmXfho0zW0+TNkgQBTUQCqD1S/INUnZlSxNBaIz8atN/y7ttx3bFuzugEOav5Z2wV4CuCrt+BJ+GuWw+0nnX9U9HD7FeLS4ozRvkKn3heaU5jO+n0kaDb/xNI/Prxd+mQq0PDpNFwCDd9IDV/2J6C3S58ZBho+pYa3QcM3UsOX/CXu7dKnuYGGT6nhEmj4Rmr4kr+HXgMe3gAiPlcVb4CO142JV/hd+hrQ7gbwblDo9SbeFZ4PUAOW3QCaDQq93jy7wnMaakCqG8CqQaHXm1ZvzFH1p74cs7MzPOmCFMCapADCG36Q9gdv+C0vs6+zkMy+NhxYD6l7cGB9Yed23jFzELSC1LJFn70KEriOErj83KfS/kMQQcjNWdZxryCBa2oEl5w8Uj0SBEfrJuc2VI/7QN42OfRePcgDedvkyHD1kA7kDQKXywpc9rQfGIKW9QtaqlhhdgVilxC7hNjl0mKX4uFCgpeHELyE4CUELwu4rPxvG4LbFKKXy/JigQhC+HLZ4UuQQYhfLjl+CSIIAcxlBzDnJYPgcYUIZqXYDwQOQpiVAj0QOIhhVgrrQOAgiFlREDM8ZRGClxC8hOAlBC8heMk/WpF7pkJn1uBla6FHK8LhiBBanCG0yD+X3FHGBOoqWLvHqg91V/n4v9mA7nqdSL7sUwArELgGSFzlQrWuB9qdUFlEl75oolvSEgqTQq+6F6gbiiGwqOWyqBM4m37OBCo7DuBPwJ9KHk0/O4HaBwIFBGr1CJRN5jWBZUtQpHzYC1B2Rihb81PS5yJPDRCoGgnUenOjHhFL5JjAioAVASuqjBVtpZqLtDId3I2WP5R/i4xjK3gMewS1SNSO0NL9pneLYCtDPJIj8Nc69C47j15LqmzdeTUJfoyEUvhItywhfLSDR2OdmEcjIkUEsNpOTIqj+YmB9ackHtSJPeFaCNLAgx9GZQwrqdhMqeKlPGYZeGhjJoYYFEV3VXysc+B8juSRPZdYN0513xoIXOIomOyE87QZFr652KIEShgTE46dIXbjmuwVfwlGFy+J1lJMXB3gh3RV+04b/8vSe4+Gwul2IC+xbm+l5i62crKuB91JTUtsTeOTXoRV02XTDI1zBPzUaxZoiFBi4ezsLS+tGmuu79n0bQ7I2lnIkmX3SqqcbF3sfwZdE5KlmTHQa/zKvkB4g4wVuDa+8h4U57ZTS0l5zQ+nN1f7oxpcC7AsYbz0gTlPGpPjDYXxGwHzU0tiQAQWKn6t1Ra/CuUitcShWHR16qcLN3H0KeATJeTEthTDLiMo7Bea40ie61HQ+IdbOCG5n6go5pHIOId89gifN3M8VkdPw4ha5NLdou3+r1TDZfTgzd6riNNeC9t//eda+PpqZ+/VG9CQZWtIOb34ri9PL1LOlVzR9CpSHc4T44VpzWy9KqlSoUdpNo3aB42aUaPSzHGqTYe5+RaqXAX8l5wd4AWvYjlNKutCLdYH2IrWRHHKqcsMexGak74U8PfnePPzBXvBKjWvXm70ZsX+p87mZ89TpzGh9H10t17kQDDM76/FwF0uOKZ/TUjcNtaUOxbL9W/2Z7IfBFviPmuhPYp8kmL8j/1mvCRy0Qut2O+i6jH4r0GPaLAnpjYvthJ/8GH8wfFWWgfxPyKVFo7UeH+DviQm44dpRE652McEqSKEL9+cyvbwxpQt73GCy3IdhPPf+u5vx/cj37F6Hzpfxa3n/wMhOAi8d7EBAA== + H4sIAAAAAAAC/+1dbVPbSLb+zq9otHNvQYoQy2CwpjKpAkNms8skXMxmqnbIdQmpsXWRJUcvBEKxv/12t96llpFsS5bt8yEE1LLU6j7n9POc51j9vIWQMBhoxsR1bOFX9Bf5G6Fn9pO0GPIYk6PCWX9wefXlj/Prv5//qy/sBc26fIt12n5pmWPsjLBrR40qthVLmziaadBTogbnacIuqsqObJuupeCobaK7Q834pNL2CeeiXvtnv1ux27ITXsjPb3veI1n4u6tZmPNQwf2HlnwnG3J0cU3lHg4G4fd0wwO2bP/pjvYP9w/8TuzxbzeRDTJY2ZtNRtxbxQ/HbjT9Hrwh1XIH08gOI++Wnf3WfmuGZ3PkWx1n73adPJx9tnASZcMwHZkaEJ1F76aCrtlOOKdRV0jLravpzid6JXEvOhobEv6jknOwwfr0K3IsF8eOjzSVc1RTTKNn6qZFL2gNb+Wd1h5qiyL50ensIXE3fungqU+iZ0H/jU50bDmJLkQzaI9uTdlSBb/thf3/bcsf/LRbfbLJb6iPrQdNwegs+DTyhxWJ+xIxTPZJrGqO/5x3sm6zRxKGBnaYux0fHRx5R6jtXZum7mgTcrzFDmoOtmT/nmJH6nQksS2Jx9Ixa9U14545mjdvzBo4jqeYui5PbKymxpTcUlMvzWiSvcFPTeQP8nf7MHbgMeid//cT/TsYtaQtiq2jKIQke5c0ItZLw8EGtTHhvao9IEWXbfu3m2hm3o6wrGILOfjReauQU7F1I3y4Md7b5NIf+udXXz/1zn9Fv9jepLx/x46T9nfkch9ik05udqdhXe2Zxp02TDy9H0HvZFd37EwL7aVrO+aYtrwkml72ktcwiSGQ0fXi4LetnBO5MxDOwkHqmumZ4M1GOCNi7j3ZzHSlxLGkJYWHxyZzRGHkjPXkAJqTZHxYyjSme5j7yN7y9TUKecf7IomtiXMczWE+KqQOW7JBemB5z5QMSbH4QZ8mun/s7kkbl5URvtbG2HTp1QxX1/eSPkDC26ms3A8t0zXUeMRInvNV1l2c35x0M3acBcz2IQmUnUPvX2tf2k0Pp3fWwTEJp21pDx226GndnPM6pFU8bpPTOuxqx7vxGUhaUoHloEFueWdaY5nZsDlJdXEou0Oc7dxYfgzmRGyl/XFMbM9vTDfZI/MHbx6paY0IjBqZunpB4Z792ll/yNY9ZjNPjXS2iHPIiThHBQPO4fSAI7aTx6jfP8g6zwtygtGUkENM/g8/EDywgU71j62twRnEl+X0Cf9H7Ee7ewpPcR0zfYppaSQGBIuxMCJ//ySxTtYzvoFVV8FfcnrLooCuZD3UR/a289l0PpMhEVKt3/bS12HeYmdClgfv6DCENjPVJ2joSjx47kSO5clEM4bXXswT89qyz5YZggCesW4ix2SrQc5TpBazzAPkXpxE7mGBi7eTF88NYcTFz0gUuzSJ7dpZPxeoHbPGYDDJUmhgxSG4K3PeNe0Rx/Qnpu3caY/ZZchv+Egsrq/9ZJfvtP4rdY6F+Z9lx6d+lA3VHwQkvj51dxYLtOwxsoPqeA8mfH53wmk0ww8WHHO68t6TeMCJuHearifpwAGhAqLYJT8IuiELktjNrFt39N7ZZdwLOjh+Oe9qbbrESQeZ6/hhe2qkZaCfXNIdGxxYIVtD7BQYbvw48bpEAcGO7Y53NMII8I5GCQhj29h27IFDWI7+bOGJaZHASvDWL9/JHzfCHlnrHM1gYWvgQ6rf/kOa/d9vhJe/OuNvu7t0wFotcTc7a9Fi6BDkQi+iYTt7WhDRP8qKw0ZRzJxC7NBLMXBsw3bwJLWKTDOMcNmzOWMbILmeTkM2uvIGCX2lc4EFLn6zSezSyUlyMlB4QSLuOt2067AzirmOSZ9Q+K2k1wRxqpTjsA8F+RrFtRh+bTw+5SDKeQBqFu7OiU9VrGhjmWGxzNg0D7tOsEX5lWtozkIxbLcOEJvGLRWg2KMZUOwhoFhAsYBiAcUCip0FxVaAX/fI0jEhMQoPFGKx26Sts/8mhLXoHaoOM9eElWeCxBKBPpJE/+13psPjvqso2LbRFRkitGOYxtvO4yMKRtXe3UDALD8MXwfLsq7Jdi+As89Jt7mVLS7YYXnwC2wMHaaztDJtOO9jK5U/1ViAEjMHf7dkVfNS6a3FA7pkjvFVRDfSVBUbfc9HeYPuKSXHSViHh5gxn1Q/iTUMjRP7Oq30xU95GOa0BASN30rl0PPxxHma0v5vbJk5zWTFz2vRjJwWSxuOiAdzZNj0wpGGYTSm5lw0F829pIEztnlaC234oanOKGNaOVg7g2Uyq33AkeQhn60WEY4m9A7Url3ahU62jW9cZMJVbGG2LN7pZiqMeivHl4S7pTGFgvMCCQnLyj33rjTNMsHqhQdGsu2lV/SdEVlVTUJNxoPvrmw4mo53WvuUyOesuwPV9STtwZgEBM3GBGCq9uDWVe6xM+NqLLKl/vYJ7eiYrfkE1bZ2kWmh2XtXQcfmxgt8llQATXhR62N4t8tOSygOOfbmMQWp0aYgrZop+LUxnPg6oyVI0yzhdKGWIDXaEqSNtwRpiiX0ZqIhqUWLPs9Hj3lnkga07QoP/XQT54P9kXbn8D/Jz/ef+bOWIjBhidVzGtTIVqZGKmgj08zRvRmiGQSUSDNU7UFTXXlKJUqsCDJWaeghWPlR4wBgz9R4KedY+QsdnzS79xNInE/lw7cQok0B90/yY5HkWGTFHNsNKliznaON5vBUtjHXaD0wy/2Yh2a5TfxEx+vxLPYMI2oBDXkOTjY01x2f+FbF6EoeT6FtF/ghfLIt3o0aqCH9rS1JyuHRrJrR39TDQ/lAXr8iplMoYuJlC7rl9Z8u6D+g/4D+A/oP6D8L0X8cZUJIloK1B6wObp8cPEWVQUWpVWMYlbBwVSdHybnuXRLW4w0jOqXDCKINVOBDBf5mVeB3p2NX6RiwK2BXwK6AXQG71lGBH4OrG16HT78mjS2ow4c6fKjDhzr8QnX4r2HZLmBZwLKAZQHLApattg6/JIpdYDX+bPh5xWvyfagMNflQk7/GNfndOWvyJQlq8qEmH2rya6/JL70m11uZP2f3oD6/oWYhraZZbHitfuVWIYFVrEXdfkofgLp9qNuHun2o24e6fSh9Klq3/4pelM6ug2AEghEIRiAYgWA0R+G+TQISFO3PVbTfp99XhoL92Ge+RfvVBAP1OzawFVvowtGxiBdw90epYccLMW/Hi8O6drzoXXw6/3yN/vxy9c+LLydn/Q3f8KI9HQEedpq54UWZWVy5/S5Ajn1Njm0tSI494rhORnrNc53OnHJsu1NMjl2A1sqBTrNoqassliLZRj+ptAyi6SJF00Ll/T6n8oUF19Kcp8HE1DXliUT8seu4sj5wdLtgpVQGLXvha/DDJKzDlFXvA5byQ8+2DSjNZGMXnGXYUZ0VFSlSH9hDuVfYQ4mqrVq+g5CG3c/P6f6hl5f97NGo16QdkRjznOw8Pbpz46rdA5X8VMXW+PqivyvM8h2I9kx62px2tJ02pNcZFJ2+LPGaZktolYxp0QRvgZY2VY0taFZNFOQ+GYpJFs9hIMnZhKOiPhsedGKo5LA3DqhH16WSMh13Pd44nS6bnHLkFI1eoJSX/hJ1xSKYwKlTAC2vPi0PeFddZbCFeRe3DnZu4nW0TOJVQY0rFLECH5t5qwhUnJ2hWb/IsjxcPdtWFosdk+U9/Iqzh6Uy0oVazTa40oJIO/jSTL609pz7te/+RVwcCmSXWyCb92KHRdBSgcBBUQB+Dfy66CtRbMXSJkF94MaS725BzVMU5+XejdoXqh56Hv8KKtB3oO9L3ReqgdRWnEkvq3PPqrUatAYRmVRPGY3J7f3LC7rstJabFKh+s67N8k8J/HOt/FMq6Z+n9fhnB/xzRv/sgH+ulX92yvlnrx7/lMA/Z/RPCfxzrfxTKuefZ7X4Z038cxsI6GaM2moz0CnueL5OdHMb+Ca4Y/MJ5xR3/LhO7HIb6CW4Y/P55RR3/H2dyOQ2sElwx+bTySnu+Pd1+q5V+PpDqPPayBchQm0X1HbZUL4VK9/qVlK+1W14+RbUZ0F9VoX1WaVzvN6L3kB5gcqiWSxLAstaectaflFM6dQbmBaUc8xhWRJY1hoErSVXItSJtCArub4Sep24CuxofbXfOkEU2NH6ipZ1QiawI1DbqlHb6Av0QWlritKmYuU2sxsCCG4guIHgVqPgJh5VobgdgeIGihsobkXyQP7r0iDHCJLbolNDYFqguVWVLgLbAtGtogwSmBaoblWpbvPbFqQnQXYDQwLdrTlACgwJhDcwJFDemqO8hXuGgeIGihsobqC4wfZgMcmN+4ry4zkVt26r0u3BYIMv0MPm0MP4G+U6yoTgVgVrD1j1cesqbW41C2pdhy1ym7nr8oKMaXuzrKkmg1nXjZxOqZ2hK9/s0B25EwrLDq97l6gXGhqQn+WSn1PYFnnBvCf7HEB7gPaU2xV5ft7TBt4DvGf1eI9NxjUBU0vwGx6iBZS6vrSmhK1sg7EApVkApekTk0OOCWQGyAyQmdrIzFbqdpFX9nSKzNGffiCKZj80f4s8xlZsAoLeCYqp6/LEzvghF9YLKfjI4Hz7MHYgpWAwFB+EtXDuPWwutkJwLkxkA+tJG08QNrKkOR4MF94Tn0eKLts2WbMo7bo1ZUt9O8IygYTIwY/OW4pKsXUjfLgx3pN4bHzon199/dQ7R39+ufrnxZeTs/77d+w4aX9HLvehmVRsKrE64BCr+ExM1ZMOcu/pfT9LKsIQglA4csYpYmCyzYc5Ha9zGtM9zH3kIjwl9LTUYUs2SA98hpghXYHz0acRICkxd1Ki1QQt9mjOlIRUrRRbwTcbVzljgWQb/aRbX0PmovrMhf96j/I5i9cZZ4ynxk5OfTRBR1XbYXSUd0KKkpJTPUraCSkp71NJZsyrOQzLLBVigsviqrxO+oR1ev/JSYjEoefkY6BmZj4K29pMkm2uuaFX7Q2tsMFVUD2weGtc19RK+oU0Ni2NPYvGKKR36MRQUVhE26OrXcmEC3eV37iMS9pUPDSTOLrApIwJSRmorAU21zSJeW46Jy6TztHmf1N6A/o06NMLZ3mvYO58kLzQatwE/tsmbZ39N01D4nQfumaMVrOGZW2YyFIZcAWWtQ2OWPVwgSdW4Ylrz/77rqJg20ZXxIDRjmEabzuPj2ESxN7NywpA0cVyiy58tOka2uIJskCAqSgA0wemX4zpkyiqWBpT4bOi9Ya/0pb7/dr2vFmAw4a/0baCRAG88BYSCX/Ns6v9tJ2Piu5qv0JUWpxT6yu03X2hMYXhbN77geZLTyz9vcM7pfekAg+fzcMl8PCN9PBlv/15p/RuYeDhM3p4Bzx8Iz18yS/h3im9jxt4+IweLoGHb6SHL/lV6A3g4dtAxBfq4tvg401j4jW+kr4BtHsbeDc49HoT7xq3BmgAy94Gmg0Ovd48u8YtGhpAqreBVYNDrzet3phN6s98O2bbZnjWBRWADakAhO/3QdUffL9vaYV93UoK+zqwVT1U7sFW9YVz23kbzIFmBZVlVe+6Cha4jha4UhvfgwlCaU4FpTml811ggWsXBJdcO1I/EoQ86yaXNtSP+8DeNll5rx/kgb1tsjBcP6QDewPdclm6ZV/7iUGzbJ5mqWKFxRWQLkG6BOlyWdKleFSJdnkE2iVol6BdFshY+S82hKwpiJfLSmKBCYJ6uWz1EmwQ5Msly5dggqBfLlu/XJQNQsIVBMxasR8YHCiYtQI9MDiQMGuFdWBwoGHWpGGGOyyCdgnaJWiXoF2CdsndVpG7oUJ3Xu2yXem2irAxIiiLcyiL/C3JHWVCkK6CtQes+kh3lbf+mw/nrtdm5MveAbAGg9sGi6vdqNZ1M7tTaovoyjdNdEfuhMKS0OveJeqFZggkarkk6hS2pV8wf8o+B9AnoE/ldqWfnz8dAH8C/rR6/Mkm45qAsiUYUj7qBSQ7J5Jt+AbpC7GnbTCoBhnUelOjPjFL5JhAioAUASmqjRRtpW4XeWVa2o2mP7R/izzHVnAZdgkakWgcoa0HLe8jgq2M8FiOwF/7yDvsPHl3UmXr3juT4MfIKIVPdMkSwks7eDzRSXg0IlJEAKvtxKw4Gp8YWH9O4kGdxBNuhCA3ePRFVMawko7NnCreyiOWQYI2FmJIQFF0V8UnOgfO51geWXNJdOOc7keDOI2MtZpsd/N0GBa+u9iiBEqY8Oim1/w1eLp4SzSXYuLoED+mT7Xvtcm/LL3/ZCicbgf2Euv2VmrsYjMn63rQndSwxOY0PugFSDWdNc3QOLu/zzxlgYMIJebNzn7ktUljtxt4IX2Hg7F2K5mx7FJJfZNNi/0/QdeEZGvmGegx/sm+PXgPGWtwbXztXShObWc2kvKOHw5vrvNHZ3ADwJJs8cqH5TxjTD5uaIvfCZSf2RADGlCp9bVX2/rqM4vUDIdW0dNpki5cwdHngEyUMBPbUgy7jJ2wX2h5I7muxz/jr2zhyHG/UEvMY5BxAvnisT1vHHmUjm6DEd2Ry3WL3vc/pW5cxg3e7b+JCO2NsPPX/94I397s7r95Bw6yZAcp5xY/9OW5RSqxkmuZ3onUhfOsuDKnma9XJT0qzCbN51AH4FDzOVSaNM605LAMX6W+VSB1yYn/ryQUyzlS2expsT7AQrQeflPOW+ZYidCC3KVApj8nj59v1xV71KJ6udFLFfufpplfvBydxozSz87deZqBYJg/3opBolxwTP+YkPjYRFPumYrrf9gfyUEgs8Sz1UJnHGUjxfgfB614S5ScF9qx30XVI+/fgh5RmSfmNq/eJX7ho/iF43dpH8b/iFxaOFbj/Q36khiMn6YRpeNi7xCkjhB+6eZMtke3pmx5lxNcVuUgXPxj4P7j5GHsp1QfwrSruPXy/5MKGTNasQEA istio-workload.json.gz: - H4sIAAAAAAAC/+1da3PbOLL97l8Bc7O35FzHEfWwralMqmI7mUndTOxrOZmqGWdVNAlJXFOkwodjx+X97QuAbxKUST1ISuoP44kIEASB7sY53U3gcQchYTBQ9aljW8Iv6G/yG6FH9peU6NIEk6vCWX9wcXn+x/ur399/6Qv7frEm3WCNll+YxgTbY+xYYaGCLdlUp7Zq6LRKWGA/TFmjimRLluGYMg7LppozUvWPCi2fchp1yz973Yo8llV4In+/7buvZOLvjmpizkv5zx+Z0lDSpbBxVeFe9gfht2TBHTYt7+0ODzoHba8T+/zHTSWdDFb6YdMx91HRy5EHzX4Gb0jVzMHU08PIe2T3oHnQnOPdbOlGw+mnXcUvp98tmERJ1w1bogJEZ9F9qKCplh3MadgVUnLjqJr9kbYk7odXI0NCHvGCM0KkEtZZp35BtungyPWxqnCuqrKhnxqaYdIWzdGN1Gjuo5Yokj/d7j4S96JN+6/9LnwZ9D/onYZN24rWC6fQGt8YkqkIXtkT+/+3HW/0k3r10SL/Qn8a5q1mSAo6829H3sAi8aBHRJPdihXV9l50KGkWeydhpGObKdzRYbvpXqHSd2UYmq1OyXX3ompjU/IeKnbbYrvT7RyKncMuK9VU/ZapmjtzTB44qicbmiZNLawkBpU8UlUujHCa3dFPTOUP8rvViVy493vn/X6gv/1hi0tjrxvakHjn4lLEOqnbWKdCJrxR1Dska5Jl/XodzsyrMZYUbCIb39uvZFIVm9fC22v9jUWafvvn+eX/fTp/d/YLevHDm5WDF1QKSKmM37xmlUjl16TttxEJIE8eqlhTTg19qI5iI+HZ06HkaLaVKqFddizbmNCSp1jR0368DYMIBRlp1yp+28moyJ2NYEbaiTaTs8KbmWB2xMxnslk67sWuxaUquDwxmFYKY3uixQfQmMatRfVzmuxu5vu7K9vX0BoeHYjE7Mbq2KrNlFdIXDYlnfTAdF8wbqwiloW+Wvj8yNPj0i/JY3ylTrDh0NZ0R9P249pBDN+JJN+OTMPRlagpidf5KmkOzi6OKyC7zkxpq0NMaLfj/tc86O0lh9Ot1T4ihrbV20edJq12nFGvS0rFoxap1mWtHe1FZyAuVnlWihop6dAwJxKTaGNqxbs4kpwRTnduIt37kyI2k9o5IcLnFSaLrLHxgzeRVLbGBGKNDU35RKGg9VytPyTzFrOpp1I6n/3pcOzPcU7z05ltfsRW/Bq1AneSxlODDNM0wwARmf/DswR3bKAT/WOrrl+DKLOUrPBvIj/q8CGo4thGsophqsQI+Mu0MCa/fxLLJ2kp5cCKI+PzjN4yM6DJaRX1UL9lfzbsz2RIhETpt/1kO0xbrJTNcqEfHYZAZmbqBLVdsRfPnMiJNJ2q+ujKNXpiVln63VJD4CM31k1kG2xtyHiLxNKWeoHMxonpHuVovBVvPNOGERU/I3brwiCya6X1XKByzAr9wSQLo45lmyCyVL0r2iOO6E8Nyx6q9+l1yCv4QCSur/5kzXeb/0zUMTH/XnZ95q1sqP4g8PH5qRuazNCy10gPqu2+mPD59TtOoRHcmHPM6dJ7S+wBx+IOVU2LM4U2YQmieEz+EKxDViTxOLVwDemz0+u4a3RwtDm3tRZd43rtVDue2Z5paRkdIE06E52DKyRzhO0cw43vp26XKCJoWM6koRKugBsq5SaMiWPLtgY2IUDao4mnhkkMK0FfL76TH9fCPlnrbFVnZmvg46tBAK9+/Q+pGfzKqM4q+T+uhae/u5Nve3t0hJtNcS89zeHqaROsM7AwsZxWupq/BHyQZJsNu5iqQgTX9VdwhMmy8TSx7MySpGCdtDiT4WO/j7pskMV6hC7dcUVf6fQlQIqP+SxSUSOVpLhtce1KVNuOk9rGauTTNoO+o/BrQUXzTVshXWM3+e4f2TEZ5q09puWg0EVAbRoiL4ppFSyrE4nht9Tg1A/vTrFJGZqjq/ZSce9xGcA3iXVWgHyP50C+HUC+gHwB+QLyBeQ7D/KtA+bdJ2vNlBg1PJCJiO+Swu7BywAKo9eoRsC8JEA+F+7uEXTV69H/DrrPYfC+I8vYstAlGVXU0A39Vff+HvkTYe1tISqX7kbPI3JJUyXr1MfMj3HVvJFMLqBi3vpPWB/ZLDLUTJXhrNvWy7GrMisopi7+ZkqK6jr5m8tHjeJhIdg4VhUF631XTXmj7gZ0juLYEY8w41eJfhJxGOnvrKtkcDJa5W6UUeLTQH4pDeG+n0zthxnlf2HTyCgmsCKrRNUzSkx1NCYqzAkdJ1enJNajdjij0UzI+JRE59jiRYFowQ9Vsccp0coA9CnAlIIUPhGTRnxOnCekNaVPoHLt0C5002V84SITrmATs7V3qBkJO+ouHucxdUsCFxlnWRJil+Vb7lOpO2eKlU8u4kmXF4YNjTFZiQ3CfyaD746k26qGG80D6i7IWKsHiuNG4QcTYhBUCxMUq1iDG0e+xXbeFTyJG1CxZf7pb5EBipsH1NAwQxYEbDf3kGGi+d+nFq+yMCzh070coMW1jB+Cp110m0J+ZLO/iLj1Nkzcepsvbl5eEmedmFPaerOk7WSp0tbbMGnrgbQVlrbeDGk7nYu1JRZ4+j4fXFdIyotDyy7xyPP/cW7sj9Whzb/T439+6OXMm+AEzQtS5x6TyE8yU7lvfhmRCE7WAoN9A584qrqi3qmKI81IJIqkt0ZySF2YL92rHJbgSiXP+R/JXqIDk/SzeK48zl3ZGDfAsTMY0IN0n8dNGYovR2j93OR052ihMTqRLMyVVhfxc29zIT+3iO9yet5YRt5hTCWgJu/B8Utn6uEDX6oYp8sic7TsE74L3myH96AahvP+0er15M7hvOG7fyidjtSWNjAH7WTrc9DiyWMzktCOn8mBhVAchOIgFAehOAjFLSUUZ8tTwmx0e3DzYOMZsa6C7IpfPxXtcinY/yJer0wsY/UOKxX2rDacUFh6GC8jdHd1eoH6pHfYRFemNByqMsToAGYDzF5rmJ26mAdndwFnA84GnA04G3B2qTjbXWlzAdlE1dWg62X3Zzsx9alGVyPA1PF7voW7nviD9RvWsRlZpIPRMYkGc3fZKGHXBDFj14R2WbsmfPx8cv7l8xnyv7Tvb/tGCe3Z6LXTredOCYXmce02R4C01WfTVptLSls9XCTEIh4umLfa6ubLW11CUioH+s2TdLrOWaVIstBPmoML2aXLzC7N9Tm2xwm9LCXHVO2HwdTQVPmBGP2JYzuSNrA1a0Vu6H2UA5GzeyxT/qE9D9dJNd0KP3qhuU+JG2a0sI9in9CU8vl4ErQ/Pib7h56eDtJXw16TckQs02O88/Rq49pRjtsK+auIzcnVp/6eMM/n66250v8WlL5dEL+FxW/ZhHKJsjkz3TSnINYxMzC5O4OFTh5Qnw0Peqcr5LI7DuiUrn8F0wa56/7W5Q2mnXi2lKDsS0wtTG6uteKkPIGTiA25heXlFgLDK+3DxMOFonuLU7zDKineCj47hO8KgfnNvR0BqpgH8vcuqA6Lz7dfQtWjWN1wrTlHqZQpL1XOdkFd12cYQV/n09eN9x48t69M6FWATw+r/fQwazPCZRBsgcBaUQBPAXgKcu/jyT9zZ9vcCHn3g2+1FnUi1Gp/o3L8DNHtjcAPAX6ISvc3WpRxL7jHxnIQvDhXsLDMvZe2fJhrxJYSPWVcKbP3T0/ootus1rux+k2nwAaUsiEW2IC1tQG9gjbgpBwb0AUbUJoN6IIN2HIb0C1mA07LsQE9sAGl2YAe2IAttwG9YjbgrBQbUJI/YBeMQCkOgV2wAnX3CMxQ+febRP9B5cvh/6DytXcAzFD5D5vE9kHly6H7oPK15/szVP63TSL3oPLlsHtQ+drT+xkq//smfVYZnLwAiZBbeQYDJD9C8uMWfyaZ99TvxfMbj2ue3wgJjJDAuMIExsIudHezSQieQSLd0hPpCvt2QRY3VRarz+gq7HUEYYTMohVlFhV2h4EsbrBhrDjFpUzECA5ZyLSoBh+C5EHAvxowCJIHcedqoB9IHoQ/6xL+pAelQOizLqFPBcvMrkAEFCKgEAGtLAIqHq4iBHoIIVAIgUIINI9Dy9sTEtyrEAOtPgYKwghB0PoEQUEaIQpamygoCCOEQesTBl1cGsEzC3HQijAiiB4EQisChCB6EAmtCP6B6EEotKJQaHCAJoRAIQQKIVAIgcJZmdEYKPeUi+MFQ6DHzZUelQmHXUKAcoEAJf+geVueEqgrY/UOKx7U3exjG+cBuptwxHzVpzeuVPx2Qf5my19JIrapBw6eUMlEl56goiF5EgqSTa9OL9BpIJrAsKplWCdTIFfLJVfp9wBuBdwqm1ulLi6HXLWAXAG5Wj9yZZFxjSHbAiSKB4IB2AJ3mku6dkG8gDdVwpv6REiRbQBjAsYEjKk0xrSTeFz0M9kbuqCgPz1LFE5/IP8meY+dyAz43RNkQ9OkqZVSRC53EBIYlXGGVidyIRGLcanCTmLyXQLQCxoSppKOtbiMx1ghWQRtF+oLb4jOI1mTLIuscpTb3RiSqbwaY4nATmTje/sVRb7YvBbeXutviD3W355/uTo5//L5DPXfX379ePq+/+Y1u07KX5Pm3taT7s0kb20OeYtOxMzAmNjJfKj76V8vDw3xbeHYniTYhzG1vUUm0fMy5zHZw8xXzkOGAlVLXDYlnfTAo6EpZucrH30bAVwfi7s+mnUIK4tHC3o+jlYbVl7BZ7Pr7BhBkoV+YtMAB0kJDhJvu5k5XCOzWeOz6ZFRdhqhwW7VJK8lg3ynei0rpLd3MiWk3YCQcurSVr38UZkIV1VElNMzQkYRsR2P8Q6iero4csvH7tYLyAoyDIpIz6Z6MM4de2REthSyaC7tWTgw6B0hUkGy7SldSAq6M7gL6Nb5M5IS4gKF2NUlujwMcHlABi5QpdpFiRfnSmKVXIkW/0W5A8SYIca8dAr1DDjmotRKGFUMG+6Swu7ByyVAZnq249qMwcIvuwZEoFLCuFxJ2AV1WDGJ3nx92HgK3HdkGVsWuiQChxq6ob/q3t8HngBrL0GNIa5fbVzfA1yOri6fJQoEm4kC0F2guznpLjGZsqmyQG86LrrlG/JyP0ZtL8qEWzXfj3cFZBm26wUynZNMNwofQKV4R6kPJsQgqAQ6G7rC2xdmGcTzuQzavBm5fKAtzg5M0U1dCLcQm83mHjJMNP9IbfggrQFFr3wn4kbh47ZAzwqPFOhZ5XpW9R7LjcKHi4GeFR4p0LPq9azi7aMbhY9SAz0rPFKgZ9XrWcUbY1fLz3aBoG3EKK0HQytx0+9q6dgu8DFQq9IIWYkbmlfLvnaBfoFalca/StysvVqytQtsC9SqNLq1qRvRJz+gQWee3EG2UK2yheCDGMgQgg9iqksCOl5JElAbDuWGLB84lDu3ZzTrsCeIO2xJvkphpx9IzJZnXhT2Z4HEbHkOQWFXDUjMlkfDS8Ux4NFbu7BuqagF5GPt4pOlYhSQj7ULtJWKSEA+IGJUKGLkH2EM0SI4xxiCRhA0gqCRHzQSD1cSNepA1AiiRhA1yuFt8XZ2AgcdhI3yOmBAZCBuVNAnAyIDgaOCbhoQmVqIzFqFjpYgM+Db2+TYEQgIBI9WDVRAQDY5egQCAuGjFYePgsN4IGwEYSMIG0HYCA7fScWNuDsO9xYNGx2u9OwdOD0HgjoLBHX4B0za8pTAIt1PaCp8DCkXu5Z6asZMPFqzc0dreMJoQQHYBQmAI1LSnOOECg/qEzlCho6CBLar0wt0GsgSkI9qyccJHPq5ZN6Rfg+gHUA7Cp75uTjvOAbeAbxj/XiHiWWs3mElBj3X6gRL4B77dRCCXZAC4B+Ef1x6soSG5EnAQoCFAAspjYXsJB4X+5zthpp/1HfNUDj9gfyb5D12/GZYE9QuUWtCS9tN9xbBksd4IoVgq3XoXrYf3Ccpknnr1iR4LRRK4SNdbYSgaRtPphqxjHpIQghAtOyIFIfjEwHHj3H8pRF7wrUQ5AH3XrSPMZq4YjOlipbymJwfCY+YGGJQZM1R8DuNA58zJI+sisS6cap71kDgEjXBYMdUJo2x8N3B5gM7WJWYV2yPsRPVZLf4q/920ZJwLsXY1RG+T1a1btXpF1PrP+gyp9u+vES6vZMYu8jMSZrmdycxLJE5jQ56HhZLp03VVc45nnPPmfDZxyNCganTeTc9N3PsHzQNhTTuwjHOsdlpLJPGUWzL3bABnhc5VzN7BeTk9cHLCJC7Fhp//+ta+PZy7+Dl6/wilF67qbFgcmL9vz9MQrw0NZ70Gr+yJ6DugEcKHAtfuQ1Fue1aS+2f3mwWEdofnHsKyWym0D7OFrQk7n/Kls6EaCcek5tZeE9I1N9jrTdma87K3oX7sIXfqKAOB0xrMQ0W11uDi2OJjCSuCJ4Ia3AxRVWG4tJj8zxDEX/fwE58NwnzLGAj2LMGLl9p+N6DvZVgEE/8WrCA5JSLxBQHYvFRd2mBv46gmTAoQ1AsU9atpSOgqAPqBZXFpCMoYsMynUXzGtRCIKsuPS24BiwLxoEWLlsLi+neD21dde85HzEzLJmYpzaKudhrVIPc2qC0CyptJKPf96oV0Fs3crFSxc3MBJv/25Sn7BBLMW0st28FVYwXbVpM2zr10Db2fxpGeHJ9sCqTRc/7OnQjQ4Ju/Hgl+oEQwTa8a0Lstqkq37KouHezN6gDP04WjUYI3UnobRajP9rNaEkYfBFakX+Lisukvvk9onG6iLI8+5Row4fRhqNPaXWiP8IVWDhSov31+xIbjJ+GHrpbQ58783iHyPpMssY3hmS67QkOSxsRvtxY/b+uzlTPZ34X+NXFnaf/AluDM8vxjQEA + H4sIAAAAAAAC/+1dbXPbNhL+7l8B83o3cs5xRL3YVifNTGwnbebS2Gc56UzrnIYmIYlnilT44tjx+H77AeA7CcqkXkhK2g91IwIEQWB38Ty7S+BxByFhMFD1qWNbws/oL/IboUf2l5To0gSTq8JZf3Bxef77u6vf3n3uC/t+sSbdYI2WX5jGBNtj7FhhoYIt2VSntmrotEpYYD9MWaOKZEuW4ZgyDsummjNS9Q8KLZ9yGnXLP3ndijyWVXgif7/uu69k4m+OamLOS/nPH5nSUNKlsHFV4V72B+HXZMEdNi3v7Q4POgdtrxP7/MdNJZ0MVvph0zH3UdHLkQfNfgZvSNXMwdTTw8h7ZPegedCc491s6UbD6addxS+n3y2YREnXDVuiAkRn0X2ooKmWHcxp2BVScuOomv2BtiTuh1cjQ8J/VVIH66xPPyPbdHDk+lhVOFdV2dBPDc0waYPm6EZqNPdRSxTJn253H4l70ab9t34bvgv6B3qrYdOOdSGcQWt8Y0imInhlT+z/X3e8wU+q1QeL/Av9YZi3miEp6My/HXnjisSDHpFMditWVNt70aGkWeydhJGObaZvR4ftpnuFCt+VYWi2OiXX3YuqjU3Je6jYbYvtTrdzKHYOu6xUU/VbpmnuxDFx4GiebGiaNLWwkhhU8khVuTDCWXZHPzGT38nvVidy4d7vnff7gf72hy0ujL1uaELinYsLEeukbmOdypjwWlHvkKxJlvXLdTgzL8dYUrCJbHxvv5RJVWxeC2+u9dcWafrNH+eX//p4/vbsZ/TTd29WDn6iUkBKZfz6FatEKr8ibb+JSAB58lDFmnJq6EN1FBsJz5wOJUezrVQJ7bJj2caEljzFip72420YRCjISLtG8etORkXubAQz0k60mZwV3swEsyNmPpPN0nEvdi0uVcHlicG0UhjbEy0+gMY0biyqn9NkdzPf313YvoTG8OhAJFY3VsdWbaa8QuKyKemkB6b7gnFjFbEs9NXC50eeHpd+SR7jK3WCDYe2pjuath/XDmL4TiT5dmQajq5ETUm8zhdJc3B2cVwB2XVmSlsdYkK7Hfe/5kFvLzmcbq32ETG0rd4+6jRpteOMel1SKh61SLUua+1oLzoDcbHKsVDUSEeHhjmRmEAb00QXR5IzwunOTaR7f07EZlI5J0T2vMJkkTU2vvPmkYrWmACssaEpHykQtJ6r9btk3mI281RI5zM/HY75Oc5pfTqzrY/Yil+jRuBO0nhakGGZZtgfIvK/e4bgjg10on9s0fVrEF2WkhX+S+RHHT4EVRzbSFYxTJXYAH+VFsbk9w9i+CQtpRtYcWR8ntFbZgU0Oa2hHua37E+G/YkMiZAo/bqfbIdpi5UyWS7wo8MQyMxMnaCmK/bimRM5kaZTVR9duTZPzCpLv1tqCHzgxrqJbIMtDRlvkVjZUi+Q2Tix3KMcjbfijWeaMKLiZ8SKXRhEdq20ngtUjlmhP5hkXdSxbBNAlqp3RXvEEf2pYdlD9T69DHkF74nE9dUfrPlu8++JOibm38uuz7yVDdXvBD0+P3VDkxla9hrpQbXdFxM+vXrLKTSCG3OOOV15b4k94FjcoappcaLQJiRBFI/JHwJ1yIIkHqfWrSF9dnoZd40OjjbnttaiS1yvnWrHM9szLS1jA6RJZ6JzYIVkjrCdY7jx/dTtEgUEDcuZNFRCFXBDpdSE8XBs2dbAJvxHezTx1DCJYSXg66dv5Me1sE/WOlvVmdka+PBqEKCrX/5Haga/MqqzSv6Pa+Hpr+7k694eHeFmU9xLT3O4etoE6gwsTCynla7mLwHvJdlmwy6mqhDBdb0VHGGybDxNLDuzJClYJy3OZPjQ74MuG2SxHqFLd1zRFzp9WOBCPotU1EglKW5bXLsS1bbjpLaxGvm0zaDvKPxSUNF801ZI19hNvvNHdkwGeWsPaTkgdBFMm0bIC0JaBcvqRGLwLTU29YO7U2xSfuboqr1U2HtcBu5NQp0VAN/jOYBvB4AvAF8AvgB8AfjOA3zrAHn3yVozJUYND2Qi4ruksHvwIkDC6BWqES4vCY/PBbt7BFz1evS/g+5zELzvyDK2LHRJRhU1dEN/2b2/R/5EWHtbCMqlu9HzgFzSVMk69SHzY1w1bySTC6iYr/4j1kc2iws1U2U467a1cuuqzAiKqYu/mpKiuh7+5vJBo3hYCDWOVUXBet/VUt6gu9Gcozh0xCPM2FWin0QaRvpb6yoZmYxWuRtllPgkkF9K47fvJlP7YUb5n9g0MooJqsgqUfWMElMdjYkGc+LGycUpCfWoGc5oNBMxPiXBObZ4ISBa8F1V7HFKtDLwfAovpRCFz8OkEZ8R54lnTekTqFw7tAvddBlfuMiEK9jEbOkdakbCjLprx3lM3ZK4RcZZhoSYZfmW+1TqzJli5aMLeNLlhVFDY0wWYoPQn8ngmyPptqrhRvOAOgsyluqB4rgh+MGEGATVwgTEKtbgxpFvsZ13AU/CBlRslX/6S2R44uYBNTTMgAXB2s09ZJho/vepxassjEr4bC8HZnEt4/vgaRfdppAf2OwvIm69DRO33uaLm5eUxFkn5pS23ixpO1mqtPU2TNp6IG2Fpa03Q9pO5yJtiQWevs971xOScuLQsks88tx/nBv7Y3Vo8+/06J8feDnzJjjB8oK8ucck8pPMVOKbX0YkgpOzwGDfwOeNqq6od6riSDOyiCKprZH8URfmS/cqhyW4Usnz/UdSl+jAJN0sniePc1c2xg1w7AwG9CDd5/FShuLLEVo/LzndOVpojE4kC3Ol1UX83NtcyM8t4nucnjeWkXcYUwmoyXtw3NKZevjAlyrG6bLIHC37iO+CN9vhPaiGwby/tXo9uXM4b/Dub0qnI7WlzUtAO9n6BLR45tiMDLTjZ/JfIRAHgTgIxEEgDgJxSwnE2fKUEBvdHtw82HhGpKsgueLXT8W6XAb2T8TrlYllrN5hpcKe1YYSCksP4mUE7q5OL1Cf9A6b6MqUhkNVhggdoGxA2euMslMX88DsLsBsgNkAswFmA8wuFWa7624uHJuouhpwvez+bCekPtXoagSQOn7P13C/E3+wfsU6NiOLdDA6JtFg7v4aJWyYIGZsmNAua8OED59Ozj9/OkP+R/b9bd8joT0bvXa69dwkodA8rt2+CJCz+lzOanNJOauHiwRYxMMFk1Zb3XxJq0vISOUgv3kyTtc5pRRJFvpBE3AhtXSZqaW5vsT2KKGXouSYqv0wmBqaKj8Qmz9xbEfSBrZmrcgJvY9yAHJ2j2XK37Xn0TqpplvhBy808Slxw4wW9lHs85lSvhxPYvbHx2T/0NPTQfpq2GtSjohleox3nl5tXDvKcVshfxWxObn62N8T5vlyvTVX7t+C0rcL4rew+C2bTy5RNmfmmuYUxDqmBSY3ZrDQyQPqs+FBb3WFXHbHAZ3S9a9gziB33d+6pMG0D8+WEox9iXmFyX21VpyRJ3CysCGxsLzEQiB4ZX2UeLhQbG9xhndYJcNbwSeH8E0hEL+5dyJAFdNA/rYF1UHx+bZKqHoUqxuuNacolRLlpcrZLqjr+gwj6Ot8+rrxzoPntpQJnQrw2WG1nx1m7UO4DH4tEFgrCuAoAEdB3h08+UftbJsXIe9G8K3Woj6EWm1tVI6bIbqzEbghwA1R6dZGixLuBbfXWA6AF+cKFZa57dKWD3ONyFKip4wqZfb+6QlddJvVOjdWv98U2IBS9sICG7C2NqBX0AaclGMDumADSrMBXbABW24DusVswGk5NqAHNqA0G9ADG7DlNqBXzAaclWIDSvIH7IIRKMUhsAtWoO4egRkq/26T6D+ofDn8H1S+9g6AGSr/fpPYPqh8OXQfVL72fH+Gyv+6SeQeVL4cdg8qX3t6P0Plf9ukjyqDQxcgD3Irj1+A3EfIfbQgvTGS3ni8kvTG45qnN0L+IuQvrjB/sbAH3d1oEmJnkEe39Dy6wq5dkMVNlcXqE7oKOx1BGCGxaEWJRYW9YSCLG2wYK85wKRMxgj8WEi2qwYcgeRDvrwYMguRB2Lka6AeSB9HPukQ/6SEpEPmsS+RTwTKzKxAAhQAoBECrCoCKh6uIgB5CBBQioBABzePP8jaEBO8qhECrD4GCMEIMtD4xUJBGCILWJggKwghR0PpEQReXRnDMQhi0IowIogdx0IoAIYgeBEIrgn8gehAJrSgSGhyeCRFQiIBCBBQioHBOZiQEyj3i4njBCOhxc6XHZMJBlxCfXCA+yT9j3panBOnKWL3Diod0N/vIxnlw7iacLl/1yY0rFb9dkL/Z8leSiG3qYYMnVDLRpSeoaEiehIJU06vTC3QaiCYQrGoJ1skUuNVyuVX6PYBaAbXKpFapi8vhVi3gVsCt1o9bWWRcY8C2AIfiYWDAtUCd5pKuXRAvoE2V0KY+EVJkG0CYgDABYSqNMO0kHhf9RvaGLijoD88ShdMfyL9J3mMnMgN+9wTZ0DRpaqUUkcsdhARGZZyh1YlcSIRiXKqwk5h8lwD0goaEqaRjLS7jMVJIFkHbhfrCa6LzSNYkyyKrHKV2N4ZkKi/HWCKwE9n43n5JkS82r4U31/prYo/1N+efr07OP386Q/13l18+nL7rv37FrpPyV6S5N/WkezPJW5tD3qITMTMuJnYyH+p++NfLQ0N8Wzi2Jwn2YUxtb5FJ9LzMeUz2MPOV85ChQNUSl01JJz3waGiK2fnKR99GAM/Hwp6PZh2CyuLRgo6Po9UGlVfwzew6+0WQZKEf2DTAP1KCf8TbamYOz8hs0vhsbmSUnEZYsFs1SWvJIN+pXssK6e2dTPloN+CjnLq0VS95VCbCVRUP5fSMcFFEbMdjvIOonh6O3PKxu/UCsoL8giLSs6kOjHPHHhmR7YQsmkh7Fg4Mekt4VJBpe0oXkoLeDO4CunXujKSEuEAhdnWJHg8DPB6QfgtMqW4x4sWpklglVaLFf1LqABFmiDAvnUE9g425ILUSQhWDhruksHvwYgmImZ7quDZjsPDLrgEPqJQvLlcSdkEdVsyhN18fNp4B9x1ZxpaFLonAoYZu6C+79/eBI8DaSzBjiOpXG9X3AJejq8sniQLBZqIAbBfYbj62SyymbKosypsOim75XrzcD1HbixLhVs234l0BV4adeoFL5+TSjcJHTyneGeqDCTEIKkHOhq7wtoRZBu98Ln02bzouH2eLs8NSdD8XQi3EZrO5hwwTzT9SGz5Ia8DQK9+EuFH4oC3Qs8IjBXpWuZ5Vvb1yo/CxYqBnhUcK9Kx6Pat45+hG4UPUQM8KjxToWfV6VvGe2NXys10gaBsxSuvB0Erc77taOrYLfAzUqjRCVuJe5tWyr12gX6BWpfGvEvdpr5Zs7QLbArUqjW5t6h70yc9n0Jknd5AsVKtkIfgcBhKE4HOYynKAjleSA9SG47ghyQeO487tGM065gnCDluSrlLY5wcSs+WJF4XdWSAxW55CUNhTAxKz5cHwUnEMOPTWLqpbKmoB+Vi78GSpGAXkY+3ibKUiEpAPCBgVChj5hxdDsAhOMIaYEcSMIGbk76B2uJKgUQeCRhA0gqBRDmeLt60T+OcgapTX/wIiA2Gjgi4ZEBmIGxX00oDI1EJk1ipytASZAdfeJoeOQEAgdrRqoAICssnBIxAQiB6tOHoUHMQDUSOIGkHUCKJGcPBOMmzE3W64t2jU6HCl5+7AyTkQ01kgpsM/W9KWpwQV6X46U+ETSLnQtdQTM2bC0ZodOVrDw0ULCsAuSAAcj5KmHCdUeFCfyBEydBSkr12dXqDTQJaAe1TLPU7gvM8l0470ewDrANZR7LjPxWnHMdAOoB3rRztMLGP1Disx5LlWh1cC9divgxDsghQA/SD049KTJTQkTwISAiQESEhpJGQn8bjYt2w31PyjvmuGwukP5N8k77HjN8OaoHaJWhNa2m66twiWPMYTKQRbrUP3sv3gPkmRzFu3JsFroVAKH+hqIwRN23gy1Yhl1EMSQgCiZUekOByfCDh+jOMvjdgTroUgD7j3Yn2M0cQVmylVtJRH5Pw4eMTEEIMia46C32oc+JwheWRVJNaNU92zBlHaFik12BGVSWMsfHOw+cDOVOXRO7f4i/920ZJwLsXY1RG+T1a1btXpZ1PrP+gyp9u+vES6vZMYu8jMSZrmdycxLJE5jQ56DhJLZ03VVc4RnnNPmfDJhyNCgZnTeTc9N3HsHzQHhTTuojHOgdlpKJOGUWy33bABng85VzN7BcTk1cGLCI67Fhp//eda+Ppi7+DFq/wSlF66qa1gYmL92x8mIV6aGk96jV/Zk093wCMFjoWv3Iai1HadhfYPbzKLyOx3zj2FRDZTZh9ny1kS9T9lC2dCshOPyc0rvCck6u+x1huzFWdl78J92MJvVFCFA561mAKL663AxZFERgJXBE2ENbiIoiI7celReZ6diL9uYCa+EfpfxESwZw1cstLwXQd7KwEgnvS1YPnIJxaJGQ6k4oPuUgJ/FUEzMVCGnFimrFtLhz9R59NPVBSTTqCIBct0FM1rTgshrLr0tOAKsCwMB0q4ZCUspnrftXVVvefcw8yuZAKe2ujlYq9RDWxrg84uprORTH7fn1ZAbd2YxUr1NjMFbP5vUp6ygyvFlLHcvhXUMF6caTFl69RD2dj/aQDhyfW+qkwWPb/r0I0JCbrx/aXoh0AE2/CuCbHbpqp8y+Lh3s3eoA78CFk0DiF0J6GfWYz+aDejJWHYRWhF/i0qLov66veIRugiyvLsU6INH0Ybjj6l1Yn+CBdg4UiJ9tfvS2wwfhh66GgNve3M1x3i6jPJGt8Ykum2JzgsYUT4fGP1/7w6Uz1v+V3gURd3nv4Ps0DK0NeNAQA= {{- end }} diff --git a/charts/kubezero-istio/update.sh b/charts/kubezero-istio/update.sh index f7b2313..63f5bb9 100755 --- a/charts/kubezero-istio/update.sh +++ b/charts/kubezero-istio/update.sh @@ -4,14 +4,14 @@ set -ex ### TODO # - https://istio.io/latest/docs/ops/configuration/security/harden-docker-images/ -export ISTIO_VERSION=1.9.3 +export ISTIO_VERSION=1.10.2 rm -rf istio curl -sL "https://github.com/istio/istio/releases/download/${ISTIO_VERSION}/istio-${ISTIO_VERSION}-linux-amd64.tar.gz" | tar xz mv istio-${ISTIO_VERSION} istio # remove unused old telemetry filters -rm -f istio/manifests/charts/istio-control/istio-discovery/templates/telemetryv2_1.[678].yaml +rm -f istio/manifests/charts/istio-control/istio-discovery/templates/telemetryv2_1.[6789].yaml # Patch patch -p0 -i zdt.patch --no-backup-if-mismatch diff --git a/charts/kubezero-istio/values.yaml b/charts/kubezero-istio/values.yaml index e1531d4..e248fd9 100644 --- a/charts/kubezero-istio/values.yaml +++ b/charts/kubezero-istio/values.yaml @@ -1,9 +1,8 @@ global: # hub: docker.io/istio - # tag: 1.9.3 + # tag: 1.10.2 logAsJson: true - jwtPolicy: first-party-jwt defaultPodDisruptionBudget: enabled: false -- 2.40.1 From eab6f8253c864b82306e4446d77277e6de4193b6 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 1 Jul 2021 16:42:39 +0200 Subject: [PATCH 21/45] chore: update docs --- README.md | 12 ++- .../templates/resources/10-runtimeClass.yaml | 2 - charts/kubezero-logging/README.md | 14 +-- charts/kubezero-metrics/README.md | 22 ++--- charts/kubezero-redis/README.md | 2 +- docs/Upgrade-2.20.md | 98 ++++++++++--------- 6 files changed, 76 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 8f48dbf..7b1a644 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,14 @@ KubeZero is a Kubernetes distribution providing an integrated container platform # Version / Support Matrix -| KubeZero \ Kubernetes Version | v1.18 | v1.19 | v1.20 | EOL | +| KubeZero \ Kubernetes Version | v1.19 | v1.20 | v1.21 | EOL | |----------------------------------------|-------|-------|-------|-------------| -| master branch | yes | yes | beta | | -| stable branch | yes | yes | no | | -| v2.19.0 | yes | yes | no | 30 Jun 2021 | -| v2.18.0 | yes | no | no | 30 Apr 2021 | +| master branch | no | yes | alpha | | +| stable branch | yes | no | no | | +| v2.20.0 | no | yes | no | 30 Aug 2021 | +| v2.19.0 | yes | no | no | 30 Aug 2021 | + +[Upstream release policy](https://kubernetes.io/releases/) # Architecure ![aws_architecture](docs/aws_architecture.png) diff --git a/charts/kubeadm/templates/resources/10-runtimeClass.yaml b/charts/kubeadm/templates/resources/10-runtimeClass.yaml index 54610a4..ed979d2 100644 --- a/charts/kubeadm/templates/resources/10-runtimeClass.yaml +++ b/charts/kubeadm/templates/resources/10-runtimeClass.yaml @@ -1,4 +1,3 @@ -{{- if .Values.disabledfor120 }} apiVersion: node.k8s.io/v1 kind: RuntimeClass metadata: @@ -7,4 +6,3 @@ handler: runc overhead: podFixed: memory: 16Mi -{{- end }} diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md index 47b0b7c..8ccdec5 100644 --- a/charts/kubezero-logging/README.md +++ b/charts/kubezero-logging/README.md @@ -1,6 +1,6 @@ # kubezero-logging -![Version: 0.6.5](https://img.shields.io/badge/Version-0.6.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.5.0](https://img.shields.io/badge/AppVersion-1.5.0-informational?style=flat-square) +![Version: 0.7.0](https://img.shields.io/badge/Version-0.7.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6.0](https://img.shields.io/badge/AppVersion-1.6.0-informational?style=flat-square) KubeZero Umbrella Chart for complete EFK stack @@ -18,9 +18,9 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| -| | eck-operator | 1.5.0 | -| | fluent-bit | 0.15.4 | -| | fluentd | 0.2.2 | +| | eck-operator | 1.6.0 | +| | fluent-bit | 0.15.14 | +| | fluentd | 0.2.6 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Changes from upstream @@ -108,9 +108,9 @@ Kubernetes: `>= 1.18.0` | fluentd.fileConfigs."00_system.conf" | string | `"\n root_dir /var/log/fluentd\n # log_level debug\n workers 2\n"` | | | fluentd.fileConfigs."01_sources.conf" | string | `"\n @type http\n @label @KUBERNETES\n port 9880\n bind 0.0.0.0\n keepalive_timeout 30\n\n\n\n @type forward\n @label @KUBERNETES\n port 24224\n bind 0.0.0.0\n # skip_invalid_event true\n send_keepalive_packet true\n \n self_hostname \"#{ENV['HOSTNAME']}\"\n shared_key {{ .Values.shared_key }}\n \n"` | | | fluentd.fileConfigs."02_filters.conf" | string | `""` | | -| fluentd.fileConfigs."04_outputs.conf" | string | `""` | | +| fluentd.fileConfigs."04_outputs.conf" | string | `""` | | | fluentd.image.repository | string | `"fluent/fluentd-kubernetes-daemonset"` | | -| fluentd.image.tag | string | `"v1.12-debian-elasticsearch7-1"` | | +| fluentd.image.tag | string | `"v1-debian-elasticsearch"` | | | fluentd.istio.enabled | bool | `false` | | | fluentd.kind | string | `"Deployment"` | | | fluentd.metrics.serviceMonitor.additionalLabels.release | string | `"metrics"` | | @@ -141,7 +141,7 @@ Kubernetes: `>= 1.18.0` | kibana.istio.enabled | bool | `false` | | | kibana.istio.gateway | string | `"istio-system/ingressgateway"` | | | kibana.istio.url | string | `""` | | -| version | string | `"7.11.1"` | | +| version | string | `"7.13.2"` | | ## Resources: diff --git a/charts/kubezero-metrics/README.md b/charts/kubezero-metrics/README.md index 16c8e81..a0e5279 100644 --- a/charts/kubezero-metrics/README.md +++ b/charts/kubezero-metrics/README.md @@ -18,8 +18,8 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| -| | kube-prometheus-stack | 15.4.4 | -| https://prometheus-community.github.io/helm-charts | prometheus-adapter | 2.12.3 | +| | kube-prometheus-stack | 16.12.0 | +| https://prometheus-community.github.io/helm-charts | prometheus-adapter | 2.14.2 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Values @@ -67,7 +67,6 @@ Kubernetes: `>= 1.18.0` | kube-prometheus-stack.grafana.service.portName | string | `"http-grafana"` | | | kube-prometheus-stack.grafana.sidecar.dashboards.provider.foldersFromFilesStructure | bool | `true` | | | kube-prometheus-stack.grafana.sidecar.dashboards.searchNamespace | string | `"ALL"` | | -| kube-prometheus-stack.grafana.sidecar.image.tag | string | `"1.12.0"` | | | kube-prometheus-stack.grafana.testFramework.enabled | bool | `false` | | | kube-prometheus-stack.kube-state-metrics.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | kube-prometheus-stack.kube-state-metrics.podSecurityPolicy.enabled | bool | `false` | | @@ -110,6 +109,7 @@ Kubernetes: `>= 1.18.0` | kube-prometheus-stack.prometheus.prometheusSpec.resources.requests.cpu | string | `"500m"` | | | kube-prometheus-stack.prometheus.prometheusSpec.resources.requests.memory | string | `"512Mi"` | | | kube-prometheus-stack.prometheus.prometheusSpec.retention | string | `"8d"` | | +| kube-prometheus-stack.prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues | bool | `false` | | | kube-prometheus-stack.prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues | bool | `false` | | | kube-prometheus-stack.prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.accessModes[0] | string | `"ReadWriteOnce"` | | | kube-prometheus-stack.prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage | string | `"16Gi"` | | @@ -132,8 +132,8 @@ Kubernetes: `>= 1.18.0` | prometheus-adapter.prometheus.url | string | `"http://metrics-kube-prometheus-st-prometheus"` | | | prometheus-adapter.rules.default | bool | `false` | | | prometheus-adapter.rules.resource.cpu.containerLabel | string | `"container"` | | -| prometheus-adapter.rules.resource.cpu.containerQuery | string | `"sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!=\"POD\",container!=\"\",pod!=\"\"}[3m])) by (<<.GroupBy>>)"` | | -| prometheus-adapter.rules.resource.cpu.nodeQuery | string | `"sum(1 - irate(node_cpu_seconds_total{mode=\"idle\"}[3m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>)"` | | +| prometheus-adapter.rules.resource.cpu.containerQuery | string | `"sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!=\"POD\",container!=\"\",pod!=\"\"}[5m])) by (<<.GroupBy>>)"` | | +| prometheus-adapter.rules.resource.cpu.nodeQuery | string | `"sum(1 - irate(node_cpu_seconds_total{mode=\"idle\"}[5m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>)"` | | | prometheus-adapter.rules.resource.cpu.resources.overrides.namespace.resource | string | `"namespace"` | | | prometheus-adapter.rules.resource.cpu.resources.overrides.node.resource | string | `"node"` | | | prometheus-adapter.rules.resource.cpu.resources.overrides.pod.resource | string | `"pod"` | | @@ -143,15 +143,13 @@ Kubernetes: `>= 1.18.0` | prometheus-adapter.rules.resource.memory.resources.overrides.namespace.resource | string | `"namespace"` | | | prometheus-adapter.rules.resource.memory.resources.overrides.node.resource | string | `"node"` | | | prometheus-adapter.rules.resource.memory.resources.overrides.pod.resource | string | `"pod"` | | -| prometheus-adapter.rules.resource.window | string | `"3m"` | | +| prometheus-adapter.rules.resource.window | string | `"5m"` | | | prometheus-adapter.tolerations[0].effect | string | `"NoSchedule"` | | | prometheus-adapter.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | # Dashboards -## Etcs -- https://grafana.com/grafana/dashboards/3070 - -## ElasticSearch -- https://grafana.com/grafana/dashboards/266 - +## Alertmanager +- https://grafana.com/api/dashboards/9578/revisions/4/download +## Prometheus +- https://grafana.com/api/dashboards/3662/revisions/2/download diff --git a/charts/kubezero-redis/README.md b/charts/kubezero-redis/README.md index 2617a26..a648f7b 100644 --- a/charts/kubezero-redis/README.md +++ b/charts/kubezero-redis/README.md @@ -1,6 +1,6 @@ # kubezero-redis -![Version: 0.2.1](https://img.shields.io/badge/Version-0.2.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.2.2](https://img.shields.io/badge/Version-0.2.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero Umbrella Chart for Redis HA diff --git a/docs/Upgrade-2.20.md b/docs/Upgrade-2.20.md index 14fd510..34ed405 100644 --- a/docs/Upgrade-2.20.md +++ b/docs/Upgrade-2.20.md @@ -1,68 +1,71 @@ -# Upgrade to KubeZero V2.20 / Kubernetes 1.20 - -# CloudBender -## Changes -### Single node control plane -- Control - -## Upgrade -- Set the specific wanted Kubernetes version in the controller config to eg. `v1.20.2` -- configure your AWS CLI profile as well as your kubectl context to cluster you want to upgrade. -- verify your config ... - -- run ./scripts/upgrade_120.sh -- update the CFN stack for kube-control-plane - -### Single node control plane -- will automatically be upgraded and the controller node replaced as part of the CFN update - -### Clustered control plane -- replace controller instances one by one in no particular order -- once confirmed that the upgraded 1.20 control plane is working as expected update the clustered control plane CFN stack once more with `LBType: none` to remove the AWS NLB fronting the Kubernetes API which is not required anymore. - -- replace worker nodes in a rolling fashion via. drain / terminate / rinse-repeat - -# KubeZero +# KubeZero V2.20 / Kubernetes 1.20 ## New features +- Support for [Service Account Tokens](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#service-account-token-volume-projection) incl. federation with AWS IAM +This allows pods to assume IAM roles without the need of additional services like kiam. +- Cert-manager integration now supports [cross-account issuer](https://cert-manager.io/docs/configuration/acme/dns01/route53/#cross-account-access) for AWS route53 +- Optional Proxy Protocol support for Ingress Loadbalancers, which allows preserving the real client IP and at the same time solves the hairpin routing issues of the AWS NLBs, see [Istio blog](https://istio.io/v1.9/blog/2020/show-source-ip/) -### NATS -Deploy NATS services +## New modules +### MQ / NATS +Deploy [NATS](https://docs.nats.io/jetstream/jetstream) services incl. jetstream engine, Grafana dashboards etc. ### TimeCapsule -Providing backup solutions for KubeZero clusters: - -- scheduled snapshots for EBS backed PVCs incl. custome retention and restore +Provides backup solutions for KubeZero clusters, like +Scheduled snapshots for EBS backed PVCs incl. custom retention and restore. - -## Changes +## Changelog ### General -- various version bumps -- removed deprecated nodeLabels from `failure-domain.beta.kubernetes.io/[region|zone]` to `topology.kubernetes.io/[region|zone]` please adapt existing affinity rules ! +- version bumps of all modules +- cert-manager, ebs-csi and efs-csi driver now leverage service account tokens and do not rely on kiam anymore + +### Logging +- version bumps for ElasticSearch, Kibana, ECK, fluentd and fluent-bit +- various fixes and tuning to improve reliability of the fluentd aggregator layer ### Istio - hardened and optimized settings for Envoy gateway proxies - improved deployment strategy to reduce errors during upgrades - Added various Grafana Dashboards +- version bump to 1.10.2 -## Metrics +### Metrics - Added various dashboards for KubeZero modules - Updated / improved dashboard organization incl. folders and tags -- Grafana Dashboards are now all provided via configmaps, no more state required, no manual changes persisted -- Grafana allows anonymous read-only access -- all dashboards ndefault to now-1h and prohibit less than 30s refresh +- Grafana Dashboards are now all provided via configmaps, no more state required, also no more manual changes persisted +- Grafana now allows anonymous read-only access +- all dashboards default to `now-1h` and prohibit less than 30s refresh cycles - Custom dashboards can easily be provided by simple installing a ConfigMap along with workloads in any namespace -## Upgrade - Without ArgoCD +# Upgrade - CloudBender +- Set the specific wanted Kubernetes version in the controller config to eg. `v1.20.8` +- configure your AWS CLI profile as well as your kubectl context to point to the cluster you want to upgrade +and verify your config via `aws sts get-caller-identity` and `kubectl cluster-info` + +- run `./scripts/upgrade_120.sh` +- update the CFN stack kube-control-plane for your cluster + +### Single node control plane +- a new controller instance will automatically be launched and replace the current controller as part of the CFN update + +### Clustered control plane +- replace controller instances one by one in no particular order +- once confirmed that the upgraded 1.20 control plane is working as expected update the clustered control plane CFN stack once more with `LBType: none` to remove the AWS NLB fronting the Kubernetes API which is not required anymore. + +## Upgrade Cloudbender continue +- upgrade all `kube-worker*` CFN stacks +- replace worker nodes in a rolling fashion via. drain / terminate and rinse-repeat + +# Upgrade KubeZero 1. Update CRDs of all enabled components: - `./bootstrap.sh crds all clusters/$CLUSTER` +`./bootstrap.sh crds all clusters/$CLUSTER` 2. Prepare upgrade - Remove legacy monitoring configmaps - Remove previous Grafana stateful config -- Remove legacy Istio Enovyfilter +- Remove legacy Istio Envoyfilter ``` kubectl delete cm -n monitoring -l grafana_dashboard=1 @@ -70,13 +73,14 @@ kubectl delete pvc metrics-grafana -n monitoring kubectl delete envoyfilter -A -l operator.istio.io/version=1.6.9 ``` -3. Upgrade all components -`./bootstrap.sh deploy all clusters/$CLUSTER` +3. Upgrade all KubeZero modules: +- without ArgoCD: + - `./bootstrap.sh deploy all clusters/$CLUSTER` +- with ArgoCD: -## Upgrade - ArgoCD -- ArgoCD itself: `./bootstrap.sh deploy argocd clusters/$CLUSTER` -- push latest cluster config to your git repo -- trigger sync in ArgoCD incl. *prune* starting with the KubeZero root app + - ArgoCD itself: `./bootstrap.sh deploy argocd clusters/$CLUSTER` + - push latest cluster config to your git repo + - trigger sync in ArgoCD incl. *prune* starting with the KubeZero root app ( only if auto-sync is not enabled ) ## Verification / Tests -- 2.40.1 From 540f7be36844c5ac36ea262c789451b70486bb67 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 14 Jul 2021 17:38:32 +0200 Subject: [PATCH 22/45] fix: more fine tuning for fluent pipeline --- charts/kubezero-logging/Chart.yaml | 4 ++-- charts/kubezero-logging/charts/fluent-bit/Chart.yaml | 6 +++--- charts/kubezero-logging/update.sh | 2 +- charts/kubezero-logging/values.yaml | 12 +++++++----- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index ced132b..e846099 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application -version: 0.7.0 +version: 0.7.1 appVersion: 1.6.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -26,6 +26,6 @@ dependencies: version: 0.2.6 condition: fluentd.enabled - name: fluent-bit - version: 0.15.14 + version: 0.15.15 condition: fluent-bit.enabled kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml index e3e351d..140229e 100644 --- a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml +++ b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml @@ -1,8 +1,8 @@ annotations: artifacthub.io/changes: | - - Upgrade fluent-bit image to v1.7.8 + - Upgrade fluent-bit image to v1.7.9 apiVersion: v1 -appVersion: 1.7.8 +appVersion: 1.7.9 description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems. home: https://fluentbit.io/ @@ -21,4 +21,4 @@ maintainers: name: fluent-bit sources: - https://github.com/fluent/fluent-bit/ -version: 0.15.14 +version: 0.15.15 diff --git a/charts/kubezero-logging/update.sh b/charts/kubezero-logging/update.sh index 9c0dd23..19473dc 100755 --- a/charts/kubezero-logging/update.sh +++ b/charts/kubezero-logging/update.sh @@ -1,7 +1,7 @@ #!/bin/bash ECK_VERSION=1.6.0 -FLUENT_BIT_VERSION=0.15.14 +FLUENT_BIT_VERSION=0.15.15 FLUENTD_VERSION=0.2.6 # fix ECK crds handling to adhere to proper helm v3 support which also fixes ArgoCD applyong updates on upgrades diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index df80173..72a0999 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -120,7 +120,9 @@ fluentd: 00_system.conf: |- root_dir /var/log/fluentd - # log_level debug + log_level info + ignore_repeated_log_interval 60s + ignore_same_log_interval 60s workers 2 01_sources.conf: |- @@ -165,7 +167,7 @@ fluentd: @id out_es @type elasticsearch - @log_level info + # @log_level debug include_tag_key true id_key id remove_keys id @@ -185,7 +187,7 @@ fluentd: reload_on_failure true request_timeout 60s suppress_type_name true - slow_flush_log_threshold 50.0 + slow_flush_log_threshold 55.0 # Retry failed bulk requests # https://github.com/uken/fluent-plugin-elasticsearch#unrecoverable-error-types @@ -199,12 +201,12 @@ fluentd: flush_thread_count 1 flush_interval 30s - chunk_limit_size 4MB + chunk_limit_size 8MB total_limit_size 2GB flush_at_shutdown true retry_type exponential_backoff - retry_timeout 2h + retry_timeout 6h overflow_action drop_oldest_chunk disable_chunk_backup true -- 2.40.1 From bf688adf60507f8304d3c88daf703c144053ad5e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 14 Jul 2021 18:15:47 +0200 Subject: [PATCH 23/45] chore: metrics version bump --- .../charts/kube-prometheus-stack/Chart.yaml | 2 +- .../admission-webhooks/job-patch/clusterrole.yaml | 2 +- .../job-patch/clusterrolebinding.yaml | 2 +- .../job-patch/job-createSecret.yaml | 2 +- .../job-patch/job-patchWebhook.yaml | 2 +- .../admission-webhooks/job-patch/psp.yaml | 2 +- .../admission-webhooks/job-patch/role.yaml | 2 +- .../admission-webhooks/job-patch/rolebinding.yaml | 2 +- .../job-patch/serviceaccount.yaml | 2 +- .../mutatingWebhookConfiguration.yaml | 2 +- .../validatingWebhookConfiguration.yaml | 2 +- .../templates/prometheus/prometheus.yaml | 8 +++++++- .../prometheus/rules-1.14/alertmanager.rules.yaml | 2 +- .../prometheus/rules-1.14/kubernetes-apps.yaml | 6 +++--- .../prometheus/rules-1.14/kubernetes-storage.yaml | 13 +++++++++---- .../templates/prometheus/rules-1.14/prometheus.yaml | 12 ++++++++++++ .../charts/kube-prometheus-stack/values.yaml | 8 ++++++++ .../dashboards/jsonnetfile.lock.json | 6 +++--- charts/kubezero-metrics/update.sh | 2 +- 19 files changed, 55 insertions(+), 24 deletions(-) diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml index fdad40a..9ab2967 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml @@ -47,4 +47,4 @@ sources: - https://github.com/prometheus-community/helm-charts - https://github.com/prometheus-operator/kube-prometheus type: application -version: 16.12.0 +version: 16.13.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml index 249af77..6c91ee0 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml index 31fd2de..b909d14 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml index de81974..566fc6e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: batch/v1 kind: Job metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml index 23dc9f4..968d474 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: batch/v1 kind: Job metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml index 5834c48..3bc451d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml index d229f76..a64e982 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml index f4b1fbf..d713629 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml index a91889b..804d94a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml index f42e33e..9cb8993 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml index 1439ed5..9242659 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled }} +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.admissionWebhooks.enabled }} apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml index 6f00a8f..b2b6890 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml @@ -254,11 +254,17 @@ spec: name: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.name }} key: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.key }} {{- end }} -{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} +{{- if or .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} additionalAlertManagerConfigs: +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg key: additional-alertmanager-configs.yaml {{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret }} + name: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalAlertManagerConfigsSecret.key }} +{{- end }} +{{- end }} {{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} additionalAlertRelabelConfigs: name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/alertmanager.rules.yaml index 49e78b5..34eecbe 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/alertmanager.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -52,7 +52,7 @@ spec: max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m]) < on (namespace,service) group_left count by (namespace,service) (max_over_time(alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}[5m])) - for: 10m + for: 15m labels: severity: critical {{- if .Values.defaultRules.additionalRuleLabels }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml index 94c16cd..5caae97 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -82,7 +82,7 @@ spec: expr: |- ( kube_deployment_spec_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - != + > kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} ) and ( changes(kube_deployment_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[10m]) @@ -260,7 +260,7 @@ spec: {{- end }} - alert: KubeHpaReplicasMismatch annotations: - description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.hpa {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes. + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes. runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubehpareplicasmismatch summary: HPA has not matched descired number of replicas. expr: |- @@ -285,7 +285,7 @@ spec: {{- end }} - alert: KubeHpaMaxedOut annotations: - description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.hpa {{`}}`}} has been running at max replicas for longer than 15 minutes. + description: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.horizontalpodautoscaler {{`}}`}} has been running at max replicas for longer than 15 minutes. runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubehpamaxedout summary: HPA is running at max replicas expr: |- diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml index 3e06766..96d3c6f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -31,10 +31,13 @@ spec: runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumefillingup summary: PersistentVolume is filling up. expr: |- - kubelet_volume_stats_available_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} - / - kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} - < 0.03 + ( + kubelet_volume_stats_available_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_used_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 for: 1m labels: severity: critical @@ -53,6 +56,8 @@ spec: kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} ) < 0.15 and + kubelet_volume_stats_used_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} > 0 + and predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 for: 1h labels: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/prometheus.yaml index 52cc0d8..1047942 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/prometheus.yaml @@ -272,6 +272,18 @@ spec: severity: warning {{- if .Values.defaultRules.additionalRuleLabels }} {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} +{{- end }} + - alert: PrometheusTargetSyncFailure + annotations: + description: '{{`{{`}} printf "%.0f" $value {{`}}`}} targets in Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} have failed to sync because invalid configuration was supplied.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheustargetsyncfailure + summary: Prometheus has failed to sync targets. + expr: increase(prometheus_target_sync_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[30m]) > 0 + for: 5m + labels: + severity: critical +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} {{- end }} - alert: PrometheusErrorSendingAlertsToAnyAlertmanager annotations: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml index c3a9ffd..3dddd43 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml @@ -2299,6 +2299,14 @@ prometheus: # services: # - metrics-prometheus-alertmanager + ## If additional alertmanager configurations are already deployed in a single secret, or you want to manage + ## them separately from the helm deployment, you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalAlertManagerConfigs + additionalAlertManagerConfigsSecret: {} + # name: + # key: + ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. diff --git a/charts/kubezero-metrics/dashboards/jsonnetfile.lock.json b/charts/kubezero-metrics/dashboards/jsonnetfile.lock.json index 2926b20..7749a90 100644 --- a/charts/kubezero-metrics/dashboards/jsonnetfile.lock.json +++ b/charts/kubezero-metrics/dashboards/jsonnetfile.lock.json @@ -18,7 +18,7 @@ "subdir": "grafana-builder" } }, - "version": "136b7e4fa204d6b4f9f3bfc5d8ace8834d2b4aae", + "version": "e5cc5d604c74e887b4e56ee3fd30334bf6cef19a", "sum": "GRf2GvwEU4jhXV+JOonXSZ4wdDv8mnHBPCQ6TUVd+g8=" }, { @@ -28,8 +28,8 @@ "subdir": "" } }, - "version": "baf5e20d3275ebed214eaa26a0377ec3bf2b8897", - "sum": "U1JPaRwc6xDBkE30bbEpT2j0vAJvrm5GXhCIZqLLLWU=" + "version": "b710a868a95621aa93e0b661954f63f4db82aaea", + "sum": "W3jM1f3BUCc2mursXapw0OwsU2raoCuRy9897aA9R9k=" } ], "legacyImports": false diff --git a/charts/kubezero-metrics/update.sh b/charts/kubezero-metrics/update.sh index fcbc7d9..d0ddfae 100755 --- a/charts/kubezero-metrics/update.sh +++ b/charts/kubezero-metrics/update.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION=16.12.0 +VERSION=16.13.0 rm -rf charts/kube-prometheus-stack curl -L -s -o - https://github.com/prometheus-community/helm-charts/releases/download/kube-prometheus-stack-${VERSION}/kube-prometheus-stack-${VERSION}.tgz | tar xfz - -C charts -- 2.40.1 From 7ac7c43b9ead412bf98071b3979402a9f14d82e3 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 14 Jul 2021 18:18:57 +0200 Subject: [PATCH 24/45] chore: metrics version bump --- charts/kubezero-metrics/Chart.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index 0c2a44c..cb9a2c2 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-metrics description: KubeZero Umbrella Chart for prometheus-operator type: application -version: 0.4.1 +version: 0.4.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -16,11 +16,11 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: kube-prometheus-stack - version: 16.12.0 + version: 16.13.0 # Switch back to upstream once all alerts are fixed eg. etcd gpcr # repository: https://prometheus-community.github.io/helm-charts - name: prometheus-adapter - version: 2.14.2 + version: 2.15.0 repository: https://prometheus-community.github.io/helm-charts condition: prometheus-adapter.enabled kubeVersion: ">= 1.18.0" -- 2.40.1 From c52258dd9d833b971882db7a002e29820304bcff Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 14 Jul 2021 23:36:57 +0200 Subject: [PATCH 25/45] Fix: aws-ebs version bump --- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 6 +- .../charts/aws-ebs-csi-driver/CHANGELOG.md | 6 + .../charts/aws-ebs-csi-driver/Chart.yaml | 4 +- .../crds/snapshot-controller-crds.yaml | 628 ++++++++++++++++++ .../templates/controller.yaml | 2 + .../aws-ebs-csi-driver/templates/crds.yml | 6 + .../templates/node-windows.yaml | 185 ++++++ .../charts/aws-ebs-csi-driver/values.yaml | 2 +- charts/kubezero-aws-ebs-csi-driver/update.sh | 2 +- 9 files changed, 834 insertions(+), 7 deletions(-) create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/CHANGELOG.md create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/crds/snapshot-controller-crds.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/crds.yml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node-windows.yaml diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index 44b3e93..0532c70 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.6.0 -appVersion: 1.2.3 +version: 0.6.1 +appVersion: 1.2.4 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png sources: @@ -18,7 +18,7 @@ maintainers: - name: Quarky9 dependencies: - name: aws-ebs-csi-driver - version: 1.2.3 + version: 1.2.4 # repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: kubezero-lib version: ">= 0.1.3" diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/CHANGELOG.md b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/CHANGELOG.md new file mode 100644 index 0000000..b82cf86 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/CHANGELOG.md @@ -0,0 +1,6 @@ +# Helm chart + +# v1.2.4 +* Bump app/driver version to `v1.1.1` +* Install VolumeSnapshotClass, VolumeSnapshotContent, VolumeSnapshot CRDs if enableVolumeSnapshot is true +* Only run csi-snapshotter sidecar if enableVolumeSnapshot is true or if CRDs are already installed diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml index b773b00..80496ee 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: 1.1.0 +appVersion: 1.1.1 description: A Helm chart for AWS EBS CSI Driver home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver keywords: @@ -15,4 +15,4 @@ maintainers: name: aws-ebs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver -version: 1.2.3 +version: 1.2.4 diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/crds/snapshot-controller-crds.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/crds/snapshot-controller-crds.yaml new file mode 100644 index 0000000..6ca2443 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/crds/snapshot-controller-crds.yaml @@ -0,0 +1,628 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: false + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + oneOf: + - required: ["snapshotHandle"] + - required: ["volumeHandle"] + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + name: + description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names" + type: string + namespace: + description: "Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/" + type: string + resourceVersion: + description: "Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency" + type: string + uid: + description: "UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids" + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: "message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information." + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + name: + description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names" + type: string + namespace: + description: "Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/" + type: string + resourceVersion: + description: "Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency" + type: string + uid: + description: "UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids" + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: "message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information." + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + spec: + description: "spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required." + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + oneOf: + - required: ["persistentVolumeClaimName"] + - required: ["volumeSnapshotContentName"] + volumeSnapshotClassName: + description: "VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field." + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: "boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object." + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: "message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information." + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + spec: + description: "spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required." + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: "VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field." + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: "boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object." + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: "message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information." + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml index 48420c8..6934902 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -192,6 +192,7 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} +{{- if or .Values.enableVolumeSnapshot (.Capabilities.APIVersions.Has "snapshot.storage.k8s.io/v1") }} - name: csi-snapshotter image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} args: @@ -213,6 +214,7 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} +{{- end }} - name: csi-resizer image: {{ printf "%s:%s" .Values.sidecars.resizerImage.repository .Values.sidecars.resizerImage.tag }} imagePullPolicy: Always diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/crds.yml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/crds.yml new file mode 100644 index 0000000..ebd5f49 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/crds.yml @@ -0,0 +1,6 @@ +{{- if and .Values.enableVolumeSnapshot (ne (.Capabilities.APIVersions.Has "snapshot.storage.k8s.io/v1") true) }} +{{- range $path, $bytes := .Files.Glob "crds/*.yaml" }} +{{ $.Files.Get $path }} +--- +{{- end }} +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node-windows.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node-windows.yaml new file mode 100644 index 0000000..b6cde40 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node-windows.yaml @@ -0,0 +1,185 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ebs-csi-node-windows + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.node.podAnnotations }} + annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- end }} + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + nodeSelector: + kubernetes.io/os: windows + {{- with .Values.node.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + serviceAccountName: {{ .Values.serviceAccount.node.name }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-cluster-critical" }} + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- else }} + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 + {{- end }} + {{- with .Values.node.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: ebs-plugin + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + args: + - node + - --endpoint=$(CSI_ENDPOINT) + {{- if .Values.volumeAttachLimit }} + - --volume-attach-limit={{ .Values.volumeAttachLimit }} + {{- end }} + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: C:\var\lib\kubelet + mountPropagation: "None" + - name: plugin-dir + mountPath: C:\csi + - name: csi-proxy-disk-pipe + mountPath: \\.\pipe\csi-proxy-disk-v1beta2 + - name: csi-proxy-volume-pipe + mountPath: \\.\pipe\csi-proxy-volume-v1beta2 + - name: csi-proxy-filesystem-pipe + mountPath: \\.\pipe\csi-proxy-filesystem-v1beta1 + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: node-driver-registrar + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + env: + - name: ADDRESS + value: unix:/csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: C:\var\lib\kubelet\plugins\ebs.csi.aws.com\csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} + volumeMounts: + - name: plugin-dir + mountPath: C:\csi + - name: registration-dir + mountPath: C:\registration + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + args: + - --csi-address=unix:/csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: C:\csi + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: C:\var\lib\kubelet + type: Directory + - name: plugin-dir + hostPath: + path: C:\var\lib\kubelet\plugins\ebs.csi.aws.com + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: C:\var\lib\kubelet\plugins_registry + type: Directory + - name: csi-proxy-disk-pipe + hostPath: + path: \\.\pipe\csi-proxy-disk-v1beta2 + type: "" + - name: csi-proxy-volume-pipe + hostPath: + path: \\.\pipe\csi-proxy-volume-v1beta2 + type: "" + - name: csi-proxy-filesystem-pipe + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1beta1 + type: "" diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml index a33fda9..8a6e62a 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml @@ -4,7 +4,7 @@ image: repository: k8s.gcr.io/provider-aws/aws-ebs-csi-driver - tag: "v1.1.0" + tag: "v1.1.1" pullPolicy: IfNotPresent sidecars: diff --git a/charts/kubezero-aws-ebs-csi-driver/update.sh b/charts/kubezero-aws-ebs-csi-driver/update.sh index 12e4d9d..cfb9377 100755 --- a/charts/kubezero-aws-ebs-csi-driver/update.sh +++ b/charts/kubezero-aws-ebs-csi-driver/update.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION=1.2.3 +VERSION=1.2.4 rm -rf charts/aws-ebs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts -- 2.40.1 From ed848a4ce4d98c46c72f598278a31bb562421210 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 15 Jul 2021 01:26:25 +0200 Subject: [PATCH 26/45] fix: update aws-efs driver --- charts/kubezero-aws-efs-csi-driver/Chart.yaml | 6 +-- .../charts/aws-efs-csi-driver/CHANGELOG.md | 8 +++- .../charts/aws-efs-csi-driver/Chart.yaml | 4 +- .../templates/controller-deployment.yaml | 10 ++-- .../charts/aws-efs-csi-driver/values.yaml | 4 +- charts/kubezero-aws-efs-csi-driver/update.sh | 4 +- charts/kubezero-aws-efs-csi-driver/zdt.patch | 47 +++++++++++++++++-- charts/kubezero-logging/update.sh | 2 +- charts/kubezero/bootstrap.sh | 4 +- .../templates/aws-efs-csi-driver.yaml | 4 +- 10 files changed, 70 insertions(+), 23 deletions(-) diff --git a/charts/kubezero-aws-efs-csi-driver/Chart.yaml b/charts/kubezero-aws-efs-csi-driver/Chart.yaml index 2e00e92..2313a5c 100644 --- a/charts/kubezero-aws-efs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-efs-csi-driver/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: kubezero-aws-efs-csi-driver description: KubeZero Umbrella Chart for aws-efs-csi-driver -version: 0.4.0 -appVersion: 1.3.1 +version: 0.4.1 +appVersion: 1.3.2 kubeVersion: ">=1.18.0-0" home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -21,5 +21,5 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: aws-efs-csi-driver - version: 2.1.1 + version: 2.1.3 # repository: https://kubernetes-sigs.github.io/aws-efs-csi-driver/ diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/CHANGELOG.md b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/CHANGELOG.md index a842698..0e499d8 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/CHANGELOG.md +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/CHANGELOG.md @@ -1,7 +1,13 @@ # Helm chart +# v2.1.3 +* Bump app/driver version to `v1.3.2` + +# v2.1.2 +* Add extra-create-metadata + # v2.1.1 -* Update app/driver version to `v1.3.0` +* Update app/driver version to `v1.3.1` # v2.1.0 diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml index 29c91a7..9c0b3b2 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.3.1 +appVersion: 1.3.2 description: A Helm chart for AWS EFS CSI Driver home: https://github.com/kubernetes-sigs/aws-efs-csi-driver keywords: @@ -15,4 +15,4 @@ maintainers: name: aws-efs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-efs-csi-driver -version: 2.1.1 +version: 2.1.3 diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml index 67f3ada..c1beb69 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml @@ -23,7 +23,6 @@ spec: annotations: {{- toYaml . | nindent 8 }} {{- end }} spec: - hostNetwork: true {{- if .Values.imagePullSecrets }} imagePullSecrets: {{- range .Values.imagePullSecrets }} @@ -57,6 +56,9 @@ spec: env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + {{- if .Values.controller.extraEnv }} +{{ toYaml .Values.controller.extraEnv | indent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -84,13 +86,13 @@ spec: - --csi-address=$(ADDRESS) - --v={{ .Values.controller.logLevel }} - --feature-gates=Topology=true + {{- if .Values.controller.extraCreateMetadata }} + - --extra-create-metadata + {{- end }} - --leader-election env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock - {{- if .Values.controller.extraEnv }} -{{ toYaml .Values.extraEnv | indent 12 }} - {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml index e85d3fb..6afb5d6 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml @@ -9,7 +9,7 @@ replicaCount: 2 image: repository: amazon/aws-efs-csi-driver - tag: "v1.3.1" + tag: "v1.3.2" pullPolicy: IfNotPresent sidecars: @@ -41,6 +41,8 @@ controller: create: true # Number for the log level verbosity logLevel: 2 + # If set, add pv/pvc metadata to plugin create requests as parameters. + extraCreateMetadata: true # Add additional tags to access points tags: {} # environment: prod diff --git a/charts/kubezero-aws-efs-csi-driver/update.sh b/charts/kubezero-aws-efs-csi-driver/update.sh index a0cc453..ea3e8fb 100755 --- a/charts/kubezero-aws-efs-csi-driver/update.sh +++ b/charts/kubezero-aws-efs-csi-driver/update.sh @@ -1,8 +1,8 @@ #!/bin/bash -VERSION=2.1.1 +VERSION=2.1.3 rm -rf charts/aws-efs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-efs-csi-driver/releases/download/helm-chart-aws-efs-csi-driver-${VERSION}/aws-efs-csi-driver-${VERSION}.tgz | tar xfz - -C charts -# patch -i zdt.patch -p0 --no-backup-if-mismatch +patch -i zdt.patch -p0 --no-backup-if-mismatch diff --git a/charts/kubezero-aws-efs-csi-driver/zdt.patch b/charts/kubezero-aws-efs-csi-driver/zdt.patch index 36637ef..d5978c8 100644 --- a/charts/kubezero-aws-efs-csi-driver/zdt.patch +++ b/charts/kubezero-aws-efs-csi-driver/zdt.patch @@ -1,7 +1,46 @@ -Only in charts/aws-efs-csi-driver.zdt/: .values.yaml.swp -diff -rtub charts/aws-efs-csi-driver/templates/node-daemonset.yaml charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml ---- charts/aws-efs-csi-driver/templates/node-daemonset.yaml 2021-06-14 23:12:20.000000000 +0200 -+++ charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml 2021-06-25 12:29:05.279549688 +0200 +diff -tubNr charts/aws-efs-csi-driver/templates/controller-deployment.yaml charts/aws-efs-csi-driver.zdt/templates/controller-deployment.yaml +--- charts/aws-efs-csi-driver/templates/controller-deployment.yaml 2021-06-25 22:43:06.000000000 +0200 ++++ charts/aws-efs-csi-driver.zdt/templates/controller-deployment.yaml 2021-07-15 00:37:28.960305011 +0200 +@@ -23,7 +23,6 @@ + annotations: {{- toYaml . | nindent 8 }} + {{- end }} + spec: +- hostNetwork: true + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} +@@ -57,9 +56,14 @@ + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock ++ {{- if .Values.controller.extraEnv }} ++{{ toYaml .Values.controller.extraEnv | indent 12 }} ++ {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ ++ - name: aws-token ++ mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" + ports: + - name: healthz + containerPort: 9909 +@@ -110,6 +114,13 @@ + volumes: + - name: socket-dir + emptyDir: {} ++ - name: aws-token ++ projected: ++ sources: ++ - serviceAccountToken: ++ path: token ++ expirationSeconds: 86400 ++ audience: "sts.amazonaws.com" + {{- with .Values.controller.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} +diff -tubNr charts/aws-efs-csi-driver/templates/node-daemonset.yaml charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml +--- charts/aws-efs-csi-driver/templates/node-daemonset.yaml 2021-06-25 22:43:06.000000000 +0200 ++++ charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml 2021-07-15 00:14:58.680295564 +0200 @@ -40,15 +40,10 @@ {{- with .Values.node.nodeSelector }} {{- toYaml . | nindent 8 }} diff --git a/charts/kubezero-logging/update.sh b/charts/kubezero-logging/update.sh index 19473dc..c81f4e5 100755 --- a/charts/kubezero-logging/update.sh +++ b/charts/kubezero-logging/update.sh @@ -4,7 +4,7 @@ ECK_VERSION=1.6.0 FLUENT_BIT_VERSION=0.15.15 FLUENTD_VERSION=0.2.6 -# fix ECK crds handling to adhere to proper helm v3 support which also fixes ArgoCD applyong updates on upgrades +# fix ECK crds handling to adhere to proper helm v3 support which also fixes ArgoCD applying updates on upgrades helm repo list | grep elastic -qc || { helm repo add elastic https://helm.elastic.co; helm repo update; } rm -rf charts/eck-operator && helm pull elastic/eck-operator --untar --untardir charts --version $ECK_VERSION diff --git a/charts/kubezero/bootstrap.sh b/charts/kubezero/bootstrap.sh index 869f28f..5e2da6a 100755 --- a/charts/kubezero/bootstrap.sh +++ b/charts/kubezero/bootstrap.sh @@ -72,8 +72,8 @@ function delete_ns() { # Extract crds via helm calls and apply delta=crds only function _crds() { - helm template $(chart_location $chart) --namespace $namespace --name-template $release --skip-crds --set ${release}.installCRDs=false > $TMPDIR/helm-no-crds.yaml - helm template $(chart_location $chart) --namespace $namespace --name-template $release --include-crds --set ${release}.installCRDs=true > $TMPDIR/helm-crds.yaml + helm template $(chart_location $chart) --namespace $namespace --name-template $release --skip-crds --set ${release}.installCRDs=false -f $TMPDIR/values.yaml > $TMPDIR/helm-no-crds.yaml + helm template $(chart_location $chart) --namespace $namespace --name-template $release --include-crds --set ${release}.installCRDs=true -f $TMPDIR/values.yaml > $TMPDIR/helm-crds.yaml diff -e $TMPDIR/helm-no-crds.yaml $TMPDIR/helm-crds.yaml | head -n-1 | tail -n+2 > $TMPDIR/crds.yaml [ -s $TMPDIR/crds.yaml ] && kubectl apply -f $TMPDIR/crds.yaml } diff --git a/charts/kubezero/templates/aws-efs-csi-driver.yaml b/charts/kubezero/templates/aws-efs-csi-driver.yaml index 1f87e93..1d5f808 100644 --- a/charts/kubezero/templates/aws-efs-csi-driver.yaml +++ b/charts/kubezero/templates/aws-efs-csi-driver.yaml @@ -1,15 +1,13 @@ {{- define "aws-efs-csi-driver-values" }} -{{ with index .Values "aws-efs-csi-driver" "IamArn" }} aws-efs-csi-driver: controller: extraEnv: - name: AWS_ROLE_ARN - value: "{{ . }}" + value: {{ index .Values "aws-efs-csi-driver" "IamArn" | quote }} - name: AWS_WEB_IDENTITY_TOKEN_FILE value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" - name: AWS_STS_REGIONAL_ENDPOINTS value: regional -{{- end }} {{- with index .Values "aws-efs-csi-driver" "PersistentVolumes" }} PersistentVolumes: {{- toYaml . | nindent 2 }} -- 2.40.1 From 660c1a0d8885326336198e50af46b6086537708b Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 15 Jul 2021 12:14:01 +0200 Subject: [PATCH 27/45] fix: remove duplicated CRDs --- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 2 +- .../aws-ebs-csi-driver/templates/crds.yml | 6 - .../crds/crd_snapshotter.yaml | 420 ------------------ charts/kubezero-aws-ebs-csi-driver/update.sh | 3 + 4 files changed, 4 insertions(+), 427 deletions(-) delete mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/crds.yml delete mode 100644 charts/kubezero-aws-ebs-csi-driver/crds/crd_snapshotter.yaml diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index 0532c70..2db9c92 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.6.1 +version: 0.6.2 appVersion: 1.2.4 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/crds.yml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/crds.yml deleted file mode 100644 index ebd5f49..0000000 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/crds.yml +++ /dev/null @@ -1,6 +0,0 @@ -{{- if and .Values.enableVolumeSnapshot (ne (.Capabilities.APIVersions.Has "snapshot.storage.k8s.io/v1") true) }} -{{- range $path, $bytes := .Files.Glob "crds/*.yaml" }} -{{ $.Files.Get $path }} ---- -{{- end }} -{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/crds/crd_snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/crds/crd_snapshotter.yaml deleted file mode 100644 index aff837a..0000000 --- a/charts/kubezero-aws-ebs-csi-driver/crds/crd_snapshotter.yaml +++ /dev/null @@ -1,420 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" - creationTimestamp: null - name: volumesnapshotclasses.snapshot.storage.k8s.io -spec: - group: snapshot.storage.k8s.io - names: - kind: VolumeSnapshotClass - listKind: VolumeSnapshotClassList - plural: volumesnapshotclasses - singular: volumesnapshotclass - scope: Cluster - preserveUnknownFields: false - validation: - openAPIV3Schema: - description: VolumeSnapshotClass specifies parameters that a underlying storage - system uses when creating a volume snapshot. A specific VolumeSnapshotClass - is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses - are non-namespaced - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - deletionPolicy: - description: deletionPolicy determines whether a VolumeSnapshotContent created - through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot - is deleted. Supported values are "Retain" and "Delete". "Retain" means - that the VolumeSnapshotContent and its physical snapshot on underlying - storage system are kept. "Delete" means that the VolumeSnapshotContent - and its physical snapshot on underlying storage system are deleted. Required. - enum: - - Delete - - Retain - type: string - driver: - description: driver is the name of the storage driver that handles this - VolumeSnapshotClass. Required. - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - parameters: - additionalProperties: - type: string - description: parameters is a key-value map with storage driver specific - parameters for creating snapshots. These values are opaque to Kubernetes. - type: object - required: - - deletionPolicy - - driver - type: object - version: v1beta1 - versions: - - name: v1beta1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" - creationTimestamp: null - name: volumesnapshotcontents.snapshot.storage.k8s.io -spec: - group: snapshot.storage.k8s.io - names: - kind: VolumeSnapshotContent - listKind: VolumeSnapshotContentList - plural: volumesnapshotcontents - singular: volumesnapshotcontent - scope: Cluster - subresources: - status: {} - preserveUnknownFields: false - validation: - openAPIV3Schema: - description: VolumeSnapshotContent represents the actual "on-disk" snapshot - object in the underlying storage system - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - spec: - description: spec defines properties of a VolumeSnapshotContent created - by the underlying storage system. Required. - properties: - deletionPolicy: - description: deletionPolicy determines whether this VolumeSnapshotContent - and its physical snapshot on the underlying storage system should - be deleted when its bound VolumeSnapshot is deleted. Supported values - are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent - and its physical snapshot on underlying storage system are kept. "Delete" - means that the VolumeSnapshotContent and its physical snapshot on - underlying storage system are deleted. In dynamic snapshot creation - case, this field will be filled in with the "DeletionPolicy" field - defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For - pre-existing snapshots, users MUST specify this field when creating - the VolumeSnapshotContent object. Required. - enum: - - Delete - - Retain - type: string - driver: - description: driver is the name of the CSI driver used to create the - physical snapshot on the underlying storage system. This MUST be the - same as the name returned by the CSI GetPluginName() call for that - driver. Required. - type: string - source: - description: source specifies from where a snapshot will be created. - This field is immutable after creation. Required. - properties: - snapshotHandle: - description: snapshotHandle specifies the CSI "snapshot_id" of a - pre-existing snapshot on the underlying storage system. This field - is immutable. - type: string - volumeHandle: - description: volumeHandle specifies the CSI "volume_id" of the volume - from which a snapshot should be dynamically taken from. This field - is immutable. - type: string - type: object - volumeSnapshotClassName: - description: name of the VolumeSnapshotClass to which this snapshot - belongs. - type: string - volumeSnapshotRef: - description: volumeSnapshotRef specifies the VolumeSnapshot object to - which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName - field must reference to this VolumeSnapshotContent's name for the - bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent - object, name and namespace of the VolumeSnapshot object MUST be provided - for binding to happen. This field is immutable after creation. Required. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of an - entire object, this string should contain a valid JSON/Go field - access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen only - to have some well-defined way of referencing a part of an object. - TODO: this design is not final and this field is subject to change - in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference is - made, if any. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - required: - - deletionPolicy - - driver - - source - - volumeSnapshotRef - type: object - status: - description: status represents the current information of a snapshot. - properties: - creationTime: - description: creationTime is the timestamp when the point-in-time snapshot - is taken by the underlying storage system. In dynamic snapshot creation - case, this field will be filled in with the "creation_time" value - returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing - snapshot, this field will be filled with the "creation_time" value - returned from the CSI "ListSnapshots" gRPC call if the driver supports - it. If not specified, it indicates the creation time is unknown. The - format of this field is a Unix nanoseconds time encoded as an int64. - On Unix, the command `date +%s%N` returns the current time in nanoseconds - since 1970-01-01 00:00:00 UTC. - format: int64 - type: integer - error: - description: error is the latest observed error during snapshot creation, - if any. - properties: - message: - description: 'message is a string detailing the encountered error - during snapshot creation if specified. NOTE: message may be logged, - and it should not contain sensitive information.' - type: string - time: - description: time is the timestamp when the error was encountered. - format: date-time - type: string - type: object - readyToUse: - description: readyToUse indicates if a snapshot is ready to be used - to restore a volume. In dynamic snapshot creation case, this field - will be filled in with the "ready_to_use" value returned from CSI - "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this - field will be filled with the "ready_to_use" value returned from the - CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, - this field will be set to "True". If not specified, it means the readiness - of a snapshot is unknown. - type: boolean - restoreSize: - description: restoreSize represents the complete size of the snapshot - in bytes. In dynamic snapshot creation case, this field will be filled - in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" - gRPC call. For a pre-existing snapshot, this field will be filled - with the "size_bytes" value returned from the CSI "ListSnapshots" - gRPC call if the driver supports it. When restoring a volume from - this snapshot, the size of the volume MUST NOT be smaller than the - restoreSize if it is specified, otherwise the restoration will fail. - If not specified, it indicates that the size is unknown. - format: int64 - minimum: 0 - type: integer - snapshotHandle: - description: snapshotHandle is the CSI "snapshot_id" of a snapshot on - the underlying storage system. If not specified, it indicates that - dynamic snapshot creation has either failed or it is still in progress. - type: string - type: object - required: - - spec - type: object - version: v1beta1 - versions: - - name: v1beta1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (devel) - api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" - creationTimestamp: null - name: volumesnapshots.snapshot.storage.k8s.io -spec: - group: snapshot.storage.k8s.io - names: - kind: VolumeSnapshot - listKind: VolumeSnapshotList - plural: volumesnapshots - singular: volumesnapshot - scope: Namespaced - subresources: - status: {} - preserveUnknownFields: false - validation: - openAPIV3Schema: - description: VolumeSnapshot is a user's request for either creating a point-in-time - snapshot of a persistent volume, or binding to a pre-existing snapshot. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - spec: - description: 'spec defines the desired characteristics of a snapshot requested - by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots - Required.' - properties: - source: - description: source specifies where a snapshot will be created from. - This field is immutable after creation. Required. - properties: - persistentVolumeClaimName: - description: persistentVolumeClaimName specifies the name of the - PersistentVolumeClaim object in the same namespace as the VolumeSnapshot - object where the snapshot should be dynamically taken from. This - field is immutable. - type: string - volumeSnapshotContentName: - description: volumeSnapshotContentName specifies the name of a pre-existing - VolumeSnapshotContent object. This field is immutable. - type: string - type: object - volumeSnapshotClassName: - description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass - requested by the VolumeSnapshot. If not specified, the default snapshot - class will be used if one exists. If not specified, and there is no - default snapshot class, dynamic snapshot creation will fail. Empty - string is not allowed for this field. TODO(xiangqian): a webhook validation - on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes' - type: string - required: - - source - type: object - status: - description: 'status represents the current information of a snapshot. NOTE: - status can be modified by sources other than system controllers, and must - not be depended upon for accuracy. Controllers should only use information - from the VolumeSnapshotContent object after verifying that the binding - is accurate and complete.' - properties: - boundVolumeSnapshotContentName: - description: 'boundVolumeSnapshotContentName represents the name of - the VolumeSnapshotContent object to which the VolumeSnapshot object - is bound. If not specified, it indicates that the VolumeSnapshot object - has not been successfully bound to a VolumeSnapshotContent object - yet. NOTE: Specified boundVolumeSnapshotContentName alone does not - mean binding is valid. Controllers MUST always verify bidirectional - binding between VolumeSnapshot and VolumeSnapshotContent to - avoid possible security issues.' - type: string - creationTime: - description: creationTime is the timestamp when the point-in-time snapshot - is taken by the underlying storage system. In dynamic snapshot creation - case, this field will be filled in with the "creation_time" value - returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing - snapshot, this field will be filled with the "creation_time" value - returned from the CSI "ListSnapshots" gRPC call if the driver supports - it. If not specified, it indicates that the creation time of the snapshot - is unknown. - format: date-time - type: string - error: - description: error is the last observed error during snapshot creation, - if any. This field could be helpful to upper level controllers(i.e., - application controller) to decide whether they should continue on - waiting for the snapshot to be created based on the type of error - reported. - properties: - message: - description: 'message is a string detailing the encountered error - during snapshot creation if specified. NOTE: message may be logged, - and it should not contain sensitive information.' - type: string - time: - description: time is the timestamp when the error was encountered. - format: date-time - type: string - type: object - readyToUse: - description: readyToUse indicates if a snapshot is ready to be used - to restore a volume. In dynamic snapshot creation case, this field - will be filled in with the "ready_to_use" value returned from CSI - "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this - field will be filled with the "ready_to_use" value returned from the - CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, - this field will be set to "True". If not specified, it means the readiness - of a snapshot is unknown. - type: boolean - restoreSize: - description: restoreSize represents the complete size of the snapshot - in bytes. In dynamic snapshot creation case, this field will be filled - in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" - gRPC call. For a pre-existing snapshot, this field will be filled - with the "size_bytes" value returned from the CSI "ListSnapshots" - gRPC call if the driver supports it. When restoring a volume from - this snapshot, the size of the volume MUST NOT be smaller than the - restoreSize if it is specified, otherwise the restoration will fail. - If not specified, it indicates that the size is unknown. - type: string - type: object - required: - - spec - type: object - version: v1beta1 - versions: - - name: v1beta1 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/charts/kubezero-aws-ebs-csi-driver/update.sh b/charts/kubezero-aws-ebs-csi-driver/update.sh index cfb9377..3908a3d 100755 --- a/charts/kubezero-aws-ebs-csi-driver/update.sh +++ b/charts/kubezero-aws-ebs-csi-driver/update.sh @@ -6,3 +6,6 @@ rm -rf charts/aws-ebs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts patch -i zdt.patch -p0 --no-backup-if-mismatch + +# Remove duplicated CRDs +rm -f charts/aws-ebs-csi-driver/templates/crds.yml -- 2.40.1 From a62399aebebc137f83bc09923e7864b8ee555ebe Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 15 Jul 2021 13:54:49 +0200 Subject: [PATCH 28/45] Fix: fix argocd warnings for istio, upgrades doc --- charts/kubezero/templates/istio.yaml | 5 ++++- docs/Upgrade-2.20.md | 22 +++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/charts/kubezero/templates/istio.yaml b/charts/kubezero/templates/istio.yaml index 021a36a..abd6a4c 100644 --- a/charts/kubezero/templates/istio.yaml +++ b/charts/kubezero/templates/istio.yaml @@ -28,7 +28,10 @@ global: kind: MutatingWebhookConfiguration jsonPointers: - /webhooks/0/clientConfig/caBundle - - /webhooks/0/failurePolicy + - /webhooks/1/clientConfig/caBundle + - /webhooks/2/clientConfig/caBundle + - /webhooks/3/clientConfig/caBundle + {{- end }} {{ include "kubezero-app.app" . }} diff --git a/docs/Upgrade-2.20.md b/docs/Upgrade-2.20.md index 34ed405..31583d2 100644 --- a/docs/Upgrade-2.20.md +++ b/docs/Upgrade-2.20.md @@ -59,20 +59,28 @@ and verify your config via `aws sts get-caller-identity` and `kubectl cluster-in - replace worker nodes in a rolling fashion via. drain / terminate and rinse-repeat # Upgrade KubeZero -1. Update CRDs of all enabled components: -`./bootstrap.sh crds all clusters/$CLUSTER` - -2. Prepare upgrade +1. Prepare upgrade - Remove legacy monitoring configmaps -- Remove previous Grafana stateful config -- Remove legacy Istio Envoyfilter - ``` kubectl delete cm -n monitoring -l grafana_dashboard=1 +``` + +- Remove previous Grafana stateful config +``` kubectl delete pvc metrics-grafana -n monitoring +``` + +- Remove legacy Istio Envoyfilter +``` kubectl delete envoyfilter -A -l operator.istio.io/version=1.6.9 ``` +- ensure that the latest kubezero.yaml output from CloudBender is present under `clusters/$CLUSTER` and no legacy cloudbender.yaml is around anymore. +If ArgoCD is used make sure the `valuesFiles` settings in the top-level values.yaml matches the files under `clusters/$CLUSTER` + +2. Update CRDs of all enabled components: +`./bootstrap.sh crds all clusters/$CLUSTER` + 3. Upgrade all KubeZero modules: - without ArgoCD: - `./bootstrap.sh deploy all clusters/$CLUSTER` -- 2.40.1 From d4aaa88643d06c41d3ed3b0c88cc693f4a01b0f7 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 16 Jul 2021 22:41:02 +0200 Subject: [PATCH 29/45] fix: all upgrade_120.sh to run more than once --- charts/kubeadm/templates/ClusterConfiguration.yaml | 2 ++ scripts/upgrade_120.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/charts/kubeadm/templates/ClusterConfiguration.yaml b/charts/kubeadm/templates/ClusterConfiguration.yaml index 9d424bc..985b048 100644 --- a/charts/kubeadm/templates/ClusterConfiguration.yaml +++ b/charts/kubeadm/templates/ClusterConfiguration.yaml @@ -68,6 +68,8 @@ apiServer: api-audiences: "istio-ca" {{- end }} feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" "platform" .Values.platform ) | trimSuffix "," | quote }} + # for 1.21 + # enable-admission-plugins: DenyServiceExternalIPs,NodeRestriction,EventRateLimit enable-admission-plugins: NodeRestriction,EventRateLimit # {{- if .Values.highAvailable }} # goaway-chance: ".001" diff --git a/scripts/upgrade_120.sh b/scripts/upgrade_120.sh index eef12c9..a4dcb30 100755 --- a/scripts/upgrade_120.sh +++ b/scripts/upgrade_120.sh @@ -79,6 +79,7 @@ spec: echo "Patching ClusterConfig to re-create new etcd server certificates" yq w /etc/kubezero/kubeadm/templates/ClusterConfiguration.yaml etcd.local.serverCertSANs[+] $nodename > /etc/kubernetes/kubeadm-recert.yaml + yq w -i /etc/kubernetes/kubeadm-recert.yaml etcd.local.serverCertSANs[+] $nodename.$zone_name rm -f /etc/kubernetes/pki/etcd/server.* kubeadm init phase certs etcd-server --config=/etc/kubernetes/kubeadm-recert.yaml 2>/dev/null kill -s HUP $(ps -e | grep etcd | awk '{print $1}') -- 2.40.1 From 2a8f77bf959e44aa6ad94fd7b01eeef7c2b2535f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 19 Jul 2021 11:03:49 +0200 Subject: [PATCH 30/45] feat: Istio version bump --- charts/kubezero-istio-ingress/Chart.yaml | 8 ++++---- .../charts/istio-ingress/Chart.yaml | 2 +- .../charts/istio-ingress/values.yaml | 2 +- .../charts/istio-private-ingress/Chart.yaml | 2 +- .../charts/istio-private-ingress/values.yaml | 2 +- charts/kubezero-istio/Chart.yaml | 8 ++++---- charts/kubezero-istio/charts/base/Chart.yaml | 2 +- charts/kubezero-istio/charts/istio-discovery/Chart.yaml | 2 +- charts/kubezero-istio/charts/istio-discovery/values.yaml | 2 +- charts/kubezero-istio/update.sh | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/charts/kubezero-istio-ingress/Chart.yaml b/charts/kubezero-istio-ingress/Chart.yaml index d8ac819..9905182 100644 --- a/charts/kubezero-istio-ingress/Chart.yaml +++ b/charts/kubezero-istio-ingress/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-istio-ingress description: KubeZero Umbrella Chart for Istio based Ingress type: application -version: 0.6.0 -appVersion: 1.10.2 +version: 0.6.1 +appVersion: 1.10.3 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -16,9 +16,9 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: istio-ingress - version: 1.10.2 + version: 1.10.3 condition: istio-ingress.enabled - name: istio-private-ingress - version: 1.10.2 + version: 1.10.3 condition: istio-private-ingress.enabled kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml b/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml index 43ca564..0c7fc03 100644 --- a/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: istio-ingress -version: 1.10.2 +version: 1.10.3 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio gateways keywords: diff --git a/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml b/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml index 8a3ef0a..22bdc65 100644 --- a/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml @@ -165,7 +165,7 @@ global: hub: docker.io/istio # Default tag for Istio images. - tag: 1.10.2 + tag: 1.10.3 # Specify image pull policy if default behavior isn't desired. # Default behavior: latest images will be Always else IfNotPresent. diff --git a/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml b/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml index b0d3b2e..e5f551f 100644 --- a/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: istio-private-ingress -version: 1.10.2 +version: 1.10.3 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio gateways keywords: diff --git a/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml b/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml index 8a3ef0a..22bdc65 100644 --- a/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml @@ -165,7 +165,7 @@ global: hub: docker.io/istio # Default tag for Istio images. - tag: 1.10.2 + tag: 1.10.3 # Specify image pull policy if default behavior isn't desired. # Default behavior: latest images will be Always else IfNotPresent. diff --git a/charts/kubezero-istio/Chart.yaml b/charts/kubezero-istio/Chart.yaml index deebed3..ec52659 100644 --- a/charts/kubezero-istio/Chart.yaml +++ b/charts/kubezero-istio/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-istio description: KubeZero Umbrella Chart for Istio type: application -version: 0.6.0 -appVersion: 1.10.2 +version: 0.6.1 +appVersion: 1.10.3 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -16,7 +16,7 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: base - version: 1.10.2 + version: 1.10.3 - name: istio-discovery - version: 1.10.2 + version: 1.10.3 kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-istio/charts/base/Chart.yaml b/charts/kubezero-istio/charts/base/Chart.yaml index eba13b9..50041cd 100644 --- a/charts/kubezero-istio/charts/base/Chart.yaml +++ b/charts/kubezero-istio/charts/base/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: base -version: 1.10.2 +version: 1.10.3 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio cluster resources and CRDs keywords: diff --git a/charts/kubezero-istio/charts/istio-discovery/Chart.yaml b/charts/kubezero-istio/charts/istio-discovery/Chart.yaml index 6afbc9b..17a1f8e 100644 --- a/charts/kubezero-istio/charts/istio-discovery/Chart.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: istio-discovery -version: 1.10.2 +version: 1.10.3 tillerVersion: ">=2.7.2" description: Helm chart for istio control plane keywords: diff --git a/charts/kubezero-istio/charts/istio-discovery/values.yaml b/charts/kubezero-istio/charts/istio-discovery/values.yaml index fc595d4..b159557 100644 --- a/charts/kubezero-istio/charts/istio-discovery/values.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/values.yaml @@ -237,7 +237,7 @@ global: # Dev builds from prow are on gcr.io hub: docker.io/istio # Default tag for Istio images. - tag: 1.10.2 + tag: 1.10.3 # Specify image pull policy if default behavior isn't desired. # Default behavior: latest images will be Always else IfNotPresent. diff --git a/charts/kubezero-istio/update.sh b/charts/kubezero-istio/update.sh index 63f5bb9..2af9b65 100755 --- a/charts/kubezero-istio/update.sh +++ b/charts/kubezero-istio/update.sh @@ -4,7 +4,7 @@ set -ex ### TODO # - https://istio.io/latest/docs/ops/configuration/security/harden-docker-images/ -export ISTIO_VERSION=1.10.2 +export ISTIO_VERSION=1.10.3 rm -rf istio curl -sL "https://github.com/istio/istio/releases/download/${ISTIO_VERSION}/istio-${ISTIO_VERSION}-linux-amd64.tar.gz" | tar xz -- 2.40.1 From 1de2aa4ef1682dfdfaa428c6f50893491d6816b4 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 19 Jul 2021 16:17:46 +0200 Subject: [PATCH 31/45] Feat: add workergroup toleration to ebs driver --- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 2 +- charts/kubezero-aws-ebs-csi-driver/values.yaml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index 2db9c92..c9737eb 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.6.2 +version: 0.6.3 appVersion: 1.2.4 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/values.yaml index b7340b8..cf8979b 100644 --- a/charts/kubezero-aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/values.yaml @@ -29,6 +29,11 @@ aws-ebs-csi-driver: # cpu: 50m memory: 40Mi + node: + tolerations: + - key: kubezero-workergroup + effect: NoSchedule + storageClasses: - name: ebs-sc-gp2-xfs volumeBindingMode: WaitForFirstConsumer -- 2.40.1 From 4348462ebb668d61b337381f7b28a5999657c0a5 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 19 Jul 2021 16:19:23 +0200 Subject: [PATCH 32/45] Feat: add workergroup toleration to efs driver --- charts/kubezero-aws-efs-csi-driver/Chart.yaml | 2 +- charts/kubezero-aws-efs-csi-driver/values.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/charts/kubezero-aws-efs-csi-driver/Chart.yaml b/charts/kubezero-aws-efs-csi-driver/Chart.yaml index 2313a5c..7978af0 100644 --- a/charts/kubezero-aws-efs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-efs-csi-driver/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: kubezero-aws-efs-csi-driver description: KubeZero Umbrella Chart for aws-efs-csi-driver -version: 0.4.1 +version: 0.4.2 appVersion: 1.3.2 kubeVersion: ">=1.18.0-0" home: https://kubezero.com diff --git a/charts/kubezero-aws-efs-csi-driver/values.yaml b/charts/kubezero-aws-efs-csi-driver/values.yaml index 288a630..e91df9e 100644 --- a/charts/kubezero-aws-efs-csi-driver/values.yaml +++ b/charts/kubezero-aws-efs-csi-driver/values.yaml @@ -36,6 +36,9 @@ aws-efs-csi-driver: - matchExpressions: - key: node.kubernetes.io/csi.efs.fs operator: Exists + tolerations: + - key: kubezero-workergroup + effect: NoSchedule storageClasses: - name: efs-sc -- 2.40.1 From b0de45d360a68d01ce15204409162d671a87d457 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 19 Jul 2021 16:20:25 +0200 Subject: [PATCH 33/45] Feat: add workergroup toleration to fluent-bit --- charts/kubezero-logging/Chart.yaml | 2 +- charts/kubezero-logging/values.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index e846099..651b5dc 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application -version: 0.7.1 +version: 0.7.2 appVersion: 1.6.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 72a0999..f50d814 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -224,6 +224,8 @@ fluent-bit: tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule + - key: kubezero-workergroup + effect: NoSchedule resources: requests: -- 2.40.1 From 96cdc5b6fcc7c14178a8a46a741e7ee73c7752cf Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 19 Jul 2021 16:21:20 +0200 Subject: [PATCH 34/45] Feat: add workergroup toleration to kiam-agents --- charts/kubezero-kiam/Chart.yaml | 2 +- charts/kubezero-kiam/values.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index 45defec..14debca 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-kiam description: KubeZero Umbrella Chart for Kiam type: application -version: 0.3.4 +version: 0.3.5 appVersion: "4.0" home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-kiam/values.yaml b/charts/kubezero-kiam/values.yaml index 8f1b8da..dc4d2b4 100644 --- a/charts/kubezero-kiam/values.yaml +++ b/charts/kubezero-kiam/values.yaml @@ -65,6 +65,8 @@ kiam: tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule + - key: kubezero-workergroup + effect: NoSchedule priorityClassName: system-node-critical prometheus: servicemonitor: -- 2.40.1 From f4ff4bcab001f932365c7c0bd5a370058198674f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 19 Jul 2021 16:36:07 +0200 Subject: [PATCH 35/45] fix: fix toleration --- charts/kubezero-kiam/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/kubezero-kiam/values.yaml b/charts/kubezero-kiam/values.yaml index dc4d2b4..82a050a 100644 --- a/charts/kubezero-kiam/values.yaml +++ b/charts/kubezero-kiam/values.yaml @@ -67,6 +67,7 @@ kiam: effect: NoSchedule - key: kubezero-workergroup effect: NoSchedule + operator: Exists priorityClassName: system-node-critical prometheus: servicemonitor: -- 2.40.1 From 84b31c3ddc91320ad14bc24bb5cb58746a1e2e9d Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 19 Jul 2021 16:38:32 +0200 Subject: [PATCH 36/45] fix: fix toleration --- charts/kubezero-aws-ebs-csi-driver/values.yaml | 1 + charts/kubezero-aws-efs-csi-driver/values.yaml | 1 + charts/kubezero-logging/values.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/charts/kubezero-aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/values.yaml index cf8979b..44489ef 100644 --- a/charts/kubezero-aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/values.yaml @@ -33,6 +33,7 @@ aws-ebs-csi-driver: tolerations: - key: kubezero-workergroup effect: NoSchedule + operator: Exists storageClasses: - name: ebs-sc-gp2-xfs diff --git a/charts/kubezero-aws-efs-csi-driver/values.yaml b/charts/kubezero-aws-efs-csi-driver/values.yaml index e91df9e..8a33534 100644 --- a/charts/kubezero-aws-efs-csi-driver/values.yaml +++ b/charts/kubezero-aws-efs-csi-driver/values.yaml @@ -39,6 +39,7 @@ aws-efs-csi-driver: tolerations: - key: kubezero-workergroup effect: NoSchedule + operator: Exists storageClasses: - name: efs-sc diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index f50d814..68562ce 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -226,6 +226,7 @@ fluent-bit: effect: NoSchedule - key: kubezero-workergroup effect: NoSchedule + operator: Exists resources: requests: -- 2.40.1 From 99b42e27815cb61dd9c60b191c7582b6df3a3199 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 20 Jul 2021 13:28:37 +0200 Subject: [PATCH 37/45] fix: prepare for fluent-bit multiline support, still broken though, minor tuning --- charts/kubezero-logging/Chart.yaml | 2 +- .../charts/fluent-bit/templates/_pod.tpl | 4 +- .../fluent-bit/templates/clusterrole.yaml | 4 +- charts/kubezero-logging/fluent-bit.patch | 38 +++++++++---------- charts/kubezero-logging/values.yaml | 12 ++++-- 5 files changed, 33 insertions(+), 27 deletions(-) diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index 651b5dc..5b62cd7 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application -version: 0.7.2 +version: 0.7.3 appVersion: 1.6.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl b/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl index 22cd951..3f13b47 100644 --- a/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl +++ b/charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl @@ -21,8 +21,8 @@ hostAliases: initContainers: {{- toYaml .Values.initContainers | nindent 2 }} {{- end }} -#hostNetwork: true -#dnsPolicy: ClusterFirstWithHostNet +hostNetwork: true +dnsPolicy: ClusterFirstWithHostNet containers: - name: {{ .Chart.Name }} securityContext: diff --git a/charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml b/charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml index 9414222..818c5ee 100644 --- a/charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml +++ b/charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml @@ -11,8 +11,8 @@ rules: resources: - pods - namespaces - #- nodes - #- nodes/proxy + - nodes + - nodes/proxy verbs: - get - list diff --git a/charts/kubezero-logging/fluent-bit.patch b/charts/kubezero-logging/fluent-bit.patch index 40e1844..8698f25 100644 --- a/charts/kubezero-logging/fluent-bit.patch +++ b/charts/kubezero-logging/fluent-bit.patch @@ -1,30 +1,30 @@ -diff -tubrN charts/fluent-bit/templates/_pod.tpl charts/fluent-bit.zdt/templates/_pod.tpl ---- charts/fluent-bit/templates/_pod.tpl 2021-04-06 13:55:48.000000000 +0200 -+++ charts/fluent-bit.zdt/templates/_pod.tpl 2021-04-07 11:53:45.609030378 +0200 -@@ -13,6 +13,8 @@ - dnsConfig: - {{- toYaml . | nindent 2 }} +diff -tubr charts/fluent-bit/templates/_pod.tpl charts/fluent-bit.zdt/templates/_pod.tpl +--- charts/fluent-bit/templates/_pod.tpl 2021-06-22 12:25:10.000000000 +0200 ++++ charts/fluent-bit.zdt/templates/_pod.tpl 2021-07-20 12:44:55.349873171 +0200 +@@ -21,6 +21,8 @@ + initContainers: + {{- toYaml .Values.initContainers | nindent 2 }} {{- end }} -+#hostNetwork: true -+#dnsPolicy: ClusterFirstWithHostNet ++hostNetwork: true ++dnsPolicy: ClusterFirstWithHostNet containers: - name: {{ .Chart.Name }} securityContext: -diff -tubrN charts/fluent-bit/templates/clusterrole.yaml charts/fluent-bit.zdt/templates/clusterrole.yaml ---- charts/fluent-bit/templates/clusterrole.yaml 2021-04-06 13:55:48.000000000 +0200 -+++ charts/fluent-bit.zdt/templates/clusterrole.yaml 2021-04-07 11:53:26.049029491 +0200 +diff -tubr charts/fluent-bit/templates/clusterrole.yaml charts/fluent-bit.zdt/templates/clusterrole.yaml +--- charts/fluent-bit/templates/clusterrole.yaml 2021-06-22 12:25:10.000000000 +0200 ++++ charts/fluent-bit.zdt/templates/clusterrole.yaml 2021-07-20 12:44:29.936278716 +0200 @@ -11,6 +11,8 @@ resources: - pods - namespaces -+ #- nodes -+ #- nodes/proxy ++ - nodes ++ - nodes/proxy verbs: - get - list -diff -tubrN charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent-bit.zdt/templates/tests/test-connection.yaml ---- charts/fluent-bit/templates/tests/test-connection.yaml 2021-04-06 13:55:48.000000000 +0200 -+++ charts/fluent-bit.zdt/templates/tests/test-connection.yaml 2021-04-07 11:52:34.199027132 +0200 +diff -tubr charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent-bit.zdt/templates/tests/test-connection.yaml +--- charts/fluent-bit/templates/tests/test-connection.yaml 2021-06-22 12:25:10.000000000 +0200 ++++ charts/fluent-bit.zdt/templates/tests/test-connection.yaml 2021-05-17 12:09:02.724057438 +0200 @@ -1,3 +1,4 @@ +{{- if .Values.testFramework.enabled }} apiVersion: v1 @@ -35,9 +35,9 @@ diff -tubrN charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent {{- end }} restartPolicy: Never +{{- end }} -diff -tubrN charts/fluent-bit/values.yaml charts/fluent-bit.zdt/values.yaml ---- charts/fluent-bit/values.yaml 2021-04-06 13:55:48.000000000 +0200 -+++ charts/fluent-bit.zdt/values.yaml 2021-04-07 11:52:54.742361399 +0200 +diff -tubr charts/fluent-bit/values.yaml charts/fluent-bit.zdt/values.yaml +--- charts/fluent-bit/values.yaml 2021-06-22 12:25:10.000000000 +0200 ++++ charts/fluent-bit.zdt/values.yaml 2021-07-19 10:23:01.383398153 +0200 @@ -12,6 +12,7 @@ # tag: diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 68562ce..f8afa01 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -216,6 +216,9 @@ fluentd: fluent-bit: enabled: false + image: + tag: 1.8.1 + serviceMonitor: enabled: false selector: @@ -245,7 +248,7 @@ fluent-bit: memBufLimit: 4MB refreshInterval: 10 - logLevel: warn + logLevel: info flushInterval: 5 #extraRecords: @@ -266,10 +269,12 @@ fluent-bit: Name tail Path /var/log/containers/*.log Parser cri-log + #multiline.parser cri Tag cri.* Skip_Long_Lines On DB /var/log/flb_kube.db DB.Sync Normal + DB.locking true {{- with .Values.config.input }} Mem_Buf_Limit {{ default "4MB" .memBufLimit }} Refresh_Interval {{ default 10 .refreshInterval }} @@ -291,8 +296,9 @@ fluent-bit: Keep_Log Off K8S-Logging.Parser Off K8S-Logging.Exclude Off + Kube_Meta_Cache_TTL 3600s + Buffer_Size 0 #Use_Kubelet true - #Kubelet_Port 10250 {{- if index .Values "config" "extraRecords" }} @@ -331,7 +337,7 @@ fluent-bit: [PARSER] Name cri-log Format regex - Regex ^(?