From 9504f874a02f8de6ab6fbf484af05ea727bbefc8 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 8 Nov 2024 16:45:10 +0000 Subject: [PATCH] feat: nvidia runtimeClass, various cleanups for 1.30 --- admin/kubezero.sh | 22 +- admin/libhelm.sh | 2 + admin/upgrade_cluster.sh | 2 +- .../kubeadm/templates/InitConfiguration.yaml | 2 +- .../kubeadm/templates/JoinConfiguration.yaml | 6 - .../templates/KubeletConfiguration.yaml | 4 +- .../templates/resources/10-runtimeClass.yaml | 4 +- charts/kubezero-addons/Chart.yaml | 2 +- charts/kubezero-addons/README.md | 8 +- .../device-plugins/nvidia-runtimeClass.yaml | 10 + charts/kubezero-addons/values.yaml | 5 + .../jsonnet/jsonnetfile.lock.json | 34 +- .../jsonnet/zdt-dashboards.yaml | 4 - charts/kubezero-metrics/jsonnet/zdt/crio.json | 414 ------------------ .../templates/grafana-dashboards-zdt.yaml | 2 - charts/kubezero/templates/metrics.yaml | 5 +- charts/kubezero/values.yaml | 2 +- 17 files changed, 64 insertions(+), 464 deletions(-) delete mode 100644 charts/kubeadm/templates/JoinConfiguration.yaml create mode 100644 charts/kubezero-addons/templates/device-plugins/nvidia-runtimeClass.yaml delete mode 100644 charts/kubezero-metrics/jsonnet/zdt/crio.json diff --git a/admin/kubezero.sh b/admin/kubezero.sh index a8efbde4..1b0a68e3 100755 --- a/admin/kubezero.sh +++ b/admin/kubezero.sh @@ -1,4 +1,8 @@ -#!/bin/bash -e +#!/bin/bash +set -eu -o pipefail + +DEBUG=${DEBUG:-""} +LOG="" if [ -n "$DEBUG" ]; then set -x @@ -80,9 +84,6 @@ parse_kubezero() { export NODENAME=$(yq eval '.nodeName' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml) export PROVIDER_ID=$(yq eval '.providerID // ""' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml) export AWS_IAM_AUTH=$(yq eval '.api.awsIamAuth.enabled // "false"' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml) - - # From here on bail out, allows debug_shell even in error cases - set -e } @@ -188,8 +189,15 @@ kubeadm_upgrade() { # install re-certed kubectl config for root cp ${HOSTFS}/etc/kubernetes/super-admin.conf ${HOSTFS}/root/.kube/config - # post upgrade hook - [ -f /var/lib/kubezero/post-upgrade.sh ] && . /var/lib/kubezero/post-upgrade.sh + # post upgrade + + # Update kubezero-values CM + kubectl get cm -n kube-system kubelet-config -o=yaml | \ + yq e '.data.kubelet' | yq e '.containerRuntimeEndpoint = "unix:///run/containerd/containerd.sock"' > $WORKDIR/new-kubelet.cm + + kubectl get cm -n kube-system kubelet-config -o=yaml | \ + yq e '.data.kubelet |= load_str("/tmp/kubezero/new-kubelet.cm")' | \ + kubectl apply --server-side --force-conflicts -f - # Cleanup after kubeadm on the host rm -rf ${HOSTFS}/etc/kubernetes/tmp @@ -427,7 +435,7 @@ debug_shell() { printf "For manual etcdctl commands use:\n # export ETCDCTL_ENDPOINTS=$ETCD_NODENAME:2379\n" - /bin/bash + bash } # First parse kubeadm-values.yaml diff --git a/admin/libhelm.sh b/admin/libhelm.sh index 0abb9508..c563d728 100644 --- a/admin/libhelm.sh +++ b/admin/libhelm.sh @@ -5,6 +5,8 @@ API_VERSIONS="-a monitoring.coreos.com/v1 -a snapshot.storage.k8s.io/v1 -a polic export HELM_SECRETS_BACKEND="vals" +LOCAL_DEV=${LOCAL_DEV:-""} + # Waits for max 300s and retries function wait_for() { local TRIES=0 diff --git a/admin/upgrade_cluster.sh b/admin/upgrade_cluster.sh index a2c85f26..0904bd57 100755 --- a/admin/upgrade_cluster.sh +++ b/admin/upgrade_cluster.sh @@ -29,7 +29,7 @@ read -r #[ "$ARGOCD" == "True" ] && kubectl edit app kubezero -n argocd || kubectl edit cm kubezero-values -n kubezero ### v1.30 -# +kubectl delete runtimeclass crio || true # upgrade modules # diff --git a/charts/kubeadm/templates/InitConfiguration.yaml b/charts/kubeadm/templates/InitConfiguration.yaml index dba07d92..6f8b3caa 100644 --- a/charts/kubeadm/templates/InitConfiguration.yaml +++ b/charts/kubeadm/templates/InitConfiguration.yaml @@ -8,7 +8,7 @@ patches: directory: {{ . }} {{- end }} nodeRegistration: - criSocket: "unix:///var/run/crio/crio.sock" + criSocket: "unix:///run/containerd/containerd.sock" ignorePreflightErrors: - DirAvailable--var-lib-etcd - DirAvailable--etc-kubernetes-manifests diff --git a/charts/kubeadm/templates/JoinConfiguration.yaml b/charts/kubeadm/templates/JoinConfiguration.yaml deleted file mode 100644 index 41d30297..00000000 --- a/charts/kubeadm/templates/JoinConfiguration.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta3 -kind: JoinConfiguration -nodeRegistration: - criSocket: "unix:///var/run/crio/crio.sock" -patches: - directory: /tmp/patches diff --git a/charts/kubeadm/templates/KubeletConfiguration.yaml b/charts/kubeadm/templates/KubeletConfiguration.yaml index 2e454cbd..25828191 100644 --- a/charts/kubeadm/templates/KubeletConfiguration.yaml +++ b/charts/kubeadm/templates/KubeletConfiguration.yaml @@ -6,7 +6,7 @@ cgroupDriver: cgroupfs logging: format: json hairpinMode: hairpin-veth -containerRuntimeEndpoint: "unix:///var/run/crio/crio.sock" +containerRuntimeEndpoint: "unix:///run/containerd/containerd.sock" {{- if .Values.systemd }} resolvConf: /run/systemd/resolve/resolv.conf {{- end }} @@ -21,7 +21,7 @@ featureGates: # Minimal unit is 40m per pod podsPerCore: 25 # cpuCFSQuotaPeriod: 10ms -# Basic OS incl. crio +# Basic OS incl. cri systemReserved: memory: 96Mi #ephemeral-storage: "1Gi" diff --git a/charts/kubeadm/templates/resources/10-runtimeClass.yaml b/charts/kubeadm/templates/resources/10-runtimeClass.yaml index 8ed077bf..dc6c7d8a 100644 --- a/charts/kubeadm/templates/resources/10-runtimeClass.yaml +++ b/charts/kubeadm/templates/resources/10-runtimeClass.yaml @@ -1,8 +1,8 @@ apiVersion: node.k8s.io/v1 kind: RuntimeClass metadata: - name: crio + name: crun handler: crun overhead: podFixed: - memory: 4Mi + memory: 8Mi diff --git a/charts/kubezero-addons/Chart.yaml b/charts/kubezero-addons/Chart.yaml index 92051482..569b5ca6 100644 --- a/charts/kubezero-addons/Chart.yaml +++ b/charts/kubezero-addons/Chart.yaml @@ -3,7 +3,7 @@ name: kubezero-addons description: KubeZero umbrella chart for various optional cluster addons type: application version: 0.8.10 -appVersion: v1.29 +appVersion: v1.30 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-addons/README.md b/charts/kubezero-addons/README.md index 105b0f9e..9648c81d 100644 --- a/charts/kubezero-addons/README.md +++ b/charts/kubezero-addons/README.md @@ -1,6 +1,6 @@ # kubezero-addons -![Version: 0.8.9](https://img.shields.io/badge/Version-0.8.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.29](https://img.shields.io/badge/AppVersion-v1.29-informational?style=flat-square) +![Version: 0.8.10](https://img.shields.io/badge/Version-0.8.10-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.30](https://img.shields.io/badge/AppVersion-v1.30-informational?style=flat-square) KubeZero umbrella chart for various optional cluster addons @@ -18,10 +18,10 @@ Kubernetes: `>= 1.26.0` | Repository | Name | Version | |------------|------|---------| -| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.16.1 | +| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.16.2 | | https://kubernetes-sigs.github.io/external-dns/ | external-dns | 1.15.0 | -| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.43.0 | -| https://nvidia.github.io/k8s-device-plugin | nvidia-device-plugin | 0.16.2 | +| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.43.2 | +| https://nvidia.github.io/k8s-device-plugin | nvidia-device-plugin | 0.17.0 | | https://twin.github.io/helm-charts | aws-eks-asg-rolling-update-handler | 1.5.0 | | oci://public.ecr.aws/aws-ec2/helm | aws-node-termination-handler | 0.24.1 | diff --git a/charts/kubezero-addons/templates/device-plugins/nvidia-runtimeClass.yaml b/charts/kubezero-addons/templates/device-plugins/nvidia-runtimeClass.yaml new file mode 100644 index 00000000..6e250879 --- /dev/null +++ b/charts/kubezero-addons/templates/device-plugins/nvidia-runtimeClass.yaml @@ -0,0 +1,10 @@ +{{- if index .Values "nvidia-device-plugin" "enabled" }} +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia +handler: nvidia +overhead: + podFixed: + memory: 16Mi +{{- end }} diff --git a/charts/kubezero-addons/values.yaml b/charts/kubezero-addons/values.yaml index 0a8eae69..4c921a84 100644 --- a/charts/kubezero-addons/values.yaml +++ b/charts/kubezero-addons/values.yaml @@ -164,6 +164,11 @@ awsNeuron: nvidia-device-plugin: enabled: false + + cdi: + nvidiaHookPath: /usr/bin + deviceDiscoveryStrategy: tegra + tolerations: - key: nvidia.com/gpu operator: Exists diff --git a/charts/kubezero-metrics/jsonnet/jsonnetfile.lock.json b/charts/kubezero-metrics/jsonnet/jsonnetfile.lock.json index df036829..0bfb2b4e 100644 --- a/charts/kubezero-metrics/jsonnet/jsonnetfile.lock.json +++ b/charts/kubezero-metrics/jsonnet/jsonnetfile.lock.json @@ -18,7 +18,7 @@ "subdir": "contrib/mixin" } }, - "version": "18eb5c6881d43064f4559034bf12c3ef6ce89e4b", + "version": "7b70b5f6aebe5be44691b605d5b0d179c106cbc0", "sum": "IXI3LQIT9NmTPJAk8WLUJd5+qZfcGpeNCyWIK7oEpws=" }, { @@ -58,7 +58,7 @@ "subdir": "gen/grafonnet-latest" } }, - "version": "1ce5aec95ce32336fe47c8881361847c475b5254", + "version": "82a19822e54a0a12a51e24dbd48fcde717dc0864", "sum": "64fMUPI3frXGj4X1FqFd1t7r04w3CUSmXaDcJ23EYbQ=" }, { @@ -68,7 +68,7 @@ "subdir": "gen/grafonnet-v10.0.0" } }, - "version": "1ce5aec95ce32336fe47c8881361847c475b5254", + "version": "82a19822e54a0a12a51e24dbd48fcde717dc0864", "sum": "xdcrJPJlpkq4+5LpGwN4tPAuheNNLXZjE6tDcyvFjr0=" }, { @@ -78,7 +78,7 @@ "subdir": "gen/grafonnet-v11.1.0" } }, - "version": "1ce5aec95ce32336fe47c8881361847c475b5254", + "version": "82a19822e54a0a12a51e24dbd48fcde717dc0864", "sum": "41w7p/rwrNsITqNHMXtGSJAfAyKmnflg6rFhKBduUxM=" }, { @@ -88,7 +88,7 @@ "subdir": "grafana-builder" } }, - "version": "b9761545ddeac33ffe25bd87121076eab258c6f1", + "version": "ab84b9f67c7a7f61e0c0a311afb47a1af4f5903f", "sum": "yxqWcq/N3E/a/XreeU6EuE6X7kYPnG0AspAQFKOjASo=" }, { @@ -118,8 +118,8 @@ "subdir": "" } }, - "version": "0348e09edc3961a29a55f199d1bf0060c847a608", - "sum": "kTZuZcE+pNw8ZVZECKxrZG4F9BS+ydWMcgACE9oUrRc=" + "version": "c70f03daec5d7578ef697c9f5b93ea88a41fe0d7", + "sum": "qPlmuwMEsbOsHGJT0lSHCK7jtS3k0ql6A4sudj/pnDA=" }, { "source": { @@ -128,7 +128,7 @@ "subdir": "jsonnet/kube-state-metrics" } }, - "version": "45b7e20ee3cbecd6c5c2960a581fd5a931bd7826", + "version": "c804368d08729e2d66d9f60bc1d252d0ac753a54", "sum": "lO7jUSzAIy8Yk9pOWJIWgPRhubkWzVh56W6wtYfbVH4=" }, { @@ -138,7 +138,7 @@ "subdir": "jsonnet/kube-state-metrics-mixin" } }, - "version": "45b7e20ee3cbecd6c5c2960a581fd5a931bd7826", + "version": "c804368d08729e2d66d9f60bc1d252d0ac753a54", "sum": "qclI7LwucTjBef3PkGBkKxF0mfZPbHnn4rlNWKGtR4c=" }, { @@ -158,7 +158,7 @@ "subdir": "jsonnet/mixin" } }, - "version": "d2599cfe67beb97b9208e79422457b0f7cde3c4a", + "version": "c22d1da263ace4921586cbafc658418b5c8194ba", "sum": "gi+knjdxs2T715iIQIntrimbHRgHnpM8IFBJDD1gYfs=", "name": "prometheus-operator-mixin" }, @@ -169,8 +169,8 @@ "subdir": "jsonnet/prometheus-operator" } }, - "version": "d2599cfe67beb97b9208e79422457b0f7cde3c4a", - "sum": "qZwYjsYpalWEkkS0cytnksmRz8/NkMlmytI3G29s5kA=" + "version": "c22d1da263ace4921586cbafc658418b5c8194ba", + "sum": "lJvOgTgcaWEsPtEiwyfDEE78J+0/FIkwFz3CbOMlDX8=" }, { "source": { @@ -179,8 +179,8 @@ "subdir": "doc/alertmanager-mixin" } }, - "version": "d04ef60a1675db13ffd42613bc74aec298cbe67a", - "sum": "IpF46ZXsm+0wJJAPtAre8+yxTNZA57mBqGpBP/r7/kw=", + "version": "f6b942cf9b3a503d59192eada300d2ad97cba82f", + "sum": "Mf4h1BYLle2nrgjf/HXrBbl0Zk8N+xaoEM017o0BC+k=", "name": "alertmanager" }, { @@ -190,7 +190,7 @@ "subdir": "docs/node-mixin" } }, - "version": "f35a592dd65d6b3488426a789b328c8ca5b2bc22", + "version": "07ee8efaa4f8e7260eb8611f3f42973cbbf8ce8f", "sum": "cQCW+1N0Xae5yXecCWDK2oAlN0luBS/5GrwBYSlaFms=" }, { @@ -200,7 +200,7 @@ "subdir": "documentation/prometheus-mixin" } }, - "version": "f131cdd4c5471deeda4db376d2f2b804e386dd96", + "version": "bac1fa61493a07e8f833386a802ce6baef6cb0c3", "sum": "dYLcLzGH4yF3qB7OGC/7z4nqeTNjv42L7Q3BENU8XJI=", "name": "prometheus" }, @@ -222,7 +222,7 @@ "subdir": "mixin" } }, - "version": "a31af1da03a9fb7586794ce1a94671050617cced", + "version": "df3df36986e07b21aaa88adefb5fbf0b648129b8", "sum": "ieCD4eMgGbOlrI8GmckGPHBGQDcLasE1rULYq56W/bs=", "name": "thanos-mixin" } diff --git a/charts/kubezero-metrics/jsonnet/zdt-dashboards.yaml b/charts/kubezero-metrics/jsonnet/zdt-dashboards.yaml index c5176d63..8d9da687 100644 --- a/charts/kubezero-metrics/jsonnet/zdt-dashboards.yaml +++ b/charts/kubezero-metrics/jsonnet/zdt-dashboards.yaml @@ -4,10 +4,6 @@ folder: KubeZero dashboards: - name: home url: file://zdt/home.json -- name: crio - url: file://zdt/crio.json - #url: https://raw.githubusercontent.com/cri-o/cri-o/refs/heads/main/contrib/metrics-exporter/dashboard.json - tags: ['kubernetes'] - name: docker-registry url: file://zdt/docker-registry.json tags: ['kubernetes', 'registry', 'ec2'] diff --git a/charts/kubezero-metrics/jsonnet/zdt/crio.json b/charts/kubezero-metrics/jsonnet/zdt/crio.json deleted file mode 100644 index 48b01e16..00000000 --- a/charts/kubezero-metrics/jsonnet/zdt/crio.json +++ /dev/null @@ -1,414 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "6.7.3" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - } - ], - "annotations": { - "list": [ - { - "$$hashKey": "object:173", - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "CRI-O Metrics Dashboard", - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "iteration": 1589269363200, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "description": "", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 2, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "maxPerRow": 3, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": "node", - "repeatDirection": "h", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(container_runtime_crio_operations_total{instance=~\"$node\"}[5m])", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{instance}} - {{operation}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CRI-O Operations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transparent": true, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:57", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:58", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 36 - }, - "hiddenSeries": false, - "id": 3, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "maxPerRow": 3, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": "node", - "repeatDirection": "h", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(container_runtime_crio_operations_errors_total{instance=~\"$node\"}[5m])", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{instance}} - {{operation}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CRI-O Operations Errors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transparent": true, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:57", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:58", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "${DS_PROMETHEUS}", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 68 - }, - "hiddenSeries": false, - "id": 4, - "interval": "", - "legend": { - "alignAsTable": true, - "avg": true, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "maxPerRow": 3, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": "node", - "repeatDirection": "h", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": " rate(container_runtime_crio_operations_latency_seconds_total_count{instance=~\"$node\"}[5m])", - "format": "time_series", - "instant": false, - "interval": "", - "legendFormat": "{{instance}} - {{operation}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CRI-O Operations Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transparent": true, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:57", - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:58", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "schemaVersion": 22, - "style": "dark", - "tags": ["cri-o", "container runtime", "runtime", "node"], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "container_runtime_crio_operations_total", - "hide": 0, - "includeAll": false, - "index": -1, - "label": "Selected Nodes", - "multi": true, - "name": "node", - "options": [], - "query": "container_runtime_crio_operations_total", - "refresh": 1, - "regex": "/.*instance=\"([^\"]*).*/", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "CRI-O", - "uid": "5Kh4RigGk", - "variables": { - "list": [] - }, - "version": 15 -} diff --git a/charts/kubezero-metrics/templates/grafana-dashboards-zdt.yaml b/charts/kubezero-metrics/templates/grafana-dashboards-zdt.yaml index 6b874cc8..10ff7f98 100644 --- a/charts/kubezero-metrics/templates/grafana-dashboards-zdt.yaml +++ b/charts/kubezero-metrics/templates/grafana-dashboards-zdt.yaml @@ -11,7 +11,5 @@ metadata: binaryData: home.json.gz: H4sIAAAAAAAC/9VVS2/UMBC+8yuMDwikdnezj1b0VrVCVLwKFJBoq8obTxJrHTvYTndb1P+O7byczULFDS6J55vxeB6fxz+fIISJENIQw6TQ+Aj9tJAFOdPGSpdeQjXqNcuScXMmrDLa61BKDNGyVDFYBT5XMgeTQalxYAOCLLnTG1VCgGeM7kBZLMWJ5FI5hypdkueTPTSNIvtZLPZQ9CJ0LUjuDz7uckHP0DEHZXohmLvC21Gis6UkiuJa9+D/1/b74MwxUGa2osWpAHNGLSJKzitEkSK7kJIbVlh84kHmTKYv/ZozsXJVvbz2YkEEcN3Wtalqv3qNd69JGHB6IkXC0rY51RZISMmNb9lDkKG8BaVsQatTm+xaf6nVnUvd95VZcRb4WLsE5gGwaZKr5TsnD3z7xA9aURZ9TnnQNtWAcNTCTy+/n16gtzKV188zYwp9NB7HVIzuQcl9KtfCsBxGtuZjojUYPV6VS3DKMbd79nVOON8/mI8KkaIrfE/NFX5xJb4Bjy35kJHojbX/bu3ROzCKxRo9RW0cQTK59PTDOVErdyweJFbwMmXiKyhtE3Kmh6PFaNb6wC7QV5by281z+OeMJWaoMJ5a+LUNNfBTs9PAxlRR1DH8S0SJFltMiaaPMGW2mykt47Zuye/Jk0hOQfkrGJ6Rk82ZgdwZz0L8RwnKnY/DbutMrl8DoUykejh0nPYTxJah/O4rgzXQ3TafgajYVSMhXA+Uhii1a6ch6W+q/SjDmp1DCp02o0wPieTGnJ/k/y+ZesAf2DT/ezYB0C+Kuzo142c4epigsBltcj6cCm0H3hID2qD3sNYosWMA+aFzat2gC7brfgtrWbfEfqunQUGiQLua4NmkaiXWcQY56VgxPaxgc8fr9qpVZdnjh50fecHtKyjS4YPePXEux06fVPMLC7nej7I6ZmxkjeHetoLFK1DdZvuCUxA9wjcJ3TA78NUtCZ49q1wEZI0mgTALhSjv1otgHYXCbBJqsm49DdZR/dBfN3nZJG46Vlw+ekro+CB0HJ4ynYcC7daHNIx3O5Zm6neF7RX7XgqvXyq51lZZK2ryOa45qjmmofY5waW/FPjg4wdIFy/f3VfobUul+ZOHX/6tTqj9CQAA - crio.json.gz: - H4sIAAAAAAAC/+1aW2/bNhR+768QhGJoiyS17DpNC+wha9OuaNpkcdqHNZlBS8cSF5pUKSqJZ3i/fYfUjbKYLilWdMP04ljnUOS5fudD6NU9z/OnU8rTXGX+c+8TPnveynyihpMFoNR/OZkenxy9Ozj9+eDDxN+q1IzMgGn9sRQLUAnkWaOMIAslTRUVXC9pFGqZmk0jokgmchlCo0tZHlP+JtL61LFpoX9fmmUdaxas8fN8q3BJwuecSnA4VZ0fSzInnDSb08gproLwelNxCTIrvdvdebozKo3Ych+XEo7B6h6WJs6jbLF10JfPcIWU3hhM3g2j68hgZ7Az2Aww4VwoopOrI1wY4jOaqTrejXmouX8/IVnyFpZ6QzH7HUL1PHg6qs/DJbOcMvVGnxhYUssjt6W4BjiZMa1XMgdLntDIIaWh4C8EE1JvKOMZeTDY8oZBgB/j8ZYXPLS3rkK037jr/eDtM5CqZUKTgCyZCSIjv9Stzd/ze2XONrvixcmb7SPvHShJw8x7Wb9t1kJE1YZnfsxBmfbgOWOFRNfKqRBM0RTlAyOkrSVUgSTlkcF479lw99lodzQcFGsZ5RemTYrMmjp1tA1hlGQmcCbj67pUZsRI5oRl0HQ/unIIPFaJPnPQkoNr+d+k+WY4mVPG7KIxAmygiAJXVUCMIpY0OhZNvRZFgo/PrExe4fOe9Xxtb4HPuoIHVXLrrbHUIuATkNThnEnGsHnkmI5LwtpuMIiBR23bMOQx389O3eVNLuOuMMylLPxu2VB2w8EiVUt3o/wKUnQ1C3Lt2GpBuUMqaZyoibPnskRcdaUKW4o5NsLY5CaMen0n1FiutbIlvKJRUW21FK0/Bnlizh41sIdtcSwwC++EsdUImkSItI1qdX0e1n3SsSkFrFyuSAyd5Kf6JF2NedaqAiPv1gomLwIJBpzmTCjf0qRAlLFX270hf4mzLqyawxodmSnJI4RzLH5o2rxQpiQEV5dmioQXHdsyBWkKEcah66YiMgaLQrTxX4PZdVogLlHwACFYEdxGTmXOFV3AFHtbTEVa4lQ2NbWxohwt4SH8+OeZf1+7feavP40X5zZG644XclGExuxV+NxeUuzk6IubGtJqylf19qvaovXa2/ZWq9ri9br9poR5wWL2/Vq8Lr81CVAJMpREsGgjMdqNVwiCFojX8hOIy/rceGGS0LnqvqEMdpST5qiOsMXH6tmxsjuWSIgcnSyk2kBE067TagJSHtFLGuUYzk6bYB9wrLkSoFoNbJEym/lck2u60YqzPLwoKs12VHd72c46Fo4ZvrHaDUk19DiafEmu4QsF7qI446c3FSoeL9VGsZVsum2oVoj4J5JBmxnV4NxZXqBzR2w53FTk1h182fuX+9LprmW3eMxEdYwcIz+Ey9roFn3bpNvfmQr9A4xn746MZ7T7FZRn1FOenvL0lOe2lAftRUDpmc+3Zj7egQl0T4B6AtQToP8rAdq9IwHa3fsKAvSkJ0A9AeoJkO/dkgExXMbDJdIYXBuVVGgaClzeE6JvSIgOi7j3jOj7MqKeDf1H2FB9C5qFCSzIx/qudDgsxGrJyutAeVHc5eGcaArFv8hnIDkoMBfXxV4KFikCIOXxbS5UCWMfdWF2q7hhA6v1HS9SI5hTTquJcst/0vvdG1cbCigPWR7BPnNNfUQG0Inetq98618VTIBhpUHkvRdRC8T9RY7w1G3V6ra2NSxb473BSRR/zkEuv9JTBHWN1xu31RKHhPbHf7zzqB5YZ/6DT7+d+eePHu48emzvkV3Q9INkkyUPHbFxwCoWkUl69ktlud/WdlzUMvfiElyLGFiKPIPTYiNj0A3X2AZQ6zKdF/MJ4361HVRQjUOklPmt11KKYC2bl8tITqvRa+OpP7ZYfzCwHkb2Q7Bovo+t74H9MBrYGmugDK3vQXlx3/b0D2HYj1+2cmuOFrK8+JXF+G3y5ITGr8umvySSah7v+IlEc0DzS4tgbARNaRkv763/AgepQqelIwAA docker-registry.json.gz: H4sIAAAAAAAC/+1dW3PbNhZ+76/gMN2dpKu6IiXF0s70wXHqJjNx4o2dviReDURCEtYUyZKgY9Xj/74HAC8ACeriSIrkcsZjmwCIy8E533dAHIL3PxiGORwSP0xobP7b+AzXhnHPf0OOj2YYUs3Xl0P4ufj44fy3qze/fbo0W1kJD42wlxbR5bs4diISUhL4rFSRQechr9pFFMVBEjm4yAu9ZEL8ty7LD6NghukUJ3E5/33auYuiBC/wAL+vW2JgEf4zIRHWDC1rfxKhMfJRUTlxtcmZKH4vZ9ziKE5H1zuyj6y0Ey19cyHyQV7VxsKptik5WWmofdR+RENTjOgMhdWm3pQz1mtMN4mkdvr86sRtcpAx8Scejimi1SYvNXk1TeZahHw/gBugCFMj0QfTIzHNlaroGeSMEuLRt6w6q1WkShLSjxzKYB+NPJZPowRL6VPialKJE/ingRdErMJoMkLP2y3Dtiz41eu1DOuFXHU2/pNiLMY/jRMPR1TpQjGh8XQUoMg107wH/vf6h3QuTOwSWuqtOfEx5UbrJ54nUpgGXwWBR0kI6W2eSJQihOIIpfBg9brHdmdg9XsvB1x4IGb/hhuvmAo+3xpjdgLPQ2GMWc1j5MW5nKAHxL0IiokTAi1Nzle4trtSwl3W2fR6zq4zSajqZtsFLOW9uy5gjlBPGDP2YaCecRokPgw5bhmnF59axjmeBdHcQL5rnBEPG69TvAwi4xImKq4CZhR81RuFg5wpviIzHCRUknAmoCB6hZybSQTtV8XE8/9AXoL1WbGk65D2zB4MnO5LWXO4BtqdY9A8e9AyuqCM7aP+QNHCZ263izoo06lCSkuto8wjYMERNSgM1gjGBpQ1AGgcHEt3jINohmiKTlwuZ1Dve5BeoRwomWBVNWboLhOD1ZZVYEb8LENOjqfB17LI2GRNgXSmgee+Y/wYLypxjqIbzAXMDKmiZLUK3Ckp8DL9tfT62y0umV7eIq+sO6oNplIKQ0DSK6GSli5d1Zii7xIY3TJ5GjQwKL6jkp4YaZbUaanbNZVFyJ8sqcwuKqtoIMz7a1DCiwCkEKuTbzJp8IzzgAMxmITvY4cC3ChlrljLJeGFQUzH5E71fdLEs8Cnl+QvXqVltf8hFYhw9SaeVn8PF8A5oGC94MdgAOxG1kNVRFR03Xz/y0kpI8hvWCC9OAQlBj0pGdOYeJ5KUR2gJ8vqw6/+gAGE1VcAYszaqdoKq1muR1RjA8RYg45SgWyOFX3nhAX1JLOyL4qiCaYLBIfvwij1ZhjIDDn8DBn8DGMM2uDG96D5v34xf4Q/X8yWQXwo4juYJWX/fzEfjJ+YYrVVERdAlVYYERyrRTLTPEMO5VKwlWwPT7DvnuX1qDeD3ghX+mTRFOZ4FJdkkxHYJYdchqNVTtK5XNzq6tWVZy9W14D5DOavK2pqZuVLlZUXzBYP6HayCTaVfbO1yXTw8rjTthWirE/6du68Arb0k9kIR4w7+TgNtk6C+YO/Dia3MrAV2gnaEdEnSp69Enl2VyNP62XDnnvBnr1HsGfvkNjzGUajfmewLlk+wz3ktLGGIrX28u0MGeEJrIyjOVBkEKEJHnJ8GVJYdXr3elZsMRaBpI8Cg4Amv50frYX8eH9vsDaNh4etMOUVG6xxyoE1HVT8WM7s7SFngrV/d860xsd9x1IIsj5pN5xpPJ8SGr/4u1PnYEXqtBvqbKhzJ9TZswY9B69NnW3UayPrIKjzDXkytAlDaahzq9TZH7THpbVlfdKuqHNG4oY6rVWXnc1D24Y7d7TstLtu115/2Vm5bW+58xyQ56mQJxsLoOnT5c8NPa497M3PC7ELARMcExf7NNtE/hSD5ms3QLEzmlNJgZfT6GHz6IprULvh0YZH923zsxTms/be55YYFpD3ebb9GaXAM5xx4BlycKmh2YcXB7nbuQRbnzCTIo+g+DTjvftCh0YoqqA5DxR7h/0J5TFVbSUd64qvyXUnt5j5c0YCisKjpOJ5TPGMBU+J8J84ZLNEfCPdiD+SCJBwg7KWE8pxiVD6S2Jpuno+6RTjF0qqtgSSnfgn8ZU+yo9NSCXRSaIIxlfNYA8SqonEryZKsCBzMPMXNdzMdaSWkRkYxeXnDyzxK3GpElWnI84KlShQbIYY1MKnaFJ1w0J2V4RcksQy3Yv0qpaByFwcYY5bYy+QbFPAzQdQKlAFXOofALmDdcoM9u3cVFoBPQxD7L4TyK/mrQWrEaI4x1YnTLKYkoXLl4fPvdn1i62DK7MzJebjsSCrxESKWLyyu8DSL6dkTKsZKSpnaMB6VUbkPMr0XtZ9FGFXYxPsmZJq1lzxhxmuE98lt8RNUA6oDy1N5Lgcnn2H7kgJWkaJcyO0QB4OM9TUAKgq2cyfK5XWW3BuqZ+vK12co7tFPqj0dE2d1yyaX+0Aywgmr1CMK6tcAUOV4gKIKslVD6Xeta17Avjd+llR73l1wjnCa1CVp7/Dt3mnlajq/WbgP0hEwQ44+eZekXD8jBg8HMa93AFsGUiYZ2wE8I/RM0C8Rr6w2gIt91ekZbuh5YaWv42W11ry7Iaa5SXKCvTcWnPEt8Lu92jAGRLVj/fV93FHataIjUeyskfiYofMkBfr2HjBc8zGaWmcFq3T8j7fcA5C7Btj9mqVm79aFW/DGSligZd4I93GG2m8kUe9dsJ0eTh2a2l42wT8gdnS2ev9eBYgOqO+Mxk37Ls99t0TSmuot7L3vYO3n626LdT2aq8/f0wDNMBXphFx1nmveb88CxbO9ubq6iKLYXuUL9Ffc7/Bsmuk/3IdX6KiZdyZKKcu8SbKxYVJlFN36k90Gn9iPX/CwcQTi/08bmpKaTjMVHo3Gw9L46WmyHc98OI3EDK1EY+Dm70mqqrxNB6z87BtljZLr1k3zsQBruMZ2xL/5zEMYUq3z7v9FXnX6je826zjHx+lzNmWwFqe63XOu9tb1x8c1771fz6TjL5h3YZ1G9bdJeuy5QF7fp5hlvrm1lbYt/IEvXbZu1X6rTDt9vlXf15Rs/DdwcJ39ReGdrfw3dSLQhth4vzJnfpWkfERxNjQcUPHDR1vnI63F7hu15xi2v32TentEGrE1gCX2gNwd8u1T3uxqwx7I1RLfCfCAARVpgUaBD8zD3KPk1kt4dpAuI8/M5HVo9G/9fe9gZJFpxeRsopfIEJF0pLw+BtH34fCT8Qo3gF1+87829jbbti7Ye792wtHkVuqmCVdINcl/qSqHizzY/pWuFy5+gZ4tcLiqINRdzxW4EnccukgYXzxn8oMZLlTPEuNBaAoDDwwyQ/8LVb1SPi7MPAFi7aPehojiEPsgKxnVUtjbsaZApMllFzBEdlYZL7G78i+RaD4V1Oge/7Y7zSdR4WYRPzc4j3vhScAa6h79Ye3tfulQzcRR9jnnCaw6z59qAqsNvKCESM0qw2MVkdl5a8zrLJ6XHFVqNvCTGkAVEJ6fdJ4zrr6Yik16ECVpb4B8QQA7rP6U5hTBqh8i+LupIIJi15pvnvFhSwiPcvUl2amb+AqWfNqM7XxTwuQVA+XWrDUIXqdCEOP0HyqtYA0F0N7lYKWiRIamOVcvVTmFak0GLq/GFp5HLpJEO0fIoiCue0DiH4KvQC5xipYCj1usLTB0gZLvzOWVp6DHW8MSu0DdUeHCYexAwPUouMNrja42uDqvq3zN4er3UME1hnyyVh8xOC7o+p52pdVcDXrdwOqDag2oLp3C/8Nomr7AFHVAVmD1e0DqJ6KrqyCqWmvG0j9W0Fq/tlgUCy2j8uq7bQFLJgxQy70R/7BYWHpZkznXvrZ3ehGlKRoUliVeZNAD3xcHH2Rh5Jn19jhZ/WLpimehYCJArOXfrAYeV52vG8Z3/MQjvuHNb9lnH6zWF53E9/xEhef6I8P1+5KmrMEDEZTPDv4lv3lwQ1y20GYfay5QCpIBluNGJqKf9iJPlD582Ux+coRq8Wc2krqBPPzan85+qmYqWHeN0Cw55//+8W8/teLo59+MRcHwsHE89mI/5N111RzK+NiafrCKTqIgUsZSYyvREVaWHkiuoHCcGtacV8zzz/mF6WTb1dQHfEF0UxTGj3ZlZ5kAUePVpYkVL7+WjfvVs28S8FPBwETpS/B8zCcnGbyU7aDrz9bWYBPdpC2eMO/uC0kQKHF+iCT1TBz2WS/0uxJrrXVli468oVVHA1m9qT/Lfmi05ZzpDAkW/rfcoVDeJ2NgQWaSXqxtBW54pdyxXIrdle+kM5UP3bl/mZ9UcT3V8AD9MzUY8hc1NcBk6rxUfYQzIR7/+Zp8Opy8r8++Uuk3uaeiG398PB/dY9XRg2EAAA= diff --git a/charts/kubezero/templates/metrics.yaml b/charts/kubezero/templates/metrics.yaml index 5e4305d7..6bd882db 100644 --- a/charts/kubezero/templates/metrics.yaml +++ b/charts/kubezero/templates/metrics.yaml @@ -44,13 +44,14 @@ prometheus: mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" readOnly: true additionalScrapeConfigs: - - job_name: 'crio' + - job_name: 'cri' + metrics_path: '/v1/metrics' ec2_sd_configs: - port: 9090 region: {{ .global.aws.region }} filters: - name: 'tag-key' - values: ['zdt:prometheus:crio'] + values: ['zdt:prometheus:cri'] {{- with .metrics.kubezero.prometheus.prometheusSpec.additionalScrapeConfigsEC2Filters }} {{- toYaml . | nindent 14 }} {{- end }} diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 021ba421..71be19ce 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -17,7 +17,7 @@ global: addons: enabled: true - targetRevision: 0.8.9 + targetRevision: 0.8.10 external-dns: enabled: false forseti: