feat: nvidia runtimeClass, various cleanups for 1.30

2024-11-08 16:45:10 +00:00 · 2024-11-08 16:45:10 +00:00 · 9504f874a0
commit 9504f874a0
parent d64978e5ea
17 changed files with 64 additions and 464 deletions
--- a/admin/kubezero.sh
+++ b/admin/kubezero.sh
@ -1,4 +1,8 @@
-#!/bin/bash -e
+#!/bin/bash
+set -eu -o pipefail
+
+DEBUG=${DEBUG:-""}
+LOG=""

 if [ -n "$DEBUG" ]; then
  set -x
@ -80,9 +84,6 @@ parse_kubezero() {
  export NODENAME=$(yq eval '.nodeName' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml)
  export PROVIDER_ID=$(yq eval '.providerID // ""' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml)
  export AWS_IAM_AUTH=$(yq eval '.api.awsIamAuth.enabled // "false"' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml)
-
-  # From here on bail out, allows debug_shell even in error cases
-  set -e
 }


@ -188,8 +189,15 @@ kubeadm_upgrade() {
  # install re-certed kubectl config for root
  cp ${HOSTFS}/etc/kubernetes/super-admin.conf ${HOSTFS}/root/.kube/config

-  # post upgrade hook
-  [ -f /var/lib/kubezero/post-upgrade.sh ] && . /var/lib/kubezero/post-upgrade.sh
+  # post upgrade
+
+  # Update kubezero-values CM
+  kubectl get cm -n kube-system kubelet-config -o=yaml | \
+    yq e '.data.kubelet' | yq e '.containerRuntimeEndpoint = "unix:///run/containerd/containerd.sock"' > $WORKDIR/new-kubelet.cm
+
+  kubectl get cm -n kube-system kubelet-config -o=yaml | \
+    yq e '.data.kubelet |= load_str("/tmp/kubezero/new-kubelet.cm")' | \
+    kubectl apply --server-side --force-conflicts -f -

  # Cleanup after kubeadm on the host
  rm -rf ${HOSTFS}/etc/kubernetes/tmp
@ -427,7 +435,7 @@ debug_shell() {

  printf "For manual etcdctl commands use:\n  # export ETCDCTL_ENDPOINTS=$ETCD_NODENAME:2379\n"

-  /bin/bash
+  bash
 }

 # First parse kubeadm-values.yaml
--- a/admin/libhelm.sh
+++ b/admin/libhelm.sh
@ -5,6 +5,8 @@ API_VERSIONS="-a monitoring.coreos.com/v1 -a snapshot.storage.k8s.io/v1 -a polic

 export HELM_SECRETS_BACKEND="vals"

+LOCAL_DEV=${LOCAL_DEV:-""}
+
 # Waits for max 300s and retries
 function wait_for() {
  local TRIES=0
--- a/admin/upgrade_cluster.sh
+++ b/admin/upgrade_cluster.sh
@ -29,7 +29,7 @@ read -r
 #[ "$ARGOCD" == "True" ] && kubectl edit app kubezero -n argocd || kubectl edit cm kubezero-values -n kubezero

 ### v1.30
-#
+kubectl delete runtimeclass crio || true

 # upgrade modules
 #
--- a/charts/kubeadm/templates/InitConfiguration.yaml
+++ b/charts/kubeadm/templates/InitConfiguration.yaml
@ -8,7 +8,7 @@ patches:
  directory: {{ . }}
 {{- end }}
 nodeRegistration:
-  criSocket: "unix:///var/run/crio/crio.sock"
+  criSocket: "unix:///run/containerd/containerd.sock"
  ignorePreflightErrors:
  - DirAvailable--var-lib-etcd
  - DirAvailable--etc-kubernetes-manifests
--- a/charts/kubeadm/templates/JoinConfiguration.yaml
+++ b/charts/kubeadm/templates/JoinConfiguration.yaml
@ -1,6 +0,0 @@
-apiVersion: kubeadm.k8s.io/v1beta3
-kind: JoinConfiguration
-nodeRegistration:
-  criSocket: "unix:///var/run/crio/crio.sock"
-patches:
-  directory: /tmp/patches
--- a/charts/kubeadm/templates/KubeletConfiguration.yaml
+++ b/charts/kubeadm/templates/KubeletConfiguration.yaml
@ -6,7 +6,7 @@ cgroupDriver: cgroupfs
 logging:
  format: json
 hairpinMode: hairpin-veth
-containerRuntimeEndpoint: "unix:///var/run/crio/crio.sock"
+containerRuntimeEndpoint: "unix:///run/containerd/containerd.sock"
 {{- if .Values.systemd }}
 resolvConf: /run/systemd/resolve/resolv.conf
 {{- end }}
@ -21,7 +21,7 @@ featureGates:
 # Minimal unit is 40m per pod
 podsPerCore: 25
 # cpuCFSQuotaPeriod: 10ms
-# Basic OS incl. crio
+# Basic OS incl. cri
 systemReserved:
  memory: 96Mi
  #ephemeral-storage: "1Gi"
--- a/charts/kubeadm/templates/resources/10-runtimeClass.yaml
+++ b/charts/kubeadm/templates/resources/10-runtimeClass.yaml
@ -1,8 +1,8 @@
 apiVersion: node.k8s.io/v1
 kind: RuntimeClass
 metadata:
-  name: crio
+  name: crun
 handler: crun
 overhead:
  podFixed:
-    memory: 4Mi
+    memory: 8Mi
--- a/charts/kubezero-addons/Chart.yaml
+++ b/charts/kubezero-addons/Chart.yaml
@ -3,7 +3,7 @@ name: kubezero-addons
 description: KubeZero umbrella chart for various optional cluster addons
 type: application
 version: 0.8.10
-appVersion: v1.29
+appVersion: v1.30
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
--- a/charts/kubezero-addons/README.md
+++ b/charts/kubezero-addons/README.md
@ -1,6 +1,6 @@
 # kubezero-addons

-![Version: 0.8.9](https://img.shields.io/badge/Version-0.8.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.29](https://img.shields.io/badge/AppVersion-v1.29-informational?style=flat-square)
+![Version: 0.8.10](https://img.shields.io/badge/Version-0.8.10-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.30](https://img.shields.io/badge/AppVersion-v1.30-informational?style=flat-square)

 KubeZero umbrella chart for various optional cluster addons

@ -18,10 +18,10 @@ Kubernetes: `>= 1.26.0`

 | Repository | Name | Version |
 |------------|------|---------|
-| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.16.1 |
+| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.16.2 |
 | https://kubernetes-sigs.github.io/external-dns/ | external-dns | 1.15.0 |
-| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.43.0 |
-| https://nvidia.github.io/k8s-device-plugin | nvidia-device-plugin | 0.16.2 |
+| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.43.2 |
+| https://nvidia.github.io/k8s-device-plugin | nvidia-device-plugin | 0.17.0 |
 | https://twin.github.io/helm-charts | aws-eks-asg-rolling-update-handler | 1.5.0 |
 | oci://public.ecr.aws/aws-ec2/helm | aws-node-termination-handler | 0.24.1 |

--- a/charts/kubezero-addons/templates/device-plugins/nvidia-runtimeClass.yaml
+++ b/charts/kubezero-addons/templates/device-plugins/nvidia-runtimeClass.yaml
@ -0,0 +1,10 @@
+{{- if index .Values "nvidia-device-plugin" "enabled" }}
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
+metadata:
+  name: nvidia
+handler: nvidia
+overhead:
+  podFixed:
+    memory: 16Mi
+{{- end }}
--- a/charts/kubezero-addons/values.yaml
+++ b/charts/kubezero-addons/values.yaml
@ -164,6 +164,11 @@ awsNeuron:

 nvidia-device-plugin:
  enabled: false
+
+  cdi:
+    nvidiaHookPath: /usr/bin
+  deviceDiscoveryStrategy: tegra
+
  tolerations:
  - key: nvidia.com/gpu
    operator: Exists
--- a/charts/kubezero-metrics/jsonnet/jsonnetfile.lock.json
+++ b/charts/kubezero-metrics/jsonnet/jsonnetfile.lock.json
@ -18,7 +18,7 @@
          "subdir": "contrib/mixin"
        }
      },
-      "version": "18eb5c6881d43064f4559034bf12c3ef6ce89e4b",
+      "version": "7b70b5f6aebe5be44691b605d5b0d179c106cbc0",
      "sum": "IXI3LQIT9NmTPJAk8WLUJd5+qZfcGpeNCyWIK7oEpws="
    },
    {
@ -58,7 +58,7 @@
          "subdir": "gen/grafonnet-latest"
        }
      },
-      "version": "1ce5aec95ce32336fe47c8881361847c475b5254",
+      "version": "82a19822e54a0a12a51e24dbd48fcde717dc0864",
      "sum": "64fMUPI3frXGj4X1FqFd1t7r04w3CUSmXaDcJ23EYbQ="
    },
    {
@ -68,7 +68,7 @@
          "subdir": "gen/grafonnet-v10.0.0"
        }
      },
-      "version": "1ce5aec95ce32336fe47c8881361847c475b5254",
+      "version": "82a19822e54a0a12a51e24dbd48fcde717dc0864",
      "sum": "xdcrJPJlpkq4+5LpGwN4tPAuheNNLXZjE6tDcyvFjr0="
    },
    {
@ -78,7 +78,7 @@
          "subdir": "gen/grafonnet-v11.1.0"
        }
      },
-      "version": "1ce5aec95ce32336fe47c8881361847c475b5254",
+      "version": "82a19822e54a0a12a51e24dbd48fcde717dc0864",
      "sum": "41w7p/rwrNsITqNHMXtGSJAfAyKmnflg6rFhKBduUxM="
    },
    {
@ -88,7 +88,7 @@
          "subdir": "grafana-builder"
        }
      },
-      "version": "b9761545ddeac33ffe25bd87121076eab258c6f1",
+      "version": "ab84b9f67c7a7f61e0c0a311afb47a1af4f5903f",
      "sum": "yxqWcq/N3E/a/XreeU6EuE6X7kYPnG0AspAQFKOjASo="
    },
    {
@ -118,8 +118,8 @@
          "subdir": ""
        }
      },
-      "version": "0348e09edc3961a29a55f199d1bf0060c847a608",
-      "sum": "kTZuZcE+pNw8ZVZECKxrZG4F9BS+ydWMcgACE9oUrRc="
+      "version": "c70f03daec5d7578ef697c9f5b93ea88a41fe0d7",
+      "sum": "qPlmuwMEsbOsHGJT0lSHCK7jtS3k0ql6A4sudj/pnDA="
    },
    {
      "source": {
@ -128,7 +128,7 @@
          "subdir": "jsonnet/kube-state-metrics"
        }
      },
-      "version": "45b7e20ee3cbecd6c5c2960a581fd5a931bd7826",
+      "version": "c804368d08729e2d66d9f60bc1d252d0ac753a54",
      "sum": "lO7jUSzAIy8Yk9pOWJIWgPRhubkWzVh56W6wtYfbVH4="
    },
    {
@ -138,7 +138,7 @@
          "subdir": "jsonnet/kube-state-metrics-mixin"
        }
      },
-      "version": "45b7e20ee3cbecd6c5c2960a581fd5a931bd7826",
+      "version": "c804368d08729e2d66d9f60bc1d252d0ac753a54",
      "sum": "qclI7LwucTjBef3PkGBkKxF0mfZPbHnn4rlNWKGtR4c="
    },
    {
@ -158,7 +158,7 @@
          "subdir": "jsonnet/mixin"
        }
      },
-      "version": "d2599cfe67beb97b9208e79422457b0f7cde3c4a",
+      "version": "c22d1da263ace4921586cbafc658418b5c8194ba",
      "sum": "gi+knjdxs2T715iIQIntrimbHRgHnpM8IFBJDD1gYfs=",
      "name": "prometheus-operator-mixin"
    },
@ -169,8 +169,8 @@
          "subdir": "jsonnet/prometheus-operator"
        }
      },
-      "version": "d2599cfe67beb97b9208e79422457b0f7cde3c4a",
-      "sum": "qZwYjsYpalWEkkS0cytnksmRz8/NkMlmytI3G29s5kA="
+      "version": "c22d1da263ace4921586cbafc658418b5c8194ba",
+      "sum": "lJvOgTgcaWEsPtEiwyfDEE78J+0/FIkwFz3CbOMlDX8="
    },
    {
      "source": {
@ -179,8 +179,8 @@
          "subdir": "doc/alertmanager-mixin"
        }
      },
-      "version": "d04ef60a1675db13ffd42613bc74aec298cbe67a",
-      "sum": "IpF46ZXsm+0wJJAPtAre8+yxTNZA57mBqGpBP/r7/kw=",
+      "version": "f6b942cf9b3a503d59192eada300d2ad97cba82f",
+      "sum": "Mf4h1BYLle2nrgjf/HXrBbl0Zk8N+xaoEM017o0BC+k=",
      "name": "alertmanager"
    },
    {
@ -190,7 +190,7 @@
          "subdir": "docs/node-mixin"
        }
      },
-      "version": "f35a592dd65d6b3488426a789b328c8ca5b2bc22",
+      "version": "07ee8efaa4f8e7260eb8611f3f42973cbbf8ce8f",
      "sum": "cQCW+1N0Xae5yXecCWDK2oAlN0luBS/5GrwBYSlaFms="
    },
    {
@ -200,7 +200,7 @@
          "subdir": "documentation/prometheus-mixin"
        }
      },
-      "version": "f131cdd4c5471deeda4db376d2f2b804e386dd96",
+      "version": "bac1fa61493a07e8f833386a802ce6baef6cb0c3",
      "sum": "dYLcLzGH4yF3qB7OGC/7z4nqeTNjv42L7Q3BENU8XJI=",
      "name": "prometheus"
    },
@ -222,7 +222,7 @@
          "subdir": "mixin"
        }
      },
-      "version": "a31af1da03a9fb7586794ce1a94671050617cced",
+      "version": "df3df36986e07b21aaa88adefb5fbf0b648129b8",
      "sum": "ieCD4eMgGbOlrI8GmckGPHBGQDcLasE1rULYq56W/bs=",
      "name": "thanos-mixin"
    }
--- a/charts/kubezero-metrics/jsonnet/zdt-dashboards.yaml
+++ b/charts/kubezero-metrics/jsonnet/zdt-dashboards.yaml
@ -4,10 +4,6 @@ folder: KubeZero
 dashboards:
 - name: home
  url: file://zdt/home.json
- name: crio
-  url: file://zdt/crio.json
-  #url: https://raw.githubusercontent.com/cri-o/cri-o/refs/heads/main/contrib/metrics-exporter/dashboard.json
-  tags: ['kubernetes']
 - name: docker-registry
  url: file://zdt/docker-registry.json
  tags: ['kubernetes', 'registry', 'ec2']
--- a/charts/kubezero-metrics/jsonnet/zdt/crio.json
+++ b/charts/kubezero-metrics/jsonnet/zdt/crio.json
@ -1,414 +0,0 @@
-{
-  "__inputs": [
-    {
-      "name": "DS_PROMETHEUS",
-      "label": "Prometheus",
-      "description": "",
-      "type": "datasource",
-      "pluginId": "prometheus",
-      "pluginName": "Prometheus"
-    }
-  ],
-  "__requires": [
-    {
-      "type": "grafana",
-      "id": "grafana",
-      "name": "Grafana",
-      "version": "6.7.3"
-    },
-    {
-      "type": "panel",
-      "id": "graph",
-      "name": "Graph",
-      "version": ""
-    },
-    {
-      "type": "datasource",
-      "id": "prometheus",
-      "name": "Prometheus",
-      "version": "1.0.0"
-    }
-  ],
-  "annotations": {
-    "list": [
-      {
-        "$$hashKey": "object:173",
-        "builtIn": 1,
-        "datasource": "-- Grafana --",
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "description": "CRI-O Metrics Dashboard",
-  "editable": true,
-  "gnetId": null,
-  "graphTooltip": 0,
-  "id": null,
-  "iteration": 1589269363200,
-  "links": [],
-  "panels": [
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
-      "description": "",
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 9,
-        "w": 8,
-        "x": 0,
-        "y": 0
-      },
-      "hiddenSeries": false,
-      "id": 2,
-      "interval": "",
-      "legend": {
-        "alignAsTable": true,
-        "avg": true,
-        "current": false,
-        "hideEmpty": true,
-        "hideZero": true,
-        "max": false,
-        "min": false,
-        "rightSide": true,
-        "show": true,
-        "total": false,
-        "values": true
-      },
-      "lines": true,
-      "linewidth": 1,
-      "maxPerRow": 3,
-      "nullPointMode": "null",
-      "options": {
-        "dataLinks": []
-      },
-      "percentage": false,
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "repeat": "node",
-      "repeatDirection": "h",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "rate(container_runtime_crio_operations_total{instance=~\"$node\"}[5m])",
-          "format": "time_series",
-          "instant": false,
-          "interval": "",
-          "legendFormat": "{{instance}} - {{operation}}",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "CRI-O Operations",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "transparent": true,
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:57",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "$$hashKey": "object:58",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 8,
-        "w": 8,
-        "x": 0,
-        "y": 36
-      },
-      "hiddenSeries": false,
-      "id": 3,
-      "interval": "",
-      "legend": {
-        "alignAsTable": true,
-        "avg": true,
-        "current": false,
-        "hideEmpty": true,
-        "hideZero": true,
-        "max": false,
-        "min": false,
-        "rightSide": true,
-        "show": true,
-        "total": false,
-        "values": true
-      },
-      "lines": true,
-      "linewidth": 1,
-      "maxPerRow": 3,
-      "nullPointMode": "null",
-      "options": {
-        "dataLinks": []
-      },
-      "percentage": false,
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "repeat": "node",
-      "repeatDirection": "h",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "rate(container_runtime_crio_operations_errors_total{instance=~\"$node\"}[5m])",
-          "format": "time_series",
-          "instant": false,
-          "interval": "",
-          "legendFormat": "{{instance}} - {{operation}}",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "CRI-O Operations Errors",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "transparent": true,
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:57",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "$$hashKey": "object:58",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
-      "fill": 1,
-      "fillGradient": 0,
-      "gridPos": {
-        "h": 6,
-        "w": 8,
-        "x": 0,
-        "y": 68
-      },
-      "hiddenSeries": false,
-      "id": 4,
-      "interval": "",
-      "legend": {
-        "alignAsTable": true,
-        "avg": true,
-        "current": false,
-        "hideEmpty": true,
-        "hideZero": true,
-        "max": false,
-        "min": false,
-        "rightSide": true,
-        "show": true,
-        "total": false,
-        "values": true
-      },
-      "lines": true,
-      "linewidth": 1,
-      "maxPerRow": 3,
-      "nullPointMode": "null",
-      "options": {
-        "dataLinks": []
-      },
-      "percentage": false,
-      "pointradius": 2,
-      "points": false,
-      "renderer": "flot",
-      "repeat": "node",
-      "repeatDirection": "h",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": " rate(container_runtime_crio_operations_latency_seconds_total_count{instance=~\"$node\"}[5m])",
-          "format": "time_series",
-          "instant": false,
-          "interval": "",
-          "legendFormat": "{{instance}} - {{operation}}",
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeRegions": [],
-      "timeShift": null,
-      "title": "CRI-O Operations Latency",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "transparent": true,
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "$$hashKey": "object:57",
-          "format": "s",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "$$hashKey": "object:58",
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ],
-      "yaxis": {
-        "align": false,
-        "alignLevel": null
-      }
-    }
-  ],
-  "schemaVersion": 22,
-  "style": "dark",
-  "tags": ["cri-o", "container runtime", "runtime", "node"],
-  "templating": {
-    "list": [
-      {
-        "allValue": null,
-        "current": {},
-        "datasource": "${DS_PROMETHEUS}",
-        "definition": "container_runtime_crio_operations_total",
-        "hide": 0,
-        "includeAll": false,
-        "index": -1,
-        "label": "Selected Nodes",
-        "multi": true,
-        "name": "node",
-        "options": [],
-        "query": "container_runtime_crio_operations_total",
-        "refresh": 1,
-        "regex": "/.*instance=\"([^\"]*).*/",
-        "skipUrlSync": false,
-        "sort": 0,
-        "tagValuesQuery": "",
-        "tags": [],
-        "tagsQuery": "",
-        "type": "query",
-        "useTags": false
-      }
-    ]
-  },
-  "time": {
-    "from": "now-6h",
-    "to": "now"
-  },
-  "timepicker": {
-    "refresh_intervals": [
-      "5s",
-      "10s",
-      "30s",
-      "1m",
-      "5m",
-      "15m",
-      "30m",
-      "1h",
-      "2h",
-      "1d"
-    ]
-  },
-  "timezone": "",
-  "title": "CRI-O",
-  "uid": "5Kh4RigGk",
-  "variables": {
-    "list": []
-  },
-  "version": 15
-}
--- a/charts/kubezero-metrics/templates/grafana-dashboards-zdt.yaml
+++ b/charts/kubezero-metrics/templates/grafana-dashboards-zdt.yaml
@ -11,7 +11,5 @@ metadata:
 binaryData:
  home.json.gz:
    H4sIAAAAAAAC/9VVS2/UMBC+8yuMDwikdnezj1b0VrVCVLwKFJBoq8obTxJrHTvYTndb1P+O7byczULFDS6J55vxeB6fxz+fIISJENIQw6TQ+Aj9tJAFOdPGSpdeQjXqNcuScXMmrDLa61BKDNGyVDFYBT5XMgeTQalxYAOCLLnTG1VCgGeM7kBZLMWJ5FI5hypdkueTPTSNIvtZLPZQ9CJ0LUjuDz7uckHP0DEHZXohmLvC21Gis6UkiuJa9+D/1/b74MwxUGa2osWpAHNGLSJKzitEkSK7kJIbVlh84kHmTKYv/ZozsXJVvbz2YkEEcN3Wtalqv3qNd69JGHB6IkXC0rY51RZISMmNb9lDkKG8BaVsQatTm+xaf6nVnUvd95VZcRb4WLsE5gGwaZKr5TsnD3z7xA9aURZ9TnnQNtWAcNTCTy+/n16gtzKV188zYwp9NB7HVIzuQcl9KtfCsBxGtuZjojUYPV6VS3DKMbd79nVOON8/mI8KkaIrfE/NFX5xJb4Bjy35kJHojbX/bu3ROzCKxRo9RW0cQTK59PTDOVErdyweJFbwMmXiKyhtE3Kmh6PFaNb6wC7QV5by281z+OeMJWaoMJ5a+LUNNfBTs9PAxlRR1DH8S0SJFltMiaaPMGW2mykt47Zuye/Jk0hOQfkrGJ6Rk82ZgdwZz0L8RwnKnY/DbutMrl8DoUykejh0nPYTxJah/O4rgzXQ3TafgajYVSMhXA+Uhii1a6ch6W+q/SjDmp1DCp02o0wPieTGnJ/k/y+ZesAf2DT/ezYB0C+Kuzo142c4epigsBltcj6cCm0H3hID2qD3sNYosWMA+aFzat2gC7brfgtrWbfEfqunQUGiQLua4NmkaiXWcQY56VgxPaxgc8fr9qpVZdnjh50fecHtKyjS4YPePXEux06fVPMLC7nej7I6ZmxkjeHetoLFK1DdZvuCUxA9wjcJ3TA78NUtCZ49q1wEZI0mgTALhSjv1otgHYXCbBJqsm49DdZR/dBfN3nZJG46Vlw+ekro+CB0HJ4ynYcC7daHNIx3O5Zm6neF7RX7XgqvXyq51lZZK2ryOa45qjmmofY5waW/FPjg4wdIFy/f3VfobUul+ZOHX/6tTqj9CQAA
-  crio.json.gz:
-    H4sIAAAAAAAC/+1aW2/bNhR+768QhGJoiyS17DpNC+wha9OuaNpkcdqHNZlBS8cSF5pUKSqJZ3i/fYfUjbKYLilWdMP04ljnUOS5fudD6NU9z/OnU8rTXGX+c+8TPnveynyihpMFoNR/OZkenxy9Ozj9+eDDxN+q1IzMgGn9sRQLUAnkWaOMIAslTRUVXC9pFGqZmk0jokgmchlCo0tZHlP+JtL61LFpoX9fmmUdaxas8fN8q3BJwuecSnA4VZ0fSzInnDSb08gproLwelNxCTIrvdvdebozKo3Ych+XEo7B6h6WJs6jbLF10JfPcIWU3hhM3g2j68hgZ7Az2Aww4VwoopOrI1wY4jOaqTrejXmouX8/IVnyFpZ6QzH7HUL1PHg6qs/DJbOcMvVGnxhYUssjt6W4BjiZMa1XMgdLntDIIaWh4C8EE1JvKOMZeTDY8oZBgB/j8ZYXPLS3rkK037jr/eDtM5CqZUKTgCyZCSIjv9Stzd/ze2XONrvixcmb7SPvHShJw8x7Wb9t1kJE1YZnfsxBmfbgOWOFRNfKqRBM0RTlAyOkrSVUgSTlkcF479lw99lodzQcFGsZ5RemTYrMmjp1tA1hlGQmcCbj67pUZsRI5oRl0HQ/unIIPFaJPnPQkoNr+d+k+WY4mVPG7KIxAmygiAJXVUCMIpY0OhZNvRZFgo/PrExe4fOe9Xxtb4HPuoIHVXLrrbHUIuATkNThnEnGsHnkmI5LwtpuMIiBR23bMOQx389O3eVNLuOuMMylLPxu2VB2w8EiVUt3o/wKUnQ1C3Lt2GpBuUMqaZyoibPnskRcdaUKW4o5NsLY5CaMen0n1FiutbIlvKJRUW21FK0/Bnlizh41sIdtcSwwC++EsdUImkSItI1qdX0e1n3SsSkFrFyuSAyd5Kf6JF2NedaqAiPv1gomLwIJBpzmTCjf0qRAlLFX270hf4mzLqyawxodmSnJI4RzLH5o2rxQpiQEV5dmioQXHdsyBWkKEcah66YiMgaLQrTxX4PZdVogLlHwACFYEdxGTmXOFV3AFHtbTEVa4lQ2NbWxohwt4SH8+OeZf1+7feavP40X5zZG644XclGExuxV+NxeUuzk6IubGtJqylf19qvaovXa2/ZWq9ri9br9poR5wWL2/Vq8Lr81CVAJMpREsGgjMdqNVwiCFojX8hOIy/rceGGS0LnqvqEMdpST5qiOsMXH6tmxsjuWSIgcnSyk2kBE067TagJSHtFLGuUYzk6bYB9wrLkSoFoNbJEym/lck2u60YqzPLwoKs12VHd72c46Fo4ZvrHaDUk19DiafEmu4QsF7qI446c3FSoeL9VGsZVsum2oVoj4J5JBmxnV4NxZXqBzR2w53FTk1h182fuX+9LprmW3eMxEdYwcIz+Ey9roFn3bpNvfmQr9A4xn746MZ7T7FZRn1FOenvL0lOe2lAftRUDpmc+3Zj7egQl0T4B6AtQToP8rAdq9IwHa3fsKAvSkJ0A9AeoJkO/dkgExXMbDJdIYXBuVVGgaClzeE6JvSIgOi7j3jOj7MqKeDf1H2FB9C5qFCSzIx/qudDgsxGrJyutAeVHc5eGcaArFv8hnIDkoMBfXxV4KFikCIOXxbS5UCWMfdWF2q7hhA6v1HS9SI5hTTquJcst/0vvdG1cbCigPWR7BPnNNfUQG0Inetq98618VTIBhpUHkvRdRC8T9RY7w1G3V6ra2NSxb473BSRR/zkEuv9JTBHWN1xu31RKHhPbHf7zzqB5YZ/6DT7+d+eePHu48emzvkV3Q9INkkyUPHbFxwCoWkUl69ktlud/WdlzUMvfiElyLGFiKPIPTYiNj0A3X2AZQ6zKdF/MJ4361HVRQjUOklPmt11KKYC2bl8tITqvRa+OpP7ZYfzCwHkb2Q7Bovo+t74H9MBrYGmugDK3vQXlx3/b0D2HYj1+2cmuOFrK8+JXF+G3y5ITGr8umvySSah7v+IlEc0DzS4tgbARNaRkv763/AgepQqelIwAA
  docker-registry.json.gz:
    H4sIAAAAAAAC/+1dW3PbNhZ+76/gMN2dpKu6IiXF0s70wXHqJjNx4o2dviReDURCEtYUyZKgY9Xj/74HAC8ACeriSIrkcsZjmwCIy8E533dAHIL3PxiGORwSP0xobP7b+AzXhnHPf0OOj2YYUs3Xl0P4ufj44fy3qze/fbo0W1kJD42wlxbR5bs4diISUhL4rFSRQechr9pFFMVBEjm4yAu9ZEL8ty7LD6NghukUJ3E5/33auYuiBC/wAL+vW2JgEf4zIRHWDC1rfxKhMfJRUTlxtcmZKH4vZ9ziKE5H1zuyj6y0Ey19cyHyQV7VxsKptik5WWmofdR+RENTjOgMhdWm3pQz1mtMN4mkdvr86sRtcpAx8Scejimi1SYvNXk1TeZahHw/gBugCFMj0QfTIzHNlaroGeSMEuLRt6w6q1WkShLSjxzKYB+NPJZPowRL6VPialKJE/ingRdErMJoMkLP2y3Dtiz41eu1DOuFXHU2/pNiLMY/jRMPR1TpQjGh8XQUoMg107wH/vf6h3QuTOwSWuqtOfEx5UbrJ54nUpgGXwWBR0kI6W2eSJQihOIIpfBg9brHdmdg9XsvB1x4IGb/hhuvmAo+3xpjdgLPQ2GMWc1j5MW5nKAHxL0IiokTAi1Nzle4trtSwl3W2fR6zq4zSajqZtsFLOW9uy5gjlBPGDP2YaCecRokPgw5bhmnF59axjmeBdHcQL5rnBEPG69TvAwi4xImKq4CZhR81RuFg5wpviIzHCRUknAmoCB6hZybSQTtV8XE8/9AXoL1WbGk65D2zB4MnO5LWXO4BtqdY9A8e9AyuqCM7aP+QNHCZ263izoo06lCSkuto8wjYMERNSgM1gjGBpQ1AGgcHEt3jINohmiKTlwuZ1Dve5BeoRwomWBVNWboLhOD1ZZVYEb8LENOjqfB17LI2GRNgXSmgee+Y/wYLypxjqIbzAXMDKmiZLUK3Ckp8DL9tfT62y0umV7eIq+sO6oNplIKQ0DSK6GSli5d1Zii7xIY3TJ5GjQwKL6jkp4YaZbUaanbNZVFyJ8sqcwuKqtoIMz7a1DCiwCkEKuTbzJp8IzzgAMxmITvY4cC3ChlrljLJeGFQUzH5E71fdLEs8Cnl+QvXqVltf8hFYhw9SaeVn8PF8A5oGC94MdgAOxG1kNVRFR03Xz/y0kpI8hvWCC9OAQlBj0pGdOYeJ5KUR2gJ8vqw6/+gAGE1VcAYszaqdoKq1muR1RjA8RYg45SgWyOFX3nhAX1JLOyL4qiCaYLBIfvwij1ZhjIDDn8DBn8DGMM2uDG96D5v34xf4Q/X8yWQXwo4juYJWX/fzEfjJ+YYrVVERdAlVYYERyrRTLTPEMO5VKwlWwPT7DvnuX1qDeD3ghX+mTRFOZ4FJdkkxHYJYdchqNVTtK5XNzq6tWVZy9W14D5DOavK2pqZuVLlZUXzBYP6HayCTaVfbO1yXTw8rjTthWirE/6du68Arb0k9kIR4w7+TgNtk6C+YO/Dia3MrAV2gnaEdEnSp69Enl2VyNP62XDnnvBnr1HsGfvkNjzGUajfmewLlk+wz3ktLGGIrX28u0MGeEJrIyjOVBkEKEJHnJ8GVJYdXr3elZsMRaBpI8Cg4Amv50frYX8eH9vsDaNh4etMOUVG6xxyoE1HVT8WM7s7SFngrV/d860xsd9x1IIsj5pN5xpPJ8SGr/4u1PnYEXqtBvqbKhzJ9TZswY9B69NnW3UayPrIKjzDXkytAlDaahzq9TZH7THpbVlfdKuqHNG4oY6rVWXnc1D24Y7d7TstLtu115/2Vm5bW+58xyQ56mQJxsLoOnT5c8NPa497M3PC7ELARMcExf7NNtE/hSD5ms3QLEzmlNJgZfT6GHz6IprULvh0YZH923zsxTms/be55YYFpD3ebb9GaXAM5xx4BlycKmh2YcXB7nbuQRbnzCTIo+g+DTjvftCh0YoqqA5DxR7h/0J5TFVbSUd64qvyXUnt5j5c0YCisKjpOJ5TPGMBU+J8J84ZLNEfCPdiD+SCJBwg7KWE8pxiVD6S2Jpuno+6RTjF0qqtgSSnfgn8ZU+yo9NSCXRSaIIxlfNYA8SqonEryZKsCBzMPMXNdzMdaSWkRkYxeXnDyzxK3GpElWnI84KlShQbIYY1MKnaFJ1w0J2V4RcksQy3Yv0qpaByFwcYY5bYy+QbFPAzQdQKlAFXOofALmDdcoM9u3cVFoBPQxD7L4TyK/mrQWrEaI4x1YnTLKYkoXLl4fPvdn1i62DK7MzJebjsSCrxESKWLyyu8DSL6dkTKsZKSpnaMB6VUbkPMr0XtZ9FGFXYxPsmZJq1lzxhxmuE98lt8RNUA6oDy1N5Lgcnn2H7kgJWkaJcyO0QB4OM9TUAKgq2cyfK5XWW3BuqZ+vK12co7tFPqj0dE2d1yyaX+0Aywgmr1CMK6tcAUOV4gKIKslVD6Xeta17Avjd+llR73l1wjnCa1CVp7/Dt3mnlajq/WbgP0hEwQ44+eZekXD8jBg8HMa93AFsGUiYZ2wE8I/RM0C8Rr6w2gIt91ekZbuh5YaWv42W11ry7Iaa5SXKCvTcWnPEt8Lu92jAGRLVj/fV93FHataIjUeyskfiYofMkBfr2HjBc8zGaWmcFq3T8j7fcA5C7Btj9mqVm79aFW/DGSligZd4I93GG2m8kUe9dsJ0eTh2a2l42wT8gdnS2ev9eBYgOqO+Mxk37Ls99t0TSmuot7L3vYO3n626LdT2aq8/f0wDNMBXphFx1nmveb88CxbO9ubq6iKLYXuUL9Ffc7/Bsmuk/3IdX6KiZdyZKKcu8SbKxYVJlFN36k90Gn9iPX/CwcQTi/08bmpKaTjMVHo3Gw9L46WmyHc98OI3EDK1EY+Dm70mqqrxNB6z87BtljZLr1k3zsQBruMZ2xL/5zEMYUq3z7v9FXnX6je826zjHx+lzNmWwFqe63XOu9tb1x8c1771fz6TjL5h3YZ1G9bdJeuy5QF7fp5hlvrm1lbYt/IEvXbZu1X6rTDt9vlXf15Rs/DdwcJ39ReGdrfw3dSLQhth4vzJnfpWkfERxNjQcUPHDR1vnI63F7hu15xi2v32TentEGrE1gCX2gNwd8u1T3uxqwx7I1RLfCfCAARVpgUaBD8zD3KPk1kt4dpAuI8/M5HVo9G/9fe9gZJFpxeRsopfIEJF0pLw+BtH34fCT8Qo3gF1+87829jbbti7Ye792wtHkVuqmCVdINcl/qSqHizzY/pWuFy5+gZ4tcLiqINRdzxW4EnccukgYXzxn8oMZLlTPEuNBaAoDDwwyQ/8LVb1SPi7MPAFi7aPehojiEPsgKxnVUtjbsaZApMllFzBEdlYZL7G78i+RaD4V1Oge/7Y7zSdR4WYRPzc4j3vhScAa6h79Ye3tfulQzcRR9jnnCaw6z59qAqsNvKCESM0qw2MVkdl5a8zrLJ6XHFVqNvCTGkAVEJ6fdJ4zrr6Yik16ECVpb4B8QQA7rP6U5hTBqh8i+LupIIJi15pvnvFhSwiPcvUl2amb+AqWfNqM7XxTwuQVA+XWrDUIXqdCEOP0HyqtYA0F0N7lYKWiRIamOVcvVTmFak0GLq/GFp5HLpJEO0fIoiCue0DiH4KvQC5xipYCj1usLTB0gZLvzOWVp6DHW8MSu0DdUeHCYexAwPUouMNrja42uDqvq3zN4er3UME1hnyyVh8xOC7o+p52pdVcDXrdwOqDag2oLp3C/8Nomr7AFHVAVmD1e0DqJ6KrqyCqWmvG0j9W0Fq/tlgUCy2j8uq7bQFLJgxQy70R/7BYWHpZkznXvrZ3ehGlKRoUliVeZNAD3xcHH2Rh5Jn19jhZ/WLpimehYCJArOXfrAYeV52vG8Z3/MQjvuHNb9lnH6zWF53E9/xEhef6I8P1+5KmrMEDEZTPDv4lv3lwQ1y20GYfay5QCpIBluNGJqKf9iJPlD582Ux+coRq8Wc2krqBPPzan85+qmYqWHeN0Cw55//+8W8/teLo59+MRcHwsHE89mI/5N111RzK+NiafrCKTqIgUsZSYyvREVaWHkiuoHCcGtacV8zzz/mF6WTb1dQHfEF0UxTGj3ZlZ5kAUePVpYkVL7+WjfvVs28S8FPBwETpS/B8zCcnGbyU7aDrz9bWYBPdpC2eMO/uC0kQKHF+iCT1TBz2WS/0uxJrrXVli468oVVHA1m9qT/Lfmi05ZzpDAkW/rfcoVDeJ2NgQWaSXqxtBW54pdyxXIrdle+kM5UP3bl/mZ9UcT3V8AD9MzUY8hc1NcBk6rxUfYQzIR7/+Zp8Opy8r8++Uuk3uaeiG398PB/dY9XRg2EAAA=
--- a/charts/kubezero/templates/metrics.yaml
+++ b/charts/kubezero/templates/metrics.yaml
@ -44,13 +44,14 @@ prometheus:
        mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/"
        readOnly: true
    additionalScrapeConfigs:
-      - job_name: 'crio'
+      - job_name: 'cri'
+        metrics_path: '/v1/metrics'
        ec2_sd_configs:
          - port: 9090
            region: {{ .global.aws.region }}
            filters:
              - name: 'tag-key'
-                values: ['zdt:prometheus:crio']
+                values: ['zdt:prometheus:cri']
              {{- with .metrics.kubezero.prometheus.prometheusSpec.additionalScrapeConfigsEC2Filters }}
              {{- toYaml . | nindent 14 }}
              {{- end }}
--- a/charts/kubezero/values.yaml
+++ b/charts/kubezero/values.yaml
@ -17,7 +17,7 @@ global:

 addons:
  enabled: true
-  targetRevision: 0.8.9
+  targetRevision: 0.8.10
  external-dns:
    enabled: false
  forseti: