feat: version bump for argo and all storage drivers

Reviewed-on: #39

Dockerfile

@@ -3,10 +3,10 @@ ARG ALPINE_VERSION=3.21
 FROM docker.io/alpine:${ALPINE_VERSION}
 
 ARG ALPINE_VERSION
-ARG KUBE_VERSION=1.31.4
+ARG KUBE_VERSION=1.31
 
-ARG SOPS_VERSION="3.9.1"
+ARG SOPS_VERSION="3.9.4"
-ARG VALS_VERSION="0.37.6"
+ARG VALS_VERSION="0.39.1"
 ARG HELM_SECRETS_VERSION="4.6.2"
 
 RUN cd /etc/apk/keys && \
@@ -41,8 +41,8 @@ RUN mkdir -p $(helm env HELM_PLUGINS) && \
 # vals
 RUN wget -qO - https://github.com/helmfile/vals/releases/download/v${VALS_VERSION}/vals_${VALS_VERSION}_linux_amd64.tar.gz | tar -C /usr/local/bin -xzf- vals
 
-ADD admin/kubezero.sh admin/libhelm.sh admin/migrate_argo_values.py /usr/bin
+ADD admin/kubezero.sh admin/migrate_argo_values.py /usr/bin
-ADD admin/libhelm.sh /var/lib/kubezero
+ADD admin/libhelm.sh admin/hooks-$KUBE_VERSION.sh /var/lib/kubezero
 
 ADD charts/kubeadm /charts/kubeadm
 ADD charts/kubezero /charts/kubezero

admin/dev_apply.sh

@@ -49,7 +49,6 @@ function cert-manager-post() {
     wait_for "kubectl get deployment -n $namespace cert-manager-webhook"
     kubectl rollout status deployment -n $namespace cert-manager-webhook
     wait_for 'kubectl get validatingwebhookconfigurations -o yaml | grep "caBundle: LS0"'
-    apply
   fi
 
   wait_for "kubectl get ClusterIssuer -n $namespace kubezero-local-ca-issuer"
@@ -82,11 +81,11 @@ function metrics-pre() {
 get_kubezero_values $ARGOCD
 
 # Always use embedded kubezero chart
-helm template $CHARTS/kubezero -f $WORKDIR/kubezero-values.yaml --kube-version $KUBE_VERSION --version ~$KUBE_VERSION --devel --output-dir $WORKDIR
+helm template $CHARTS/kubezero -f $WORKDIR/kubezero-values.yaml --kube-version $KUBE_VERSION --name-template kubezero --version ~$KUBE_VERSION --devel --output-dir $WORKDIR
 
 # Root KubeZero apply directly and exit
 if [ ${ARTIFACTS[0]} == "kubezero" ]; then
-  kubectl apply -f $WORKDIR/kubezero/templates
+  kubectl replace -f $WORKDIR/kubezero/templates
   exit $?
 
 # "catch all" apply all enabled modules

admin/hooks-1.31.sh

@@ -0,0 +1,43 @@
+### v1.31
+
+# All things BEFORE the first controller / control plane upgrade
+pre_control_plane_upgrade_cluster() {
+  # add kubezero version label to existing controller nodes for aws-iam migration
+  for n in $(kubectl get nodes -l "node-role.kubernetes.io/control-plane=" | grep v1.30 | awk {'print $1}'); do
+    kubectl label node $n 'node.kubernetes.io/kubezero.version=v1.30.6' || true
+  done
+
+  # patch aws-iam-authentiator DS to NOT run pods on 1.31 controllers
+  kubectl patch ds aws-iam-authentiator -p '{"spec": {"template": {"spec": {"nodeSelector": {"node.kubernetes.io/kubezero.version": "v1.30.6"}}}}}' || true
+}
+
+
+# All things after the first controller / control plane upgrade
+post_control_plane_upgrade_cluster() {
+  echo
+}
+
+
+# All things AFTER all contollers are on the new version
+pre_cluster_upgrade_final() {
+
+  if [ "$PLATFORM" == "aws" ];then
+    # cleanup aws-iam-authentiator
+    kubectl delete clusterrolebinding aws-iam-authentiator || true
+    kubectl delete clusterrole aws-iam-authentiator || true
+    kubectl delete serviceaccount aws-iam-authentiator -n kube-system || true
+    kubectl delete cm aws-iam-authentiator -n kube-system || true
+    kubectl delete ds aws-iam-authentiator -n kube-system || true
+    kubectl delete IAMIdentityMapping kubezero-worker-nodes || true
+    kubectl delete IAMIdentityMapping kubernetes-admin || true
+    kubectl delete crd iamidentitymappings.iamauthenticator.k8s.aws || true
+
+    kubectl delete secret aws-iam-certs -n kube-system || true
+  fi
+}
+
+
+# Last call
+post_cluster_upgrade_final() {
+  echo
+}

admin/kubezero.sh

@@ -29,6 +29,9 @@ export ETCDCTL_KEY=${HOSTFS}/etc/kubernetes/pki/apiserver-etcd-client.key
 
 mkdir -p ${WORKDIR}
 
+# Import version specific hooks
+. /var/lib/kubezero/hooks-${KUBE_VERSION_MINOR##v}.sh
+
 # Generic retry utility
 retry() {
   local tries=$1
@@ -64,7 +67,9 @@ render_kubeadm() {
     cat ${WORKDIR}/kubeadm/templates/${f}Configuration.yaml >> ${HOSTFS}/etc/kubernetes/kubeadm.yaml
   done
 
-  if [[ "$phase" =~ ^(bootstrap|join|restore)$ ]]; then
+  if [[ "$phase" == "upgrade" ]]; then
+    cat ${WORKDIR}/kubeadm/templates/UpgradeConfiguration.yaml >> ${HOSTFS}/etc/kubernetes/kubeadm.yaml
+  elif [[ "$phase" =~ ^(bootstrap|join|restore)$ ]]; then
     cat ${WORKDIR}/kubeadm/templates/InitConfiguration.yaml >> ${HOSTFS}/etc/kubernetes/kubeadm.yaml
   fi
 
@@ -83,7 +88,6 @@ parse_kubezero() {
   export ETCD_NODENAME=$(yq eval '.etcd.nodeName' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml)
   export NODENAME=$(yq eval '.nodeName' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml)
   export PROVIDER_ID=$(yq eval '.providerID // ""' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml)
-  export AWS_IAM_AUTH=$(yq eval '.api.awsIamAuth.enabled // "false"' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml)
 }
 
 
@@ -92,20 +96,6 @@ pre_kubeadm() {
   # update all apiserver addons first
   cp -r ${WORKDIR}/kubeadm/templates/apiserver ${HOSTFS}/etc/kubernetes
 
-  # aws-iam-authenticator enabled ?
-  if [ "$AWS_IAM_AUTH" == "true" ]; then
-
-    # Initialize webhook
-    if [ ! -f ${HOSTFS}/etc/kubernetes/pki/aws-iam-authenticator.crt ]; then
-      ${HOSTFS}/usr/bin/aws-iam-authenticator init -i ${CLUSTERNAME}
-      mv key.pem ${HOSTFS}/etc/kubernetes/pki/aws-iam-authenticator.key
-      mv cert.pem ${HOSTFS}/etc/kubernetes/pki/aws-iam-authenticator.crt
-    fi
-
-    # Patch the aws-iam-authenticator config with the actual cert.pem
-    yq eval -Mi ".clusters[0].cluster.certificate-authority-data = \"$(cat ${HOSTFS}/etc/kubernetes/pki/aws-iam-authenticator.crt| base64 -w0)\"" ${HOSTFS}/etc/kubernetes/apiserver/aws-iam-authenticator.yaml
-  fi
-
   # copy patches to host to make --rootfs of kubeadm work
   cp -r ${WORKDIR}/kubeadm/templates/patches ${HOSTFS}/etc/kubernetes
 }
@@ -120,60 +110,58 @@ post_kubeadm() {
 }
 
 
-kubeadm_upgrade() {
+# Control plane upgrade
-  # pre upgrade hook
+control_plane_upgrade() {
+  CMD=$1
 
-  # get current values, argo app over cm
-  get_kubezero_values $ARGOCD
-
-  # tumble new config through migrate.py
-  migrate_argo_values.py < "$WORKDIR"/kubezero-values.yaml > "$WORKDIR"/new-kubezero-values.yaml
-
-  # Update kubezero-values CM
-  kubectl get cm -n kubezero kubezero-values -o=yaml | \
-    yq e '.data."values.yaml" |= load_str("/tmp/kubezero/new-kubezero-values.yaml")' | \
-    kubectl apply --server-side --force-conflicts -f -
-
-  if [ "$ARGOCD" == "True" ]; then
-    # update argo app
-    export kubezero_chart_version=$(yq .version $CHARTS/kubezero/Chart.yaml)
-    kubectl get application kubezero -n argocd -o yaml | \
-      yq '.spec.source.helm.valuesObject |= load("/tmp/kubezero/new-kubezero-values.yaml") | .spec.source.targetRevision = strenv(kubezero_chart_version)' \
-      > $WORKDIR/new-argocd-app.yaml
-    kubectl apply --server-side --force-conflicts -f $WORKDIR/new-argocd-app.yaml
-
-    # finally remove annotation to allow argo to sync again
-    kubectl patch app kubezero -n argocd --type json -p='[{"op": "remove", "path": "/metadata/annotations"}]' || true
-  fi
-
-  # Local node upgrade
   render_kubeadm upgrade
 
-  pre_kubeadm
+  if [[ "$CMD" =~ ^(cluster)$ ]]; then
+    # get current values, argo app over cm
+    get_kubezero_values $ARGOCD
 
-  # Upgrade - we upload the new config first so we can use --patch during 1.30
+    # tumble new config through migrate.py
-  _kubeadm init phase upload-config kubeadm
+    migrate_argo_values.py < "$WORKDIR"/kubezero-values.yaml > "$WORKDIR"/new-kubezero-values.yaml \
+      && mv "$WORKDIR"/new-kubezero-values.yaml "$WORKDIR"/kubezero-values.yaml
 
-  kubeadm upgrade apply --yes --patches /etc/kubernetes/patches $KUBE_VERSION --rootfs ${HOSTFS} $LOG
+    update_kubezero_cm
 
-  post_kubeadm
+    if [ "$ARGOCD" == "True" ]; then
+      # update argo app
+      export kubezero_chart_version=$(yq .version $CHARTS/kubezero/Chart.yaml)
+      kubectl get application kubezero -n argocd -o yaml | \
+        yq ".spec.source.helm.valuesObject |= load(\"$WORKDIR/kubezero-values.yaml\") | .spec.source.targetRevision = strenv(kubezero_chart_version)" \
+        > $WORKDIR/new-argocd-app.yaml
+      kubectl apply --server-side --force-conflicts -f $WORKDIR/new-argocd-app.yaml
 
-  # install re-certed kubectl config for root
+      # finally remove annotation to allow argo to sync again
-  cp ${HOSTFS}/etc/kubernetes/super-admin.conf ${HOSTFS}/root/.kube/config
+      kubectl patch app kubezero -n argocd --type json -p='[{"op": "remove", "path": "/metadata/annotations"}]' || true
+    fi
 
-  # post upgrade
+    pre_kubeadm
+
+    _kubeadm init phase upload-config kubeadm
+
+    _kubeadm upgrade apply $KUBE_VERSION
+
+    post_kubeadm
+
+    # install re-certed kubectl config for root
+    cp ${HOSTFS}/etc/kubernetes/super-admin.conf ${HOSTFS}/root/.kube/config
+
+    echo "Successfully upgraded KubeZero control plane to $KUBE_VERSION using kubeadm."
+
+  elif [[ "$CMD" =~ ^(final)$ ]]; then
+    # Finally upgrade addons last, with 1.32 we can ONLY call addon phase
+    #_kubeadm upgrade apply phase addon all $KUBE_VERSION
+    _kubeadm upgrade apply $KUBE_VERSION
+
+    echo "Upgraded kubeadm addons."
+  fi
 
   # Cleanup after kubeadm on the host
   rm -rf ${HOSTFS}/etc/kubernetes/tmp
 
-  echo "Successfully upgraded kubeadm control plane."
-
-  # TODO
-  # Send Notification currently done via CloudBender -> SNS -> Slack
-  # Better deploy https://github.com/opsgenie/kubernetes-event-exporter and set proper routes and labels on this Job
-
-  # Removed:
-  # - update oidc do we need that ?
 }
 
 
@@ -200,6 +188,10 @@ control_plane_node() {
     # Put PKI in place
     cp -r ${WORKDIR}/pki ${HOSTFS}/etc/kubernetes
 
+    ### 1.31 only to clean up previous aws-iam-auth certs
+    rm -f ${HOSTFS}/etc/kubernetes/pki/aws-iam-authenticator.key ${HOSTFS}/etc/kubernetes/pki/aws-iam-authenticator.crt
+    ###
+
     # Always use kubeadm kubectl config to never run into chicken egg with custom auth hooks
     cp ${WORKDIR}/super-admin.conf ${HOSTFS}/root/.kube/config
 
@@ -220,7 +212,7 @@ control_plane_node() {
   rm -f ${HOSTFS}/etc/kubernetes/pki/etcd/peer.* ${HOSTFS}/etc/kubernetes/pki/etcd/server.* ${HOSTFS}/etc/kubernetes/pki/etcd/healthcheck-client.* \
     ${HOSTFS}/etc/kubernetes/pki/apiserver* ${HOSTFS}/etc/kubernetes/pki/front-proxy-client.*
 
-  # Issue all certs first, needed for eg. aws-iam-authenticator setup
+  # Issue all certs first
   _kubeadm init phase certs all
 
   pre_kubeadm
@@ -286,6 +278,9 @@ control_plane_node() {
         -endpoint https://${ETCD_NODENAME}:2379 \
         change-provider-id ${NODENAME} $PROVIDER_ID
     fi
+
+    # update node label for single node control plane
+    kubectl label node $NODENAME "node.kubernetes.io/kubezero.version=$KUBE_VERSION" --overwrite=true
   fi
 
   _kubeadm init phase upload-config all
@@ -305,17 +300,6 @@ control_plane_node() {
     _kubeadm init phase addon all
   fi
 
-  # Ensure aws-iam-authenticator secret is in place
-  if [ "$AWS_IAM_AUTH" == "true" ]; then
-    kubectl get secrets -n kube-system aws-iam-certs || \
-    kubectl create secret generic aws-iam-certs -n kube-system \
-      --from-file=key.pem=${HOSTFS}/etc/kubernetes/pki/aws-iam-authenticator.key \
-      --from-file=cert.pem=${HOSTFS}/etc/kubernetes/pki/aws-iam-authenticator.crt
-
-    # Store aws-iam-auth admin on SSM
-    yq eval -M ".clusters[0].cluster.certificate-authority-data = \"$(cat ${HOSTFS}/etc/kubernetes/pki/ca.crt | base64 -w0)\"" ${WORKDIR}/kubeadm/templates/admin-aws-iam.yaml > ${HOSTFS}/etc/kubernetes/admin-aws-iam.yaml
-  fi
-
   post_kubeadm
 
   echo "${CMD}ed cluster $CLUSTERNAME successfully."
@@ -413,7 +397,17 @@ for t in $@; do
     restore) control_plane_node restore;;
     kubeadm_upgrade)
       ARGOCD=$(argo_used)
-      kubeadm_upgrade;;
+      # call hooks
+      pre_control_plane_upgrade_cluster
+      control_plane_upgrade cluster
+      post_control_plane_upgrade_cluster
+      ;;
+    finalize_cluster_upgrade)
+      ARGOCD=$(argo_used)
+      pre_cluster_upgrade_final
+      control_plane_upgrade final
+      post_cluster_upgrade_final
+      ;;
     apply_*)
       ARGOCD=$(argo_used)
       apply_module "${t##apply_}";;

admin/libhelm.sh

@@ -46,13 +46,17 @@ function get_kubezero_values() {
 }
 
 
-# Update kubezero-values CM
+# Overwrite kubezero-values CM with file
 function update_kubezero_cm() {
-  kubectl get application kubezero -n argocd -o yaml | yq .spec.source.helm.valuesObject > ${WORKDIR}/kubezero-values.yaml
-
   kubectl get cm -n kubezero kubezero-values -o=yaml | \
-    yq e '.data."values.yaml" |= load_str("/tmp/kubezero/kubezero-values.yaml")' | \
+    yq e ".data.\"values.yaml\" |= load_str(\"$WORKDIR/kubezero-values.yaml\")" | \
-    kubectl apply --server-side --force-conflicts -f -
+    kubectl replace -f -
+}
+
+# sync kubezero-values CM from ArgoCD app
+function sync_kubezero_cm_from_argo() {
+  get_kubezero_values True
+  update_kubezero_cm
 }
 
 
@@ -148,7 +152,7 @@ for manifest in yaml.safe_load_all(sys.stdin):
 # helm template | kubectl apply -f -
 # confine to one namespace if possible
 function render() {
-  helm secrets --evaluate-templates template $(chart_location $chart) -n $namespace --name-template $module $targetRevision --skip-crds -f $WORKDIR/values.yaml $API_VERSIONS --kube-version $KUBE_VERSION $@ \
+  helm secrets --evaluate-templates template $(chart_location $chart) -n $namespace --name-template $module $targetRevision --skip-tests --skip-crds -f $WORKDIR/values.yaml $API_VERSIONS --kube-version $KUBE_VERSION $@ \
     | python3 -c '
 #!/usr/bin/python3
 import yaml

admin/upgrade_cluster.sh

@@ -19,22 +19,25 @@ echo "Checking that all pods in kube-system are running ..."
 
 [ "$ARGOCD" == "True" ] && disable_argo
 
-control_plane_upgrade kubeadm_upgrade
+# Check if we already have all controllers on the current version
+OLD_CONTROLLERS=$(kubectl get nodes -l "node-role.kubernetes.io/control-plane=" --no-headers=true | grep -cv $KUBE_VERSION || true)
 
-echo "Control plane upgraded, <Return> to continue"
+# All controllers already on current version
+if [ "$OLD_CONTROLLERS" == "0" ]; then
+  control_plane_upgrade finalize_cluster_upgrade
+# Otherwise run control plane upgrade
+else
+  control_plane_upgrade kubeadm_upgrade
+fi
+
+echo "<Return> to continue"
 read -r
 
 #echo "Adjust kubezero values as needed:"
 # shellcheck disable=SC2015
 #[ "$ARGOCD" == "True" ] && kubectl edit app kubezero -n argocd || kubectl edit cm kubezero-values -n kubezero
 
-### v1.31
-
 # upgrade modules
-#
-# Preload cilium images to running nodes, disabled till 1.31
-# all_nodes_upgrade "chroot /host crictl pull quay.io/cilium/cilium:v1.16.3; chroot /host crictl pull ghcr.io/k8snetworkplumbingwg/multus-cni:v3.9.3"
-
 control_plane_upgrade "apply_network, apply_addons, apply_storage, apply_operators"
 
 echo "Checking that all pods in kube-system are running ..."
@@ -45,8 +48,7 @@ echo "Applying remaining KubeZero modules..."
 control_plane_upgrade "apply_cert-manager, apply_istio, apply_istio-ingress, apply_istio-private-ingress, apply_logging, apply_metrics, apply_telemetry, apply_argo"
 
 # Final step is to commit the new argocd kubezero app
-# remove the del(.spec.source.helm.values) with 1.31
+kubectl get app kubezero -n argocd -o yaml | yq 'del(.status) | del(.metadata) | del(.operation) | .metadata.name="kubezero" | .metadata.namespace="argocd"' | yq 'sort_keys(..)' > $ARGO_APP
-kubectl get app kubezero -n argocd -o yaml | yq 'del(.spec.source.helm.values) | del(.status) | del(.metadata) | del(.operation) | .metadata.name="kubezero" | .metadata.namespace="argocd"' | yq 'sort_keys(..)' > $ARGO_APP
 
 # Trigger backup of upgraded cluster state
 kubectl create job --from=cronjob/kubezero-backup kubezero-backup-$KUBE_VERSION -n kube-system

charts/kubeadm/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubeadm
 description: KubeZero Kubeadm cluster config
 type: application
-version: 1.31.4
+version: 1.31.5
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:

charts/kubeadm/templates/ClusterConfiguration.yaml

@@ -1,4 +1,4 @@
-apiVersion: kubeadm.k8s.io/v1beta3
+apiVersion: kubeadm.k8s.io/v1beta4
 kind: ClusterConfiguration
 kubernetesVersion: {{ .Chart.Version }}
 clusterName: {{ .Values.global.clusterName }}
@@ -11,20 +11,33 @@ etcd:
   local:
     # imageTag: 3.5.12-0
     extraArgs:
+      - name: advertise-client-urls
+        value: https://{{ .Values.etcd.nodeName }}:2379
+      - name: initial-advertise-peer-urls
+        value: https://{{ .Values.etcd.nodeName }}:2380
+      - name: initial-cluster
+        value: {{ include "kubeadm.etcd.initialCluster" .Values.etcd | quote }}
+      - name: initial-cluster-state
+        value: {{ .Values.etcd.state }}
+      - name: initial-cluster-token
+        value: etcd-{{ .Values.global.clusterName }}
+      - name: name
+        value: {{ .Values.etcd.nodeName }}
+      - name: listen-peer-urls
+        value: https://{{ .Values.listenAddress }}:2380
+      - name: listen-client-urls
+        value: https://{{ .Values.listenAddress }}:2379
+      - name: listen-metrics-urls
+        value: http://0.0.0.0:2381
+      - name: logger
+        value: zap
+      - name: log-level
+        value: warn
       ### DNS discovery
-      #discovery-srv: {{ .Values.domain }}
+      #- name: discovery-srv
-      #discovery-srv-name: {{ .Values.global.clusterName }}
+      #  value: {{ .Values.domain }}
-      advertise-client-urls: https://{{ .Values.etcd.nodeName }}:2379
+      #- name: discovery-srv-name
-      initial-advertise-peer-urls: https://{{ .Values.etcd.nodeName }}:2380
+      #  value: {{ .Values.global.clusterName }}
-      initial-cluster: {{ include "kubeadm.etcd.initialCluster" .Values.etcd | quote }}
-      initial-cluster-state: {{ .Values.etcd.state }}
-      initial-cluster-token: etcd-{{ .Values.global.clusterName }}
-      name: {{ .Values.etcd.nodeName }}
-      listen-peer-urls: https://{{ .Values.listenAddress }}:2380
-      listen-client-urls: https://{{ .Values.listenAddress }}:2379
-      listen-metrics-urls: http://0.0.0.0:2381
-      logger: zap
-      # log-level: "warn"
       {{- with .Values.etcd.extraArgs }}
       {{- toYaml . | nindent 6 }}
       {{- end }}
@@ -38,49 +51,82 @@ etcd:
     - "{{ .Values.domain }}"
 controllerManager:
   extraArgs:
-    profiling: "false"
+    - name: profiling
-    terminated-pod-gc-threshold: "300"
+      value: "false"
-    leader-elect: {{ .Values.global.highAvailable | quote }}
+    - name: terminated-pod-gc-threshold
-    logging-format: json
+      value: "300"
-    feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }}
+    - name: leader-elect
+      value: {{ .Values.global.highAvailable | quote }}
+    - name: logging-format
+      value: json
+    - name: feature-gates
+      value: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }}
 scheduler:
   extraArgs:
-    profiling: "false"
+    - name: feature-gates
-    leader-elect: {{ .Values.global.highAvailable | quote }}
+      value: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }}
-    logging-format: json
+    - name: leader-elect
-    feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }}
+      value: {{ .Values.global.highAvailable | quote }}
+    - name: logging-format
+      value: json
+    - name: profiling
+      value: "false"
 apiServer:
   certSANs:
   -  {{ regexSplit ":" .Values.api.endpoint -1 | first }}
   extraArgs:
-    etcd-servers: {{ .Values.api.etcdServers }}
+    - name: profiling
-    profiling: "false"
+      value: "false"
-    audit-log-path: "/var/log/kubernetes/audit.log"
+    - name: etcd-servers
-    audit-policy-file: /etc/kubernetes/apiserver/audit-policy.yaml
+      value: {{ .Values.api.etcdServers }}
-    audit-log-maxage: "7"
+    - name: audit-log-path
-    audit-log-maxsize: "100"
+      value: /var/log/kubernetes/audit.log
-    audit-log-maxbackup: "1"
+    - name: audit-policy-file
-    audit-log-compress: "true"
+      value: /etc/kubernetes/apiserver/audit-policy.yaml
+    - name: audit-log-maxage
+      value: "7"
+    - name: audit-log-maxsize
+      value: "100"
+    - name: audit-log-maxbackup
+      value: "1"
+    - name: audit-log-compress
+      value: "true"
     {{- if .Values.api.falco.enabled }}
-    audit-webhook-config-file: /etc/kubernetes/apiserver/audit-webhook.yaml
+    - name: audit-webhook-config-file
+      value: /etc/kubernetes/apiserver/audit-webhook.yaml
     {{- end }}
-    tls-cipher-suites: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"
+    - name: tls-cipher-suites
-    admission-control-config-file: /etc/kubernetes/apiserver/admission-configuration.yaml
+      value: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"
-    api-audiences: {{ .Values.api.apiAudiences }}
+    - name: admission-control-config-file
+      value: /etc/kubernetes/apiserver/admission-configuration.yaml
+    - name: api-audiences
+      value: {{ .Values.api.apiAudiences }}
     {{- if .Values.api.serviceAccountIssuer }}
-    service-account-issuer: "{{ .Values.api.serviceAccountIssuer }}"
+    - name: service-account-issuer
-    service-account-jwks-uri: "{{ .Values.api.serviceAccountIssuer }}/openid/v1/jwks"
+      value: "{{ .Values.api.serviceAccountIssuer }}"
+    - name: service-account-jwks-uri
+      value: "{{ .Values.api.serviceAccountIssuer }}/openid/v1/jwks"
     {{- end }}
-    {{- if .Values.api.awsIamAuth.enabled }}
+    {{- if .Values.api.awsIamAuth }}
-    authentication-token-webhook-config-file: /etc/kubernetes/apiserver/aws-iam-authenticator.yaml
+    - name: authentication-token-webhook-config-file
-    authentication-token-webhook-cache-ttl: 3600s
+      value: /etc/kubernetes/apiserver/aws-iam-authenticator.yaml
+    - name: authentication-token-webhook-cache-ttl
+      value: 3600s
+    - name: authentication-token-webhook-version
+      value: v1
     {{- end }}
-    feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }}
+    - name: feature-gates
-    enable-admission-plugins: DenyServiceExternalIPs,NodeRestriction,EventRateLimit,ExtendedResourceToleration
+      value: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }}
+    - name: authorization-config
+      value: /etc/kubernetes/apiserver/authz-config.yaml
+    - name: enable-admission-plugins
+      value: DenyServiceExternalIPs,NodeRestriction,EventRateLimit,ExtendedResourceToleration
     {{- if .Values.global.highAvailable }}
-    goaway-chance: ".001"
+    - name: goaway-chance
+      value: ".001"
     {{- end }}
-    logging-format: json
+    - name: logging-format
+      value: json
     {{- with .Values.api.extraArgs }}
     {{- toYaml . | nindent 4 }}
     {{- end }}

charts/kubeadm/templates/InitConfiguration.yaml

@@ -1,4 +1,4 @@
-apiVersion: kubeadm.k8s.io/v1beta3
+apiVersion: kubeadm.k8s.io/v1beta4
 kind: InitConfiguration
 localAPIEndpoint:
   advertiseAddress: {{ .Values.listenAddress }}
@@ -17,10 +17,13 @@ nodeRegistration:
   - Swap
   - KubeletVersion
   kubeletExtraArgs:
-    node-labels: {{ .Values.nodeLabels | quote }}
+    - name: node-labels
+      value: {{ .Values.nodeLabels | quote }}
     {{- with .Values.providerID }}
-    provider-id: {{ . }}
+    - name: provider-id
+      value: {{ . }}
     {{- end }}
     {{- if ne .Values.listenAddress "0.0.0.0" }}
-    node-ip: {{ .Values.listenAddress }}
+    - name: node-ip
+      value: {{ .Values.listenAddress }}
     {{- end }}

charts/kubeadm/templates/UpgradeConfiguration.yaml

@@ -0,0 +1,16 @@
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: UpgradeConfiguration
+apply:
+  forceUpgrade: true
+  {{- with .Values.patches }}
+  patches:
+    directory: {{ . }}
+  {{- end }}
+  ignorePreflightErrors:
+  - DirAvailable--var-lib-etcd
+  - DirAvailable--etc-kubernetes-manifests
+  - FileAvailable--etc-kubernetes-pki-ca.crt
+  - FileAvailable--etc-kubernetes-manifests-etcd.yaml
+  - Swap
+  - KubeletVersion
+  skipPhases: []

charts/kubeadm/templates/admin-aws-iam.yaml

@@ -1,27 +0,0 @@
-{{- if .Values.api.awsIamAuth.enabled }}
-apiVersion: v1
-kind: Config
-clusters:
-- cluster:
-    server: https://{{ .Values.api.endpoint }}
-  name: {{ .Values.global.clusterName }}
-contexts:
-- context:
-    cluster: {{ .Values.global.clusterName }}
-    user: kubernetes-admin
-  name: kubernetes-admin@{{ .Values.global.clusterName }}
-current-context: kubernetes-admin@{{ .Values.global.clusterName }}
-preferences: {}
-users:
-- name: kubernetes-admin
-  user:
-    exec:
-      apiVersion: client.authentication.k8s.io/v1beta1
-      command: aws-iam-authenticator
-      args:
-        - "token"
-        - "-i"
-        - "{{ .Values.global.clusterName }}"
-        - "-r"
-        - "{{ .Values.api.awsIamAuth.kubeAdminRole }}"
-{{- end }}

charts/kubeadm/templates/apiserver/authz-config.yaml

@@ -0,0 +1,32 @@
+apiVersion: apiserver.config.k8s.io/v1beta1
+kind: AuthorizationConfiguration
+authorizers:
+  - type: Node
+    name: node
+  - type: RBAC
+    name: rbac
+# - type: Webhook
+#   name: Example
+#   webhook:
+#     authorizedTTL: 300s
+#     unauthorizedTTL: 30s
+#     timeout: 3s
+#     subjectAccessReviewVersion: v1
+#     matchConditionSubjectAccessReviewVersion: v1
+#     failurePolicy: NoOpinion
+#     connectionInfo:
+#       type: KubeConfigFile
+#       kubeConfigFile: /etc/kubernetes/apiserver/example.yaml
+#     matchConditions:
+#     # only send resource requests to the webhook
+#     - expression: has(request.resourceAttributes)
+#     # Don't intercept requests from kube-system service accounts
+#     - expression: "!('system:serviceaccounts:kube-system' in request.groups)"
+#     ## Below expressions avoid issues with kubeadm init and other system components that should be authorized by Node and RBAC
+#     # Don't process node and bootstrap token requests with the webhook
+#     - expression: "!('system:nodes' in request.groups)"
+#     - expression: "!('system:bootstrappers' in request.groups)"
+#     - expression: "!('system:bootstrappers:kubeadm:default-node-token' in request.groups)"
+#     # Don't process kubeadm requests with the webhook
+#     - expression: "!('kubeadm:cluster-admins' in request.groups)"
+#     - expression: "!('system:masters' in request.groups)"

charts/kubeadm/templates/apiserver/aws-iam-authenticator.yaml

@@ -1,19 +0,0 @@
-{{- if .Values.api.awsIamAuth.enabled }}
-# clusters refers to the remote service.
-clusters:
-  - name: aws-iam-authenticator
-    cluster:
-      certificate-authority-data: "replaced at runtime"
-      server: https://localhost:21362/authenticate
-# users refers to the API Server's webhook configuration
-# (we don't need to authenticate the API server).
-users:
-  - name: apiserver
-# kubeconfig files require a context. Provide one for the API Server.
-current-context: webhook
-contexts:
-- name: webhook
-  context:
-    cluster: aws-iam-authenticator
-    user: apiserver
-{{- end }}

charts/kubeadm/templates/resources/50-aws-iam-authenticator-crds.yaml

@@ -1,46 +0,0 @@
-{{- if .Values.api.awsIamAuth.enabled }}
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: iamidentitymappings.iamauthenticator.k8s.aws
-spec:
-  group: iamauthenticator.k8s.aws
-  scope: Cluster
-  names:
-    plural: iamidentitymappings
-    singular: iamidentitymapping
-    kind: IAMIdentityMapping
-    categories:
-    - all
-  versions:
-    - name: v1alpha1
-      served: true
-      storage: true
-      schema:
-        openAPIV3Schema:
-          type: object
-          properties:
-            spec:
-              type: object
-              required:
-              - arn
-              - username
-              properties:
-                arn:
-                  type: string
-                username:
-                  type: string
-                groups:
-                  type: array
-                  items:
-                    type: string
-            status:
-              type: object
-              properties:
-                canonicalARN:
-                  type: string
-                userID:
-                  type: string
-      subresources:
-        status: {}
-{{- end }}

charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml

@@ -1,155 +0,0 @@
-{{- if .Values.api.awsIamAuth.enabled }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: aws-iam-authenticator
-rules:
-- apiGroups:
-  - iamauthenticator.k8s.aws
-  resources:
-  - iamidentitymappings
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - iamauthenticator.k8s.aws
-  resources:
-  - iamidentitymappings/status
-  verbs:
-  - patch
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - create
-  - update
-  - patch
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  resourceNames:
-  - aws-auth
-  verbs:
-  - get
-
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: aws-iam-authenticator
-  namespace: kube-system
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: aws-iam-authenticator
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: aws-iam-authenticator
-subjects:
-- kind: ServiceAccount
-  name: aws-iam-authenticator
-  namespace: kube-system
-
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  namespace: kube-system
-  name: aws-iam-authenticator
-  labels:
-    k8s-app: aws-iam-authenticator
-data:
-  config.yaml: |
-    clusterID: {{ .Values.global.clusterName }}
-
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  namespace: kube-system
-  name: aws-iam-authenticator
-  labels:
-    k8s-app: aws-iam-authenticator
-spec:
-  selector:
-    matchLabels:
-      k8s-app: aws-iam-authenticator
-  updateStrategy:
-    type: RollingUpdate
-  template:
-    metadata:
-      labels:
-        k8s-app: aws-iam-authenticator
-    spec:
-      securityContext:
-        seccompProfile:
-          type: RuntimeDefault
-
-      priorityClassName: system-cluster-critical
-
-      # use service account with access to
-      serviceAccountName: aws-iam-authenticator
-
-      # run on the host network (don't depend on CNI)
-      hostNetwork: true
-
-      # run on each controller
-      nodeSelector:
-        node-role.kubernetes.io/control-plane: ""
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/control-plane
-
-      containers:
-      - name: aws-iam-authenticator
-        image: public.ecr.aws/zero-downtime/aws-iam-authenticator:v0.6.27
-        args:
-        - server
-        - --backend-mode=CRD,MountedFile
-        - --config=/etc/aws-iam-authenticator/config.yaml
-        - --state-dir=/var/aws-iam-authenticator
-        - --kubeconfig-pregenerated=true
-
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-
-        resources:
-          requests:
-            memory: 32Mi
-            cpu: 10m
-          limits:
-            memory: 64Mi
-            #cpu: 100m
-
-        volumeMounts:
-        - name: config
-          mountPath: /etc/aws-iam-authenticator/
-        - name: state
-          mountPath: /var/aws-iam-authenticator/
-
-      volumes:
-      - name: config
-        configMap:
-          name: aws-iam-authenticator
-      - name: state
-        secret:
-          secretName: aws-iam-certs
-{{- end }}

charts/kubeadm/templates/resources/52-aws-iam-authenticator-mappings.yaml

@@ -1,23 +0,0 @@
-{{- if .Values.api.awsIamAuth.enabled }}
-apiVersion: iamauthenticator.k8s.aws/v1alpha1
-kind: IAMIdentityMapping
-metadata:
-  name: kubezero-worker-nodes
-spec:
-  arn: {{ .Values.api.awsIamAuth.workerNodeRole }}
-  username: system:node:{{ "{{" }}EC2PrivateDNSName{{ "}}" }}
-  groups:
-  - system:bootstrappers:kubeadm:default-node-token
----
-
-# Admin Role for remote access
-apiVersion: iamauthenticator.k8s.aws/v1alpha1
-kind: IAMIdentityMapping
-metadata:
-  name: kubernetes-admin
-spec:
-  arn: {{ .Values.api.awsIamAuth.kubeAdminRole }}
-  username: kubernetes-admin
-  groups:
-  - system:masters
-{{- end }}

charts/kubeadm/values.yaml

@@ -20,10 +20,7 @@ api:
   oidcEndpoint: ""
   apiAudiences: "istio-ca"
 
-  awsIamAuth:
+  awsIamAuth: false
-    enabled: false
-    workerNodeRole: "arn:aws:iam::000000000000:role/KubernetesNode"
-    kubeAdminRole: "arn:aws:iam::000000000000:role/KubernetesNode"
 
   falco:
     enabled: false

charts/kubezero-addons/Chart.yaml

@@ -54,4 +54,4 @@ dependencies:
     version: 0.2.12
     repository: https://caas-team.github.io/helm-charts/
     condition: py-kube-downscaler.enabled
-kubeVersion: ">= 1.26.0"
+kubeVersion: ">= 1.30.0-0"

charts/kubezero-addons/README.md

@@ -1,6 +1,6 @@
 # kubezero-addons
 
-![Version: 0.8.11](https://img.shields.io/badge/Version-0.8.11-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.30](https://img.shields.io/badge/AppVersion-v1.30-informational?style=flat-square)
+![Version: 0.8.13](https://img.shields.io/badge/Version-0.8.13-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.30](https://img.shields.io/badge/AppVersion-v1.30-informational?style=flat-square)
 
 KubeZero umbrella chart for various optional cluster addons
 
@@ -18,13 +18,13 @@ Kubernetes: `>= 1.26.0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.16.2 |
+| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.17.1 |
-| https://caas-team.github.io/helm-charts/ | py-kube-downscaler | 0.2.11 |
+| https://caas-team.github.io/helm-charts/ | py-kube-downscaler | 0.2.12 |
-| https://kubernetes-sigs.github.io/external-dns/ | external-dns | 1.15.0 |
+| https://kubernetes-sigs.github.io/external-dns/ | external-dns | 1.15.1 |
-| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.43.2 |
+| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.46.0 |
 | https://nvidia.github.io/k8s-device-plugin | nvidia-device-plugin | 0.17.0 |
 | https://twin.github.io/helm-charts | aws-eks-asg-rolling-update-handler | 1.5.0 |
-| oci://public.ecr.aws/aws-ec2/helm | aws-node-termination-handler | 0.24.1 |
+| oci://public.ecr.aws/aws-ec2/helm | aws-node-termination-handler | 0.26.0 |
 | oci://public.ecr.aws/neuron | neuron-helm-chart | 1.1.1 |
 
 # MetalLB   

charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 1.22.1
+appVersion: 1.24.0
 description: A Helm chart for the AWS Node Termination Handler.
 home: https://github.com/aws/aws-node-termination-handler/
 icon: https://raw.githubusercontent.com/aws/eks-charts/master/docs/logo/aws.png
@@ -21,4 +21,4 @@ name: aws-node-termination-handler
 sources:
 - https://github.com/aws/aws-node-termination-handler/
 type: application
-version: 0.24.1
+version: 0.26.0

charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.linux.yaml

@@ -143,6 +143,8 @@ spec:
             {{- end }}
             - name: ENABLE_SPOT_INTERRUPTION_DRAINING
               value: {{ .Values.enableSpotInterruptionDraining | quote }}
+            - name: ENABLE_ASG_LIFECYCLE_DRAINING
+              value: {{ .Values.enableASGLifecycleDraining | quote }}
             - name: ENABLE_SCHEDULED_EVENT_DRAINING
               value: {{ .Values.enableScheduledEventDraining | quote }}
             - name: ENABLE_REBALANCE_MONITORING

charts/kubezero-addons/charts/aws-node-termination-handler/templates/daemonset.windows.yaml

@@ -143,6 +143,8 @@ spec:
             {{- end }}
             - name: ENABLE_SPOT_INTERRUPTION_DRAINING
               value: {{ .Values.enableSpotInterruptionDraining | quote }}
+            - name: ENABLE_ASG_LIFECYCLE_DRAINING
+              value: {{ .Values.enableASGLifecycleDraining | quote }}
             - name: ENABLE_SCHEDULED_EVENT_DRAINING
               value: {{ .Values.enableScheduledEventDraining | quote }}
             - name: ENABLE_REBALANCE_MONITORING

charts/kubezero-addons/charts/aws-node-termination-handler/templates/deployment.yaml

@@ -168,6 +168,10 @@ spec:
               value: {{ .Values.deleteSqsMsgIfNodeNotFound | quote }}
             - name: WORKERS
               value: {{ .Values.workers | quote }}
+            - name: HEARTBEAT_INTERVAL
+              value: {{ .Values.heartbeatInterval | quote }}
+            - name: HEARTBEAT_UNTIL
+              value: {{ .Values.heartbeatUntil | quote }}
             {{- with .Values.extraEnv }}
               {{- toYaml . | nindent 12 }}
             {{- end }}

charts/kubezero-addons/charts/aws-node-termination-handler/values.yaml

@@ -270,6 +270,9 @@ metadataTries: 3
 # enableSpotInterruptionDraining If false, do not drain nodes when the spot interruption termination notice is received. Only used in IMDS mode.
 enableSpotInterruptionDraining: true
 
+# enableASGLifecycleDraining If false, do not drain nodes when ASG target lifecycle state Terminated is received. Only used in IMDS mode.
+enableASGLifecycleDraining: true
+
 # enableScheduledEventDraining If false, do not drain nodes before the maintenance window starts for an EC2 instance scheduled event. Only used in IMDS mode.
 enableScheduledEventDraining: true
 

charts/kubezero-addons/values.yaml

@@ -134,7 +134,7 @@ aws-node-termination-handler:
     value: "regional"
 
   enablePrometheusServer: false
-  podMonitor:
+  serviceMonitor:
     create: false
 
   jsonLogging: true
@@ -146,9 +146,6 @@ aws-node-termination-handler:
   nodeSelector:
     node-role.kubernetes.io/control-plane: ""
 
-  rbac:
-    pspEnabled: false
-
 fuseDevicePlugin:
   enabled: false
   image:
@@ -206,7 +203,7 @@ cluster-autoscaler:
 
   image:
     repository: registry.k8s.io/autoscaling/cluster-autoscaler
-    tag: v1.30.2
+    tag: v1.31.1
 
   autoDiscovery:
     clusterName: ""

charts/kubezero-argo/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 description: KubeZero Argo - Events, Workflow, CD
 name: kubezero-argo
-version: 0.2.7
+version: 0.2.8
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -18,11 +18,11 @@ dependencies:
     version: ">= 0.1.6"
     repository: https://cdn.zero-downtime.net/charts/
   - name: argo-events
-    version: 2.4.9
+    version: 2.4.13
     repository: https://argoproj.github.io/argo-helm
     condition: argo-events.enabled
   - name: argo-cd
-    version: 7.7.7
+    version: 7.8.2
     repository: https://argoproj.github.io/argo-helm
     condition: argo-cd.enabled
   - name: argocd-apps
@@ -30,7 +30,7 @@ dependencies:
     repository: https://argoproj.github.io/argo-helm
     condition: argo-cd.enabled
   - name: argocd-image-updater
-    version: 0.11.2
+    version: 0.12.0
     repository: https://argoproj.github.io/argo-helm
     condition: argocd-image-updater.enabled
 kubeVersion: ">= 1.26.0-0"

charts/kubezero-argo/README.md

@@ -1,6 +1,6 @@
 # kubezero-argo
 
-![Version: 0.2.7](https://img.shields.io/badge/Version-0.2.7-informational?style=flat-square)
+![Version: 0.2.8](https://img.shields.io/badge/Version-0.2.8-informational?style=flat-square)
 
 KubeZero Argo - Events, Workflow, CD
 
@@ -18,16 +18,17 @@ Kubernetes: `>= 1.26.0-0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://argoproj.github.io/argo-helm | argo-cd | 7.7.7 |
+| https://argoproj.github.io/argo-helm | argo-cd | 7.8.2 |
-| https://argoproj.github.io/argo-helm | argo-events | 2.4.9 |
+| https://argoproj.github.io/argo-helm | argo-events | 2.4.13 |
 | https://argoproj.github.io/argo-helm | argocd-apps | 2.0.2 |
-| https://argoproj.github.io/argo-helm | argocd-image-updater | 0.11.2 |
+| https://argoproj.github.io/argo-helm | argocd-image-updater | 0.12.0 |
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
 
 ## Values
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
+| argo-cd.configs.cm."application.instanceLabelKey" | string | `nil` |  |
 | argo-cd.configs.cm."application.resourceTrackingMethod" | string | `"annotation"` |  |
 | argo-cd.configs.cm."resource.customizations" | string | `"cert-manager.io/Certificate:\n  # Lua script for customizing the health status assessment\n  health.lua: |\n    hs = {}\n    if obj.status ~= nil then\n      if obj.status.conditions ~= nil then\n        for i, condition in ipairs(obj.status.conditions) do\n          if condition.type == \"Ready\" and condition.status == \"False\" then\n            hs.status = \"Degraded\"\n            hs.message = condition.message\n            return hs\n          end\n          if condition.type == \"Ready\" and condition.status == \"True\" then\n            hs.status = \"Healthy\"\n            hs.message = condition.message\n            return hs\n          end\n        end\n      end\n    end\n    hs.status = \"Progressing\"\n    hs.message = \"Waiting for certificate\"\n    return hs\n"` |  |
 | argo-cd.configs.cm."timeout.reconciliation" | string | `"300s"` |  |
@@ -35,6 +36,7 @@ Kubernetes: `>= 1.26.0-0`
 | argo-cd.configs.cm."ui.bannerpermanent" | string | `"true"` |  |
 | argo-cd.configs.cm."ui.bannerposition" | string | `"bottom"` |  |
 | argo-cd.configs.cm."ui.bannerurl" | string | `"https://kubezero.com/releases/v1.31"` |  |
+| argo-cd.configs.cm.installationID | string | `"KubeZero-ArgoCD"` |  |
 | argo-cd.configs.cm.url | string | `"https://argocd.example.com"` |  |
 | argo-cd.configs.params."controller.diff.server.side" | string | `"true"` |  |
 | argo-cd.configs.params."controller.operation.processors" | string | `"5"` |  |
@@ -94,7 +96,7 @@ Kubernetes: `>= 1.26.0-0`
 | argo-events.configs.jetstream.streamConfig.maxMsgs | int | `1000000` | Maximum number of messages before expiring oldest message |
 | argo-events.configs.jetstream.streamConfig.replicas | int | `1` | Number of replicas, defaults to 3 and requires minimal 3 |
 | argo-events.configs.jetstream.versions[0].configReloaderImage | string | `"natsio/nats-server-config-reloader:0.14.1"` |  |
-| argo-events.configs.jetstream.versions[0].metricsExporterImage | string | `"natsio/prometheus-nats-exporter:0.14.0"` |  |
+| argo-events.configs.jetstream.versions[0].metricsExporterImage | string | `"natsio/prometheus-nats-exporter:0.16.0"` |  |
 | argo-events.configs.jetstream.versions[0].natsImage | string | `"nats:2.10.11-scratch"` |  |
 | argo-events.configs.jetstream.versions[0].startCommand | string | `"/nats-server"` |  |
 | argo-events.configs.jetstream.versions[0].version | string | `"2.10.11"` |  |

charts/kubezero-argo/values.yaml

@@ -45,7 +45,7 @@ argo-cd:
       format: json
     image:
       repository: public.ecr.aws/zero-downtime/zdt-argocd
-      tag: v2.13.1
+      tag: v2.14.2
     networkPolicy:
       create: true
 

charts/kubezero-cert-manager/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-cert-manager
 description: KubeZero Umbrella Chart for cert-manager
 type: application
-version: 0.9.10
+version: 0.9.12
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -16,6 +16,6 @@ dependencies:
     version: ">= 0.1.6"
     repository: https://cdn.zero-downtime.net/charts/
   - name: cert-manager
-    version: v1.16.1
+    version: v1.17.1
     repository: https://charts.jetstack.io
-kubeVersion: ">= 1.26.0-0"
+kubeVersion: ">= 1.30.0-0"

charts/kubezero-cert-manager/README.md

@@ -1,6 +1,6 @@
 # kubezero-cert-manager
 
-![Version: 0.9.9](https://img.shields.io/badge/Version-0.9.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.9.12](https://img.shields.io/badge/Version-0.9.12-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero Umbrella Chart for cert-manager
 
@@ -14,12 +14,12 @@ KubeZero Umbrella Chart for cert-manager
 
 ## Requirements
 
-Kubernetes: `>= 1.26.0-0`
+Kubernetes: `>= 1.30.0-0`
 
 | Repository | Name | Version |
 |------------|------|---------|
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
-| https://charts.jetstack.io | cert-manager | v1.15.2 |
+| https://charts.jetstack.io | cert-manager | v1.17.1 |
 
 ## AWS - OIDC IAM roles
 

charts/kubezero-cert-manager/jsonnetfile.lock.json

@@ -18,8 +18,8 @@
           "subdir": "contrib/mixin"
         }
       },
-      "version": "df4e472a2d09813560ba44b21a29c0453dbec18c",
+      "version": "8c52b414f324d6369b77096af98d8f0416fe20cb",
-      "sum": "IXI3LQIT9NmTPJAk8WLUJd5+qZfcGpeNCyWIK7oEpws="
+      "sum": "XmXkOCriQIZmXwlIIFhqlJMa0e6qGWdxZD+ZDYaN0Po="
     },
     {
       "source": {
@@ -41,16 +41,6 @@
       "version": "a1d61cce1da59c71409b99b5c7568511fec661ea",
       "sum": "342u++/7rViR/zj2jeJOjshzglkZ1SY+hFNuyCBFMdc="
     },
-    {
-      "source": {
-        "git": {
-          "remote": "https://github.com/grafana/grafonnet-lib.git",
-          "subdir": "grafonnet-7.0"
-        }
-      },
-      "version": "a1d61cce1da59c71409b99b5c7568511fec661ea",
-      "sum": "gCtR9s/4D5fxU9aKXg0Bru+/njZhA0YjLjPiASc61FM="
-    },
     {
       "source": {
         "git": {
@@ -58,8 +48,8 @@
           "subdir": "gen/grafonnet-latest"
         }
       },
-      "version": "733beadbc8dab55c5fe1bcdcf0d8a2d215759a55",
+      "version": "d20e609202733790caf5b554c9945d049f243ae3",
-      "sum": "eyuJ0jOXeA4MrobbNgU4/v5a7ASDHslHZ0eS6hDdWoI="
+      "sum": "V9vAj21qJOc2DlMPDgB1eEjSQU4A+sAA4AXuJ6bd4xc="
     },
     {
       "source": {
@@ -68,18 +58,18 @@
           "subdir": "gen/grafonnet-v10.0.0"
         }
       },
-      "version": "733beadbc8dab55c5fe1bcdcf0d8a2d215759a55",
+      "version": "d20e609202733790caf5b554c9945d049f243ae3",
       "sum": "xdcrJPJlpkq4+5LpGwN4tPAuheNNLXZjE6tDcyvFjr0="
     },
     {
       "source": {
         "git": {
           "remote": "https://github.com/grafana/grafonnet.git",
-          "subdir": "gen/grafonnet-v11.0.0"
+          "subdir": "gen/grafonnet-v11.4.0"
         }
       },
-      "version": "733beadbc8dab55c5fe1bcdcf0d8a2d215759a55",
+      "version": "d20e609202733790caf5b554c9945d049f243ae3",
-      "sum": "0BvzR0i4bS4hc2O3xDv6i9m52z7mPrjvqxtcPrGhynA="
+      "sum": "aVAX09paQYNOoCSKVpuk1exVIyBoMt/C50QJI+Q/3nA="
     },
     {
       "source": {
@@ -88,7 +78,7 @@
           "subdir": "grafana-builder"
         }
       },
-      "version": "d9ba581fb27aa6689e911f288d4df06948eb8aad",
+      "version": "393630ca7ba9b25258c098f1fd4c81962e3ca046",
       "sum": "yxqWcq/N3E/a/XreeU6EuE6X7kYPnG0AspAQFKOjASo="
     },
     {
@@ -98,8 +88,8 @@
           "subdir": ""
         }
       },
-      "version": "b6a425db5c72b0159e3b8666b373e21bf487df9a",
+      "version": "de46a6811837f9750ef9c6df29dcae314f22da81",
-      "sum": "6KgWSzBFX09sycxfM3mYsY9Ua16ydCVpyFGYS0ZkpIo="
+      "sum": "TMt8tZMdzt2JL7Wt6cvxxdwEY9FbQ3OOKxAzLsTu5/w="
     },
     {
       "source": {
@@ -118,8 +108,8 @@
           "subdir": ""
         }
       },
-      "version": "63d430b69a95741061c2f7fc9d84b1a778511d9c",
+      "version": "1199b50e9d2ff53d4bb5fb2304ad1fb69d38e609",
-      "sum": "qiZi3axUSXCVzKUF83zSAxklwrnitMmrDK4XAfjPMdE="
+      "sum": "LfbgcJbilu4uBdKYZSvmkoOTPwEAzg10L3/VqKAIWtA="
     },
     {
       "source": {
@@ -128,8 +118,8 @@
           "subdir": ""
         }
       },
-      "version": "1b71e399caee334af8ba2d15d0dd615043a652d0",
+      "version": "4ff562d5e8145940cf355f62cf2308895c4dca81",
-      "sum": "qcRxavmCpuWQuwCMqYaOZ+soA8jxwWLrK7LYqohN5NA="
+      "sum": "kiL19fTbXOtNglsmT62kOzIf/Xpu+YwoiMPAApDXhkE="
     },
     {
       "source": {
@@ -138,8 +128,8 @@
           "subdir": "jsonnet/kube-state-metrics"
         }
       },
-      "version": "f8aa7d9bb9d8e29876e19f4859391a54a7e61d63",
+      "version": "2a95d4649b2fea55799032fb9c0b571c4ba7f776",
-      "sum": "lO7jUSzAIy8Yk9pOWJIWgPRhubkWzVh56W6wtYfbVH4="
+      "sum": "3bioG7CfTfY9zeu5xU4yon6Zt3kYvNkyl492nOhQxnM="
     },
     {
       "source": {
@@ -148,7 +138,7 @@
           "subdir": "jsonnet/kube-state-metrics-mixin"
         }
       },
-      "version": "f8aa7d9bb9d8e29876e19f4859391a54a7e61d63",
+      "version": "2a95d4649b2fea55799032fb9c0b571c4ba7f776",
       "sum": "qclI7LwucTjBef3PkGBkKxF0mfZPbHnn4rlNWKGtR4c="
     },
     {
@@ -158,8 +148,8 @@
           "subdir": "jsonnet/kube-prometheus"
         }
       },
-      "version": "33c43a4067a174a99529e41d537eef290a7028ea",
+      "version": "1eea946a1532f1e8cccfceea98d907bf3a10b1d9",
-      "sum": "/jU8uXWR202aR7K/3zOefhc4JBUAUkTdHvE9rhfzI/g="
+      "sum": "17LhiwefVfoNDsF3DcFZw/UL4PMU7YpNNUaOdaYd1gE="
     },
     {
       "source": {
@@ -168,7 +158,7 @@
           "subdir": "jsonnet/mixin"
         }
       },
-      "version": "aa74b0d377d32648ca50f2531fe2253895629d9f",
+      "version": "7deab71d6d5921eeaf8c79e3ae8e31efe63783a9",
       "sum": "gi+knjdxs2T715iIQIntrimbHRgHnpM8IFBJDD1gYfs=",
       "name": "prometheus-operator-mixin"
     },
@@ -179,8 +169,8 @@
           "subdir": "jsonnet/prometheus-operator"
         }
       },
-      "version": "aa74b0d377d32648ca50f2531fe2253895629d9f",
+      "version": "7deab71d6d5921eeaf8c79e3ae8e31efe63783a9",
-      "sum": "EZR4sBAtmFRsUR7U4SybuBUhK9ncMCvEu9xHtu8B9KA="
+      "sum": "LctDdofQostvviE5y8vpRKWGGO1ZKO3dgJe7P9xifW0="
     },
     {
       "source": {
@@ -189,8 +179,8 @@
           "subdir": "doc/alertmanager-mixin"
         }
       },
-      "version": "27b6eb7ce02680c84b9a06503edbddc9213f586d",
+      "version": "b5d1a64ad5bb0ff879705714d1e40cea82efbd5c",
-      "sum": "IpF46ZXsm+0wJJAPtAre8+yxTNZA57mBqGpBP/r7/kw=",
+      "sum": "Mf4h1BYLle2nrgjf/HXrBbl0Zk8N+xaoEM017o0BC+k=",
       "name": "alertmanager"
     },
     {
@@ -200,8 +190,8 @@
           "subdir": "docs/node-mixin"
         }
       },
-      "version": "b9d0932179a0c5b3a8863f3d6cdafe8584cedc8e",
+      "version": "11365f97bef6cb0e6259d536a7e21c49e3f5c065",
-      "sum": "rhUvbqviGjQ2mwsRhHKMN0TiS3YvnYpUXHew3XlQ+Wg="
+      "sum": "xYj6VYFT/eafsbleNlC+Z2VfLy1CndyYrJs9BcTmnX8="
     },
     {
       "source": {
@@ -210,19 +200,19 @@
           "subdir": "documentation/prometheus-mixin"
         }
       },
-      "version": "616038f2b64656b2c9c6053f02aee544c5b8bb17",
+      "version": "a5ffa83be83be22e2ec9fd1d4765299d8d16119e",
-      "sum": "dYLcLzGH4yF3qB7OGC/7z4nqeTNjv42L7Q3BENU8XJI=",
+      "sum": "2c+wttfee9TwuQJZIkNV7Tekem74Qgc7iZ842P28rNw=",
       "name": "prometheus"
     },
     {
       "source": {
         "git": {
           "remote": "https://github.com/pyrra-dev/pyrra.git",
-          "subdir": "config/crd/bases"
+          "subdir": "jsonnet/controller-gen"
         }
       },
-      "version": "551856d42dff02ec38c5b0ea6a2d99c4cb127e82",
+      "version": "d723f4d1a066dd657e9d09c46a158519dda0faa8",
-      "sum": "bY/Pcrrbynguq8/HaI88cQ3B2hLv/xc+76QILY7IL+g=",
+      "sum": "cxAPQovFkM16zNB5/94O+sk/n3SETk6ao6Oas2Sa6RE=",
       "name": "pyrra"
     },
     {
@@ -232,8 +222,8 @@
           "subdir": "mixin"
         }
       },
-      "version": "dcadaae80fcce1fb05452b37ca8d3b2809d7cef9",
+      "version": "346d18bb0f8011c63d7106de494cf3b9253161a1",
-      "sum": "HhSSbGGCNHCMy1ee5jElYDm0yS9Vesa7QB2/SHKdjsY=",
+      "sum": "ieCD4eMgGbOlrI8GmckGPHBGQDcLasE1rULYq56W/bs=",
       "name": "thanos-mixin"
     }
   ],

charts/kubezero-ci/values.yaml

@@ -192,6 +192,8 @@ jenkins:
     annotations:
       container.apparmor.security.beta.kubernetes.io/jnlp: "unconfined"
       cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+    garbageCollection:
+      enabled: true
     customJenkinsLabels:
     - podman-aws-trivy
     idleMinutes: 30

charts/kubezero-graph/templates/neo4j/secrets.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: neo4j-admin
+  labels:
+    {{- include "kubezero-lib.labels" . | nindent 4 }}
+type: Opaque
+data:
+  NEO4J_AUTH: {{.Values.neo4j.neo4j.password | printf "neo4j/%v" | b64enc -}}

charts/kubezero-graph/values.yaml

@@ -6,6 +6,9 @@ neo4j:
   neo4j:
     name: test-db
 
+    password: secret
+    passwordFromSecret: "neo4j-admin"
+
   volumes:
     data:
       mode: defaultStorageClass

charts/kubezero-istio-gateway/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-istio-gateway
 description: KubeZero Umbrella Chart for Istio gateways
 type: application
-version: 0.23.2
+version: 0.24.2
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -17,6 +17,6 @@ dependencies:
     version: ">= 0.1.6"
     repository: https://cdn.zero-downtime.net/charts/
   - name: gateway
-    version: 1.23.2
+    version: 1.24.2
     repository: https://istio-release.storage.googleapis.com/charts
-kubeVersion: ">= 1.26.0-0"
+kubeVersion: ">= 1.30.0-0"

charts/kubezero-istio-gateway/README.md

@@ -1,6 +1,6 @@
 # kubezero-istio-gateway
 
-![Version: 0.23.2](https://img.shields.io/badge/Version-0.23.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.24.2](https://img.shields.io/badge/Version-0.24.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero Umbrella Chart for Istio gateways
 
@@ -16,12 +16,12 @@ Installs Istio Ingress Gateways, requires kubezero-istio to be installed !
 
 ## Requirements
 
-Kubernetes: `>= 1.26.0-0`
+Kubernetes: `>= 1.30.0-0`
 
 | Repository | Name | Version |
 |------------|------|---------|
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
-| https://istio-release.storage.googleapis.com/charts | gateway | 1.23.2 |
+| https://istio-release.storage.googleapis.com/charts | gateway | 1.24.2 |
 
 ## Values
 
@@ -32,6 +32,7 @@ Kubernetes: `>= 1.26.0-0`
 | gateway.autoscaling.maxReplicas | int | `4` |  |
 | gateway.autoscaling.minReplicas | int | `1` |  |
 | gateway.autoscaling.targetCPUUtilizationPercentage | int | `80` |  |
+| gateway.minReadySeconds | int | `120` |  |
 | gateway.podAnnotations."proxy.istio.io/config" | string | `"{ \"terminationDrainDuration\": \"20s\" }"` |  |
 | gateway.replicaCount | int | `1` |  |
 | gateway.resources.limits.memory | string | `"512Mi"` |  |
@@ -47,5 +48,6 @@ Kubernetes: `>= 1.26.0-0`
 
 ## Resources
 
+### Envoy Listener Filter - TCP KeepAlive
 - https://github.com/cilium/cilium/blob/main/operator/pkg/model/translation/envoy_listener.go#L134
 

charts/kubezero-istio-gateway/charts/gateway/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 1.23.2
+appVersion: 1.24.2
 description: Helm chart for deploying Istio gateways
 icon: https://istio.io/latest/favicons/android-192x192.png
 keywords:
@@ -9,4 +9,4 @@ name: gateway
 sources:
 - https://github.com/istio/istio
 type: application
-version: 1.23.2
+version: 1.24.2

charts/kubezero-istio-gateway/charts/gateway/files/profile-ambient.yaml

@@ -15,6 +15,3 @@ pilot:
 cni:
   ambient:
     enabled: true
-
-# Ztunnel doesn't use a namespace, so everything here is mostly for ztunnel
-variant: distroless

charts/kubezero-istio-gateway/charts/gateway/files/profile-compatibility-version-1.20.yaml

@@ -1,26 +0,0 @@
-# WARNING: DO NOT EDIT, THIS FILE IS A COPY.
-# The original version of this file is located at /manifests/helm-profiles directory.
-# If you want to make a change in this file, edit the original one and run "make gen".
-
-pilot:
-  env:
-    # 1.21 behavioral changes
-    ENABLE_EXTERNAL_NAME_ALIAS: "false"
-    PERSIST_OLDEST_FIRST_HEURISTIC_FOR_VIRTUAL_SERVICE_HOST_MATCHING: "true"
-    VERIFY_CERTIFICATE_AT_CLIENT: "false"
-    ENABLE_AUTO_SNI: "false"
-
-    # 1.22 behavioral changes
-    ENABLE_ENHANCED_RESOURCE_SCOPING: "false"
-    ENABLE_RESOLUTION_NONE_TARGET_PORT: "false"
-
-meshConfig:
-  defaultConfig:
-    proxyMetadata:
-      # 1.22 behavioral changes
-      ISTIO_DELTA_XDS: "false"
-      # 1.23 behavioral changes
-      ENABLE_DELIMITED_STATS_TAG_REGEX: "false"
-    tracing:
-      zipkin:
-        address: zipkin.istio-system:9411

charts/kubezero-istio-gateway/charts/gateway/files/profile-compatibility-version-1.21.yaml

@@ -11,6 +11,13 @@ pilot:
     # 1.23 behavioral changes
     ENABLE_DELIMITED_STATS_TAG_REGEX: "false"
 
+    # 1.24 behavioral changes
+    ENABLE_INBOUND_RETRY_POLICY: "false"
+    EXCLUDE_UNSAFE_503_FROM_DEFAULT_RETRY: "false"
+    PREFER_DESTINATIONRULE_TLS_FOR_EXTERNAL_SERVICES: "false"
+    ENABLE_ENHANCED_DESTINATIONRULE_MERGE: "false"
+    PILOT_UNIFIED_SIDECAR_SCOPE: "false"
+
 meshConfig:
   # 1.22 behavioral changes
   defaultConfig:
@@ -18,6 +25,9 @@ meshConfig:
       ISTIO_DELTA_XDS: "false"
       # 1.23 behavioral changes
       ENABLE_DELIMITED_STATS_TAG_REGEX: "false"
+      # 1.24 behaviour changes
+      ENABLE_DEFERRED_STATS_CREATION: "false"
+      BYPASS_OVERLOAD_MANAGER_FOR_STATIC_LISTENERS: "false"
     tracing:
       zipkin:
         address: zipkin.istio-system:9411

charts/kubezero-istio-gateway/charts/gateway/files/profile-compatibility-version-1.22.yaml

@@ -6,7 +6,14 @@ pilot:
   env:
     # 1.23 behavioral changes
     ENABLE_DELIMITED_STATS_TAG_REGEX: "false"
-  
+    
+    # 1.24 behavioral changes
+    ENABLE_INBOUND_RETRY_POLICY: "false"
+    EXCLUDE_UNSAFE_503_FROM_DEFAULT_RETRY: "false"
+    PREFER_DESTINATIONRULE_TLS_FOR_EXTERNAL_SERVICES: "false"
+    ENABLE_ENHANCED_DESTINATIONRULE_MERGE: "false"
+    PILOT_UNIFIED_SIDECAR_SCOPE: "false"
+
 meshConfig:
   defaultConfig:
     proxyMetadata:
@@ -14,3 +21,6 @@ meshConfig:
       ENABLE_DEFERRED_CLUSTER_CREATION: "false"
       # 1.23 behavioral changes
       ENABLE_DELIMITED_STATS_TAG_REGEX: "false"
+      # 1.24 behaviour changes
+      ENABLE_DEFERRED_STATS_CREATION: "false"
+      BYPASS_OVERLOAD_MANAGER_FOR_STATIC_LISTENERS: "false"

charts/kubezero-istio-gateway/charts/gateway/files/profile-compatibility-version-1.23.yaml

@@ -0,0 +1,19 @@
+# WARNING: DO NOT EDIT, THIS FILE IS A COPY.
+# The original version of this file is located at /manifests/helm-profiles directory.
+# If you want to make a change in this file, edit the original one and run "make gen".
+
+pilot:
+  env:
+    # 1.24 behavioral changes
+    ENABLE_INBOUND_RETRY_POLICY: "false"
+    EXCLUDE_UNSAFE_503_FROM_DEFAULT_RETRY: "false"
+    PREFER_DESTINATIONRULE_TLS_FOR_EXTERNAL_SERVICES: "false"
+    ENABLE_ENHANCED_DESTINATIONRULE_MERGE: "false"
+    PILOT_UNIFIED_SIDECAR_SCOPE: "false"
+
+meshConfig:
+  defaultConfig:
+    proxyMetadata:
+      # 1.24 behaviour changes
+      ENABLE_DEFERRED_STATS_CREATION: "false"
+      BYPASS_OVERLOAD_MANAGER_FOR_STATIC_LISTENERS: "false"

charts/kubezero-istio-gateway/charts/gateway/files/profile-demo.yaml

@@ -22,12 +22,29 @@ meshConfig:
         port: 4317
         service: opentelemetry-collector.observability.svc.cluster.local
 
+cni:
+  resources:
+    requests:
+      cpu: 10m
+      memory: 40Mi
+
+ztunnel:
+  resources:
+    requests:
+      cpu: 10m
+      memory: 40Mi
+
 global:
   proxy:
     resources:
       requests:
         cpu: 10m
         memory: 40Mi
+  waypoint:
+    resources:
+      requests:
+        cpu: 10m
+        memory: 40Mi
 
 pilot:
   autoscaleEnabled: false

charts/kubezero-istio-gateway/charts/gateway/files/profile-openshift-ambient.yaml

@@ -1,33 +0,0 @@
-# WARNING: DO NOT EDIT, THIS FILE IS A COPY.
-# The original version of this file is located at /manifests/helm-profiles directory.
-# If you want to make a change in this file, edit the original one and run "make gen".
-
-meshConfig:
-  defaultConfig:
-    proxyMetadata:
-      ISTIO_META_ENABLE_HBONE: "true"
-global:
-  platform: openshift
-cni:
-  ambient:
-    enabled: true
-  cniBinDir: /var/lib/cni/bin
-  cniConfDir: /etc/cni/multus/net.d
-  chained: false
-  cniConfFileName: "istio-cni.conf"
-  logLevel: info
-  provider: "multus"
-pilot:
-  cni:
-    enabled: true
-    provider: "multus"
-  variant: distroless
-  env:
-    PILOT_ENABLE_AMBIENT: "true"
-    # Allow sidecars/ingress to send/receive HBONE. This is required for interop.
-    PILOT_ENABLE_SENDING_HBONE: "true"
-    PILOT_ENABLE_SIDECAR_LISTENING_HBONE: "true"
-platform: openshift
-variant: distroless
-seLinuxOptions:
-  type: spc_t

charts/kubezero-istio-gateway/charts/gateway/files/profile-platform-k3d.yaml

@@ -0,0 +1,7 @@
+# WARNING: DO NOT EDIT, THIS FILE IS A COPY.
+# The original version of this file is located at /manifests/helm-profiles directory.
+# If you want to make a change in this file, edit the original one and run "make gen".
+
+cni:
+  cniConfDir: /var/lib/rancher/k3s/agent/etc/cni/net.d
+  cniBinDir: /bin

charts/kubezero-istio-gateway/charts/gateway/files/profile-platform-k3s.yaml

@@ -0,0 +1,7 @@
+# WARNING: DO NOT EDIT, THIS FILE IS A COPY.
+# The original version of this file is located at /manifests/helm-profiles directory.
+# If you want to make a change in this file, edit the original one and run "make gen".
+
+cni:
+  cniConfDir: /var/lib/rancher/k3s/agent/etc/cni/net.d
+  cniBinDir: /var/lib/rancher/k3s/data/current/bin/

charts/kubezero-istio-gateway/charts/gateway/files/profile-platform-microk8s.yaml

@@ -0,0 +1,7 @@
+# WARNING: DO NOT EDIT, THIS FILE IS A COPY.
+# The original version of this file is located at /manifests/helm-profiles directory.
+# If you want to make a change in this file, edit the original one and run "make gen".
+
+cni:
+  cniConfDir: /var/snap/microk8s/current/args/cni-network
+  cniBinDir: /var/snap/microk8s/current/opt/cni/bin

charts/kubezero-istio-gateway/charts/gateway/files/profile-platform-minikube.yaml

@@ -0,0 +1,6 @@
+# WARNING: DO NOT EDIT, THIS FILE IS A COPY.
+# The original version of this file is located at /manifests/helm-profiles directory.
+# If you want to make a change in this file, edit the original one and run "make gen".
+
+cni:
+  cniNetnsDir: /var/run/docker/netns

charts/kubezero-istio-gateway/charts/gateway/files/profile-platform-openshift.yaml

@@ -3,18 +3,17 @@
 # If you want to make a change in this file, edit the original one and run "make gen".
 
 # The OpenShift profile provides a basic set of settings to run Istio on OpenShift
-# CNI must be installed.
 cni:
   cniBinDir: /var/lib/cni/bin
   cniConfDir: /etc/cni/multus/net.d
   chained: false
   cniConfFileName: "istio-cni.conf"
-  logLevel: info
   provider: "multus"
-global:
-  platform: openshift
 pilot:
   cni:
     enabled: true
     provider: "multus"
-platform: openshift
+seLinuxOptions:
+  type: spc_t
+# Openshift requires privileged pods to run in kube-system
+trustedZtunnelNamespace: "kube-system"

charts/kubezero-istio-gateway/charts/gateway/files/profile-remote.yaml

@@ -0,0 +1,13 @@
+# WARNING: DO NOT EDIT, THIS FILE IS A COPY.
+# The original version of this file is located at /manifests/helm-profiles directory.
+# If you want to make a change in this file, edit the original one and run "make gen".
+
+# The remote profile enables installing istio with a remote control plane. The `base` and `istio-discovery` charts must be deployed with this profile.
+istiodRemote:
+  enabled: true
+configMap: false
+telemetry:
+  enabled: false
+global:
+  # TODO BML maybe a different profile for a configcluster/revisit this
+  omitSidecarInjectorConfigMap: true

charts/kubezero-istio-gateway/charts/gateway/templates/_helpers.tpl

@@ -6,19 +6,8 @@
 {{- end -}}
 {{- end }}
 
-{{/*
-Create chart name and version as used by the helm.sh/chart label.
-*/}}
-{{- define "gateway.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
 {{- define "gateway.labels" -}}
-helm.sh/chart: {{ include "gateway.chart" . }}
 {{ include "gateway.selectorLabels" . }}
-app.kubernetes.io/name: {{ include "gateway.name" . }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- range $key, $val := .Values.labels }}
 {{- if and (ne $key "app") (ne $key "istio") }}
 {{ $key | quote }}: {{ $val | quote }}

charts/kubezero-istio-gateway/charts/gateway/templates/deployment.yaml

@@ -4,6 +4,8 @@ metadata:
   name: {{ include "gateway.name" . }}
   namespace: {{ .Release.Namespace }}
   labels:
+    app.kubernetes.io/name: {{ include "gateway.name" . }}
+    {{- include "istio.labels" . | nindent 4}}
     {{- include "gateway.labels" . | nindent 4}}
   annotations:
     {{- .Values.annotations | toYaml | nindent 4 }}
@@ -13,9 +15,13 @@ spec:
   replicas: {{ . }}
   {{- end }}
   {{- end }}
-  # Give the LB 120s to detect and take into service
+  {{- with .Values.strategy }}
-  # should only be 40s by we are on AWS so ...
+  strategy:
-  minReadySeconds: 120
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.minReadySeconds }}
+  minReadySeconds: {{ . }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "gateway.selectorLabels" . | nindent 6 }}
@@ -29,7 +35,7 @@ spec:
         {{- include "gateway.sidecarInjectionLabels" . | nindent 8 }}
         {{- include "gateway.selectorLabels" . | nindent 8 }}
         app.kubernetes.io/name: {{ include "gateway.name" . }}
-        app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+        {{- include "istio.labels" .  | nindent 8}}
         {{- range $key, $val := .Values.labels }}
         {{- if and (ne $key "app") (ne $key "istio") }}
         {{ $key | quote }}: {{ $val | quote }}
@@ -94,7 +100,11 @@ spec:
             {{- toYaml .Values.resources | nindent 12 }}
           {{- with .Values.volumeMounts }}
           volumeMounts:
-            {{ toYaml . | nindent 12 }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.readinessProbe }}
+          readinessProbe:
+            {{- toYaml . | nindent 12 }}
           {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:

charts/kubezero-istio-gateway/charts/gateway/templates/hpa.yaml

@@ -5,6 +5,8 @@ metadata:
   name: {{ include "gateway.name" . }}
   namespace: {{ .Release.Namespace }}
   labels:
+    app.kubernetes.io/name: {{ include "gateway.name" . }}
+    {{- include "istio.labels" . | nindent 4}}
     {{- include "gateway.labels" . | nindent 4 }}
   annotations:
     {{- .Values.annotations | toYaml | nindent 4 }}

charts/kubezero-istio-gateway/charts/gateway/templates/poddisruptionbudget.yaml

@@ -5,6 +5,8 @@ metadata:
   name: {{ include "gateway.name" . }}
   namespace: {{ .Release.Namespace }}
   labels:
+    app.kubernetes.io/name: {{ include "gateway.name" . }}
+    {{- include "istio.labels" . | nindent 4}}
     {{- include "gateway.labels" . | nindent 4}}
 spec:
   selector:

charts/kubezero-istio-gateway/charts/gateway/templates/role.yaml

@@ -6,6 +6,8 @@ metadata:
   name: {{ include "gateway.serviceAccountName" . }}
   namespace: {{ .Release.Namespace }}
   labels:
+    app.kubernetes.io/name: {{ include "gateway.name" . }}
+    {{- include "istio.labels" . | nindent 4}}
     {{- include "gateway.labels" . | nindent 4}}
   annotations:
     {{- .Values.annotations | toYaml | nindent 4 }}
@@ -20,6 +22,8 @@ metadata:
   name: {{ include "gateway.serviceAccountName" . }}
   namespace: {{ .Release.Namespace }}
   labels:
+    app.kubernetes.io/name: {{ include "gateway.name" . }}
+    {{- include "istio.labels" . | nindent 4}}
     {{- include "gateway.labels" . | nindent 4}}
   annotations:
     {{- .Values.annotations | toYaml | nindent 4 }}

charts/kubezero-istio-gateway/charts/gateway/templates/service.yaml

@@ -5,6 +5,8 @@ metadata:
   name: {{ include "gateway.name" . }}
   namespace: {{ .Release.Namespace }}
   labels:
+    app.kubernetes.io/name: {{ include "gateway.name" . }}
+    {{- include "istio.labels" . | nindent 4}}
     {{- include "gateway.labels" . | nindent 4 }}
     {{- with .Values.networkGateway }}
     topology.istio.io/network: "{{.}}"

charts/kubezero-istio-gateway/charts/gateway/templates/serviceaccount.yaml

@@ -5,6 +5,8 @@ metadata:
   name: {{ include "gateway.serviceAccountName" . }}
   namespace: {{ .Release.Namespace }}
   labels:
+    app.kubernetes.io/name: {{ include "gateway.name" . }}
+    {{- include "istio.labels" . | nindent 4}}
     {{- include "gateway.labels" . | nindent 4 }}
   {{- with .Values.serviceAccount.annotations }}
   annotations:

charts/kubezero-istio-gateway/charts/gateway/templates/zzz_profile.yaml

@@ -15,9 +15,14 @@ However, we can workaround this by placing all of (1) under a specific key (.Val
 We can then merge the profile onto the defaults, then the user settings onto that.
 Finally, we can set all of that under .Values so the chart behaves without awareness.
 */}}
-{{- $globals := $.Values.global | default dict | deepCopy }}
+{{- if $.Values.defaults}}
-{{- $defaults := $.Values.defaults }}
+{{ fail (cat
-{{- $_ := unset $.Values "defaults" }}
+  "Setting with .default prefix found; remove it. For example, replace `--set defaults.hub=foo` with `--set hub=foo`. Defaults set:\n"
+  ($.Values.defaults | toYaml |nindent 4)
+) }}
+{{- end }}
+{{- $defaults := $.Values._internal_defaults_do_not_set }}
+{{- $_ := unset $.Values "_internal_defaults_do_not_set" }}
 {{- $profile := dict }}
 {{- with .Values.profile }}
 {{- with $.Files.Get (printf "files/profile-%s.yaml" .)}}
@@ -33,11 +38,37 @@ Finally, we can set all of that under .Values so the chart behaves without aware
 {{ fail (cat "unknown compatibility version" $.Values.compatibilityVersion) }}
 {{- end }}
 {{- end }}
+{{- if ($.Values.global).platform }}
+{{- with $.Files.Get (printf "files/profile-platform-%s.yaml" ($.Values.global).platform) }}
+{{- $ignore := mustMergeOverwrite $profile (. | fromYaml) }}
+{{- else }}
+{{ fail (cat "unknown platform" ($.Values.global).platform) }}
+{{- end }}
+{{- end }}
 {{- if $profile }}
 {{- $a := mustMergeOverwrite $defaults $profile }}
 {{- end }}
 #  Flatten globals, if defined on a per-chart basis
 {{- if false }}
-{{- $a := mustMergeOverwrite $defaults $globals  }}
+{{- $a := mustMergeOverwrite $defaults ($profile.global) ($.Values.global | default dict)  }}
 {{- end }}
 {{- $b := set $ "Values" (mustMergeOverwrite $defaults $.Values) }}
+
+{{/*
+Labels that should be applied to ALL resources.
+*/}}
+{{- define "istio.labels" -}}
+{{- if .Release.Service -}}
+app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+{{- end }}
+{{- if .Release.Name }}
+app.kubernetes.io/instance: {{ .Release.Name | quote }}
+{{- end }}
+app.kubernetes.io/part-of: "istio"
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+{{- if and .Chart.Name .Chart.Version }}
+helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end -}}

charts/kubezero-istio-gateway/charts/gateway/values.schema.json

@@ -60,6 +60,15 @@
         "env": {
           "type": "object"
         },
+        "strategy": {
+          "type": "object"
+        },
+        "minReadySeconds": {
+          "type": [ "null", "integer" ]
+        },
+        "readinessProbe": {
+          "type": [ "null", "object" ]
+        },
         "labels": {
           "type": "object"
         },

charts/kubezero-istio-gateway/charts/gateway/values.yaml

@@ -1,6 +1,6 @@
-# "defaults" is a workaround for Helm limitations. Users should NOT set ".defaults" explicitly, but rather directly set the fields internally.
+# "_internal_defaults_do_not_set" is a workaround for Helm limitations. Users should NOT set "._internal_defaults_do_not_set" explicitly, but rather directly set the fields internally.
-# For instance, instead of `--set defaults.foo=bar`, just set `--set foo=bar`.
+# For instance, instead of `--set _internal_defaults_do_not_set.foo=bar``, just set `--set foo=bar`.
-defaults:
+_internal_defaults_do_not_set:
   # Name allows overriding the release name. Generally this should not be set
   name: ""
   # revision declares which revision this gateway is a part of
@@ -84,6 +84,17 @@ defaults:
   # Pod environment variables
   env: {}
 
+  # Deployment Update strategy
+  strategy: {}
+  
+  # Sets the Deployment minReadySeconds value
+  minReadySeconds:
+  
+  # Optionally configure a custom readinessProbe. By default the control plane
+  # automatically injects the readinessProbe. If you wish to override that
+  # behavior, you may define your own readinessProbe here.
+  readinessProbe: {}
+
   # Labels to apply to all resources
   labels: {}
 
@@ -137,6 +148,7 @@ defaults:
   #
   podDisruptionBudget: {}
 
+  # Sets the per-pod terminationGracePeriodSeconds setting.
   terminationGracePeriodSeconds: 30
 
   # A list of `Volumes` added into the Gateway Pods. See

charts/kubezero-istio-gateway/gateway.patch

@@ -1,16 +1,3 @@
-diff -tubr charts/gateway.orig/templates/deployment.yaml charts/gateway/templates/deployment.yaml
---- charts/gateway.orig/templates/deployment.yaml	2022-12-09 14:58:33.000000000 +0000
-+++ charts/gateway/templates/deployment.yaml	2022-12-13 11:43:02.196667885 +0000
-@@ -11,6 +11,9 @@
-   {{- if not .Values.autoscaling.enabled }}
-   replicas: {{ .Values.replicaCount }}
-   {{- end }}
-+  # Give the LB 120s to detect and take into service
-+  # should only be 40s by we are on AWS so ...
-+  minReadySeconds: 120
-   selector:
-     matchLabels:
-       {{- include "gateway.selectorLabels" . | nindent 6 }}
 diff -tubr charts/gateway.orig/templates/service.yaml charts/gateway/templates/service.yaml
 --- charts/gateway.orig/templates/service.yaml	2022-12-09 14:58:33.000000000 +0000
 +++ charts/gateway/templates/service.yaml	2022-12-12 22:52:27.629670669 +0000

charts/kubezero-istio-gateway/templates/bootstrap-config.yaml

@@ -51,7 +51,7 @@ data:
           {
             "name": "static_layer_0",
             "staticLayer": {
-              "overload.global_downstream_max_connections": 50000
+              "envoy.resource_monitors.downstream_connections": 50000
             }
           }
         ]

charts/kubezero-istio-gateway/values.yaml

@@ -8,6 +8,7 @@ gateway:
   replicaCount: 1
 
   terminationGracePeriodSeconds: 120
+  minReadySeconds: 10
 
   resources:
     requests:
@@ -28,7 +29,7 @@ gateway:
     # gatewayProtocol: Loadbalancer protocol which is NOT the same as Container Procotol !
 
   podAnnotations:
-    proxy.istio.io/config: '{ "terminationDrainDuration": "20s" }'
+    proxy.istio.io/config: '{ "terminationDrainDuration": "90s" }'
 
 certificates: []
 #- name: ingress-cert

charts/kubezero-istio/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-istio
 description: KubeZero Umbrella Chart for Istio
 type: application
-version: 0.23.2
+version: 0.24.2
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -16,13 +16,13 @@ dependencies:
     version: ">= 0.1.6"
     repository: https://cdn.zero-downtime.net/charts/
   - name: base
-    version: 1.23.2
+    version: 1.24.2
     repository: https://istio-release.storage.googleapis.com/charts
   - name: istiod
-    version: 1.23.2
+    version: 1.24.2
     repository: https://istio-release.storage.googleapis.com/charts
   - name: kiali-server
-    version: "1.89.7"
+    version: "2.5.0"
     repository: https://kiali.org/helm-charts
     condition: kiali-server.enabled
-kubeVersion: ">= 1.26.0-0"
+kubeVersion: ">= 1.30.0-0"

charts/kubezero-istio/README.md

@@ -1,6 +1,6 @@
 # kubezero-istio
 
-![Version: 0.23.2](https://img.shields.io/badge/Version-0.23.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.24.2](https://img.shields.io/badge/Version-0.24.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero Umbrella Chart for Istio
 
@@ -16,14 +16,14 @@ Installs the Istio control plane
 
 ## Requirements
 
-Kubernetes: `>= 1.26.0-0`
+Kubernetes: `>= 1.30.0-0`
 
 | Repository | Name | Version |
 |------------|------|---------|
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
-| https://istio-release.storage.googleapis.com/charts | base | 1.23.2 |
+| https://istio-release.storage.googleapis.com/charts | base | 1.24.2 |
-| https://istio-release.storage.googleapis.com/charts | istiod | 1.23.2 |
+| https://istio-release.storage.googleapis.com/charts | istiod | 1.24.2 |
-| https://kiali.org/helm-charts | kiali-server | 1.89.7 |
+| https://kiali.org/helm-charts | kiali-server | 2.5.0 |
 
 ## Values
 
@@ -64,12 +64,11 @@ Kubernetes: `>= 1.26.0-0`
 
 ## Resources
 
-- https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/#IstioOperatorSpec
 - https://github.com/istio/istio/blob/master/manifests/profiles/default.yaml
-- https://istio.io/latest/docs/setup/install/standalone-operator/
 
 ### Grafana
 - https://grafana.com/grafana/dashboards/7645
 - https://grafana.com/grafana/dashboards/7639
 - https://grafana.com/grafana/dashboards/7636
 - https://grafana.com/grafana/dashboards/7630
+- https://grafana.com/grafana/dashboards/11829

charts/kubezero-istio/dashboards.yaml

@@ -5,22 +5,22 @@ folder: Istio
 condition: '.Values.istiod.telemetry.enabled'
 dashboards:
 - name: istio-control-plane
-  url: https://grafana.com/api/dashboards/7645/revisions/229/download
+  url: https://grafana.com/api/dashboards/7645/revisions/239/download
   tags:
   - Istio
 - name: istio-mesh
-  url: https://grafana.com/api/dashboards/7639/revisions/229/download
+  url: https://grafana.com/api/dashboards/7639/revisions/239/download
   tags:
   - Istio
 - name: istio-service
-  url: https://grafana.com/api/dashboards/7636/revisions/229/download
+  url: https://grafana.com/api/dashboards/7636/revisions/239/download
   tags:
   - Istio
 - name: istio-workload
-  url: https://grafana.com/api/dashboards/7630/revisions/229/download
+  url: https://grafana.com/api/dashboards/7630/revisions/239/download
   tags:
   - Istio
 - name: istio-performance
-  url: https://grafana.com/api/dashboards/11829/revisions/229/download
+  url: https://grafana.com/api/dashboards/11829/revisions/239/download
   tags:
   - Istio

charts/kubezero-logging/charts/fluentd/templates/tests/test-connection.yaml

@@ -4,7 +4,6 @@ fluentd is deployed with the default values
 If the fluentd config is overriden and the metrics server removed
 this will fail.
 */}}
-{{- if .Values.testFramework.enabled }}
 {{ if empty .Values.service.ports }}
 apiVersion: v1
 kind: Pod
@@ -27,5 +26,4 @@ spec:
           while :; do nc -vz {{ include "fluentd.fullname" . }}:24231 && break; sleep 1; done
           wget '{{ include "fluentd.fullname" . }}:24231/metrics'
   restartPolicy: Never
-{{ end }}
+{{ end }}
-{{- end }}

charts/kubezero-logging/charts/fluentd/values.yaml

@@ -13,9 +13,6 @@ image:
   pullPolicy: "IfNotPresent"
   tag: ""
 
-testFramework:
-  enabled: false
-
 ## Optional array of imagePullSecrets containing private registry credentials
 ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
 imagePullSecrets: []

charts/kubezero-logging/fluentd.patch

@@ -9,36 +9,3 @@ diff -rtuN charts/fluentd.orig/templates/fluentd-configurations-cm.yaml charts/f
 +    {{- (tpl $value $) | nindent 4 }}
  {{- end }}
  {{- end }}
- 
-diff -rtuN charts/fluentd.orig/templates/tests/test-connection.yaml charts/fluentd/templates/tests/test-connection.yaml
---- charts/fluentd.orig/templates/tests/test-connection.yaml	2024-04-08 11:00:03.030515998 +0000
-+++ charts/fluentd/templates/tests/test-connection.yaml	2024-04-08 11:03:16.254774985 +0000
-@@ -4,6 +4,7 @@
- If the fluentd config is overriden and the metrics server removed
- this will fail.
- */}}
-+{{- if .Values.testFramework.enabled }}
- {{ if empty .Values.service.ports }}
- apiVersion: v1
- kind: Pod
-@@ -26,4 +27,5 @@
-           while :; do nc -vz {{ include "fluentd.fullname" . }}:24231 && break; sleep 1; done
-           wget '{{ include "fluentd.fullname" . }}:24231/metrics'
-   restartPolicy: Never
--{{ end }}
-\ No newline at end of file
-+{{ end }}
-+{{- end }}
-diff -rtuN charts/fluentd.orig/values.yaml charts/fluentd/values.yaml
---- charts/fluentd.orig/values.yaml	2024-04-08 11:00:03.030515998 +0000
-+++ charts/fluentd/values.yaml	2024-04-08 11:00:03.040516045 +0000
-@@ -13,6 +13,9 @@
-   pullPolicy: "IfNotPresent"
-   tag: ""
- 
-+testFramework:
-+  enabled: false
-+
- ## Optional array of imagePullSecrets containing private registry credentials
- ## Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
- imagePullSecrets: []

charts/kubezero-metrics/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-metrics
 description: KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all Kubernetes integrations.
 type: application
-version: 0.10.2
+version: 0.11.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -19,14 +19,14 @@ dependencies:
     version: ">= 0.1.6"
     repository: https://cdn.zero-downtime.net/charts/
   - name: kube-prometheus-stack
-    version: 66.1.1
+    version: 69.2.3
     repository: https://prometheus-community.github.io/helm-charts
   - name: prometheus-adapter
     version: 4.11.0
     repository: https://prometheus-community.github.io/helm-charts
     condition: prometheus-adapter.enabled
   - name: prometheus-pushgateway
-    version: 2.15.0
+    version: 3.0.0
     repository: https://prometheus-community.github.io/helm-charts
     condition: prometheus-pushgateway.enabled
-kubeVersion: ">= 1.26.0"
+kubeVersion: ">= 1.30.0-0"

charts/kubezero-metrics/README.md

@@ -1,6 +1,6 @@
 # kubezero-metrics
 
-![Version: 0.10.2](https://img.shields.io/badge/Version-0.10.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.10.4](https://img.shields.io/badge/Version-0.10.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all Kubernetes integrations.
 
@@ -14,14 +14,14 @@ KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all
 
 ## Requirements
 
-Kubernetes: `>= 1.26.0`
+Kubernetes: `>= 1.30.0-0`
 
 | Repository | Name | Version |
 |------------|------|---------|
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
-| https://prometheus-community.github.io/helm-charts | kube-prometheus-stack | 66.1.1 |
+| https://prometheus-community.github.io/helm-charts | kube-prometheus-stack | 69.2.3 |
 | https://prometheus-community.github.io/helm-charts | prometheus-adapter | 4.11.0 |
-| https://prometheus-community.github.io/helm-charts | prometheus-pushgateway | 2.15.0 |
+| https://prometheus-community.github.io/helm-charts | prometheus-pushgateway | 3.0.0 |
 
 ## Values
 

charts/kubezero-metrics/jsonnet/dashboards/apiserver.json

@@ -30,7 +30,7 @@
          "options": {
             "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only."
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "title": "Notice",
          "type": "text"
       },
@@ -54,7 +54,7 @@
          },
          "id": 2,
          "interval": "1m",
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -100,7 +100,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -134,7 +134,7 @@
          },
          "id": 4,
          "interval": "1m",
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -232,7 +232,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -276,7 +276,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -319,7 +319,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -353,7 +353,7 @@
          },
          "id": 8,
          "interval": "1m",
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -451,7 +451,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -495,7 +495,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -538,7 +538,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -581,7 +581,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -624,7 +624,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -670,7 +670,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -712,7 +712,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -755,7 +755,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -797,7 +797,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/cluster-total.json

@@ -46,7 +46,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -92,7 +92,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -163,7 +163,7 @@
             "y": 9
          },
          "id": 3,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -328,7 +328,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -374,7 +374,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -420,7 +420,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -466,7 +466,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -512,7 +512,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -558,7 +558,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -604,7 +604,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -650,7 +650,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -696,7 +696,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -742,7 +742,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/controller-manager.json

@@ -35,7 +35,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -86,7 +86,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -137,7 +137,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -188,7 +188,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -239,7 +239,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -314,15 +314,15 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
                   "type": "prometheus",
                   "uid": "${datasource}"
                },
-               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))",
+               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, le))",
-               "legendFormat": "{{verb}} {{url}}"
+               "legendFormat": "{{verb}}"
             }
          ],
          "title": "Post Request Latency 99th Quantile",
@@ -365,15 +365,15 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
                   "type": "prometheus",
                   "uid": "${datasource}"
                },
-               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))",
+               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, le))",
-               "legendFormat": "{{verb}} {{url}}"
+               "legendFormat": "{{verb}}"
             }
          ],
          "title": "Get Request Latency 99th Quantile",
@@ -416,7 +416,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -467,7 +467,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -518,7 +518,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-cluster.json

@@ -35,7 +35,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -70,7 +70,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -105,7 +105,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -140,7 +140,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -175,7 +175,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -210,7 +210,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -260,7 +260,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -319,7 +319,7 @@
             "y": 12
          },
          "id": 8,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -476,7 +476,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -571,7 +571,7 @@
             "y": 24
          },
          "id": 10,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -748,7 +748,7 @@
             "y": 30
          },
          "id": 11,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -892,7 +892,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -943,7 +943,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -994,7 +994,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1045,7 +1045,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1096,7 +1096,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1147,7 +1147,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1198,7 +1198,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1249,7 +1249,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1300,7 +1300,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1351,7 +1351,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1422,7 +1422,7 @@
             "y": 96
          },
          "id": 22,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-namespace.json

@@ -35,7 +35,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -70,7 +70,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -105,7 +105,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -140,7 +140,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -242,7 +242,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -317,7 +317,7 @@
             "y": 14
          },
          "id": 6,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -500,7 +500,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -578,7 +578,7 @@
             "y": 28
          },
          "id": 8,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -768,7 +768,7 @@
             "y": 35
          },
          "id": 9,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -912,7 +912,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -963,7 +963,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1014,7 +1014,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1065,7 +1065,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1116,7 +1116,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1167,7 +1167,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1218,7 +1218,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1269,7 +1269,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1340,7 +1340,7 @@
             "y": 70
          },
          "id": 18,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-node.json

@@ -94,7 +94,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -161,7 +161,7 @@
             "y": 6
          },
          "id": 2,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -323,7 +323,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -342,6 +342,109 @@
                "legendFormat": "{{pod}}"
             }
          ],
+         "title": "Memory Usage (w/cache)",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true,
+                  "stacking": {
+                     "mode": "normal"
+                  }
+               },
+               "unit": "bytes"
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "max capacity"
+                  },
+                  "properties": [
+                     {
+                        "id": "color",
+                        "value": {
+                           "fixedColor": "red",
+                           "mode": "fixed"
+                        }
+                     },
+                     {
+                        "id": "custom.stacking",
+                        "value": {
+                           "mode": "none"
+                        }
+                     },
+                     {
+                        "id": "custom.hideFrom",
+                        "value": {
+                           "legend": false,
+                           "tooltip": true,
+                           "viz": false
+                        }
+                     },
+                     {
+                        "id": "custom.lineStyle",
+                        "value": {
+                           "dash": [
+                              10,
+                              10
+                           ],
+                           "fill": "dash"
+                        }
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 6,
+            "w": 24,
+            "x": 0,
+            "y": 18
+         },
+         "id": 4,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_node_status_capacity{cluster=\"$cluster\", job=\"kube-state-metrics\", node=~\"$node\", resource=\"memory\"})",
+               "legendFormat": "max capacity"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)",
+               "legendFormat": "{{pod}}"
+            }
+         ],
          "title": "Memory Usage (w/o cache)",
          "type": "timeseries"
       },
@@ -390,10 +493,10 @@
             "h": 6,
             "w": 24,
             "x": 0,
-            "y": 18
+            "y": 24
          },
-         "id": 4,
+         "id": 5,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-pod.json

@@ -102,7 +102,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -208,7 +208,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -250,7 +250,7 @@
             "y": 14
          },
          "id": 3,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -433,7 +433,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -494,7 +494,7 @@
             "y": 28
          },
          "id": 5,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -664,7 +664,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -715,7 +715,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -766,7 +766,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -817,7 +817,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -868,7 +868,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -919,7 +919,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -970,7 +970,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1029,7 +1029,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1088,7 +1088,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1139,7 +1139,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1193,7 +1193,7 @@
             "y": 70
          },
          "id": 16,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-windows-cluster.json

@@ -0,0 +1,675 @@
+{
+   "editable": false,
+   "panels": [
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "unit": "none"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 4,
+            "x": 0,
+            "y": 0
+         },
+         "id": 1,
+         "interval": "1m",
+         "options": {
+            "colorMode": "none"
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "1 - avg(rate(windows_cpu_time_total{cluster=\"$cluster\", job=\"kubernetes-windows-exporter\", mode=\"idle\"}[$__rate_interval]))",
+               "instant": true
+            }
+         ],
+         "title": "CPU Utilisation",
+         "type": "stat"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 4,
+            "x": 4,
+            "y": 0
+         },
+         "id": 2,
+         "interval": "1m",
+         "options": {
+            "colorMode": "none"
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})",
+               "instant": true
+            }
+         ],
+         "title": "CPU Requests Commitment",
+         "type": "stat"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 4,
+            "x": 8,
+            "y": 0
+         },
+         "id": 3,
+         "interval": "1m",
+         "options": {
+            "colorMode": "none"
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) / sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"})",
+               "instant": true
+            }
+         ],
+         "title": "CPU Limits Commitment",
+         "type": "stat"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 4,
+            "x": 12,
+            "y": 0
+         },
+         "id": 4,
+         "interval": "1m",
+         "options": {
+            "colorMode": "none"
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "1 - sum(:windows_node_memory_MemFreeCached_bytes:sum{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})",
+               "instant": true
+            }
+         ],
+         "title": "Memory Utilisation",
+         "type": "stat"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 4,
+            "x": 16,
+            "y": 0
+         },
+         "id": 5,
+         "interval": "1m",
+         "options": {
+            "colorMode": "none"
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})",
+               "instant": true
+            }
+         ],
+         "title": "Memory Requests Commitment",
+         "type": "stat"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 3,
+            "w": 4,
+            "x": 20,
+            "y": 0
+         },
+         "id": 6,
+         "interval": "1m",
+         "options": {
+            "colorMode": "none"
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) / sum(:windows_node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})",
+               "instant": true
+            }
+         ],
+         "title": "Memory Limits Commitment",
+         "type": "stat"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               }
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 7
+         },
+         "id": 7,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)",
+               "legendFormat": "__auto"
+            }
+         ],
+         "title": "CPU Usage",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byRegexp",
+                     "options": "/%/"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "percentunit"
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Namespace"
+                  },
+                  "properties": [
+                     {
+                        "id": "links",
+                        "value": [
+                           {
+                              "title": "Drill down to pods",
+                              "url": "/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?${datasource:queryparam}&var-cluster=$cluster&var-namespace=${__data.fields.Namespace}"
+                           }
+                        ]
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 14
+         },
+         "id": 8,
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\"}) by (namespace)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\"}) by (namespace)",
+               "format": "table",
+               "instant": true
+            }
+         ],
+         "title": "CPU Quota",
+         "transformations": [
+            {
+               "id": "joinByField",
+               "options": {
+                  "byField": "namespace",
+                  "mode": "outer"
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time": true,
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true
+                  },
+                  "indexByName": {
+                     "Time 1": 0,
+                     "Time 2": 1,
+                     "Time 3": 2,
+                     "Time 4": 3,
+                     "Time 5": 4,
+                     "Value #A": 6,
+                     "Value #B": 7,
+                     "Value #C": 8,
+                     "Value #D": 9,
+                     "Value #E": 10,
+                     "namespace": 5
+                  },
+                  "renameByName": {
+                     "Value #A": "CPU Usage",
+                     "Value #B": "CPU Requests",
+                     "Value #C": "CPU Requests %",
+                     "Value #D": "CPU Limits",
+                     "Value #E": "CPU Limits %",
+                     "namespace": "Namespace"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "decbytes"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 21
+         },
+         "id": 9,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)",
+               "legendFormat": "__auto"
+            }
+         ],
+         "title": "Memory Usage (Private Working Set)",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "unit": "bytes"
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byRegexp",
+                     "options": "/%/"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "percentunit"
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Memory Usage"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "decbytes"
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Memory Requests"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "decbytes"
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Memory Limits"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "decbytes"
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Namespace"
+                  },
+                  "properties": [
+                     {
+                        "id": "links",
+                        "value": [
+                           {
+                              "title": "Drill down to pods",
+                              "url": "/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?${datasource:queryparam}&var-cluster=$cluster&var-namespace=${__data.fields.Namespace}"
+                           }
+                        ]
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 28
+         },
+         "id": 10,
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\"}) by (namespace)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(windows_container_private_working_set_usage{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\"}) by (namespace)",
+               "format": "table",
+               "instant": true
+            }
+         ],
+         "title": "Memory Requests by Namespace",
+         "transformations": [
+            {
+               "id": "joinByField",
+               "options": {
+                  "byField": "namespace",
+                  "mode": "outer"
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time": true,
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true
+                  },
+                  "indexByName": {
+                     "Time 1": 0,
+                     "Time 2": 1,
+                     "Time 3": 2,
+                     "Time 4": 3,
+                     "Time 5": 4,
+                     "Value #A": 6,
+                     "Value #B": 7,
+                     "Value #C": 8,
+                     "Value #D": 9,
+                     "Value #E": 10,
+                     "namespace": 5
+                  },
+                  "renameByName": {
+                     "Value #A": "Memory Usage",
+                     "Value #B": "Memory Requests",
+                     "Value #C": "Memory Requests %",
+                     "Value #D": "Memory Limits",
+                     "Value #E": "Memory Limits %",
+                     "namespace": "Namespace"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      }
+   ],
+   "refresh": "10s",
+   "schemaVersion": 39,
+   "tags": [
+      "kubernetes"
+   ],
+   "templating": {
+      "list": [
+         {
+            "current": {
+               "selected": true,
+               "text": "default",
+               "value": "default"
+            },
+            "hide": 0,
+            "label": "Data source",
+            "name": "datasource",
+            "query": "prometheus",
+            "regex": "",
+            "type": "datasource"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "hide": 2,
+            "label": "cluster",
+            "name": "cluster",
+            "query": "label_values(up{job=\"kubernetes-windows-exporter\"}, cluster)",
+            "refresh": 2,
+            "sort": 1,
+            "type": "query"
+         }
+      ]
+   },
+   "time": {
+      "from": "now-1h",
+      "to": "now"
+   },
+   "timezone": "utc",
+   "title": "Compute Resources / Cluster(Windows)",
+   "uid": "4d08557fd9391b100730f2494bccac68"
+}

charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-windows-namespace.json

@@ -0,0 +1,442 @@
+{
+   "editable": false,
+   "panels": [
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               }
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 0
+         },
+         "id": 1,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
+               "legendFormat": "__auto"
+            }
+         ],
+         "title": "CPU Usage",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byRegexp",
+                     "options": "/%/"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "percentunit"
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Pod"
+                  },
+                  "properties": [
+                     {
+                        "id": "links",
+                        "value": [
+                           {
+                              "title": "Drill down to pods",
+                              "url": "/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}"
+                           }
+                        ]
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 7
+         },
+         "id": 2,
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
+               "format": "table",
+               "instant": true
+            }
+         ],
+         "title": "CPU Quota",
+         "transformations": [
+            {
+               "id": "joinByField",
+               "options": {
+                  "byField": "pod",
+                  "mode": "outer"
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time": true,
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true
+                  },
+                  "indexByName": {
+                     "Time 1": 0,
+                     "Time 2": 1,
+                     "Time 3": 2,
+                     "Time 4": 3,
+                     "Time 5": 4,
+                     "Value #A": 6,
+                     "Value #B": 7,
+                     "Value #C": 8,
+                     "Value #D": 9,
+                     "Value #E": 10,
+                     "pod": 5
+                  },
+                  "renameByName": {
+                     "Value #A": "CPU Usage",
+                     "Value #B": "CPU Requests",
+                     "Value #C": "CPU Requests %",
+                     "Value #D": "CPU Limits",
+                     "Value #E": "CPU Limits %",
+                     "pod": "Pod"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "decbytes"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 14
+         },
+         "id": 3,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
+               "legendFormat": "__auto"
+            }
+         ],
+         "title": "Memory Usage (Private Working Set)",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "unit": "bytes"
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byRegexp",
+                     "options": "/%/"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "percentunit"
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Pod"
+                  },
+                  "properties": [
+                     {
+                        "id": "links",
+                        "value": [
+                           {
+                              "title": "Drill down to pods",
+                              "url": "/d/40597a704a610e936dc6ed374a7ce023/k8s-resources-windows-pod?${datasource:queryparam}&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__data.fields.Pod}"
+                           }
+                        ]
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 21
+         },
+         "id": 4,
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)",
+               "format": "table",
+               "instant": true
+            }
+         ],
+         "title": "Memory Quota",
+         "transformations": [
+            {
+               "id": "joinByField",
+               "options": {
+                  "byField": "pod",
+                  "mode": "outer"
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time": true,
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true
+                  },
+                  "indexByName": {
+                     "Time 1": 0,
+                     "Time 2": 1,
+                     "Time 3": 2,
+                     "Time 4": 3,
+                     "Time 5": 4,
+                     "Value #A": 6,
+                     "Value #B": 7,
+                     "Value #C": 8,
+                     "Value #D": 9,
+                     "Value #E": 10,
+                     "pod": 5
+                  },
+                  "renameByName": {
+                     "Value #A": "Memory Usage",
+                     "Value #B": "Memory Requests",
+                     "Value #C": "Memory Requests %",
+                     "Value #D": "Memory Limits",
+                     "Value #E": "Memory Limits %",
+                     "pod": "Pod"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      }
+   ],
+   "refresh": "10s",
+   "schemaVersion": 39,
+   "tags": [
+      "kubernetes"
+   ],
+   "templating": {
+      "list": [
+         {
+            "current": {
+               "selected": true,
+               "text": "default",
+               "value": "default"
+            },
+            "hide": 0,
+            "label": "Data source",
+            "name": "datasource",
+            "query": "prometheus",
+            "regex": "",
+            "type": "datasource"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "hide": 2,
+            "label": "cluster",
+            "name": "cluster",
+            "query": "label_values(up{job=\"kubernetes-windows-exporter\"}, cluster)",
+            "refresh": 2,
+            "sort": 1,
+            "type": "query"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "hide": 0,
+            "label": "namespace",
+            "name": "namespace",
+            "query": "label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)",
+            "refresh": 2,
+            "sort": 1,
+            "type": "query"
+         }
+      ]
+   },
+   "time": {
+      "from": "now-1h",
+      "to": "now"
+   },
+   "timezone": "utc",
+   "title": "Compute Resources / Namespace(Windows)",
+   "uid": "490b402361724ab1d4c45666c1fa9b6f"
+}

charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-windows-pod.json

@@ -0,0 +1,497 @@
+{
+   "editable": false,
+   "panels": [
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               }
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 0
+         },
+         "id": 1,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)",
+               "legendFormat": "__auto"
+            }
+         ],
+         "title": "CPU Usage",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byRegexp",
+                     "options": "/%/"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "percentunit"
+                     }
+                  ]
+               },
+               {
+                  "matcher": {
+                     "id": "byName",
+                     "options": "Namespace"
+                  },
+                  "properties": [
+                     {
+                        "id": "links",
+                        "value": [
+                           {
+                              "title": "Drill down to pods",
+                              "url": "/d/490b402361724ab1d4c45666c1fa9b6f/k8s-resources-windows-namespace?${datasource:queryparam}&var-cluster=$cluster&var-namespace=${__data.fields.Namespace}"
+                           }
+                        ]
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 7
+         },
+         "id": 2,
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_cpu_cores_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)",
+               "format": "table",
+               "instant": true
+            }
+         ],
+         "title": "CPU Quota",
+         "transformations": [
+            {
+               "id": "joinByField",
+               "options": {
+                  "byField": "container",
+                  "mode": "outer"
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time": true,
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true
+                  },
+                  "indexByName": {
+                     "Time 1": 0,
+                     "Time 2": 1,
+                     "Time 3": 2,
+                     "Time 4": 3,
+                     "Time 5": 4,
+                     "Value #A": 6,
+                     "Value #B": 7,
+                     "Value #C": 8,
+                     "Value #D": 9,
+                     "Value #E": 10,
+                     "container": 5
+                  },
+                  "renameByName": {
+                     "Value #A": "CPU Usage",
+                     "Value #B": "CPU Requests",
+                     "Value #C": "CPU Requests %",
+                     "Value #D": "CPU Limits",
+                     "Value #E": "CPU Limits %",
+                     "container": "Container"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "decbytes"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 14
+         },
+         "id": 3,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)",
+               "legendFormat": "__auto"
+            }
+         ],
+         "title": "Memory Usage",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "unit": "bytes"
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byRegexp",
+                     "options": "/%/"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "percentunit"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 21
+         },
+         "id": 4,
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_request{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)",
+               "format": "table",
+               "instant": true
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum(windows_container_private_working_set_usage{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_windows_container_resource_memory_limit{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)",
+               "format": "table",
+               "instant": true
+            }
+         ],
+         "title": "Memory Quota",
+         "transformations": [
+            {
+               "id": "joinByField",
+               "options": {
+                  "byField": "container",
+                  "mode": "outer"
+               }
+            },
+            {
+               "id": "organize",
+               "options": {
+                  "excludeByName": {
+                     "Time": true,
+                     "Time 1": true,
+                     "Time 2": true,
+                     "Time 3": true,
+                     "Time 4": true,
+                     "Time 5": true
+                  },
+                  "indexByName": {
+                     "Time 1": 0,
+                     "Time 2": 1,
+                     "Time 3": 2,
+                     "Time 4": 3,
+                     "Time 5": 4,
+                     "Value #A": 6,
+                     "Value #B": 7,
+                     "Value #C": 8,
+                     "Value #D": 9,
+                     "Value #E": 10,
+                     "container": 5
+                  },
+                  "renameByName": {
+                     "Value #A": "Memory Usage",
+                     "Value #B": "Memory Requests",
+                     "Value #C": "Memory Requests %",
+                     "Value #D": "Memory Limits",
+                     "Value #E": "Memory Limits %",
+                     "container": "Container"
+                  }
+               }
+            }
+         ],
+         "type": "table"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "bytes"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 28
+         },
+         "id": 5,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sort_desc(sum by (container) (rate(windows_container_network_received_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))",
+               "legendFormat": "Received : {{ container }}"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sort_desc(sum by (container) (rate(windows_container_network_transmitted_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))",
+               "legendFormat": "Transmitted : {{ container }}"
+            }
+         ],
+         "title": "Network I/O",
+         "type": "timeseries"
+      }
+   ],
+   "refresh": "10s",
+   "schemaVersion": 39,
+   "tags": [
+      "kubernetes"
+   ],
+   "templating": {
+      "list": [
+         {
+            "current": {
+               "selected": true,
+               "text": "default",
+               "value": "default"
+            },
+            "hide": 0,
+            "label": "Data source",
+            "name": "datasource",
+            "query": "prometheus",
+            "regex": "",
+            "type": "datasource"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "hide": 2,
+            "label": "cluster",
+            "name": "cluster",
+            "query": "label_values(up{job=\"kubernetes-windows-exporter\"}, cluster)",
+            "refresh": 2,
+            "sort": 1,
+            "type": "query"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "hide": 0,
+            "label": "namespace",
+            "name": "namespace",
+            "query": "label_values(windows_pod_container_available{cluster=\"$cluster\"}, namespace)",
+            "refresh": 2,
+            "sort": 1,
+            "type": "query"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "hide": 0,
+            "label": "pod",
+            "name": "pod",
+            "query": "label_values(windows_pod_container_available{cluster=\"$cluster\",namespace=\"$namespace\"}, pod)",
+            "refresh": 2,
+            "sort": 1,
+            "type": "query"
+         }
+      ]
+   },
+   "time": {
+      "from": "now-1h",
+      "to": "now"
+   },
+   "timezone": "utc",
+   "title": "Compute Resources / Pod(Windows)",
+   "uid": "40597a704a610e936dc6ed374a7ce023"
+}

charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-workload.json

@@ -50,7 +50,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -109,7 +109,7 @@
             "y": 7
          },
          "id": 2,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -240,7 +240,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -302,7 +302,7 @@
             "y": 21
          },
          "id": 4,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -453,7 +453,7 @@
             "y": 28
          },
          "id": 5,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -597,7 +597,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -648,7 +648,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -699,7 +699,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -750,7 +750,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -801,7 +801,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -852,7 +852,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -903,7 +903,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -954,7 +954,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/k8s-resources-workloads-namespace.json

@@ -102,7 +102,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -189,7 +189,7 @@
             "y": 7
          },
          "id": 2,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -397,7 +397,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -487,7 +487,7 @@
             "y": 21
          },
          "id": 4,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -663,7 +663,7 @@
             "y": 28
          },
          "id": 5,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -807,7 +807,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -858,7 +858,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -909,7 +909,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -960,7 +960,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1011,7 +1011,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1062,7 +1062,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1113,7 +1113,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1164,7 +1164,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/k8s-windows-cluster-rsrc-use.json

@@ -0,0 +1,404 @@
+{
+   "editable": false,
+   "panels": [
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 0
+         },
+         "id": 1,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\"} * node:windows_node_num_cpu:sum{cluster=\"$cluster\"} / scalar(sum(node:windows_node_num_cpu:sum{cluster=\"$cluster\"}))",
+               "legendFormat": "{{instance}}"
+            }
+         ],
+         "title": "CPU Utilisation",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 7
+         },
+         "id": 2,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "node:windows_node_memory_utilisation:ratio{cluster=\"$cluster\"}",
+               "legendFormat": "{{instance}}"
+            }
+         ],
+         "title": "Memory Utilisation",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "short"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 7
+         },
+         "id": 3,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\"}",
+               "legendFormat": "{{instance}}"
+            }
+         ],
+         "title": "Memory Saturation (Swap I/O Pages)",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 14
+         },
+         "id": 4,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(node:windows_node:sum{cluster=\"$cluster\"})",
+               "legendFormat": "{{instance}}"
+            }
+         ],
+         "title": "Disk IO Utilisation",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "Bps"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 21
+         },
+         "id": 5,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\"}",
+               "legendFormat": "{{instance}}"
+            }
+         ],
+         "title": "Net Utilisation (Transmitted)",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "Bps"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 21
+         },
+         "id": 6,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\"}",
+               "legendFormat": "{{instance}}"
+            }
+         ],
+         "title": "Net Utilisation (Dropped)",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 28
+         },
+         "id": 7,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum by (instance)(node:windows_node_filesystem_usage:{cluster=\"$cluster\"})",
+               "legendFormat": "{{instance}}"
+            }
+         ],
+         "title": "Disk Capacity",
+         "type": "timeseries"
+      }
+   ],
+   "refresh": "10s",
+   "schemaVersion": 39,
+   "tags": [
+      "kubernetes"
+   ],
+   "templating": {
+      "list": [
+         {
+            "current": {
+               "selected": true,
+               "text": "default",
+               "value": "default"
+            },
+            "hide": 0,
+            "label": "Data source",
+            "name": "datasource",
+            "query": "prometheus",
+            "regex": "",
+            "type": "datasource"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "hide": 2,
+            "label": "cluster",
+            "name": "cluster",
+            "query": "label_values(up{job=\"kubernetes-windows-exporter\"}, cluster)",
+            "refresh": 2,
+            "sort": 1,
+            "type": "query"
+         }
+      ]
+   },
+   "time": {
+      "from": "now-1h",
+      "to": "now"
+   },
+   "timezone": "utc",
+   "title": "USE Method / Cluster(Windows)",
+   "uid": "53a43377ec9aaf2ff64dfc7a1f539334"
+}

charts/kubezero-metrics/jsonnet/dashboards/k8s-windows-node-rsrc-use.json

@@ -0,0 +1,615 @@
+{
+   "editable": false,
+   "panels": [
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 0
+         },
+         "id": 1,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "node:windows_node_cpu_utilisation:avg1m{cluster=\"$cluster\", instance=\"$instance\"}",
+               "legendFormat": "Utilisation"
+            }
+         ],
+         "title": "CPU Utilisation",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 0
+         },
+         "id": 2,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "sum by (core) (irate(windows_cpu_time_total{cluster=\"$cluster\", job=\"kubernetes-windows-exporter\", mode!=\"idle\", instance=\"$instance\"}[$__rate_interval]))",
+               "legendFormat": "{{core}}"
+            }
+         ],
+         "title": "CPU Usage Per Core",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 0,
+            "y": 7
+         },
+         "id": 3,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "node:windows_node_memory_utilisation:{cluster=\"$cluster\", instance=\"$instance\"}",
+               "legendFormat": "Memory"
+            }
+         ],
+         "title": "Memory Utilisation %",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "bytes"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 8,
+            "y": 7
+         },
+         "id": 4,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "max(\n  windows_os_visible_memory_bytes{cluster=\"$cluster\", job=\"kubernetes-windows-exporter\", instance=\"$instance\"}\n  - windows_memory_available_bytes{cluster=\"$cluster\", job=\"kubernetes-windows-exporter\", instance=\"$instance\"}\n)\n",
+               "legendFormat": "memory used"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "max(node:windows_node_memory_totalCached_bytes:sum{cluster=\"$cluster\", instance=\"$instance\"})",
+               "legendFormat": "memory cached"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "max(windows_memory_available_bytes{cluster=\"$cluster\", job=\"kubernetes-windows-exporter\", instance=\"$instance\"})",
+               "legendFormat": "memory free"
+            }
+         ],
+         "title": "Memory Usage",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "short"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 16,
+            "y": 7
+         },
+         "id": 5,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "node:windows_node_memory_swap_io_pages:irate{cluster=\"$cluster\", instance=\"$instance\"}",
+               "legendFormat": "Swap IO"
+            }
+         ],
+         "title": "Memory Saturation (Swap I/O) Pages",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 14
+         },
+         "id": 6,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "node:windows_node_disk_utilisation:avg_irate{cluster=\"$cluster\", instance=\"$instance\"}",
+               "legendFormat": "Utilisation"
+            }
+         ],
+         "title": "Disk IO Utilisation",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "bytes"
+            },
+            "overrides": [
+               {
+                  "matcher": {
+                     "id": "byRegexp",
+                     "options": "/io time/"
+                  },
+                  "properties": [
+                     {
+                        "id": "unit",
+                        "value": "ms"
+                     }
+                  ]
+               }
+            ]
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 14
+         },
+         "id": 7,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "max(rate(windows_logical_disk_read_bytes_total{cluster=\"$cluster\", job=\"kubernetes-windows-exporter\", instance=\"$instance\"}[$__rate_interval]))",
+               "legendFormat": "read"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "max(rate(windows_logical_disk_write_bytes_total{cluster=\"$cluster\", job=\"kubernetes-windows-exporter\", instance=\"$instance\"}[$__rate_interval]))",
+               "legendFormat": "written"
+            },
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "max(rate(windows_logical_disk_read_seconds_total{cluster=\"$cluster\", job=\"kubernetes-windows-exporter\",  instance=\"$instance\"}[$__rate_interval]) + rate(windows_logical_disk_write_seconds_total{cluster=\"$cluster\", job=\"kubernetes-windows-exporter\", instance=\"$instance\"}[$__rate_interval]))",
+               "legendFormat": "io time"
+            }
+         ],
+         "title": "Disk IO",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "percentunit"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 21
+         },
+         "id": 8,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "node:windows_node_filesystem_usage:{cluster=\"$cluster\", instance=\"$instance\"}",
+               "legendFormat": "{{volume}}"
+            }
+         ],
+         "title": "Disk Utilisation",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "Bps"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 28
+         },
+         "id": 9,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "node:windows_node_net_utilisation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}",
+               "legendFormat": "Utilisation"
+            }
+         ],
+         "title": "Net Utilisation (Transmitted)",
+         "type": "timeseries"
+      },
+      {
+         "datasource": {
+            "type": "datasource",
+            "uid": "-- Mixed --"
+         },
+         "fieldConfig": {
+            "defaults": {
+               "custom": {
+                  "fillOpacity": 10,
+                  "showPoints": "never",
+                  "spanNulls": true
+               },
+               "unit": "Bps"
+            }
+         },
+         "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 28
+         },
+         "id": 10,
+         "interval": "1m",
+         "options": {
+            "legend": {
+               "asTable": true,
+               "calcs": [
+                  "lastNotNull"
+               ],
+               "displayMode": "table",
+               "placement": "right",
+               "showLegend": true
+            },
+            "tooltip": {
+               "mode": "single"
+            }
+         },
+         "pluginVersion": "v11.4.0",
+         "targets": [
+            {
+               "datasource": {
+                  "type": "prometheus",
+                  "uid": "${datasource}"
+               },
+               "expr": "node:windows_node_net_saturation:sum_irate{cluster=\"$cluster\", instance=\"$instance\"}",
+               "legendFormat": "Saturation"
+            }
+         ],
+         "title": "Net Saturation (Dropped)",
+         "type": "timeseries"
+      }
+   ],
+   "refresh": "10s",
+   "schemaVersion": 39,
+   "tags": [
+      "kubernetes"
+   ],
+   "templating": {
+      "list": [
+         {
+            "current": {
+               "selected": true,
+               "text": "default",
+               "value": "default"
+            },
+            "hide": 0,
+            "label": "Data source",
+            "name": "datasource",
+            "query": "prometheus",
+            "regex": "",
+            "type": "datasource"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "hide": 2,
+            "label": "cluster",
+            "name": "cluster",
+            "query": "label_values(up{job=\"kubernetes-windows-exporter\"}, cluster)",
+            "refresh": 2,
+            "sort": 1,
+            "type": "query"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "hide": 0,
+            "label": "instance",
+            "name": "instance",
+            "query": "label_values(windows_system_system_up_time{cluster=\"$cluster\"}, instance)",
+            "refresh": 2,
+            "type": "query"
+         }
+      ]
+   },
+   "time": {
+      "from": "now-1h",
+      "to": "now"
+   },
+   "timezone": "utc",
+   "title": "USE Method / Node(Windows)",
+   "uid": "96e7484b0bb53b74fbc2bcb7723cd40b"
+}

charts/kubezero-metrics/jsonnet/dashboards/kubelet.json

@@ -35,7 +35,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -70,7 +70,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -105,7 +105,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -140,7 +140,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -175,7 +175,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -210,7 +210,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -261,7 +261,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -312,7 +312,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -363,7 +363,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -414,7 +414,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -473,7 +473,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -532,7 +532,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -583,7 +583,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -634,7 +634,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -685,7 +685,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -736,7 +736,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -787,7 +787,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -838,7 +838,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -889,7 +889,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -940,7 +940,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1015,15 +1015,15 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
                   "type": "prometheus",
                   "uid": "${datasource}"
                },
-               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))",
+               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, le))",
-               "legendFormat": "{{instance}} {{verb}} {{url}}"
+               "legendFormat": "{{instance}} {{verb}}"
             }
          ],
          "title": "Request duration 99th quantile",
@@ -1066,7 +1066,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1117,7 +1117,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -1168,7 +1168,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/namespace-by-pod.json

@@ -53,7 +53,7 @@
             "y": 0
          },
          "id": 1,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -106,7 +106,7 @@
             "y": 0
          },
          "id": 2,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -177,7 +177,7 @@
             "y": 9
          },
          "id": 3,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -316,7 +316,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -362,7 +362,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -408,7 +408,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -454,7 +454,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -500,7 +500,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -546,7 +546,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/namespace-by-workload.json

@@ -39,7 +39,7 @@
             "displayMode": "basic",
             "showUnfilled": false
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -78,7 +78,7 @@
             "displayMode": "basic",
             "showUnfilled": false
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -149,7 +149,7 @@
             "y": 9
          },
          "id": 3,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -335,7 +335,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -386,7 +386,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -437,7 +437,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -488,7 +488,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -539,7 +539,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -590,7 +590,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -641,7 +641,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -692,7 +692,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/persistentvolumesusage.json

@@ -50,7 +50,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -112,7 +112,7 @@
          },
          "id": 2,
          "interval": "1m",
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -162,7 +162,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -224,7 +224,7 @@
          },
          "id": 4,
          "interval": "1m",
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/pod-total.json

@@ -53,7 +53,7 @@
             "y": 0
          },
          "id": 1,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -106,7 +106,7 @@
             "y": 0
          },
          "id": 2,
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -152,7 +152,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -198,7 +198,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -244,7 +244,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -290,7 +290,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -336,7 +336,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -382,7 +382,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/proxy.json

@@ -35,7 +35,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -86,7 +86,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -137,7 +137,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -188,7 +188,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -239,7 +239,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -290,7 +290,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -365,15 +365,15 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
                   "type": "prometheus",
                   "uid": "${datasource}"
                },
-               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, url, le))",
+               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval])) by (verb, le))",
-               "legendFormat": "{{verb}} {{url}}"
+               "legendFormat": "{{verb}}"
             }
          ],
          "title": "Post Request Latency 99th Quantile",
@@ -416,15 +416,15 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
                   "type": "prometheus",
                   "uid": "${datasource}"
                },
-               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))",
+               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, le))",
-               "legendFormat": "{{verb}} {{url}}"
+               "legendFormat": "{{verb}}"
             }
          ],
          "title": "Get Request Latency 99th Quantile",
@@ -467,7 +467,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -518,7 +518,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -569,7 +569,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/scheduler.json

@@ -35,7 +35,7 @@
          "options": {
             "colorMode": "none"
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -86,7 +86,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -161,7 +161,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -236,7 +236,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -311,15 +311,15 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
                   "type": "prometheus",
                   "uid": "${datasource}"
                },
-               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))",
+               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, le))",
-               "legendFormat": "{{verb}} {{url}}"
+               "legendFormat": "{{verb}}"
             }
          ],
          "title": "Post Request Latency 99th Quantile",
@@ -362,15 +362,15 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
                   "type": "prometheus",
                   "uid": "${datasource}"
                },
-               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))",
+               "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, le))",
-               "legendFormat": "{{verb}} {{url}}"
+               "legendFormat": "{{verb}}"
             }
          ],
          "title": "Get Request Latency 99th Quantile",
@@ -413,7 +413,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -464,7 +464,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -515,7 +515,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/dashboards/workload-total.json

@@ -39,7 +39,7 @@
             "displayMode": "basic",
             "showUnfilled": false
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -78,7 +78,7 @@
             "displayMode": "basic",
             "showUnfilled": false
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -117,7 +117,7 @@
             "displayMode": "basic",
             "showUnfilled": false
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -156,7 +156,7 @@
             "displayMode": "basic",
             "showUnfilled": false
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -207,7 +207,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -258,7 +258,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -309,7 +309,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -360,7 +360,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -411,7 +411,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -462,7 +462,7 @@
                "mode": "single"
             }
          },
-         "pluginVersion": "v11.1.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

charts/kubezero-metrics/jsonnet/jsonnetfile.lock.json

@@ -18,8 +18,8 @@
           "subdir": "contrib/mixin"
         }
       },
-      "version": "2b323071a8bd4f02ddaf63e0dfa1fd98c221dccb",
+      "version": "f30cbaac111aa01a310fe75360c759cdd4d9cd14",
-      "sum": "IXI3LQIT9NmTPJAk8WLUJd5+qZfcGpeNCyWIK7oEpws="
+      "sum": "XmXkOCriQIZmXwlIIFhqlJMa0e6qGWdxZD+ZDYaN0Po="
     },
     {
       "source": {
@@ -41,16 +41,6 @@
       "version": "a1d61cce1da59c71409b99b5c7568511fec661ea",
       "sum": "342u++/7rViR/zj2jeJOjshzglkZ1SY+hFNuyCBFMdc="
     },
-    {
-      "source": {
-        "git": {
-          "remote": "https://github.com/grafana/grafonnet-lib.git",
-          "subdir": "grafonnet-7.0"
-        }
-      },
-      "version": "a1d61cce1da59c71409b99b5c7568511fec661ea",
-      "sum": "gCtR9s/4D5fxU9aKXg0Bru+/njZhA0YjLjPiASc61FM="
-    },
     {
       "source": {
         "git": {
@@ -58,8 +48,8 @@
           "subdir": "gen/grafonnet-latest"
         }
       },
-      "version": "82a19822e54a0a12a51e24dbd48fcde717dc0864",
+      "version": "d20e609202733790caf5b554c9945d049f243ae3",
-      "sum": "64fMUPI3frXGj4X1FqFd1t7r04w3CUSmXaDcJ23EYbQ="
+      "sum": "V9vAj21qJOc2DlMPDgB1eEjSQU4A+sAA4AXuJ6bd4xc="
     },
     {
       "source": {
@@ -68,18 +58,18 @@
           "subdir": "gen/grafonnet-v10.0.0"
         }
       },
-      "version": "82a19822e54a0a12a51e24dbd48fcde717dc0864",
+      "version": "d20e609202733790caf5b554c9945d049f243ae3",
       "sum": "xdcrJPJlpkq4+5LpGwN4tPAuheNNLXZjE6tDcyvFjr0="
     },
     {
       "source": {
         "git": {
           "remote": "https://github.com/grafana/grafonnet.git",
-          "subdir": "gen/grafonnet-v11.1.0"
+          "subdir": "gen/grafonnet-v11.4.0"
         }
       },
-      "version": "82a19822e54a0a12a51e24dbd48fcde717dc0864",
+      "version": "d20e609202733790caf5b554c9945d049f243ae3",
-      "sum": "41w7p/rwrNsITqNHMXtGSJAfAyKmnflg6rFhKBduUxM="
+      "sum": "aVAX09paQYNOoCSKVpuk1exVIyBoMt/C50QJI+Q/3nA="
     },
     {
       "source": {
@@ -88,7 +78,7 @@
           "subdir": "grafana-builder"
         }
       },
-      "version": "767befa8fb46a07be516dec2777d7d89909a529d",
+      "version": "d6c38bb26f576b128cadca4137d73a037afdd872",
       "sum": "yxqWcq/N3E/a/XreeU6EuE6X7kYPnG0AspAQFKOjASo="
     },
     {
@@ -108,8 +98,8 @@
           "subdir": ""
         }
       },
-      "version": "63d430b69a95741061c2f7fc9d84b1a778511d9c",
+      "version": "1199b50e9d2ff53d4bb5fb2304ad1fb69d38e609",
-      "sum": "qiZi3axUSXCVzKUF83zSAxklwrnitMmrDK4XAfjPMdE="
+      "sum": "LfbgcJbilu4uBdKYZSvmkoOTPwEAzg10L3/VqKAIWtA="
     },
     {
       "source": {
@@ -118,8 +108,8 @@
           "subdir": ""
         }
       },
-      "version": "a3fbf21977deb89b7d843eb8371170c011ea6835",
+      "version": "e27267571be06c2bdc3d2fd8dbd70161cd709cb4",
-      "sum": "57zW2IGJ9zbYd8BI0qe6JkoWTRSMNiBUWC6+YcnEsWo="
+      "sum": "je1RPCp2aFNefYs5Q57Q5wDm93p8pL4pdBtA5rC7jLA="
     },
     {
       "source": {
@@ -128,8 +118,8 @@
           "subdir": "jsonnet/kube-state-metrics"
         }
       },
-      "version": "32e7727ff4613b0f55dfc18aff15afb8c04d03c5",
+      "version": "2a95d4649b2fea55799032fb9c0b571c4ba7f776",
-      "sum": "lO7jUSzAIy8Yk9pOWJIWgPRhubkWzVh56W6wtYfbVH4="
+      "sum": "3bioG7CfTfY9zeu5xU4yon6Zt3kYvNkyl492nOhQxnM="
     },
     {
       "source": {
@@ -138,7 +128,7 @@
           "subdir": "jsonnet/kube-state-metrics-mixin"
         }
       },
-      "version": "32e7727ff4613b0f55dfc18aff15afb8c04d03c5",
+      "version": "2a95d4649b2fea55799032fb9c0b571c4ba7f776",
       "sum": "qclI7LwucTjBef3PkGBkKxF0mfZPbHnn4rlNWKGtR4c="
     },
     {
@@ -148,8 +138,8 @@
           "subdir": "jsonnet/kube-prometheus"
         }
       },
-      "version": "7e5a571a3fb735c78e17c76a637eb7e8bb5dd086",
+      "version": "1eea946a1532f1e8cccfceea98d907bf3a10b1d9",
-      "sum": "uTw/Mj+X91S+oqUpAX81xcfWPDlox0tdSZY/YBw7nGE="
+      "sum": "17LhiwefVfoNDsF3DcFZw/UL4PMU7YpNNUaOdaYd1gE="
     },
     {
       "source": {
@@ -158,7 +148,7 @@
           "subdir": "jsonnet/mixin"
         }
       },
-      "version": "a366602bacb2c8d773a9cee058b6971b8d2e3732",
+      "version": "465bcbaf2a727c942e7f923aacfb9dff9af8d4a1",
       "sum": "gi+knjdxs2T715iIQIntrimbHRgHnpM8IFBJDD1gYfs=",
       "name": "prometheus-operator-mixin"
     },
@@ -169,8 +159,8 @@
           "subdir": "jsonnet/prometheus-operator"
         }
       },
-      "version": "a366602bacb2c8d773a9cee058b6971b8d2e3732",
+      "version": "465bcbaf2a727c942e7f923aacfb9dff9af8d4a1",
-      "sum": "z0/lCiMusMHTqntsosMVGYkVcSZjCpyZBmUMVUsK5nA="
+      "sum": "LctDdofQostvviE5y8vpRKWGGO1ZKO3dgJe7P9xifW0="
     },
     {
       "source": {
@@ -179,7 +169,7 @@
           "subdir": "doc/alertmanager-mixin"
         }
       },
-      "version": "0f65e8fa5fc72d2678655105c0213b416ca6f34c",
+      "version": "b5d1a64ad5bb0ff879705714d1e40cea82efbd5c",
       "sum": "Mf4h1BYLle2nrgjf/HXrBbl0Zk8N+xaoEM017o0BC+k=",
       "name": "alertmanager"
     },
@@ -190,8 +180,8 @@
           "subdir": "docs/node-mixin"
         }
       },
-      "version": "cf8c6891cc610e54f70383addd4bb6079f0add35",
+      "version": "11365f97bef6cb0e6259d536a7e21c49e3f5c065",
-      "sum": "cQCW+1N0Xae5yXecCWDK2oAlN0luBS/5GrwBYSlaFms="
+      "sum": "xYj6VYFT/eafsbleNlC+Z2VfLy1CndyYrJs9BcTmnX8="
     },
     {
       "source": {
@@ -200,8 +190,8 @@
           "subdir": "documentation/prometheus-mixin"
         }
       },
-      "version": "b407c2930da4f50c0d17fc39404c6302a9eb740b",
+      "version": "509b978f0d675b4c9b3ccf8c0fc06961b0f03e8f",
-      "sum": "OYT5u3S8DbamuJV/v3gbWSteOvFzMeNwMj+u4Apk7jM=",
+      "sum": "2c+wttfee9TwuQJZIkNV7Tekem74Qgc7iZ842P28rNw=",
       "name": "prometheus"
     },
     {
@@ -222,7 +212,7 @@
           "subdir": "mixin"
         }
       },
-      "version": "7037331e6ea7dbe85a1b7af37bf8ea277a80663d",
+      "version": "346d18bb0f8011c63d7106de494cf3b9253161a1",
       "sum": "ieCD4eMgGbOlrI8GmckGPHBGQDcLasE1rULYq56W/bs=",
       "name": "thanos-mixin"
     }

charts/kubezero-metrics/jsonnet/metrics-dashboards.yaml

@@ -6,5 +6,5 @@ dashboards:
   url: https://grafana.com/api/dashboards/9578/revisions/4/download
   tags: []
 - name: Prometheus
-  url: https://grafana.com/api/dashboards/3662/revisions/2/download
+  url: https://grafana.com/api/dashboards/19105/revisions/7/download
   tags: []

charts/kubezero-metrics/jsonnet/rules/alertmanager-prometheusRule

@@ -7,7 +7,7 @@
          "app.kubernetes.io/instance": "main",
          "app.kubernetes.io/name": "alertmanager",
          "app.kubernetes.io/part-of": "kube-prometheus",
-         "app.kubernetes.io/version": "0.27.0",
+         "app.kubernetes.io/version": "0.28.0",
          "prometheus": "k8s",
          "role": "alert-rules"
       },

charts/kubezero-metrics/jsonnet/rules/etcd-mixin-prometheusRule

@@ -20,9 +20,9 @@
                      "summary": "etcd cluster members are down."
                   },
                   "expr": "max without (endpoint) (\n  sum without (instance, pod) (up{job=~\".*etcd.*\"} == bool 0)\nor\n  count without (To) (\n    sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job=~\".*etcd.*\"}[120s])) > 0.01\n  )\n)\n> 0\n",
-                  "for": "10m",
+                  "for": "20m",
                   "labels": {
-                     "severity": "critical"
+                     "severity": "warning"
                   }
                },
                {

charts/kubezero-metrics/jsonnet/rules/kube-state-metrics-prometheusRule

@@ -6,7 +6,7 @@
          "app.kubernetes.io/component": "exporter",
          "app.kubernetes.io/name": "kube-state-metrics",
          "app.kubernetes.io/part-of": "kube-prometheus",
-         "app.kubernetes.io/version": "2.14.0",
+         "app.kubernetes.io/version": "2.15.0",
          "prometheus": "k8s",
          "role": "alert-rules"
       },

charts/kubezero-metrics/jsonnet/rules/kubernetes-prometheusRule

@@ -19,7 +19,7 @@
                {
                   "alert": "KubePodCrashLooping",
                   "annotations": {
-                     "description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason: \"CrashLoopBackOff\").",
+                     "description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason: \"CrashLoopBackOff\") on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodcrashlooping",
                      "summary": "Pod is crash looping."
                   },
@@ -32,7 +32,7 @@
                {
                   "alert": "KubePodNotReady",
                   "annotations": {
-                     "description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.",
+                     "description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready",
                      "summary": "Pod has been in a non-ready state for more than 15 minutes."
                   },
@@ -45,7 +45,7 @@
                {
                   "alert": "KubeDeploymentGenerationMismatch",
                   "annotations": {
-                     "description": "Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.",
+                     "description": "Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentgenerationmismatch",
                      "summary": "Deployment generation mismatch due to possible roll-back"
                   },
@@ -58,7 +58,7 @@
                {
                   "alert": "KubeDeploymentReplicasMismatch",
                   "annotations": {
-                     "description": "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes.",
+                     "description": "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch",
                      "summary": "Deployment has not matched the expected number of replicas."
                   },
@@ -71,7 +71,7 @@
                {
                   "alert": "KubeDeploymentRolloutStuck",
                   "annotations": {
-                     "description": "Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment }} is not progressing for longer than 15 minutes.",
+                     "description": "Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment }} is not progressing for longer than 15 minutes on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentrolloutstuck",
                      "summary": "Deployment rollout is not progressing."
                   },
@@ -84,7 +84,7 @@
                {
                   "alert": "KubeStatefulSetReplicasMismatch",
                   "annotations": {
-                     "description": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.",
+                     "description": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetreplicasmismatch",
                      "summary": "StatefulSet has not matched the expected number of replicas."
                   },
@@ -97,7 +97,7 @@
                {
                   "alert": "KubeStatefulSetGenerationMismatch",
                   "annotations": {
-                     "description": "StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.",
+                     "description": "StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetgenerationmismatch",
                      "summary": "StatefulSet generation mismatch due to possible roll-back"
                   },
@@ -110,7 +110,7 @@
                {
                   "alert": "KubeStatefulSetUpdateNotRolledOut",
                   "annotations": {
-                     "description": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.",
+                     "description": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetupdatenotrolledout",
                      "summary": "StatefulSet update has not been rolled out."
                   },
@@ -123,7 +123,7 @@
                {
                   "alert": "KubeDaemonSetRolloutStuck",
                   "annotations": {
-                     "description": "DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15m.",
+                     "description": "DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15m on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetrolloutstuck",
                      "summary": "DaemonSet rollout is stuck."
                   },
@@ -136,7 +136,7 @@
                {
                   "alert": "KubeContainerWaiting",
                   "annotations": {
-                     "description": "pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour. (reason: \"{{ $labels.reason }}\").",
+                     "description": "pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour. (reason: \"{{ $labels.reason }}\") on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting",
                      "summary": "Pod container waiting longer than 1 hour"
                   },
@@ -149,7 +149,7 @@
                {
                   "alert": "KubeDaemonSetNotScheduled",
                   "annotations": {
-                     "description": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.",
+                     "description": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetnotscheduled",
                      "summary": "DaemonSet pods are not scheduled."
                   },
@@ -162,7 +162,7 @@
                {
                   "alert": "KubeDaemonSetMisScheduled",
                   "annotations": {
-                     "description": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.",
+                     "description": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetmisscheduled",
                      "summary": "DaemonSet pods are misscheduled."
                   },
@@ -175,7 +175,7 @@
                {
                   "alert": "KubeJobNotCompleted",
                   "annotations": {
-                     "description": "Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ \"43200\" | humanizeDuration }} to complete.",
+                     "description": "Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ \"43200\" | humanizeDuration }} to complete on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobnotcompleted",
                      "summary": "Job did not complete in time"
                   },
@@ -187,7 +187,7 @@
                {
                   "alert": "KubeJobFailed",
                   "annotations": {
-                     "description": "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert.",
+                     "description": "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobfailed",
                      "summary": "Job failed to complete."
                   },
@@ -200,7 +200,7 @@
                {
                   "alert": "KubeHpaReplicasMismatch",
                   "annotations": {
-                     "description": "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }} has not matched the desired number of replicas for longer than 15 minutes.",
+                     "description": "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }} has not matched the desired number of replicas for longer than 15 minutes on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpareplicasmismatch",
                      "summary": "HPA has not matched desired number of replicas."
                   },
@@ -213,7 +213,7 @@
                {
                   "alert": "KubeHpaMaxedOut",
                   "annotations": {
-                     "description": "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }} has been running at max replicas for longer than 15 minutes.",
+                     "description": "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }} has been running at max replicas for longer than 15 minutes on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpamaxedout",
                      "summary": "HPA is running at max replicas"
                   },
@@ -257,7 +257,7 @@
                {
                   "alert": "KubeQuotaAlmostFull",
                   "annotations": {
-                     "description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.",
+                     "description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaalmostfull",
                      "summary": "Namespace quota is going to be full."
                   },
@@ -270,7 +270,7 @@
                {
                   "alert": "KubeQuotaFullyUsed",
                   "annotations": {
-                     "description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.",
+                     "description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotafullyused",
                      "summary": "Namespace quota is fully used."
                   },
@@ -283,7 +283,7 @@
                {
                   "alert": "KubeQuotaExceeded",
                   "annotations": {
-                     "description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.",
+                     "description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaexceeded",
                      "summary": "Namespace quota has exceeded the limits."
                   },
@@ -296,7 +296,7 @@
                {
                   "alert": "CPUThrottlingHigh",
                   "annotations": {
-                     "description": "{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.",
+                     "description": "{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }} on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/cputhrottlinghigh",
                      "summary": "Processes experience elevated CPU throttling."
                   },
@@ -396,7 +396,7 @@
                {
                   "alert": "KubeVersionMismatch",
                   "annotations": {
-                     "description": "There are {{ $value }} different semantic versions of Kubernetes components running.",
+                     "description": "There are {{ $value }} different semantic versions of Kubernetes components running on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeversionmismatch",
                      "summary": "Different semantic versions of Kubernetes components running."
                   },
@@ -409,7 +409,7 @@
                {
                   "alert": "KubeClientErrors",
                   "annotations": {
-                     "description": "Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'",
+                     "description": "Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclienterrors",
                      "summary": "Kubernetes API server client is experiencing errors."
                   },
@@ -427,7 +427,7 @@
                {
                   "alert": "KubeAPIErrorBudgetBurn",
                   "annotations": {
-                     "description": "The API server is burning too much error budget.",
+                     "description": "The API server is burning too much error budget on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn",
                      "summary": "The API server is burning too much error budget."
                   },
@@ -442,7 +442,7 @@
                {
                   "alert": "KubeAPIErrorBudgetBurn",
                   "annotations": {
-                     "description": "The API server is burning too much error budget.",
+                     "description": "The API server is burning too much error budget on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn",
                      "summary": "The API server is burning too much error budget."
                   },
@@ -457,7 +457,7 @@
                {
                   "alert": "KubeAPIErrorBudgetBurn",
                   "annotations": {
-                     "description": "The API server is burning too much error budget.",
+                     "description": "The API server is burning too much error budget on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn",
                      "summary": "The API server is burning too much error budget."
                   },
@@ -472,7 +472,7 @@
                {
                   "alert": "KubeAPIErrorBudgetBurn",
                   "annotations": {
-                     "description": "The API server is burning too much error budget.",
+                     "description": "The API server is burning too much error budget on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn",
                      "summary": "The API server is burning too much error budget."
                   },
@@ -518,11 +518,12 @@
                {
                   "alert": "KubeAggregatedAPIErrors",
                   "annotations": {
-                     "description": "Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.",
+                     "description": "Kubernetes aggregated API {{ $labels.instance }}/{{ $labels.name }} has reported {{ $labels.reason }} errors on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors",
                      "summary": "Kubernetes aggregated API has reported errors."
                   },
-                  "expr": "sum by(name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job=\"apiserver\"}[10m])) > 4\n",
+                  "expr": "sum by(cluster, instance, name, reason)(increase(aggregator_unavailable_apiservice_total{job=\"apiserver\"}[1m])) > 0\n",
+                  "for": "10m",
                   "labels": {
                      "severity": "warning"
                   }
@@ -530,7 +531,7 @@
                {
                   "alert": "KubeAggregatedAPIDown",
                   "annotations": {
-                     "description": "Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.",
+                     "description": "Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown",
                      "summary": "Kubernetes aggregated API is down."
                   },
@@ -556,7 +557,7 @@
                {
                   "alert": "KubeAPITerminatedRequests",
                   "annotations": {
-                     "description": "The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.",
+                     "description": "The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests",
                      "summary": "The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests."
                   },
@@ -574,11 +575,11 @@
                {
                   "alert": "KubeNodeNotReady",
                   "annotations": {
-                     "description": "{{ $labels.node }} has been unready for more than 15 minutes.",
+                     "description": "{{ $labels.node }} has been unready for more than 15 minutes on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodenotready",
                      "summary": "Node is not ready."
                   },
-                  "expr": "kube_node_status_condition{job=\"kube-state-metrics\",condition=\"Ready\",status=\"true\"} == 0\n",
+                  "expr": "kube_node_status_condition{job=\"kube-state-metrics\",condition=\"Ready\",status=\"true\"} == 0\nand on (cluster, node)\nkube_node_spec_unschedulable{job=\"kube-state-metrics\"} == 0\n",
                   "for": "15m",
                   "labels": {
                      "severity": "warning"
@@ -587,7 +588,7 @@
                {
                   "alert": "KubeNodeUnreachable",
                   "annotations": {
-                     "description": "{{ $labels.node }} is unreachable and some workloads may be rescheduled.",
+                     "description": "{{ $labels.node }} is unreachable and some workloads may be rescheduled on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodeunreachable",
                      "summary": "Node is unreachable."
                   },
@@ -600,11 +601,11 @@
                {
                   "alert": "KubeletTooManyPods",
                   "annotations": {
-                     "description": "Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.",
+                     "description": "Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubelettoomanypods",
                      "summary": "Kubelet is running at capacity."
                   },
-                  "expr": "count by(cluster, node) (\n  (kube_pod_status_phase{job=\"kube-state-metrics\",phase=\"Running\"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job=\"kube-state-metrics\"})\n)\n/\nmax by(cluster, node) (\n  kube_node_status_capacity{job=\"kube-state-metrics\",resource=\"pods\"} != 1\n) > 0.95\n",
+                  "expr": "(\n  max by (cluster, instance) (\n    kubelet_running_pods{job=\"kubelet\", metrics_path=\"/metrics\"} > 1\n  )\n  * on (cluster, instance) group_left(node)\n  max by (cluster, instance, node) (\n    kubelet_node_name{job=\"kubelet\", metrics_path=\"/metrics\"}\n  )\n)\n/ on (cluster, node) group_left()\nmax by (cluster, node) (\n  kube_node_status_capacity{job=\"kube-state-metrics\", resource=\"pods\"} != 1\n) > 0.95\n",
                   "for": "15m",
                   "labels": {
                      "severity": "info"
@@ -613,11 +614,11 @@
                {
                   "alert": "KubeNodeReadinessFlapping",
                   "annotations": {
-                     "description": "The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.",
+                     "description": "The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodereadinessflapping",
                      "summary": "Node readiness status is flapping."
                   },
-                  "expr": "sum(changes(kube_node_status_condition{job=\"kube-state-metrics\",status=\"true\",condition=\"Ready\"}[15m])) by (cluster, node) > 2\n",
+                  "expr": "sum(changes(kube_node_status_condition{job=\"kube-state-metrics\",status=\"true\",condition=\"Ready\"}[15m])) by (cluster, node) > 2\nand on (cluster, node)\nkube_node_spec_unschedulable{job=\"kube-state-metrics\"} == 0\n",
                   "for": "15m",
                   "labels": {
                      "severity": "warning"
@@ -626,7 +627,7 @@
                {
                   "alert": "KubeletPlegDurationHigh",
                   "annotations": {
-                     "description": "The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.",
+                     "description": "The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }} on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletplegdurationhigh",
                      "summary": "Kubelet Pod Lifecycle Event Generator is taking too long to relist."
                   },
@@ -639,7 +640,7 @@
                {
                   "alert": "KubeletPodStartUpLatencyHigh",
                   "annotations": {
-                     "description": "Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.",
+                     "description": "Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }} on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletpodstartuplatencyhigh",
                      "summary": "Kubelet Pod startup latency is too high."
                   },
@@ -652,7 +653,7 @@
                {
                   "alert": "KubeletClientCertificateExpiration",
                   "annotations": {
-                     "description": "Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.",
+                     "description": "Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }} on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration",
                      "summary": "Kubelet client certificate is about to expire."
                   },
@@ -664,7 +665,7 @@
                {
                   "alert": "KubeletClientCertificateExpiration",
                   "annotations": {
-                     "description": "Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.",
+                     "description": "Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }} on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration",
                      "summary": "Kubelet client certificate is about to expire."
                   },
@@ -676,7 +677,7 @@
                {
                   "alert": "KubeletServerCertificateExpiration",
                   "annotations": {
-                     "description": "Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.",
+                     "description": "Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }} on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration",
                      "summary": "Kubelet server certificate is about to expire."
                   },
@@ -688,7 +689,7 @@
                {
                   "alert": "KubeletServerCertificateExpiration",
                   "annotations": {
-                     "description": "Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.",
+                     "description": "Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }} on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration",
                      "summary": "Kubelet server certificate is about to expire."
                   },
@@ -700,7 +701,7 @@
                {
                   "alert": "KubeletClientCertificateRenewalErrors",
                   "annotations": {
-                     "description": "Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes).",
+                     "description": "Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes) on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificaterenewalerrors",
                      "summary": "Kubelet has failed to renew its client certificate."
                   },
@@ -713,7 +714,7 @@
                {
                   "alert": "KubeletServerCertificateRenewalErrors",
                   "annotations": {
-                     "description": "Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes).",
+                     "description": "Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes) on cluster {{ $labels.cluster }}.",
                      "runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificaterenewalerrors",
                      "summary": "Kubelet has failed to renew its server certificate."
                   },
@@ -809,25 +810,25 @@
                   "record": "cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h"
                },
                {
-                  "expr": "sum by (cluster, verb, scope) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{le=\"+Inf\"} * 24 * 30)\n",
+                  "expr": "sum by (cluster, verb, scope) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{le=\"+Inf\"})\n",
                   "record": "cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d"
                },
                {
-                  "expr": "1 - (\n  (\n    # write too slow\n    sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n    -\n    sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n  ) +\n  (\n    # read too slow\n    sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n    -\n    (\n      (\n        sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n      +\n      sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"})\n    )\n  ) +\n  # errors\n  sum by (cluster) (code:apiserver_request_total:increase30d{code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d)\n",
+                  "expr": "1 - (\n  (\n    # write too slow\n    sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n    -\n    sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=~\"1(\\\\.0)?\"})\n  ) +\n  (\n    # read too slow\n    sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n    -\n    (\n      (\n        sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=~\"1(\\\\.0)?\"})\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=~\"5(\\\\.0)?\"})\n      +\n      sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=~\"30(\\\\.0)?\"})\n    )\n  ) +\n  # errors\n  sum by (cluster) (code:apiserver_request_total:increase30d{code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d)\n",
                   "labels": {
                      "verb": "all"
                   },
                   "record": "apiserver_request:availability30d"
                },
                {
-                  "expr": "1 - (\n  sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n  -\n  (\n    # too slow\n    (\n      sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=\"1\"})\n      or\n      vector(0)\n    )\n    +\n    sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=\"5\"})\n    +\n    sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=\"30\"})\n  )\n  +\n  # errors\n  sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n",
+                  "expr": "1 - (\n  sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~\"LIST|GET\"})\n  -\n  (\n    # too slow\n    (\n      sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=~\"resource|\",le=~\"1(\\\\.0)?\"})\n      or\n      vector(0)\n    )\n    +\n    sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"namespace\",le=~\"5(\\\\.0)?\"})\n    +\n    sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"LIST|GET\",scope=\"cluster\",le=~\"30(\\\\.0)?\"})\n  )\n  +\n  # errors\n  sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"read\"})\n",
                   "labels": {
                      "verb": "read"
                   },
                   "record": "apiserver_request:availability30d"
                },
                {
-                  "expr": "1 - (\n  (\n    # too slow\n    sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n    -\n    sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=\"1\"})\n  )\n  +\n  # errors\n  sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\"})\n",
+                  "expr": "1 - (\n  (\n    # too slow\n    sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~\"POST|PUT|PATCH|DELETE\"})\n    -\n    sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~\"POST|PUT|PATCH|DELETE\",le=~\"1(\\\\.0)?\"})\n  )\n  +\n  # errors\n  sum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\",code=~\"5..\"} or vector(0))\n)\n/\nsum by (cluster) (code:apiserver_request_total:increase30d{verb=\"write\"})\n",
                   "labels": {
                      "verb": "write"
                   },
@@ -869,98 +870,98 @@
             "name": "kube-apiserver-burnrate.rules",
             "rules": [
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[1d]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[1d]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[1d]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[1d]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[1d]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=~\"1(\\\\.0)?\"}[1d]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=~\"5(\\\\.0)?\"}[1d]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=~\"30(\\\\.0)?\"}[1d]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1d]))\n",
                   "labels": {
                      "verb": "read"
                   },
                   "record": "apiserver_request:burnrate1d"
                },
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[1h]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[1h]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[1h]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[1h]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[1h]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=~\"1(\\\\.0)?\"}[1h]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=~\"5(\\\\.0)?\"}[1h]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=~\"30(\\\\.0)?\"}[1h]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[1h]))\n",
                   "labels": {
                      "verb": "read"
                   },
                   "record": "apiserver_request:burnrate1h"
                },
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[2h]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[2h]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[2h]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[2h]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[2h]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=~\"1(\\\\.0)?\"}[2h]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=~\"5(\\\\.0)?\"}[2h]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=~\"30(\\\\.0)?\"}[2h]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[2h]))\n",
                   "labels": {
                      "verb": "read"
                   },
                   "record": "apiserver_request:burnrate2h"
                },
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[30m]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[30m]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[30m]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[30m]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[30m]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=~\"1(\\\\.0)?\"}[30m]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=~\"5(\\\\.0)?\"}[30m]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=~\"30(\\\\.0)?\"}[30m]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[30m]))\n",
                   "labels": {
                      "verb": "read"
                   },
                   "record": "apiserver_request:burnrate30m"
                },
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[3d]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[3d]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[3d]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[3d]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[3d]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=~\"1(\\\\.0)?\"}[3d]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=~\"5(\\\\.0)?\"}[3d]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=~\"30(\\\\.0)?\"}[3d]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[3d]))\n",
                   "labels": {
                      "verb": "read"
                   },
                   "record": "apiserver_request:burnrate3d"
                },
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[5m]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[5m]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[5m]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[5m]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[5m]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=~\"1(\\\\.0)?\"}[5m]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=~\"5(\\\\.0)?\"}[5m]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=~\"30(\\\\.0)?\"}[5m]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[5m]))\n",
                   "labels": {
                      "verb": "read"
                   },
                   "record": "apiserver_request:burnrate5m"
                },
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[6h]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=\"1\"}[6h]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=\"5\"}[6h]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=\"30\"}[6h]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\"}[6h]))\n    -\n    (\n      (\n        sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=~\"resource|\",le=~\"1(\\\\.0)?\"}[6h]))\n        or\n        vector(0)\n      )\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"namespace\",le=~\"5(\\\\.0)?\"}[6h]))\n      +\n      sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"LIST|GET\",subresource!~\"proxy|attach|log|exec|portforward\",scope=\"cluster\",le=~\"30(\\\\.0)?\"}[6h]))\n    )\n  )\n  +\n  # errors\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"LIST|GET\"}[6h]))\n",
                   "labels": {
                      "verb": "read"
                   },
                   "record": "apiserver_request:burnrate6h"
                },
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[1d]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[1d]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1d]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[1d]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=~\"1(\\\\.0)?\"}[1d]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[1d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1d]))\n",
                   "labels": {
                      "verb": "write"
                   },
                   "record": "apiserver_request:burnrate1d"
                },
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[1h]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[1h]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1h]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[1h]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=~\"1(\\\\.0)?\"}[1h]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[1h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[1h]))\n",
                   "labels": {
                      "verb": "write"
                   },
                   "record": "apiserver_request:burnrate1h"
                },
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[2h]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[2h]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[2h]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[2h]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=~\"1(\\\\.0)?\"}[2h]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[2h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[2h]))\n",
                   "labels": {
                      "verb": "write"
                   },
                   "record": "apiserver_request:burnrate2h"
                },
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[30m]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[30m]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[30m]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[30m]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=~\"1(\\\\.0)?\"}[30m]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[30m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[30m]))\n",
                   "labels": {
                      "verb": "write"
                   },
                   "record": "apiserver_request:burnrate30m"
                },
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[3d]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[3d]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[3d]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[3d]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=~\"1(\\\\.0)?\"}[3d]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[3d]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[3d]))\n",
                   "labels": {
                      "verb": "write"
                   },
                   "record": "apiserver_request:burnrate3d"
                },
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[5m]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[5m]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[5m]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[5m]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=~\"1(\\\\.0)?\"}[5m]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[5m]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[5m]))\n",
                   "labels": {
                      "verb": "write"
                   },
                   "record": "apiserver_request:burnrate5m"
                },
                {
-                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[6h]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=\"1\"}[6h]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[6h]))\n",
+                  "expr": "(\n  (\n    # too slow\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\"}[6h]))\n    -\n    sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",subresource!~\"proxy|attach|log|exec|portforward\",le=~\"1(\\\\.0)?\"}[6h]))\n  )\n  +\n  sum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\",code=~\"5..\"}[6h]))\n)\n/\nsum by (cluster) (rate(apiserver_request_total{job=\"apiserver\",verb=~\"POST|PUT|PATCH|DELETE\"}[6h]))\n",
                   "labels": {
                      "verb": "write"
                   },
@@ -1237,6 +1238,144 @@
                   "record": "node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile"
                }
             ]
+         },
+         {
+            "name": "windows.node.rules",
+            "rules": [
+               {
+                  "expr": "count by (cluster) (\n  windows_system_system_up_time{job=\"kubernetes-windows-exporter\"}\n)\n",
+                  "record": "node:windows_node:sum"
+               },
+               {
+                  "expr": "count by (cluster, instance) (sum by (cluster, instance, core) (\n  windows_cpu_time_total{job=\"kubernetes-windows-exporter\"}\n))\n",
+                  "record": "node:windows_node_num_cpu:sum"
+               },
+               {
+                  "expr": "1 - avg by (cluster) (rate(windows_cpu_time_total{job=\"kubernetes-windows-exporter\",mode=\"idle\"}[1m]))\n",
+                  "record": ":windows_node_cpu_utilisation:avg1m"
+               },
+               {
+                  "expr": "1 - avg by (cluster, instance) (\n  rate(windows_cpu_time_total{job=\"kubernetes-windows-exporter\",mode=\"idle\"}[1m])\n)\n",
+                  "record": "node:windows_node_cpu_utilisation:avg1m"
+               },
+               {
+                  "expr": "1 -\nsum by (cluster) (windows_memory_available_bytes{job=\"kubernetes-windows-exporter\"})\n/\nsum by (cluster) (windows_os_visible_memory_bytes{job=\"kubernetes-windows-exporter\"})\n",
+                  "record": ":windows_node_memory_utilisation:"
+               },
+               {
+                  "expr": "sum by (cluster) (windows_memory_available_bytes{job=\"kubernetes-windows-exporter\"} + windows_memory_cache_bytes{job=\"kubernetes-windows-exporter\"})\n",
+                  "record": ":windows_node_memory_MemFreeCached_bytes:sum"
+               },
+               {
+                  "expr": "(windows_memory_cache_bytes{job=\"kubernetes-windows-exporter\"} + windows_memory_modified_page_list_bytes{job=\"kubernetes-windows-exporter\"} + windows_memory_standby_cache_core_bytes{job=\"kubernetes-windows-exporter\"} + windows_memory_standby_cache_normal_priority_bytes{job=\"kubernetes-windows-exporter\"} + windows_memory_standby_cache_reserve_bytes{job=\"kubernetes-windows-exporter\"})\n",
+                  "record": "node:windows_node_memory_totalCached_bytes:sum"
+               },
+               {
+                  "expr": "sum by (cluster) (windows_os_visible_memory_bytes{job=\"kubernetes-windows-exporter\"})\n",
+                  "record": ":windows_node_memory_MemTotal_bytes:sum"
+               },
+               {
+                  "expr": "sum by (cluster, instance) (\n  (windows_memory_available_bytes{job=\"kubernetes-windows-exporter\"})\n)\n",
+                  "record": "node:windows_node_memory_bytes_available:sum"
+               },
+               {
+                  "expr": "sum by (cluster, instance) (\n  windows_os_visible_memory_bytes{job=\"kubernetes-windows-exporter\"}\n)\n",
+                  "record": "node:windows_node_memory_bytes_total:sum"
+               },
+               {
+                  "expr": "(node:windows_node_memory_bytes_total:sum - node:windows_node_memory_bytes_available:sum)\n/\nscalar(sum(node:windows_node_memory_bytes_total:sum))\n",
+                  "record": "node:windows_node_memory_utilisation:ratio"
+               },
+               {
+                  "expr": "1 - (node:windows_node_memory_bytes_available:sum / node:windows_node_memory_bytes_total:sum)\n",
+                  "record": "node:windows_node_memory_utilisation:"
+               },
+               {
+                  "expr": "irate(windows_memory_swap_page_operations_total{job=\"kubernetes-windows-exporter\"}[5m])\n",
+                  "record": "node:windows_node_memory_swap_io_pages:irate"
+               },
+               {
+                  "expr": "avg by (cluster) (irate(windows_logical_disk_read_seconds_total{job=\"kubernetes-windows-exporter\"}[1m]) +\n    irate(windows_logical_disk_write_seconds_total{job=\"kubernetes-windows-exporter\"}[1m])\n  )\n",
+                  "record": ":windows_node_disk_utilisation:avg_irate"
+               },
+               {
+                  "expr": "avg by (cluster, instance) (\n  (irate(windows_logical_disk_read_seconds_total{job=\"kubernetes-windows-exporter\"}[1m]) +\n   irate(windows_logical_disk_write_seconds_total{job=\"kubernetes-windows-exporter\"}[1m]))\n)\n",
+                  "record": "node:windows_node_disk_utilisation:avg_irate"
+               },
+               {
+                  "expr": "max by (cluster,instance,volume)(\n  (windows_logical_disk_size_bytes{job=\"kubernetes-windows-exporter\"}\n- windows_logical_disk_free_bytes{job=\"kubernetes-windows-exporter\"})\n/ windows_logical_disk_size_bytes{job=\"kubernetes-windows-exporter\"}\n)\n",
+                  "record": "node:windows_node_filesystem_usage:"
+               },
+               {
+                  "expr": "max by (cluster, instance, volume) (windows_logical_disk_free_bytes{job=\"kubernetes-windows-exporter\"} / windows_logical_disk_size_bytes{job=\"kubernetes-windows-exporter\"})\n",
+                  "record": "node:windows_node_filesystem_avail:"
+               },
+               {
+                  "expr": "sum by (cluster) (irate(windows_net_bytes_total{job=\"kubernetes-windows-exporter\"}[1m]))\n",
+                  "record": ":windows_node_net_utilisation:sum_irate"
+               },
+               {
+                  "expr": "sum by (cluster, instance) (\n  (irate(windows_net_bytes_total{job=\"kubernetes-windows-exporter\"}[1m]))\n)\n",
+                  "record": "node:windows_node_net_utilisation:sum_irate"
+               },
+               {
+                  "expr": "sum by (cluster) (irate(windows_net_packets_received_discarded_total{job=\"kubernetes-windows-exporter\"}[1m])) +\nsum by (cluster) (irate(windows_net_packets_outbound_discarded_total{job=\"kubernetes-windows-exporter\"}[1m]))\n",
+                  "record": ":windows_node_net_saturation:sum_irate"
+               },
+               {
+                  "expr": "sum by (cluster, instance) (\n  (irate(windows_net_packets_received_discarded_total{job=\"kubernetes-windows-exporter\"}[1m]) +\n  irate(windows_net_packets_outbound_discarded_total{job=\"kubernetes-windows-exporter\"}[1m]))\n)\n",
+                  "record": "node:windows_node_net_saturation:sum_irate"
+               }
+            ]
+         },
+         {
+            "name": "windows.pod.rules",
+            "rules": [
+               {
+                  "expr": "windows_container_available{job=\"kubernetes-windows-exporter\", container_id != \"\"} * on(container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job=\"kube-state-metrics\", container_id != \"\"}) by(container, container_id, pod, namespace, cluster)\n",
+                  "record": "windows_pod_container_available"
+               },
+               {
+                  "expr": "windows_container_cpu_usage_seconds_total{job=\"kubernetes-windows-exporter\", container_id != \"\"} * on(container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job=\"kube-state-metrics\", container_id != \"\"}) by(container, container_id, pod, namespace, cluster)\n",
+                  "record": "windows_container_total_runtime"
+               },
+               {
+                  "expr": "windows_container_memory_usage_commit_bytes{job=\"kubernetes-windows-exporter\", container_id != \"\"} * on(container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job=\"kube-state-metrics\", container_id != \"\"}) by(container, container_id, pod, namespace, cluster)\n",
+                  "record": "windows_container_memory_usage"
+               },
+               {
+                  "expr": "windows_container_memory_usage_private_working_set_bytes{job=\"kubernetes-windows-exporter\", container_id != \"\"} * on(container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job=\"kube-state-metrics\", container_id != \"\"}) by(container, container_id, pod, namespace, cluster)\n",
+                  "record": "windows_container_private_working_set_usage"
+               },
+               {
+                  "expr": "windows_container_network_receive_bytes_total{job=\"kubernetes-windows-exporter\", container_id != \"\"} * on(container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job=\"kube-state-metrics\", container_id != \"\"}) by(container, container_id, pod, namespace, cluster)\n",
+                  "record": "windows_container_network_received_bytes_total"
+               },
+               {
+                  "expr": "windows_container_network_transmit_bytes_total{job=\"kubernetes-windows-exporter\", container_id != \"\"} * on(container_id, cluster) group_left(container, pod, namespace) max(kube_pod_container_info{job=\"kube-state-metrics\", container_id != \"\"}) by(container, container_id, pod, namespace, cluster)\n",
+                  "record": "windows_container_network_transmitted_bytes_total"
+               },
+               {
+                  "expr": "max by (cluster, namespace, pod, container) (\n  kube_pod_container_resource_requests{resource=\"memory\",job=\"kube-state-metrics\"}\n) * on(container,pod,namespace,cluster) (windows_pod_container_available)\n",
+                  "record": "kube_pod_windows_container_resource_memory_request"
+               },
+               {
+                  "expr": "kube_pod_container_resource_limits{resource=\"memory\",job=\"kube-state-metrics\"} * on(container,pod,namespace,cluster) (windows_pod_container_available)\n",
+                  "record": "kube_pod_windows_container_resource_memory_limit"
+               },
+               {
+                  "expr": "max by (cluster, namespace, pod, container) (\n  kube_pod_container_resource_requests{resource=\"cpu\",job=\"kube-state-metrics\"}\n) * on(container,pod,namespace,cluster) (windows_pod_container_available)\n",
+                  "record": "kube_pod_windows_container_resource_cpu_cores_request"
+               },
+               {
+                  "expr": "kube_pod_container_resource_limits{resource=\"cpu\",job=\"kube-state-metrics\"} * on(container,pod,namespace,cluster) (windows_pod_container_available)\n",
+                  "record": "kube_pod_windows_container_resource_cpu_cores_limit"
+               },
+               {
+                  "expr": "sum by (cluster, namespace, pod, container) (\n  rate(windows_container_total_runtime{}[5m])\n)\n",
+                  "record": "namespace_pod_container:windows_container_cpu_usage_seconds_total:sum_rate"
+               }
+            ]
          }
       ]
    }