Merge pull request 'Release v1.25' (#52) from v1.25 into master

Reviewed-on: ZeroDownTime/kubezero#52

.ci/podman.mk

@@ -1,11 +1,22 @@
 # Parse version from latest git semver tag
-BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
+GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
-GIT_TAG=$(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)
+GIT_TAG := $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)
-TAG ?= $(shell echo $(GIT_TAG) | awk -F '-' '{ print $$1 "-" $$2 }' | sed -e 's/-$$//')
+
+TAG := $(GIT_TAG)
+# append branch name to tag if NOT main nor master
+ifeq (,$(filter main master, $(GIT_BRANCH)))
+	# If branch is substring of tag, omit branch name
+	ifeq ($(findstring $(GIT_BRANCH), $(GIT_TAG)),)
+		# only append branch name if not equal tag
+		ifneq ($(GIT_TAG), $(GIT_BRANCH))
+			TAG = $(GIT_TAG)-$(GIT_BRANCH)
+		endif
+	endif
+endif
+
 ARCH := amd64
 ALL_ARCHS := amd64 arm64
-
+_ARCH = $(or $(filter $(ARCH),$(ALL_ARCHS)),$(error $$ARCH [$(ARCH)] must be exactly one of "$(ALL_ARCHS)"))
-# EXTRA_TAGS supposed to be set at the caller, eg. $(shell echo $(TAG) | awk -F '.' '{ print $$1 "." $$2 }')
 
 ifneq ($(TRIVY_REMOTE),)
 	TRIVY_OPTS := --server $(TRIVY_REMOTE)
@@ -22,28 +33,30 @@ help: ## Show Help
 	grep -E '^[a-zA-Z_-]+:.*?## .*$$' .ci/podman.mk | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
 build: ## Build the app
-	buildah build --rm --layers -t $(IMAGE):$(TAG)-$(ARCH) --build-arg TAG=$(TAG) --build-arg ARCH=$(ARCH) --platform linux/$(ARCH) .
+	buildah build --rm --layers -t $(IMAGE):$(TAG)-$(_ARCH) --build-arg TAG=$(TAG) --build-arg ARCH=$(_ARCH) --platform linux/$(_ARCH) .
 
 test: rm-test-image ## Execute Dockerfile.test
 	test -f Dockerfile.test && \
-		{ buildah build --rm --layers -t $(REGISTRY)/$(IMAGE):$(TAG)-test --from=$(REGISTRY)/$(IMAGE):$(TAG) -f Dockerfile.test --platform linux/$(ARCH) . && \
+		{ buildah build --rm --layers -t $(REGISTRY)/$(IMAGE):$(TAG)-test --from=$(REGISTRY)/$(IMAGE):$(TAG) -f Dockerfile.test --platform linux/$(_ARCH) . && \
-			podman run --rm --env-host -t $(REGISTRY)/$(IMAGE):$(TAG)-$(ARCH)-test; } || \
+			podman run --rm --env-host -t $(REGISTRY)/$(IMAGE):$(TAG)-$(_ARCH)-test; } || \
 		echo "No Dockerfile.test found, skipping test"
 
 scan: ## Scan image using trivy
-	echo "Scanning $(IMAGE):$(TAG)-$(ARCH) using Trivy $(TRIVY_REMOTE)"
+	echo "Scanning $(IMAGE):$(TAG)-$(_ARCH) using Trivy $(TRIVY_REMOTE)"
-	trivy image $(TRIVY_OPTS) localhost/$(IMAGE):$(TAG)-$(ARCH)
+	trivy image $(TRIVY_OPTS) localhost/$(IMAGE):$(TAG)-$(_ARCH)
 
 # first tag and push all actual images
 # create new manifest for each tag and add all available TAG-ARCH before pushing
 push: ecr-login ## push images to registry
 	for t in $(TAG) latest $(EXTRA_TAGS); do \
-		buildah tag $(IMAGE):$(TAG)-$(ARCH) $(REGISTRY)/$(IMAGE):$${t}-$(ARCH); \
+		echo "Tagging image with $(REGISTRY)/$(IMAGE):$${t}-$(ARCH)"
+		buildah tag $(IMAGE):$(TAG)-$(_ARCH) $(REGISTRY)/$(IMAGE):$${t}-$(_ARCH); \
 		buildah manifest rm $(IMAGE):$$t || true; \
 		buildah manifest create $(IMAGE):$$t; \
 		for a in $(ALL_ARCHS); do \
 			buildah manifest add $(IMAGE):$$t $(REGISTRY)/$(IMAGE):$(TAG)-$$a; \
 		done; \
+		echo "Pushing manifest $(IMAGE):$$t"
 		buildah manifest push --all $(IMAGE):$$t docker://$(REGISTRY)/$(IMAGE):$$t; \
 	done
 
@@ -58,13 +71,13 @@ rm-remote-untagged: ## delete all remote untagged images
 		[ -n "$$IMAGE_IDS" ] && aws ecr-public batch-delete-image --repository-name $(IMAGE) --region $(REGION) --image-ids $$IMAGE_IDS || echo "No image to remove"
 
 rm-image:
-	test -z "$$(docker image ls -q $(IMAGE):$(TAG)-$(ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(ARCH) > /dev/null
+	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH) > /dev/null
-	test -z "$$(docker image ls -q $(IMAGE):$(TAG)-$(ARCH))" || echo "Error: Removing image failed"
+	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH))" || echo "Error: Removing image failed"
 
 # Ensure we run the tests by removing any previous runs
 rm-test-image:
-	test -z "$$(docker image ls -q $(IMAGE):$(TAG)-$(ARCH)-test)" || podman image rm -f $(IMAGE):$(TAG)-$(ARCH)-test > /dev/null
+	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || podman image rm -f $(IMAGE):$(TAG)-$(_ARCH)-test > /dev/null
-	test -z "$$(docker image ls -q $(IMAGE):$(TAG)-$(ARCH)-test)" || echo "Error: Removing test image failed"
+	test -z "$$(podman image ls -q $(IMAGE):$(TAG)-$(_ARCH)-test)" || echo "Error: Removing test image failed"
 
 ci-pull-upstream: ## pull latest shared .ci subtree
 	git stash && git subtree pull --prefix .ci ssh://git@git.zero-downtime.net/ZeroDownTime/ci-tools-lib.git master --squash && git stash pop

.ci/vars/buildPodman.groovy

@@ -7,12 +7,14 @@ def call(Map config=[:]) {
           label 'podman-aws-trivy'
         }
       }
-
       stages {
         stage('Prepare') {
-          // get tags
           steps {
-            sh 'git fetch -q --tags ${GIT_URL} +refs/heads/${BRANCH_NAME}:refs/remotes/origin/${BRANCH_NAME}'
+            // pull tags
+            withCredentials([gitUsernamePassword(credentialsId: 'gitea-jenkins-user')]) {
+              sh 'git fetch -q --tags ${GIT_URL}'
+            }
+            sh 'make prepare || true'
           }
         }
 
@@ -36,8 +38,7 @@ def call(Map config=[:]) {
             TRIVY_OUTPUT = "reports/trivy.html"
           }
           steps {
-            sh 'mkdir -p reports'
+            sh 'mkdir -p reports && make scan'
-            sh 'make scan'
             publishHTML target: [
               allowMissing: true,
               alwaysLinkToLastBuild: true,
@@ -59,10 +60,11 @@ def call(Map config=[:]) {
           }
         }
 
-        // Push to ECR
+        // Push to container registry, skip if PR
         stage('Push') {
+          when { not { changeRequest() } }
           steps {
-            sh 'make ecr-login push'
+            sh 'make push'
           }
         }
 

Dockerfile

@@ -1,9 +1,9 @@
-ARG ALPINE_VERSION=3.16
+ARG ALPINE_VERSION=3.17
 
-FROM alpine:${ALPINE_VERSION}
+FROM docker.io/alpine:${ALPINE_VERSION}
 
 ARG ALPINE_VERSION
-ARG KUBE_VERSION=1.24
+ARG KUBE_VERSION=1.25
 
 RUN cd /etc/apk/keys && \
     wget "https://cdn.zero-downtime.net/alpine/stefan@zero-downtime.net-61bb6bfb.rsa.pub" && \
@@ -18,19 +18,19 @@ RUN cd /etc/apk/keys && \
       bash \
       python3 \
       py3-yaml \
+      restic \
+      helm \
       cri-tools@kubezero \
       kubeadm@kubezero~=${KUBE_VERSION} \
       kubectl@kubezero~=${KUBE_VERSION} \
       etcdhelper@kubezero \
-      etcd-ctl@edge-testing \
+      etcd-ctl@edge-testing
-      restic@edge-community \
-      helm@edge-community
 
 RUN helm repo add kubezero https://cdn.zero-downtime.net/charts && \
     mkdir -p /var/lib/kubezero
 
 ADD admin/kubezero.sh admin/libhelm.sh admin/migrate_argo_values.py /usr/bin
-ADD admin/libhelm.sh admin/v${KUBE_VERSION}/* /var/lib/kubezero
+ADD admin/libhelm.sh admin/pre-upgrade.sh /var/lib/kubezero
 
 ADD charts/kubeadm /charts/kubeadm
 ADD charts/kubezero /charts/kubezero

Makefile

@@ -3,7 +3,8 @@ IMAGE := kubezero-admin
 REGION := us-east-1
 
 # Also tag as Kubernetes major version
-EXTRA_TAGS = $(shell echo $(TAG) | awk -F '.' '{ print $$1 "." $$2 }')
+MY_TAG = $(shell git describe --tags --match v*.*.* 2>/dev/null || git rev-parse --short HEAD 2>/dev/null)
+EXTRA_TAGS = $(shell echo $(MY_TAG) | awk -F '.' '{ print $$1 "." $$2 }')
 
 include .ci/podman.mk
 

README.md

@@ -19,7 +19,7 @@ KubeZero is a Kubernetes distribution providing an integrated container platform
 
 # Version / Support Matrix
 KubeZero releases track the same *minor* version of Kubernetes.  
-Any 1.24.X-Y release of Kubezero supports any Kubernetes cluster 1.24.X.
+Any 1.26.X-Y release of Kubezero supports any Kubernetes cluster 1.26.X.
 
 KubeZero is distributed as a collection of versioned Helm charts, allowing custom upgrade schedules and module versions as needed.
 
@@ -28,15 +28,15 @@ KubeZero is distributed as a collection of versioned Helm charts, allowing custo
 gantt
     title KubeZero Support Timeline
     dateFormat  YYYY-MM-DD
-    section 1.23
-    beta     :123b, 2022-08-01, 2022-09-01
-    release  :after 123b, 2023-02-01
     section 1.24
     beta     :124b, 2022-11-14, 2022-12-31
     release  :after 124b, 2023-06-01
     section 1.25
     beta     :125b, 2023-03-01, 2023-03-31
     release  :after 125b, 2023-08-01
+    section 1.26
+    beta     :126b, 2023-06-01, 2023-06-30
+    release  :after 126b, 2023-10-01
 ```
 
 [Upstream release policy](https://kubernetes.io/releases/)
@@ -44,7 +44,7 @@ gantt
 # Components
 
 ## OS
-- all nodes are based on Alpine V3.16
+- all nodes are based on Alpine V3.17
 - 2 GB encrypted root filesystem
 - no 3rd party dependencies at boot ( other than container registries )
 - minimal attack surface
@@ -73,10 +73,8 @@ gantt
 - support for [Inf1 instances](https://aws.amazon.com/ec2/instance-types/inf1/) part of [AWS Neuron](https://aws.amazon.com/machine-learning/neuron/).
 
 ## Network
+- Cilium using Geneve encapsulation, incl. increased MTU allowing flexible / more containers per worker node compared to eg. AWS VPC CNI
 - Multus support for multiple network interfaces per pod, eg. additional AWS CNI
-- Calico using VxLAN incl. increased MTU  
-allows flexible / more containers per worker node compared to eg. AWS VPC CNI
-- isolates container traffic from VPC by using VxLAN overlay
 - no restrictions on IP space / sizing from the underlying VPC architecture
 
 ## Storage
@@ -95,7 +93,7 @@ allows flexible / more containers per worker node compared to eg. AWS VPC CNI
 - optional full service mesh
 
 ## Metrics
-- Prometheus support for all components
+- Prometheus support for all components, incl. out of cluster EC2 instances (node_exporter)
 - automated service discovery allowing instant access to common workload metrics
 - pre-configured Grafana dashboards and alerts
 - Alertmanager events via SNSAlertHub to Slack, Google, Matrix, etc.
@@ -105,4 +103,4 @@ allows flexible / more containers per worker node compared to eg. AWS VPC CNI
 - flexible ElasticSearch setup, leveraging the ECK operator, for easy maintenance & minimal admin knowledge required, incl. automated backups to S3
 - Kibana allowing easy search and dashboards for all logs, incl. pre configured index templates and index management
 - [fluentd-concerter](https://git.zero-downtime.net/ZeroDownTime/container-park/src/branch/master/fluentd-concenter) service providing queuing during highload as well as additional parsing options
-- lightweight fluent-bit agents on each node requiring minimal resources forwarding logs secure via TLS to fluentd-concenter
+- lightweight fluent-bit agents on each node requiring minimal resources forwarding logs secure via TLS to fluentd-concenter

admin/kubezero.sh

@@ -161,7 +161,7 @@ control_plane_node() {
 
   else
     # restore latest backup
-    retry 10 60 30 restic restore latest --no-lock -t / #Review: Use latest no matter what for now: --tag $KUBE_VERSION_MINOR
+    retry 10 60 30 restic restore latest --no-lock -t / # --tag $KUBE_VERSION_MINOR
 
     # Make last etcd snapshot available
     cp ${WORKDIR}/etcd_snapshot ${HOSTFS}/etc/kubernetes
@@ -214,30 +214,16 @@ control_plane_node() {
       sleep 3
     done
 
-    # if we are NOT member already, flush etcd to be able to join
+    # see if we are a former member and remove our former self if so
     MY_ID=$(etcdctl member list --endpoints=$etcd_endpoints | grep $ETCD_NODENAME | awk '{print $1}' | sed -e 's/,$//')
+    [ -n "$MY_ID" ] && retry 12 5 5 etcdctl member remove $MY_ID --endpoints=$etcd_endpoints
 
-    # Failsafe / etcd on ephmeral: we were a member but our dataset is missing
+    # flush etcd data directory as joining with previous storage seems flaky, especially during etcd version upgrades
-    # -> remove former self so we can re-join
+    rm -rf ${HOSTFS}/var/lib/etcd/member
-    if [ -n "$MY_ID" -a ! -d ${HOSTFS}/var/lib/etcd/member ]; then
-      # Remove former self first
-      [ -n "$MY_ID" ] && retry 12 5 5 etcdctl member remove $MY_ID --endpoints=$etcd_endpoints
-      MY_ID=""
-    fi
 
-
+    # Announce new etcd member and capture ETCD_INITIAL_CLUSTER, retry needed in case another node joining causes temp quorum loss
-    if [ -z "$MY_ID" ]; then
+    ETCD_ENVS=$(retry 12 5 5 etcdctl member add $ETCD_NODENAME --peer-urls="https://${ETCD_NODENAME}:2380" --endpoints=$etcd_endpoints)
-      # flush etcd data directory from restore
+    export $(echo "$ETCD_ENVS" | grep ETCD_INITIAL_CLUSTER= | sed -e 's/"//g')
-      rm -rf ${HOSTFS}/var/lib/etcd/member
-
-      # Announce new etcd member and capture ETCD_INITIAL_CLUSTER, retry needed in case another node joining causes temp quorum loss
-      ETCD_ENVS=$(retry 12 5 5 etcdctl member add $ETCD_NODENAME --peer-urls="https://${ETCD_NODENAME}:2380" --endpoints=$etcd_endpoints)
-      export $(echo "$ETCD_ENVS" | grep ETCD_INITIAL_CLUSTER= | sed -e 's/"//g')
-    else
-      # build initial_cluster string from running cluster
-      _cluster=$(etcdctl member list --endpoints=$etcd_endpoints -w json | jq -r '.members[] | "\(.name)=\(.peerURLs[]),"')
-      export ETCD_INITIAL_CLUSTER=$(echo ${_cluster%%,} | sed -e 's/ //g')
-    fi
 
     # Patch kubeadm-values.yaml and re-render to get etcd manifest patched
     yq eval -i '.etcd.state = "existing"
@@ -358,7 +344,6 @@ backup() {
   cp -r ${HOSTFS}/etc/kubernetes/admin.conf ${WORKDIR}
 
   # Backup via restic
-  restic snapshots || restic init
   restic backup ${WORKDIR} -H $CLUSTERNAME --tag $CLUSTER_VERSION
 
   echo "Backup complete."

admin/libhelm.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Simulate well-known CRDs being available
-API_VERSIONS="-a monitoring.coreos.com/v1 -a snapshot.storage.k8s.io/v1"
+API_VERSIONS="-a monitoring.coreos.com/v1 -a snapshot.storage.k8s.io/v1 -a policy/v1/PodDisruptionBudget"
 
 # Waits for max 300s and retries
 function wait_for() {
@@ -166,6 +166,9 @@ function _helm() {
     render
     kubectl $action -f $WORKDIR/helm.yaml --server-side --force-conflicts && rc=$? || rc=$?
 
+    # Try again without server-side, review with 1.26, required for cert-manager during 1.25
+    [ $rc -ne 0 ] && kubectl $action -f $WORKDIR/helm.yaml && rc=$? || rc=$?
+
     # Optional post hook
     declare -F ${module}-post && ${module}-post
 

admin/migrate_argo_values.py

@@ -8,27 +8,11 @@ import yaml
 def migrate(values):
     """Actual changes here"""
 
-    # ClusterBackup is enabled on AWS anyways, same with cluster-autoscaler
+    # Remove various keys as they have been merged into the metrics template
-    if "aws" in values["global"]:
+    deleteKey(values["metrics"]['kube-prometheus-stack']["alertmanager"]["alertmanagerSpec"], "podMetadata")
-        deleteKey(values["addons"], "clusterBackup")
+    deleteKey(values["metrics"]['kube-prometheus-stack']["alertmanager"], "config")
-        deleteKey(values["addons"], "cluster-autoscaler")
+    deleteKey(values["metrics"]['kube-prometheus-stack']["prometheus"]["prometheusSpec"], "externalLabels")
-
+        
-    # Remove calico and multus
-    deleteKey(values["network"], "calico")
-    deleteKey(values["network"], "multus")
-
-    # ArgoCD helm changes
-    if "argocd" in values:
-        if "server" in values["argocd"]:
-            if not "configs" in values["argocd"]:
-                values["argocd"]["configs"] = {}
-            if not "cm" in values["argocd"]["configs"]:
-                values["argocd"]["configs"]["cm"] = {}
-            values["argocd"]["configs"]["cm"]["url"] = values["argocd"]["server"]["config"][
-                "url"
-            ]
-            deleteKey(values["argocd"], "server")
-
     return values
 
 

admin/upgrade_cluster.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -e
 
 #VERSION="latest"
-VERSION="v1.24"
+VERSION="v1.25"
 ARGO_APP=${1:-/tmp/new-kubezero-argoapp.yaml}
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
@@ -10,7 +10,6 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 [ -n "$DEBUG" ] && set -x
 
-
 all_nodes_upgrade() {
   CMD="$1"
 
@@ -148,40 +147,37 @@ argo_used && disable_argo
 
 #all_nodes_upgrade ""
 
+# Cleanup
+# Remove calico CRDs
+kubectl delete -f https://git.zero-downtime.net/ZeroDownTime/kubezero/raw/tag/v1.23.11/charts/kubezero-network/charts/calico/crds/crds.yaml 2>/dev/null || true
+kubectl delete servicemonitor calico-node -n kube-system 2>/dev/null || true
+
+# delete old kubelet configs
+for cm in $(kubectl get cm -n kube-system --no-headers |  awk '{if ($1 ~ "kubelet-config-1*") print $1}'); do kubectl delete cm $cm -n kube-system; done
+for rb in $(kubectl get rolebindings -n kube-system --no-headers |  awk '{if ($1 ~ "kubelet-config-1*") print $1}'); do kubectl delete rolebindings $rb -n kube-system; done
+
 control_plane_upgrade kubeadm_upgrade
 
 echo "Adjust kubezero values as needed:"
 # shellcheck disable=SC2015
 argo_used && kubectl edit app kubezero -n argocd || kubectl edit cm kubezero-values -n kube-system
 
-# Remove calico
-#kubectl delete deployment calico-kube-controllers -n kube-system || true
-#kubectl delete daemonset calico-node -n kube-system || true
-#kubectl delete network-attachment-definitions calico -n kube-system || true
-
-# Remove previous cilium config as the helm options are additive only -> fail
-kubectl delete configmap cilium-config -n kube-system || true
-
 control_plane_upgrade "apply_network, apply_addons, apply_storage"
 
-kubectl rollout restart daemonset/kube-multus-ds -n kube-system
-kubectl rollout restart daemonset/cilium -n kube-system
-
 echo "Checking that all pods in kube-system are running ..."
 waitSystemPodsRunning
 
 echo "Applying remaining KubeZero modules..."
 
-# delete argocd deployments as various immutable things changed, also redis restart fails otherwise
-kubectl delete deployment argocd-redis -n argocd || true
-kubectl delete deployment argocd-repo-server -n argocd || true
-kubectl delete statefulset argocd-application-controller -n argocd || true
-
-# Delete prometheus-push gateway due to label changes
-kubectl delete deploy -l app=prometheus-pushgateway -n monitoring || true
-
 control_plane_upgrade "apply_cert-manager, apply_istio, apply_istio-ingress, apply_istio-private-ingress, apply_logging, apply_metrics, apply_argocd"
 
+# Trigger backup of upgraded cluster state
+kubectl create job --from=cronjob/kubezero-backup kubezero-backup-$VERSION -n kube-system
+while true; do
+  kubectl wait --for=condition=complete job/kubezero-backup-$VERSION -n kube-system 2>/dev/null && kubectl delete job kubezero-backup-$VERSION -n kube-system && break
+  sleep 1
+done
+
 # Final step is to commit the new argocd kubezero app
 kubectl get app kubezero -n argocd -o yaml | yq 'del(.status) | del(.metadata) | del(.operation) | .metadata.name="kubezero" | .metadata.namespace="argocd"' | yq 'sort_keys(..) | .spec.source.helm.values |= (from_yaml | to_yaml)' > $ARGO_APP
 

charts/kubeadm/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubeadm
 description: KubeZero Kubeadm cluster config
 type: application
-version: 1.24.9
+version: 1.25.8
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -11,4 +11,4 @@ keywords:
 maintainers:
   - name: Stefan Reimer
     email: stefan@zero-downtime.net
-kubeVersion: ">= 1.24.0"
+kubeVersion: ">= 1.25.0"

charts/kubeadm/README.md

@@ -1,6 +1,6 @@
 # kubeadm
 
-![Version: 1.24.9](https://img.shields.io/badge/Version-1.24.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 1.25.8](https://img.shields.io/badge/Version-1.25.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero Kubeadm cluster config
 
@@ -14,7 +14,7 @@ KubeZero Kubeadm cluster config
 
 ## Requirements
 
-Kubernetes: `>= 1.24.0`
+Kubernetes: `>= 1.25.0`
 
 ## Values
 

charts/kubeadm/templates/_helpers.tpl

@@ -1,6 +1,6 @@
 {{- /* Feature gates for all control plane components */ -}}
 {{- define "kubeadm.featuregates" }}
-{{- $gates := list "CustomCPUCFSQuotaPeriod" "CronJobTimeZone" "NodeOutOfServiceVolumeDetach" }}
+{{- $gates := list "CustomCPUCFSQuotaPeriod" "NodeOutOfServiceVolumeDetach" }}
 {{- if eq .return "csv" }}
 {{- range $key := $gates }}
 {{- $key }}=true,

charts/kubeadm/templates/patches/coredns0.yaml

@@ -12,13 +12,3 @@ spec:
             memory: 128Mi
       nodeSelector:
         node-role.kubernetes.io/control-plane: ""
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-              - key: k8s-app
-                operator: In
-                values:
-                - kube-dns
-            topologyKey: "kubernetes.io/hostname"

charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml

@@ -1,6 +1,6 @@
 {{- if .Values.api.awsIamAuth.enabled }}
-kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
 metadata:
   name: aws-iam-authenticator
 rules:
@@ -51,8 +51,8 @@ metadata:
   namespace: kube-system
 
 ---
-kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
 metadata:
   name: aws-iam-authenticator
   namespace: kube-system

charts/kubezero-addons/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: kubezero-addons
 description: KubeZero umbrella chart for various optional cluster addons
 type: application
-version: 0.7.3
+version: 0.7.5
-appVersion: v1.24
+appVersion: v1.25
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -20,28 +20,28 @@ maintainers:
     email: stefan@zero-downtime.net
 dependencies:
   - name: external-dns
-    version: 1.11.0
+    version: 1.12.2
     repository: https://kubernetes-sigs.github.io/external-dns/
     condition: external-dns.enabled
   - name: cluster-autoscaler
-    version: 9.21.0
+    version: 9.28.0
     repository: https://kubernetes.github.io/autoscaler
     condition: cluster-autoscaler.enabled
   - name: nvidia-device-plugin
-    version: 0.13.0
+    version: 0.14.0
     # https://github.com/NVIDIA/k8s-device-plugin
     repository: https://nvidia.github.io/k8s-device-plugin
     condition: nvidia-device-plugin.enabled
   - name: sealed-secrets
-    version: 2.7.1
+    version: 2.8.1
     repository: https://bitnami-labs.github.io/sealed-secrets
     condition: sealed-secrets.enabled
   - name: aws-node-termination-handler
-    version: 0.20.1
+    version: 0.21.0
     # repository: https://aws.github.io/eks-charts
     condition: aws-node-termination-handler.enabled
   - name: aws-eks-asg-rolling-update-handler
-    version: 1.2.7
+    version: 1.3.0
     # repository: https://twin.github.io/helm-charts
     condition: aws-eks-asg-rolling-update-handler.enabled
-kubeVersion: ">= 1.24.0"
+kubeVersion: ">= 1.25.0"

charts/kubezero-addons/README.md

@@ -1,6 +1,6 @@
 # kubezero-addons
 
-![Version: 0.7.3](https://img.shields.io/badge/Version-0.7.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.24](https://img.shields.io/badge/AppVersion-v1.24-informational?style=flat-square)
+![Version: 0.7.5](https://img.shields.io/badge/Version-0.7.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.25](https://img.shields.io/badge/AppVersion-v1.25-informational?style=flat-square)
 
 KubeZero umbrella chart for various optional cluster addons
 
@@ -14,16 +14,16 @@ KubeZero umbrella chart for various optional cluster addons
 
 ## Requirements
 
-Kubernetes: `>= 1.24.0`
+Kubernetes: `>= 1.25.0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-|  | aws-eks-asg-rolling-update-handler | 1.2.7 |
+|  | aws-eks-asg-rolling-update-handler | 1.3.0 |
-|  | aws-node-termination-handler | 0.20.1 |
+|  | aws-node-termination-handler | 0.21.0 |
-| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.7.1 |
+| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.8.1 |
-| https://kubernetes-sigs.github.io/external-dns/ | external-dns | 1.11.0 |
+| https://kubernetes-sigs.github.io/external-dns/ | external-dns | 1.12.2 |
-| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.21.0 |
+| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.28.0 |
-| https://nvidia.github.io/k8s-device-plugin | nvidia-device-plugin | 0.13.0 |
+| https://nvidia.github.io/k8s-device-plugin | nvidia-device-plugin | 0.14.0 |
 
 # MetalLB   
    
@@ -103,8 +103,11 @@ Device plugin for [AWS Neuron](https://aws.amazon.com/machine-learning/neuron/)
 | cluster-autoscaler.autoDiscovery.clusterName | string | `""` |  |
 | cluster-autoscaler.awsRegion | string | `"us-west-2"` |  |
 | cluster-autoscaler.enabled | bool | `false` |  |
+| cluster-autoscaler.extraArgs.balance-similar-node-groups | bool | `true` |  |
+| cluster-autoscaler.extraArgs.ignore-taint | string | `"node.cilium.io/agent-not-ready"` |  |
 | cluster-autoscaler.extraArgs.scan-interval | string | `"30s"` |  |
 | cluster-autoscaler.extraArgs.skip-nodes-with-local-storage | bool | `false` |  |
+| cluster-autoscaler.image.tag | string | `"v1.25.1"` |  |
 | cluster-autoscaler.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` |  |
 | cluster-autoscaler.podDisruptionBudget | bool | `false` |  |
 | cluster-autoscaler.prometheusRule.enabled | bool | `false` |  |
@@ -139,6 +142,10 @@ Device plugin for [AWS Neuron](https://aws.amazon.com/machine-learning/neuron/)
 | nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"node.kubernetes.io/instance-type"` |  |
 | nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator | string | `"In"` |  |
 | nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0] | string | `"g5.xlarge"` |  |
+| nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[10] | string | `"g4dn.4xlarge"` |  |
+| nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[11] | string | `"g4dn.8xlarge"` |  |
+| nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[12] | string | `"g4dn.12xlarge"` |  |
+| nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[13] | string | `"g4dn.16xlarge"` |  |
 | nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[1] | string | `"g5.2xlarge"` |  |
 | nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[2] | string | `"g5.4xlarge"` |  |
 | nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[3] | string | `"g5.8xlarge"` |  |
@@ -146,6 +153,8 @@ Device plugin for [AWS Neuron](https://aws.amazon.com/machine-learning/neuron/)
 | nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[5] | string | `"g5.16xlarge"` |  |
 | nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[6] | string | `"g5.24xlarge"` |  |
 | nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[7] | string | `"g5.48xlarge"` |  |
+| nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[8] | string | `"g4dn.xlarge"` |  |
+| nvidia-device-plugin.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[9] | string | `"g4dn.2xlarge"` |  |
 | nvidia-device-plugin.enabled | bool | `false` |  |
 | nvidia-device-plugin.tolerations[0].effect | string | `"NoSchedule"` |  |
 | nvidia-device-plugin.tolerations[0].key | string | `"nvidia.com/gpu"` |  |
@@ -155,7 +164,7 @@ Device plugin for [AWS Neuron](https://aws.amazon.com/machine-learning/neuron/)
 | nvidia-device-plugin.tolerations[1].operator | string | `"Exists"` |  |
 | sealed-secrets.enabled | bool | `false` |  |
 | sealed-secrets.fullnameOverride | string | `"sealed-secrets-controller"` |  |
-| sealed-secrets.keyrenewperiod | int | `0` |  |
+| sealed-secrets.keyrenewperiod | string | `"0"` |  |
 | sealed-secrets.metrics.serviceMonitor.enabled | bool | `false` |  |
 | sealed-secrets.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` |  |
 | sealed-secrets.resources.limits.memory | string | `"128Mi"` |  |

charts/kubezero-addons/charts/aws-eks-asg-rolling-update-handler/Chart.yaml

@@ -5,4 +5,4 @@ home: https://github.com/TwiN/aws-eks-asg-rolling-update-handler
 maintainers:
 - name: TwiN
 name: aws-eks-asg-rolling-update-handler
-version: 1.2.7
+version: 1.3.0

charts/kubezero-addons/charts/aws-eks-asg-rolling-update-handler/README.md

@@ -0,0 +1,13 @@
+# aws-eks-asg-rolling-update-handler
+
+## Configuration
+The following table lists the configurable parameters of the aws-eks-asg-rolling-update-handler chart and their default values.
+| Parameters | Description | Required | Default     |
+|:-----------|:------------|:---------|:------------|
+| environmentVars  | environment variables for aws-eks-asg-rolling-update-handler container, available variables are listed [here](https://github.com/TwiN/aws-eks-asg-rolling-update-handler/blob/master/README.md#usage) | yes |`[{"name":"CLUSTER_NAME","value":"cluster-name"}]`|
+| replicaCount | Number of aws-eks-asg-rolling-update-handler replicas | yes |`1` |
+| image.repository | Image repository | yes |  `twinproduction/aws-eks-asg-rolling-update-handler` |
+| image.tag | image tag | yes | `v1.4.3` |
+| image.pullPolicy | Image pull policy | yes | `IfNotPresent` |
+| resources | CPU/memory resource requests/limits | no | `{}` |
+| podAnnotations | Annotations to add to the aws-eks-asg-rolling-update-handler pod configuration | no | `{}` |

charts/kubezero-addons/charts/aws-eks-asg-rolling-update-handler/templates/deployment.yaml

@@ -15,6 +15,10 @@ spec:
     metadata:
       labels:
 {{ include "aws-eks-asg-rolling-update-handler.labels" . | indent 8 }}
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       automountServiceAccountToken: true
       serviceAccountName: {{ template "aws-eks-asg-rolling-update-handler.serviceAccountName" . }}
@@ -25,11 +29,11 @@ spec:
           image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           env:
-            {{- toYaml .Values.environmentVars | nindent 12 }}
+{{- toYaml .Values.environmentVars | nindent 12 }}
-          {{- with .Values.resources }}
+{{- with .Values.resources }}
           resources:
-            {{- toYaml . | nindent 12 }}
+{{- toYaml . | nindent 12 }}
-          {{- end }}
+{{- end }}
           volumeMounts:
             - name: aws-token
               mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/"
@@ -52,5 +56,5 @@ spec:
       {{- end }}
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
-{{- toYaml . | nindent 8 }}
+        {{- toYaml . | nindent 8 }}
       {{- end }}

charts/kubezero-addons/charts/aws-eks-asg-rolling-update-handler/values.yaml

@@ -2,7 +2,7 @@ replicaCount: 1
 
 image:
   repository: twinproduction/aws-eks-asg-rolling-update-handler
-  tag: v1.4.3
+  tag: v1.7.0
   pullPolicy: IfNotPresent
 
 #imagePullSecrets:
@@ -22,6 +22,17 @@ environmentVars:
 #- name: ENVIRONMENT
 #  value: ""
 
+resources: {}
+  # limits:
+  #   cpu: 0.3
+  #   memory: 100Mi
+  # requests:
+  #   cpu: 0.1
+  #   memory: 50Mi
+podAnnotations: {}
+  # prometheus.io/port: "8080"
+  # prometheus.io/scrape: "true"
+
 serviceAccount:
   create: true
   #name: aws-eks-asg-rolling-update-handler

charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 1.18.1
+appVersion: 1.19.0
 description: A Helm chart for the AWS Node Termination Handler.
 home: https://github.com/aws/eks-charts
 icon: https://raw.githubusercontent.com/aws/eks-charts/master/docs/logo/aws.png
@@ -22,4 +22,4 @@ sources:
 - https://github.com/aws/aws-node-termination-handler/
 - https://github.com/aws/eks-charts/
 type: application
-version: 0.20.1
+version: 0.21.0

charts/kubezero-addons/charts/aws-node-termination-handler/README.md

@@ -56,7 +56,7 @@ The configuration in this table applies to all AWS Node Termination Handler mode
 | `serviceAccount.name`              | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the full name template.                                                                                                                                                                                                                                                        | `nil`                                                 |
 | `serviceAccount.annotations`       | Annotations to add to the service account.                                                                                                                                                                                                                                                                                                                                             | `{}`                                                  |
 | `rbac.create`                      | If `true`, create the RBAC resources.                                                                                                                                                                                                                                                                                                                                                  | `true`                                                |
-| `rbac.pspEnabled`                  | If `true`, create a pod security policy resource.                                                                                                                                                                                                                                                                                                                                      | `true`                                                |
+| `rbac.pspEnabled`                  | If `true`, create a pod security policy resource. Note: `PodSecurityPolicy`s will not be created when Kubernetes version is 1.25 or later.                                                                                                                                                                                                                                                                                     | `true`                                                |
 | `customLabels`                     | Labels to add to all resource metadata.                                                                                                                                                                                                                                                                                                                                                | `{}`                                                  |
 | `podLabels`                        | Labels to add to the pod.                                                                                                                                                                                                                                                                                                                                                              | `{}`                                                  |
 | `podAnnotations`                   | Annotations to add to the pod.                                                                                                                                                                                                                                                                                                                                                         | `{}`                                                  |
@@ -123,7 +123,7 @@ The configuration in this table applies to AWS Node Termination Handler in queue
 The configuration in this table applies to AWS Node Termination Handler in IMDS mode.
 
 | Parameter                        | Description                                                                                                                                                                                                                                                   | Default                |
-| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------- |
+| -------------------------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|
 | `targetNodeOs`                   | Space separated list of node OS's to target (e.g. `"linux"`, `"windows"`, `"linux windows"`). Windows support is **EXPERIMENTAL**.                                                                                                                            | `"linux"`              |
 | `linuxPodLabels`                 | Labels to add to each Linux pod.                                                                                                                                                                                                                              | `{}`                   |
 | `windowsPodLabels`               | Labels to add to each Windows pod.                                                                                                                                                                                                                            | `{}`                   |
@@ -138,7 +138,7 @@ The configuration in this table applies to AWS Node Termination Handler in IMDS
 | `podMonitor.sampleLimit`         | Number of scraped samples accepted.                                                                                                                                                                                                                           | `5000`                 |
 | `useHostNetwork`                 | If `true`, enables `hostNetwork` for the Linux DaemonSet. NOTE: setting this to `false` may cause issues accessing IMDSv2 if your account is not configured with an IP hop count of 2 see [Metrics Endpoint Considerations](#metrics-endpoint-considerations) | `true`                 |
 | `dnsPolicy`                      | If specified, this overrides `linuxDnsPolicy` and `windowsDnsPolicy` with a single policy.                                                                                                                                                                    | `""`                   |
-| `dnsConfig`                      | If specified, this sets the dnsConfig: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config                                                                                                                                                                    | `{}`                   |
+| `dnsConfig`                      | If specified, this sets the dnsConfig: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config                                                                                                                                | `{}`                   |
 | `linuxDnsPolicy`                 | DNS policy for the Linux DaemonSet.                                                                                                                                                                                                                           | `""`                   |
 | `windowsDnsPolicy`               | DNS policy for the Windows DaemonSet.                                                                                                                                                                                                                         | `""`                   |
 | `daemonsetNodeSelector`          | Expressions to select a node by it's labels for DaemonSet pod assignment. For backwards compatibility the `nodeSelector` value has priority over this but shouldn't be used.                                                                                  | `{}`                   |
@@ -152,10 +152,10 @@ The configuration in this table applies to AWS Node Termination Handler in IMDS
 | `windowsTolerations`             | Override `daemonsetTolerations` for the Linux DaemonSet.                                                                                                                                                                                                      | `[]`                   |
 | `enableProbesServer`             | If `true`, start an http server exposing `/healthz` endpoint for probes.                                                                                                                                                                                      | `false`                |
 | `metadataTries`                  | The number of times to try requesting metadata.                                                                                                                                                                                                               | `3`                    |
-| `enableSpotInterruptionDraining` | If `true`, drain nodes when the spot interruption termination notice is received.                                                                                                                                                                             | `true`                 |
+| `enableSpotInterruptionDraining` | If `true`, drain nodes when the spot interruption termination notice is received. Only used in IMDS mode.                                                                                                                                                     | `true`                 |
-| `enableScheduledEventDraining`   | If `true`, drain nodes before the maintenance window starts for an EC2 instance scheduled event. This is **EXPERIMENTAL**.                                                                                                                                    | `false`                |
+| `enableScheduledEventDraining`   | If `true`, drain nodes before the maintenance window starts for an EC2 instance scheduled event. Only used in IMDS mode.                                                                                                                                      | `true`                 |
-| `enableRebalanceMonitoring`      | If `true`, cordon nodes when the rebalance recommendation notice is received. If you'd like to drain the node in addition to cordoning, then also set `enableRebalanceDraining`.                                                                              | `false`                |
+| `enableRebalanceMonitoring`      | If `true`, cordon nodes when the rebalance recommendation notice is received. If you'd like to drain the node in addition to cordoning, then also set `enableRebalanceDraining`. Only used in IMDS mode.                                                      | `false`                |
-| `enableRebalanceDraining`        | If `true`, drain nodes when the rebalance recommendation notice is received.                                                                                                                                                                                  | `false`                |
+| `enableRebalanceDraining`        | If `true`, drain nodes when the rebalance recommendation notice is received. Only used in IMDS mode.                                                                                                                                                          | `false`                |
 
 ### Testing Configuration
 

charts/kubezero-addons/charts/aws-node-termination-handler/templates/deployment.yaml

@@ -146,14 +146,6 @@ spec:
             - name: WEBHOOK_TEMPLATE
               value: {{ .Values.webhookTemplate | quote }}
             {{- end }}
-            - name: ENABLE_SPOT_INTERRUPTION_DRAINING
-              value: {{ .Values.enableSpotInterruptionDraining | quote }}
-            - name: ENABLE_SCHEDULED_EVENT_DRAINING
-              value: {{ .Values.enableScheduledEventDraining | quote }}
-            - name: ENABLE_REBALANCE_MONITORING
-              value: {{ .Values.enableRebalanceMonitoring | quote }}
-            - name: ENABLE_REBALANCE_DRAINING
-              value: {{ .Values.enableRebalanceDraining | quote }}
             - name: ENABLE_SQS_TERMINATION_DRAINING
               value: "true"
             {{- with .Values.awsRegion }}

charts/kubezero-addons/charts/aws-node-termination-handler/templates/psp.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.rbac.pspEnabled }}
+{{- if and (.Values.rbac.pspEnabled) (semverCompare "<1.25-0" .Capabilities.KubeVersion.GitVersion) }}
 apiVersion: policy/v1beta1
 kind: PodSecurityPolicy
 metadata:

charts/kubezero-addons/charts/aws-node-termination-handler/values.yaml

@@ -23,7 +23,7 @@ serviceAccount:
 rbac:
   # Specifies whether RBAC resources should be created
   create: true
-  # Specifies if PodSecurityPolicy resources should be created
+  # Specifies if PodSecurityPolicy resources should be created. PodSecurityPolicy will not be created when Kubernetes version is 1.25 or later.
   pspEnabled: true
 
 customLabels: {}
@@ -259,22 +259,22 @@ daemonsetTolerations:
 linuxTolerations: []
 windowsTolerations: []
 
-# If the probes server is running for the Daemonset
+# If the probes server is running.
 enableProbesServer: false
 
 # Total number of times to try making the metadata request before failing.
 metadataTries: 3
 
-# enableSpotInterruptionDraining If false, do not drain nodes when the spot interruption termination notice is received
+# enableSpotInterruptionDraining If false, do not drain nodes when the spot interruption termination notice is received. Only used in IMDS mode.
 enableSpotInterruptionDraining: true
 
-# enableScheduledEventDraining [EXPERIMENTAL] If true, drain nodes before the maintenance window starts for an EC2 instance scheduled event
+# enableScheduledEventDraining If false, do not drain nodes before the maintenance window starts for an EC2 instance scheduled event. Only used in IMDS mode.
-enableScheduledEventDraining: false
+enableScheduledEventDraining: true
 
-# enableRebalanceMonitoring If true, cordon nodes when the rebalance recommendation notice is received
+# enableRebalanceMonitoring If true, cordon nodes when the rebalance recommendation notice is received. Only used in IMDS mode.
 enableRebalanceMonitoring: false
 
-# enableRebalanceDraining If true, drain nodes when the rebalance recommendation notice is received
+# enableRebalanceDraining If true, drain nodes when the rebalance recommendation notice is received. Only used in IMDS mode.
 enableRebalanceDraining: false
 
 # ---------------------------------------------------------------------------------------------------------------------

charts/kubezero-addons/ruh.patch

@@ -1,16 +1,10 @@
 diff -tuNr charts/aws-eks-asg-rolling-update-handler.orig/templates/deployment.yaml charts/aws-eks-asg-rolling-update-handler/templates/deployment.yaml
---- charts/aws-eks-asg-rolling-update-handler.orig/templates/deployment.yaml	2022-12-16 13:10:26.049272371 +0000
+--- charts/aws-eks-asg-rolling-update-handler.orig/templates/deployment.yaml	2023-04-12 15:49:08.744242462 +0000
-+++ charts/aws-eks-asg-rolling-update-handler/templates/deployment.yaml	2022-12-16 15:56:00.880666339 +0000
++++ charts/aws-eks-asg-rolling-update-handler/templates/deployment.yaml	2023-04-12 15:55:44.399489809 +0000
-@@ -25,7 +25,31 @@
+@@ -34,6 +34,26 @@
-           image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
+           resources:
-           imagePullPolicy: {{ .Values.image.pullPolicy }}
+ {{- toYaml . | nindent 12 }}
-           env:
+ {{- end }}
--{{- toYaml .Values.environmentVars | nindent 12 }}
-+            {{- toYaml .Values.environmentVars | nindent 12 }}
-+          {{- with .Values.resources }}
-+          resources:
-+            {{- toYaml . | nindent 12 }}
-+          {{- end }}
 +          volumeMounts:
 +            - name: aws-token
 +              mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/"
@@ -33,4 +27,4 @@ diff -tuNr charts/aws-eks-asg-rolling-update-handler.orig/templates/deployment.y
 +      {{- end }}
        {{- with .Values.imagePullSecrets }}
        imagePullSecrets:
- {{- toYaml . | nindent 8 }}
+         {{- toYaml . | nindent 8 }}

charts/kubezero-addons/values.yaml

@@ -31,7 +31,7 @@ sealed-secrets:
   fullnameOverride: sealed-secrets-controller
 
   # Disable auto keyrotation for now
-  keyrenewperiod: 0
+  keyrenewperiod: "0"
 
   resources:
     requests:
@@ -179,10 +179,18 @@ nvidia-device-plugin:
                   - g5.24xlarge
                   - g5.48xlarge
                   - g4dn.xlarge
+                  - g4dn.2xlarge
+                  - g4dn.4xlarge
+                  - g4dn.8xlarge
+                  - g4dn.12xlarge
+                  - g4dn.16xlarge
 
 cluster-autoscaler:
   enabled: false
 
+  image:
+    tag: v1.25.1
+
   autoDiscovery:
     clusterName: ""
   awsRegion: "us-west-2"
@@ -201,6 +209,8 @@ cluster-autoscaler:
   extraArgs:
     scan-interval: 30s
     skip-nodes-with-local-storage: false
+    balance-similar-node-groups: true
+    ignore-taint: "node.cilium.io/agent-not-ready"
 
   #securityContext:
   #  runAsNonRoot: true

charts/kubezero-argocd/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 description: KubeZero ArgoCD - config, branding, image-updater (optional)
 name: kubezero-argocd
-version: 0.11.2
+version: 0.12.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -17,13 +17,13 @@ dependencies:
     version: ">= 0.1.6"
     repository: https://cdn.zero-downtime.net/charts/
   - name: argo-cd
-    version: 5.16.10
+    version: 5.28.2
     repository: https://argoproj.github.io/argo-helm
   - name: argocd-apps
-    version: 0.0.6
+    version: 0.0.9
     repository: https://argoproj.github.io/argo-helm
   - name: argocd-image-updater
-    version: 0.8.1
+    version: 0.8.5
     repository: https://argoproj.github.io/argo-helm
     condition: argocd-image-updater.enabled
-kubeVersion: ">= 1.24.0"
+kubeVersion: ">= 1.25.0"

charts/kubezero-argocd/README.md

@@ -1,6 +1,6 @@
 # kubezero-argocd
 
-![Version: 0.11.2](https://img.shields.io/badge/Version-0.11.2-informational?style=flat-square)
+![Version: 0.12.0](https://img.shields.io/badge/Version-0.12.0-informational?style=flat-square)
 
 KubeZero ArgoCD - config, branding, image-updater (optional)
 
@@ -14,13 +14,13 @@ KubeZero ArgoCD - config, branding, image-updater (optional)
 
 ## Requirements
 
-Kubernetes: `>= 1.24.0`
+Kubernetes: `>= 1.25.0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://argoproj.github.io/argo-helm | argo-cd | 5.16.10 |
+| https://argoproj.github.io/argo-helm | argo-cd | 5.28.2 |
-| https://argoproj.github.io/argo-helm | argocd-apps | 0.0.6 |
+| https://argoproj.github.io/argo-helm | argocd-apps | 0.0.9 |
-| https://argoproj.github.io/argo-helm | argocd-image-updater | 0.8.1 |
+| https://argoproj.github.io/argo-helm | argocd-image-updater | 0.8.5 |
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
 
 ## Values
@@ -30,12 +30,12 @@ Kubernetes: `>= 1.24.0`
 | argo-cd.applicationSet.enabled | bool | `false` |  |
 | argo-cd.configs.cm."resource.customizations" | string | `"cert-manager.io/Certificate:\n  # Lua script for customizing the health status assessment\n  health.lua: |\n    hs = {}\n    if obj.status ~= nil then\n      if obj.status.conditions ~= nil then\n        for i, condition in ipairs(obj.status.conditions) do\n          if condition.type == \"Ready\" and condition.status == \"False\" then\n            hs.status = \"Degraded\"\n            hs.message = condition.message\n            return hs\n          end\n          if condition.type == \"Ready\" and condition.status == \"True\" then\n            hs.status = \"Healthy\"\n            hs.message = condition.message\n            return hs\n          end\n        end\n      end\n    end\n    hs.status = \"Progressing\"\n    hs.message = \"Waiting for certificate\"\n    return hs\n"` |  |
 | argo-cd.configs.cm."timeout.reconciliation" | int | `300` |  |
-| argo-cd.configs.cm."ui.bannercontent" | string | `"KubeZero v1.24 - Release notes"` |  |
+| argo-cd.configs.cm."ui.bannercontent" | string | `"KubeZero v1.25 - Release notes"` |  |
 | argo-cd.configs.cm."ui.bannerpermanent" | string | `"true"` |  |
 | argo-cd.configs.cm."ui.bannerposition" | string | `"bottom"` |  |
-| argo-cd.configs.cm."ui.bannerurl" | string | `"https://kubezero.com/releases/v1.24"` |  |
+| argo-cd.configs.cm."ui.bannerurl" | string | `"https://kubezero.com/releases/v1.25"` |  |
 | argo-cd.configs.cm.url | string | `"argocd.example.com"` |  |
-| argo-cd.configs.knownHosts.data.ssh_known_hosts | string | `"bitbucket.org ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAubiN81eDcafrgMeLzaFPsw2kNvEcqTKl/VqLat/MaB33pZy0y3rJZtnqwR2qOOvbwKZYKiEO1O6VqNEBxKvJJelCq0dTXWT5pbO2gDXC6h6QDXCaHo6pOHGPUy+YBaGQRGuSusMEASYiWunYN0vCAI8QaXnWMXNMdFP3jHAJH0eDsoiGnLPBlBp4TNm6rYI74nMzgz3B9IikW4WVK+dc8KZJZWYjAuORU3jc1c/NPskD2ASinf8v3xnfXeukU0sJ5N6m5E8VLjObPEO+mN2t/FZTMZLiFqPWc/ALSqnMnnhwrNi2rbfg/rd/IpL8Le3pSBne8+seeFVBoGqzHM9yXw==\ngithub.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=\ngithub.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl\ngithub.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==\ngitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFSMqzJeV9rUzU4kWitGjeR4PWSa29SPqJ1fVkhtj3Hw9xjLVXVYrU9QlYWrOLXBpQ6KWjbjTDTdDkoohFzgbEY=\ngitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf\ngitlab.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsj2bNKTBSpIYDEGk9KxsGh3mySTRgMtXL583qmBpzeQ+jqCMRgBqB98u3z++J1sKlXHWfM9dyhSevkMwSbhoR8XIq/U0tCNyokEi/ueaBMCvbcTHhO7FcwzY92WK4Yt0aGROY5qX2UKSeOvuP4D6TPqKF1onrSzH9bx9XUf2lEdWT/ia1NEKjunUqu1xOB/StKDHMoX4/OKyIzuS0q/T1zOATthvasJFoPrAjkohTyaDUz2LN5JoH839hViyEG82yB+MjcFV5MU3N1l1QL3cVUCh93xSaua1N85qivl+siMkPGbO5xR/En4iEY6K2XPASUEMaieWVNTRCtJ4S8H+9\ngit.zero-downtime.net ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC8YdJ4YcOK7A0K7qOWsRjCS+wHTStXRcwBe7gjG43HPSNijiCKoGf/c+tfNsRhyouawg7Law6M6ahmS/jKWBpznRIM+OdOFVSuhnK/nr6h6wG3/ZfdLicyAPvx1/STGY/Fc6/zXA88i/9PV+g84gSVmhf3fGY92wokiASiu9DU4T9dT1gIkdyOX6fbMi1/mMKLSrHnAQcjyasYDvw9ISCJ95EoSwbj7O4c+7jo9fxYvdCfZZZAEZGozTRLAAO0AnjVcRah7bZV/jfHJuhOipV/TB7UVAhlVv1dfGV7hoTp9UKtKZFJF4cjIrSGxqQA/mdhSdLgkepK7yc4Jp2xGnaarhY29DfqsQqop+ugFpTbj7Xy5Rco07mXc6XssbAZhI1xtCOX20N4PufBuYippCK5AE6AiAyVtJmvfGQk4HP+TjOyhFo7PZm3wc9Hym7IBBVC0Sl30K8ddufkAgHwNGvvu1ZmD9ZWaMOXJDHBCZGMMr16QREZwVtZTwMEQalc7/yqmuqMhmcJIfs/GA2Lt91y+pq9C8XyeUL0VFPch0vkcLSRe3ghMZpRFJ/ht307xPcLzgTJqN6oQtNNDzSQglSEjwhge2K4GyWcIh+oGsWxWz5dHyk1iJmw90Y976BZIl/mYVgbTtZAJ81oGe/0k5rAe+LDL+Yq6tG28QFOg0QmiQ==\n"` |  |
+| argo-cd.configs.knownHosts.data.ssh_known_hosts | string | `"bitbucket.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPIQmuzMBuKdWeF4+a2sjSSpBK0iqitSQ+5BM9KhpexuGt20JpTVM7u5BDZngncgrqDMbWdxMWWOGtZ9UgbqgZE=\nbitbucket.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIazEu89wgQZ4bqs3d63QSMzYVa0MuJ2e2gKTKqu+UUO\nbitbucket.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDQeJzhupRu0u0cdegZIa8e86EG2qOCsIsD1Xw0xSeiPDlCr7kq97NLmMbpKTX6Esc30NuoqEEHCuc7yWtwp8dI76EEEB1VqY9QJq6vk+aySyboD5QF61I/1WeTwu+deCbgKMGbUijeXhtfbxSxm6JwGrXrhBdofTsbKRUsrN1WoNgUa8uqN1Vx6WAJw1JHPhglEGGHea6QICwJOAr/6mrui/oB7pkaWKHj3z7d1IC4KWLtY47elvjbaTlkN04Kc/5LFEirorGYVbt15kAUlqGM65pk6ZBxtaO3+30LVlORZkxOh+LKL/BvbZ/iRNhItLqNyieoQj/uh/7Iv4uyH/cV/0b4WDSd3DptigWq84lJubb9t/DnZlrJazxyDCulTmKdOR7vs9gMTo+uoIrPSb8ScTtvw65+odKAlBj59dhnVp9zd7QUojOpXlL62Aw56U4oO+FALuevvMjiWeavKhJqlR7i5n9srYcrNV7ttmDw7kf/97P5zauIhxcjX+xHv4M=\ngithub.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=\ngithub.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl\ngithub.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==\ngitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFSMqzJeV9rUzU4kWitGjeR4PWSa29SPqJ1fVkhtj3Hw9xjLVXVYrU9QlYWrOLXBpQ6KWjbjTDTdDkoohFzgbEY=\ngitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf\ngitlab.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsj2bNKTBSpIYDEGk9KxsGh3mySTRgMtXL583qmBpzeQ+jqCMRgBqB98u3z++J1sKlXHWfM9dyhSevkMwSbhoR8XIq/U0tCNyokEi/ueaBMCvbcTHhO7FcwzY92WK4Yt0aGROY5qX2UKSeOvuP4D6TPqKF1onrSzH9bx9XUf2lEdWT/ia1NEKjunUqu1xOB/StKDHMoX4/OKyIzuS0q/T1zOATthvasJFoPrAjkohTyaDUz2LN5JoH839hViyEG82yB+MjcFV5MU3N1l1QL3cVUCh93xSaua1N85qivl+siMkPGbO5xR/En4iEY6K2XPASUEMaieWVNTRCtJ4S8H+9\ngit.zero-downtime.net ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC8YdJ4YcOK7A0K7qOWsRjCS+wHTStXRcwBe7gjG43HPSNijiCKoGf/c+tfNsRhyouawg7Law6M6ahmS/jKWBpznRIM+OdOFVSuhnK/nr6h6wG3/ZfdLicyAPvx1/STGY/Fc6/zXA88i/9PV+g84gSVmhf3fGY92wokiASiu9DU4T9dT1gIkdyOX6fbMi1/mMKLSrHnAQcjyasYDvw9ISCJ95EoSwbj7O4c+7jo9fxYvdCfZZZAEZGozTRLAAO0AnjVcRah7bZV/jfHJuhOipV/TB7UVAhlVv1dfGV7hoTp9UKtKZFJF4cjIrSGxqQA/mdhSdLgkepK7yc4Jp2xGnaarhY29DfqsQqop+ugFpTbj7Xy5Rco07mXc6XssbAZhI1xtCOX20N4PufBuYippCK5AE6AiAyVtJmvfGQk4HP+TjOyhFo7PZm3wc9Hym7IBBVC0Sl30K8ddufkAgHwNGvvu1ZmD9ZWaMOXJDHBCZGMMr16QREZwVtZTwMEQalc7/yqmuqMhmcJIfs/GA2Lt91y+pq9C8XyeUL0VFPch0vkcLSRe3ghMZpRFJ/ht307xPcLzgTJqN6oQtNNDzSQglSEjwhge2K4GyWcIh+oGsWxWz5dHyk1iJmw90Y976BZIl/mYVgbTtZAJ81oGe/0k5rAe+LDL+Yq6tG28QFOg0QmiQ==\n"` |  |
 | argo-cd.configs.params."controller.operation.processors" | string | `"5"` |  |
 | argo-cd.configs.params."controller.status.processors" | string | `"10"` |  |
 | argo-cd.configs.params."server.enable.gzip" | bool | `true` |  |

charts/kubezero-argocd/values.yaml

@@ -29,8 +29,8 @@ argo-cd:
       .sidebar { background: linear-gradient(to bottom, #6A4D79, #493558, #2D1B30, #0D0711); }
 
     cm:
-      ui.bannercontent: "KubeZero v1.24 - Release notes"
+      ui.bannercontent: "KubeZero v1.25 - Release notes"
-      ui.bannerurl: "https://kubezero.com/releases/v1.24"
+      ui.bannerurl: "https://kubezero.com/releases/v1.25"
       ui.bannerpermanent: "true"
       ui.bannerposition: "bottom"
 

charts/kubezero-auth/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: kubezero-auth
 description: KubeZero umbrella chart for all things Authentication and Identity management
 type: application
-version: 0.3.4
+version: 0.3.5
-appVersion: 20.0.2
+appVersion: 21.1.1
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -20,4 +20,4 @@ dependencies:
     version: 11.8.1
     repository: https://charts.bitnami.com/bitnami
     condition: postgresql.enabled
-kubeVersion: ">= 1.20.0"
+kubeVersion: ">= 1.25.0"

charts/kubezero-auth/README.md

@@ -1,6 +1,6 @@
 # kubezero-auth
 
-![Version: 0.3.4](https://img.shields.io/badge/Version-0.3.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 20.0.2](https://img.shields.io/badge/AppVersion-20.0.2-informational?style=flat-square)
+![Version: 0.3.5](https://img.shields.io/badge/Version-0.3.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 21.1.1](https://img.shields.io/badge/AppVersion-21.1.1-informational?style=flat-square)
 
 KubeZero umbrella chart for all things Authentication and Identity management
 
@@ -14,7 +14,7 @@ KubeZero umbrella chart for all things Authentication and Identity management
 
 ## Requirements
 
-Kubernetes: `>= 1.20.0`
+Kubernetes: `>= 1.25.0`
 
 | Repository | Name | Version |
 |------------|------|---------|

charts/kubezero-auth/templates/keycloak/operator.yaml

@@ -4,20 +4,20 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   annotations:
-    app.quarkus.io/build-timestamp: 2022-12-13 - 14:29:14 +0000
+    app.quarkus.io/build-timestamp: 2023-04-26 - 10:32:03 +0000
   labels:
     app.kubernetes.io/name: keycloak-operator
-    app.kubernetes.io/version: 20.0.2
+    app.kubernetes.io/version: 21.1.1
   name: keycloak-operator
 ---
 apiVersion: v1
 kind: Service
 metadata:
   annotations:
-    app.quarkus.io/build-timestamp: 2022-12-13 - 14:29:14 +0000
+    app.quarkus.io/build-timestamp: 2023-04-26 - 10:32:03 +0000
   labels:
     app.kubernetes.io/name: keycloak-operator
-    app.kubernetes.io/version: 20.0.2
+    app.kubernetes.io/version: 21.1.1
   name: keycloak-operator
 spec:
   ports:
@@ -26,7 +26,7 @@ spec:
       targetPort: 8080
   selector:
     app.kubernetes.io/name: keycloak-operator
-    app.kubernetes.io/version: 20.0.2
+    app.kubernetes.io/version: 21.1.1
   type: ClusterIP
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -179,24 +179,24 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    app.quarkus.io/build-timestamp: 2022-12-13 - 14:29:14 +0000
+    app.quarkus.io/build-timestamp: 2023-04-26 - 10:32:03 +0000
   labels:
     app.kubernetes.io/name: keycloak-operator
-    app.kubernetes.io/version: 20.0.2
+    app.kubernetes.io/version: 21.1.1
   name: keycloak-operator
 spec:
   replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/name: keycloak-operator
-      app.kubernetes.io/version: 20.0.2
+      app.kubernetes.io/version: 21.1.1
   template:
     metadata:
       annotations:
-        app.quarkus.io/build-timestamp: 2022-12-13 - 14:29:14 +0000
+        app.quarkus.io/build-timestamp: 2023-04-26 - 10:32:03 +0000
       labels:
         app.kubernetes.io/name: keycloak-operator
-        app.kubernetes.io/version: 20.0.2
+        app.kubernetes.io/version: 21.1.1
     spec:
       containers:
         - env:
@@ -205,8 +205,8 @@ spec:
                 fieldRef:
                   fieldPath: metadata.namespace
             - name: OPERATOR_KEYCLOAK_IMAGE
-              value: quay.io/keycloak/keycloak:20.0.2
+              value: quay.io/keycloak/keycloak:21.1.1
-          image: quay.io/keycloak/keycloak-operator:20.0.2
+          image: quay.io/keycloak/keycloak-operator:21.1.1
           imagePullPolicy: Always
           livenessProbe:
             failureThreshold: 3

charts/kubezero-cert-manager/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-cert-manager
 description: KubeZero Umbrella Chart for cert-manager
 type: application
-version: 0.9.3
+version: 0.9.4
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -16,6 +16,6 @@ dependencies:
     version: ">= 0.1.6"
     repository: https://cdn.zero-downtime.net/charts/
   - name: cert-manager
-    version: 1.9.1
+    version: 1.11.1
     repository: https://charts.jetstack.io
-kubeVersion: ">= 1.24.0"
+kubeVersion: ">= 1.25.0"

charts/kubezero-cert-manager/README.md

@@ -1,6 +1,6 @@
 # kubezero-cert-manager
 
-![Version: 0.9.3](https://img.shields.io/badge/Version-0.9.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.9.4](https://img.shields.io/badge/Version-0.9.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero Umbrella Chart for cert-manager
 
@@ -14,12 +14,12 @@ KubeZero Umbrella Chart for cert-manager
 
 ## Requirements
 
-Kubernetes: `>= 1.24.0`
+Kubernetes: `>= 1.25.0`
 
 | Repository | Name | Version |
 |------------|------|---------|
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
-| https://charts.jetstack.io | cert-manager | 1.9.1 |
+| https://charts.jetstack.io | cert-manager | 1.11.1 |
 
 ## AWS - OIDC IAM roles
 

charts/kubezero-cert-manager/jsonnetfile.lock.json

@@ -18,7 +18,7 @@
           "subdir": "contrib/mixin"
         }
       },
-      "version": "f1842b6ecf67a8102766cc914eaa2a8c7ad97314",
+      "version": "9d2cda4e44a26f064d8578e258bbba2fc3cd5b73",
       "sum": "W/Azptf1PoqjyMwJON96UY69MFugDA4IAYiKURscryc="
     },
     {
@@ -28,7 +28,7 @@
           "subdir": "grafonnet"
         }
       },
-      "version": "30280196507e0fe6fa978a3e0eaca3a62844f817",
+      "version": "f0b70307b8e5f12236b277883d998af129a8211f",
       "sum": "342u++/7rViR/zj2jeJOjshzglkZ1SY+hFNuyCBFMdc="
     },
     {
@@ -38,7 +38,7 @@
           "subdir": "grafana-builder"
         }
       },
-      "version": "15484ab1cb78eb7588e6b79ac52fc04e63f552b4",
+      "version": "e0b90a4435817ad642d8d049e7dd975264cb960e",
       "sum": "tDR6yT2GVfw0wTU12iZH+m01HrbIr6g/xN+/8nzNkU0="
     },
     {
@@ -58,7 +58,7 @@
           "subdir": "lib/promgrafonnet"
         }
       },
-      "version": "3c386687c1f8ceb6b79ff887c4a934e9cee1b90a",
+      "version": "eed459199703c969afc318ea55b9361ae48180a7",
       "sum": "zv7hXGui6BfHzE9wPatHI/AGZa4A2WKo6pq7ZdqBsps="
     },
     {

charts/kubezero-cert-manager/templates/prometheus-rules.yaml

@@ -1,3 +1,4 @@
+{{- if index .Values "cert-manager" "prometheus" "servicemonitor" "enabled" }}
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
@@ -51,3 +52,4 @@ spec:
       labels:
         severity: critical
 
+{{- end }}

charts/kubezero-cert-manager/values.yaml

@@ -23,6 +23,8 @@ cert-manager:
     leaderElection:
       namespace: "cert-manager"
 
+  #enableCertificateOwnerRef: true
+
   # On AWS enable Projected Service Accounts to assume IAM role
   #extraEnv:
   #- name: AWS_ROLE_ARN

charts/kubezero-ci/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-ci
 description: KubeZero umbrella chart for all things CI
 type: application
-version: 0.5.25
+version: 0.6.2
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -22,16 +22,15 @@ dependencies:
     repository: https://gocd.github.io/helm-chart
     condition: gocd.enabled
   - name: gitea
-    version: 6.0.5
+    version: 8.2.0
     repository: https://dl.gitea.io/charts/
     condition: gitea.enabled
   - name: jenkins
-    version: 4.2.20
+    version: 4.3.20
     repository: https://charts.jenkins.io
     condition: jenkins.enabled
   - name: trivy
-    version: 0.4.17
+    version: 0.7.0
     repository: https://aquasecurity.github.io/helm-charts/
     condition: trivy.enabled
-
+kubeVersion: ">= 1.24.0"
-kubeVersion: ">= 1.20.0"

charts/kubezero-ci/README.md

@@ -1,6 +1,6 @@
 # kubezero-ci
 
-![Version: 0.5.24](https://img.shields.io/badge/Version-0.5.24-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.6.2](https://img.shields.io/badge/Version-0.6.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero umbrella chart for all things CI
 
@@ -14,14 +14,14 @@ KubeZero umbrella chart for all things CI
 
 ## Requirements
 
-Kubernetes: `>= 1.20.0`
+Kubernetes: `>= 1.24.0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://aquasecurity.github.io/helm-charts/ | trivy | 0.4.17 |
+| https://aquasecurity.github.io/helm-charts/ | trivy | 0.7.0 |
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
-| https://charts.jenkins.io | jenkins | 4.2.17 |
+| https://charts.jenkins.io | jenkins | 4.3.20 |
-| https://dl.gitea.io/charts/ | gitea | 6.0.5 |
+| https://dl.gitea.io/charts/ | gitea | 8.2.0 |
 | https://gocd.github.io/helm-chart | gocd | 1.40.8 |
 
 # Jenkins
@@ -34,8 +34,10 @@ Kubernetes: `>= 1.20.0`
 
 # Gitea
 
-## OpenSSH 8.8 RSA disabled
+# Verdaccio
-- https://github.com/go-gitea/gitea/issues/17798
+
+## Authentication sealed-secret
+```htpasswd -n -b -B -C 4 <username> <password> | kubeseal --raw --namespace verdaccio --name verdaccio-htpasswd```
 
 ## Resources
 
@@ -82,6 +84,10 @@ Kubernetes: `>= 1.20.0`
 | jenkins.agent.image | string | `"public.ecr.aws/zero-downtime/jenkins-podman"` |  |
 | jenkins.agent.podName | string | `"podman-aws"` |  |
 | jenkins.agent.podRetention | string | `"Default"` |  |
+| jenkins.agent.resources.limits.cpu | string | `""` |  |
+| jenkins.agent.resources.limits.memory | string | `""` |  |
+| jenkins.agent.resources.requests.cpu | string | `""` |  |
+| jenkins.agent.resources.requests.memory | string | `""` |  |
 | jenkins.agent.showRawYaml | bool | `false` |  |
 | jenkins.agent.tag | string | `"v0.4.1"` |  |
 | jenkins.agent.yamlMergeStrategy | string | `"merge"` |  |
@@ -92,18 +98,18 @@ Kubernetes: `>= 1.20.0`
 | jenkins.controller.initContainerResources.limits.memory | string | `"1024Mi"` |  |
 | jenkins.controller.initContainerResources.requests.cpu | string | `"50m"` |  |
 | jenkins.controller.initContainerResources.requests.memory | string | `"256Mi"` |  |
-| jenkins.controller.installPlugins[0] | string | `"kubernetes:3743.v1fa_4c724c3b_7"` |  |
+| jenkins.controller.installPlugins[0] | string | `"kubernetes:3923.v294a_d4250b_91"` |  |
 | jenkins.controller.installPlugins[10] | string | `"build-discarder:139.v05696a_7fe240"` |  |
-| jenkins.controller.installPlugins[11] | string | `"dark-theme:262.v0202a_4c8fb_6a"` |  |
+| jenkins.controller.installPlugins[11] | string | `"dark-theme:315.va_22e7d692ea_a"` |  |
-| jenkins.controller.installPlugins[12] | string | `"kubernetes-credentials-provider:1.206.v7ce2cf7b_0c8b"` |  |
+| jenkins.controller.installPlugins[12] | string | `"kubernetes-credentials-provider:1.211.vc236a_f5a_2f3c"` |  |
 | jenkins.controller.installPlugins[1] | string | `"workflow-aggregator:581.v0c46fa_697ffd"` |  |
-| jenkins.controller.installPlugins[2] | string | `"git:4.14.3"` |  |
+| jenkins.controller.installPlugins[2] | string | `"git:5.0.2"` |  |
 | jenkins.controller.installPlugins[3] | string | `"basic-branch-build-strategies:71.vc1421f89888e"` |  |
-| jenkins.controller.installPlugins[4] | string | `"pipeline-graph-view:144.vf3924feb_7e35"` |  |
+| jenkins.controller.installPlugins[4] | string | `"pipeline-graph-view:183.v9e27732d970f"` |  |
-| jenkins.controller.installPlugins[5] | string | `"pipeline-stage-view:2.28"` |  |
+| jenkins.controller.installPlugins[5] | string | `"pipeline-stage-view:2.32"` |  |
-| jenkins.controller.installPlugins[6] | string | `"configuration-as-code:1569.vb_72405b_80249"` |  |
+| jenkins.controller.installPlugins[6] | string | `"configuration-as-code:1625.v27444588cc3d"` |  |
-| jenkins.controller.installPlugins[7] | string | `"antisamy-markup-formatter:155.v795fb_8702324"` |  |
+| jenkins.controller.installPlugins[7] | string | `"antisamy-markup-formatter:159.v25b_c67cd35fb_"` |  |
-| jenkins.controller.installPlugins[8] | string | `"prometheus:2.0.11"` |  |
+| jenkins.controller.installPlugins[8] | string | `"prometheus:2.2.2"` |  |
 | jenkins.controller.installPlugins[9] | string | `"htmlpublisher:1.31"` |  |
 | jenkins.controller.javaOpts | string | `"-XX:+UseContainerSupport -XX:+UseStringDeduplication -Dhudson.model.DirectoryBrowserSupport.CSP=\"sandbox allow-popups; default-src 'none'; img-src 'self' cdn.zero-downtime.net; style-src 'unsafe-inline';\""` |  |
 | jenkins.controller.jenkinsOpts | string | `"--sessionTimeout=180 --sessionEviction=3600"` |  |
@@ -128,7 +134,7 @@ Kubernetes: `>= 1.20.0`
 | jenkins.serviceAccountAgent.create | bool | `true` |  |
 | jenkins.serviceAccountAgent.name | string | `"jenkins-podman-aws"` |  |
 | trivy.enabled | bool | `false` |  |
-| trivy.image.tag | string | `"0.35.0"` |  |
+| trivy.image.tag | string | `"0.39.1"` |  |
 | trivy.persistence.enabled | bool | `true` |  |
 | trivy.persistence.size | string | `"1Gi"` |  |
 | trivy.rbac.create | bool | `false` |  |

charts/kubezero-ci/README.md.gotmpl

@@ -23,8 +23,10 @@
 
 # Gitea
 
-## OpenSSH 8.8 RSA disabled
+# Verdaccio
-- https://github.com/go-gitea/gitea/issues/17798
+
+## Authentication sealed-secret
+```htpasswd -n -b -B -C 4 <username> <password> | kubeseal --raw --namespace verdaccio --name verdaccio-htpasswd```
 
 ## Resources
 

charts/kubezero-ci/values.yaml

@@ -121,19 +121,19 @@ jenkins:
                       numToKeepStr: "10"
 
     installPlugins:
-      - kubernetes:3802.vb_b_600831fcb_3
+      - kubernetes:3923.v294a_d4250b_91
       - workflow-aggregator:581.v0c46fa_697ffd
-      - git:5.0.0
+      - git:5.0.2
       - basic-branch-build-strategies:71.vc1421f89888e
-      - pipeline-graph-view:144.vf3924feb_7e35
+      - pipeline-graph-view:183.v9e27732d970f
-      - pipeline-stage-view:2.28
+      - pipeline-stage-view:2.32
-      - configuration-as-code:1569.vb_72405b_80249
+      - configuration-as-code:1625.v27444588cc3d
-      - antisamy-markup-formatter:155.v795fb_8702324
+      - antisamy-markup-formatter:159.v25b_c67cd35fb_
-      - prometheus:2.1.0
+      - prometheus:2.2.2
       - htmlpublisher:1.31
       - build-discarder:139.v05696a_7fe240
-      - dark-theme:262.v0202a_4c8fb_6a
+      - dark-theme:315.va_22e7d692ea_a
-      - kubernetes-credentials-provider:1.208.v128ee9800c04
+      - kubernetes-credentials-provider:1.211.vc236a_f5a_2f3c
 
   serviceAccountAgent:
     create: true
@@ -231,7 +231,7 @@ jenkins:
 trivy:
   enabled: false
   image:
-    tag: 0.35.0
+    tag: 0.39.1
   persistence:
     enabled: true
     size: 1Gi

charts/kubezero-istio-gateway/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-istio-gateway
 description: KubeZero Umbrella Chart for Istio gateways
 type: application
-version: 0.9.0
+version: 0.10.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -14,9 +14,9 @@ maintainers:
     email: stefan@zero-downtime.net
 dependencies:
   - name: kubezero-lib
-    version: ">= 0.1.5"
+    version: ">= 0.1.6"
     repository: https://cdn.zero-downtime.net/charts/
   - name: gateway
-    version: 1.16.1
+    version: 1.17.2
     repository: https://istio-release.storage.googleapis.com/charts
-kubeVersion: ">= 1.24.0"
+kubeVersion: ">= 1.25.0"

charts/kubezero-istio-gateway/README.md

@@ -1,6 +1,6 @@
 # kubezero-istio-gateway
 
-![Version: 0.9.0](https://img.shields.io/badge/Version-0.9.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.10.0](https://img.shields.io/badge/Version-0.10.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero Umbrella Chart for Istio gateways
 
@@ -16,12 +16,12 @@ Installs Istio Ingress Gateways, requires kubezero-istio to be installed !
 
 ## Requirements
 
-Kubernetes: `>= 1.24.0`
+Kubernetes: `>= 1.25.0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.5 |
+| https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
-| https://istio-release.storage.googleapis.com/charts | gateway | 1.16.1 |
+| https://istio-release.storage.googleapis.com/charts | gateway | 1.17.2 |
 
 ## Values
 

charts/kubezero-istio-gateway/charts/gateway/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 1.16.1
+appVersion: 1.17.2
 description: Helm chart for deploying Istio gateways
 icon: https://istio.io/latest/favicons/android-192x192.png
 keywords:
@@ -9,4 +9,4 @@ name: gateway
 sources:
 - http://github.com/istio/istio
 type: application
-version: 1.16.1
+version: 1.17.2

charts/kubezero-istio-gateway/charts/gateway/templates/deployment.yaml

@@ -49,6 +49,9 @@ spec:
         - name: istio-proxy
           # "auto" will be populated at runtime by the mutating webhook. See https://istio.io/latest/docs/setup/additional-setup/sidecar-injection/#customizing-injection
           image: auto
+          {{- with .Values.imagePullPolicy }}
+          imagePullPolicy: {{ . }}
+          {{- end }}
           securityContext:
           {{- if .Values.containerSecurityContext }}
             {{- toYaml .Values.containerSecurityContext | nindent 12 }}

charts/kubezero-istio-gateway/charts/gateway/values.schema.json

@@ -191,6 +191,10 @@
     "networkGateway": {
       "type": "string"
     },
+    "imagePullPolicy": {
+      "type": "string",
+      "enum": ["", "Always", "IfNotPresent", "Never"]
+    },
     "imagePullSecrets": {
       "type": "array",
       "items": {

charts/kubezero-istio-gateway/charts/gateway/values.yaml

@@ -90,4 +90,8 @@ affinity: {}
 # If specified, the gateway will act as a network gateway for the given network.
 networkGateway: ""
 
-imagePullSecrets: []
+# Specify image pull policy if default behavior isn't desired.
+# Default behavior: latest images will be Always else IfNotPresent
+imagePullPolicy: ""
+
+imagePullSecrets: []

charts/kubezero-istio/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-istio
 description: KubeZero Umbrella Chart for Istio
 type: application
-version: 0.9.0
+version: 0.10.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -16,13 +16,13 @@ dependencies:
     version: ">= 0.1.6"
     repository: https://cdn.zero-downtime.net/charts/
   - name: base
-    version: 1.16.1
+    version: 1.17.2
     repository: https://istio-release.storage.googleapis.com/charts
   - name: istiod
-    version: 1.16.1
+    version: 1.17.2
     repository: https://istio-release.storage.googleapis.com/charts
   - name: kiali-server
-    version: "1.60.0"
+    version: "1.66.0"
     repository: https://kiali.org/helm-charts
     condition: kiali-server.enabled
-kubeVersion: ">= 1.24.0"
+kubeVersion: ">= 1.25.0"

charts/kubezero-istio/README.md

@@ -1,6 +1,6 @@
 # kubezero-istio
 
-![Version: 0.9.0](https://img.shields.io/badge/Version-0.9.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.10.0](https://img.shields.io/badge/Version-0.10.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero Umbrella Chart for Istio
 
@@ -16,14 +16,14 @@ Installs the Istio control plane
 
 ## Requirements
 
-Kubernetes: `>= 1.24.0`
+Kubernetes: `>= 1.25.0`
 
 | Repository | Name | Version |
 |------------|------|---------|
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
-| https://istio-release.storage.googleapis.com/charts | base | 1.16.1 |
+| https://istio-release.storage.googleapis.com/charts | base | 1.17.2 |
-| https://istio-release.storage.googleapis.com/charts | istiod | 1.16.1 |
+| https://istio-release.storage.googleapis.com/charts | istiod | 1.17.2 |
-| https://kiali.org/helm-charts | kiali-server | 1.60.0 |
+| https://kiali.org/helm-charts | kiali-server | 1.66.0 |
 
 ## Values
 

charts/kubezero-istio/dashboards.yaml

@@ -1,22 +1,22 @@
-# Revision 148 = 1.16
+# Revision 162 = 1.17
 configmap: grafana-dashboards
 gzip: true
 folder: Istio
 condition: '.Values.istiod.telemetry.enabled'
 dashboards:
 - name: istio-control-plane
-  url: https://grafana.com/api/dashboards/7645/revisions/148/download
+  url: https://grafana.com/api/dashboards/7645/revisions/162/download
   tags:
   - Istio
 - name: istio-mesh
-  url: https://grafana.com/api/dashboards/7639/revisions/148/download
+  url: https://grafana.com/api/dashboards/7639/revisions/162/download
   tags:
   - Istio
 - name: istio-service
-  url: https://grafana.com/api/dashboards/7636/revisions/148/download
+  url: https://grafana.com/api/dashboards/7636/revisions/162/download
   tags:
   - Istio
 - name: istio-workload
-  url: https://grafana.com/api/dashboards/7630/revisions/148/download
+  url: https://grafana.com/api/dashboards/7630/revisions/162/download
   tags:
   - Istio

charts/kubezero-logging/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-logging
 description: KubeZero Umbrella Chart for complete EFK stack
 type: application
-version: 0.8.4
+version: 0.8.6
 appVersion: 1.6.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@@ -27,6 +27,6 @@ dependencies:
     version: 0.3.9
     condition: fluentd.enabled
   - name: fluent-bit
-    version: 0.20.6
+    version: 0.24.0
     condition: fluent-bit.enabled
 kubeVersion: ">= 1.24.0"

charts/kubezero-logging/README.md

@@ -1,6 +1,6 @@
 # kubezero-logging
 
-![Version: 0.8.4](https://img.shields.io/badge/Version-0.8.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6.0](https://img.shields.io/badge/AppVersion-1.6.0-informational?style=flat-square)
+![Version: 0.8.6](https://img.shields.io/badge/Version-0.8.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6.0](https://img.shields.io/badge/AppVersion-1.6.0-informational?style=flat-square)
 
 KubeZero Umbrella Chart for complete EFK stack
 
@@ -19,7 +19,7 @@ Kubernetes: `>= 1.24.0`
 | Repository | Name | Version |
 |------------|------|---------|
 |  | eck-operator | 2.4.0 |
-|  | fluent-bit | 0.20.6 |
+|  | fluent-bit | 0.24.0 |
 |  | fluentd | 0.3.9 |
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
 
@@ -72,9 +72,9 @@ Kubernetes: `>= 1.24.0`
 | fluent-bit.config.customParsers | string | `"[PARSER]\n    Name cri-log\n    Format regex\n    Regex ^(?<time>.+) (?<stream>stdout|stderr) (?<logtag>F|P) (?<log>.*)$\n    Time_Key    time\n    Time_Format %Y-%m-%dT%H:%M:%S.%L%z\n"` |  |
 | fluent-bit.config.filters | string | `"[FILTER]\n    Name parser\n    Match cri.*\n    Parser cri-log\n    Key_Name log\n\n[FILTER]\n    Name kubernetes\n    Match cri.*\n    Merge_Log On\n    Merge_Log_Key kube\n    Kube_Tag_Prefix cri.var.log.containers.\n    Keep_Log Off\n    K8S-Logging.Parser Off\n    K8S-Logging.Exclude Off\n    Kube_Meta_Cache_TTL 3600s\n    Buffer_Size 0\n    #Use_Kubelet true\n\n{{- if index .Values \"config\" \"extraRecords\" }}\n\n[FILTER]\n    Name record_modifier\n    Match cri.*\n    {{- range $k,$v := index .Values \"config\" \"extraRecords\" }}\n    Record {{ $k }} {{ $v }}\n    {{- end }}\n{{- end }}\n\n[FILTER]\n    Name rewrite_tag\n    Match cri.*\n    Emitter_Name kube_tag_rewriter\n    Rule $kubernetes['pod_id'] .* kube.$kubernetes['namespace_name'].$kubernetes['container_name'] false\n\n[FILTER]\n    Name    lua\n    Match   kube.*\n    script  /fluent-bit/scripts/kubezero.lua\n    call    nest_k8s_ns\n"` |  |
 | fluent-bit.config.flushInterval | int | `5` |  |
-| fluent-bit.config.input.memBufLimit | string | `"4MB"` |  |
+| fluent-bit.config.input.memBufLimit | string | `"16MB"` |  |
-| fluent-bit.config.input.refreshInterval | int | `10` |  |
+| fluent-bit.config.input.refreshInterval | int | `5` |  |
-| fluent-bit.config.inputs | string | `"[INPUT]\n    Name tail\n    Path /var/log/containers/*.log\n    multiline.parser cri\n    Tag cri.*\n    Skip_Long_Lines On\n    DB /var/log/flb_kube.db\n    DB.Sync Normal\n    DB.locking true\n    # Buffer_Max_Size 1M\n    {{- with .Values.config.input }}\n    Mem_Buf_Limit {{ default \"4MB\" .memBufLimit }}\n    Refresh_Interval {{ default 10 .refreshInterval }}\n    {{- end }}\n"` |  |
+| fluent-bit.config.inputs | string | `"[INPUT]\n    Name tail\n    Path /var/log/containers/*.log\n    # Exclude ourselves to current error spam, https://github.com/fluent/fluent-bit/issues/5769\n    Exclude_Path *logging-fluent-bit*\n    multiline.parser cri\n    Tag cri.*\n    Skip_Long_Lines On\n    Skip_Empty_Lines On\n    DB /var/log/flb_kube.db\n    DB.Sync Normal\n    DB.locking true\n    # Buffer_Max_Size 1M\n    {{- with .Values.config.input }}\n    Mem_Buf_Limit {{ default \"16MB\" .memBufLimit }}\n    Refresh_Interval {{ default 5 .refreshInterval }}\n    {{- end }}\n"` |  |
 | fluent-bit.config.logLevel | string | `"info"` |  |
 | fluent-bit.config.output.host | string | `"logging-fluentd"` |  |
 | fluent-bit.config.output.sharedKey | string | `"cloudbender"` |  |
@@ -90,13 +90,14 @@ Kubernetes: `>= 1.24.0`
 | fluent-bit.daemonSetVolumes[1].hostPath.path | string | `"/var/lib/containers/logs"` |  |
 | fluent-bit.daemonSetVolumes[1].name | string | `"newlog"` |  |
 | fluent-bit.enabled | bool | `false` |  |
-| fluent-bit.image.tag | string | `"1.9.8"` |  |
+| fluent-bit.image.tag | string | `"2.0.10"` |  |
 | fluent-bit.luaScripts."kubezero.lua" | string | `"function nest_k8s_ns(tag, timestamp, record)\n    if not record['kubernetes']['namespace_name'] then\n        return 0, 0, 0\n    end\n    new_record = {}\n    for key, val in pairs(record) do\n        if key == 'kube' then\n            new_record[key] = {}\n            new_record[key][record['kubernetes']['namespace_name']] = record[key]\n        else\n            new_record[key] = record[key]\n        end\n    end\n    return 1, timestamp, new_record\nend\n"` |  |
 | fluent-bit.resources.limits.memory | string | `"64Mi"` |  |
 | fluent-bit.resources.requests.cpu | string | `"20m"` |  |
 | fluent-bit.resources.requests.memory | string | `"32Mi"` |  |
 | fluent-bit.serviceMonitor.enabled | bool | `false` |  |
 | fluent-bit.serviceMonitor.selector.release | string | `"metrics"` |  |
+| fluent-bit.testFramework.enabled | bool | `false` |  |
 | fluent-bit.tolerations[0].effect | string | `"NoSchedule"` |  |
 | fluent-bit.tolerations[0].operator | string | `"Exists"` |  |
 | fluentd.dashboards.enabled | bool | `false` |  |
@@ -111,7 +112,7 @@ Kubernetes: `>= 1.24.0`
 | fluentd.fileConfigs."02_filters.conf" | string | `"<label @KUBERNETES>\n  # prevent log feedback loops eg. ES has issues etc.\n  # discard logs from our own pods\n  <match kube.logging.fluentd>\n    @type relabel\n    @label @FLUENT_LOG\n  </match>\n\n  # Exclude current fluent-bit multiline noise\n  <filter kube.logging.fluent-bit>\n    @type grep\n    <exclude>\n      key log\n      pattern /could not append content to multiline context/\n    </exclude>\n  </filter>\n\n  # Generate Hash ID to break endless loop for already ingested events during retries\n  <filter **>\n    @type elasticsearch_genid\n    use_entire_record true\n  </filter>\n\n  # Route through DISPATCH for Prometheus metrics\n  <match **>\n    @type relabel\n    @label @DISPATCH\n  </match>\n</label>"` |  |
 | fluentd.fileConfigs."04_outputs.conf" | string | `"<label @OUTPUT>\n  <match **>\n    @id out_es\n    @type elasticsearch\n    # @log_level debug\n    include_tag_key true\n\n    id_key _hash\n    remove_keys _hash\n    write_operation create\n\n    # KubeZero pipeline incl. GeoIP etc.\n    pipeline fluentd\n\n    hosts \"{{ .Values.output.host }}\"\n    port 9200\n    scheme http\n    user elastic\n    password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n    log_es_400_reason\n    logstash_format true\n    reconnect_on_error true\n    reload_on_failure true\n    request_timeout 300s\n    slow_flush_log_threshold 55.0\n\n    #with_transporter_log true\n\n    verify_es_version_at_startup false\n    default_elasticsearch_version 7\n    suppress_type_name true\n\n    # Retry failed bulk requests\n    # https://github.com/uken/fluent-plugin-elasticsearch#unrecoverable-error-types\n    unrecoverable_error_types [\"out_of_memory_error\"]\n    bulk_message_request_threshold 1048576\n\n    <buffer>\n      @type file\n\n      flush_mode interval\n      flush_thread_count 2\n      flush_interval 10s\n\n      chunk_limit_size 2MB\n      total_limit_size 1GB\n\n      flush_at_shutdown true\n      retry_type exponential_backoff\n      retry_timeout 6h\n      overflow_action drop_oldest_chunk\n      disable_chunk_backup true\n    </buffer>\n  </match>\n</label>"` |  |
 | fluentd.image.repository | string | `"public.ecr.aws/zero-downtime/fluentd-concenter"` |  |
-| fluentd.image.tag | string | `"v1.14.2"` |  |
+| fluentd.image.tag | string | `"v1.16.0"` |  |
 | fluentd.istio.enabled | bool | `false` |  |
 | fluentd.kind | string | `"Deployment"` |  |
 | fluentd.metrics.serviceMonitor.additionalLabels.release | string | `"metrics"` |  |

charts/kubezero-logging/charts/fluent-bit/Chart.yaml

@@ -1,13 +1,13 @@
 annotations:
   artifacthub.io/changes: |
     - kind: changed
-      description: "Additional upstream config option added"
+      description: "Updated Fluent Bit image to v2.0.9."
 apiVersion: v1
-appVersion: 1.9.7
+appVersion: 2.0.9
 description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD
   family operating systems.
 home: https://fluentbit.io/
-icon: https://fluentbit.io/assets/img/logo1-default.png
+icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/fluentd/fluentbit/icon/fluentbit-icon-color.svg
 keywords:
 - logging
 - fluent-bit
@@ -24,4 +24,4 @@ maintainers:
 name: fluent-bit
 sources:
 - https://github.com/fluent/fluent-bit/
-version: 0.20.6
+version: 0.24.0

charts/kubezero-logging/charts/fluent-bit/README.md

@@ -31,7 +31,7 @@ First, you should add your Lua scripts to `luaScripts` in values.yaml, for examp
 
 ```yaml
 luaScripts:
- filter_example.lua: |
+  filter_example.lua: |
     function filter_name(tag, timestamp, record)
         -- put your lua code here.
     end

charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl

@@ -11,6 +11,9 @@ priorityClassName: {{ .Values.priorityClassName }}
 securityContext:
   {{- toYaml . | nindent 2 }}
 {{- end }}
+{{- with .Values.terminationGracePeriodSeconds }}
+terminationGracePeriodSeconds: {{ . }}
+{{- end }}
 hostNetwork: {{ .Values.hostNetwork }}
 dnsPolicy: {{ .Values.dnsPolicy }}
 {{- with .Values.dnsConfig }}

charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml

@@ -19,7 +19,7 @@ rules:
       - get
       - list
       - watch
-  {{- if .Values.podSecurityPolicy.create }}
+  {{- if and .Values.podSecurityPolicy.create (semverCompare "<=1.25-0" .Capabilities.KubeVersion.GitVersion) }}
   - apiGroups:
       - policy
     resources:

charts/kubezero-logging/charts/fluent-bit/templates/daemonset.yaml

@@ -20,6 +20,9 @@ spec:
   updateStrategy:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+  {{- with .Values.minReadySeconds }}
+  minReadySeconds: {{ . }}
+  {{- end }}
   template:
     metadata:
       annotations:

charts/kubezero-logging/charts/fluent-bit/templates/deployment.yaml

@@ -23,6 +23,9 @@ spec:
   selector:
     matchLabels:
       {{- include "fluent-bit.selectorLabels" . | nindent 6 }}
+  {{- with .Values.minReadySeconds }}
+  minReadySeconds: {{ . }}
+  {{- end }}
   template:
     metadata:
       annotations:

charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.podSecurityPolicy.create }}
+{{- if and .Values.podSecurityPolicy.create (semverCompare "<=1.25-0" .Capabilities.KubeVersion.GitVersion) -}}
 apiVersion: policy/v1beta1
 kind: PodSecurityPolicy
 metadata:

charts/kubezero-logging/charts/fluent-bit/templates/service.yaml

@@ -16,6 +16,15 @@ spec:
   {{- if and (eq .Values.service.type "ClusterIP") (.Values.service.clusterIP) }}
   clusterIP: {{ .Values.service.clusterIP }}
   {{- end }}
+  {{- if (eq .Values.service.type "LoadBalancer")}}
+  {{- with .Values.service.loadBalancerClass}}
+  loadBalancerClass: {{ . }}
+  {{- end }}
+  {{- with .Values.service.loadBalancerSourceRanges}}
+  loadBalancerSourceRanges:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- end }}
   ports:
     - port: {{ .Values.service.port }}
       targetPort: http

charts/kubezero-logging/charts/fluent-bit/templates/servicemonitor.yaml

@@ -36,6 +36,16 @@ spec:
       relabelings:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- if .Values.serviceMonitor.scheme }}
+      scheme: {{ .Values.serviceMonitor.scheme }}
+      {{- end }}
+      {{- if .Values.serviceMonitor.tlsConfig }}
+      tlsConfig:
+        {{- toYaml .Values.serviceMonitor.tlsConfig | nindent 8 }}
+      {{- end }}
+    {{- with .Values.serviceMonitor.additionalEndpoints }}
+      {{- toYaml . | nindent 4 }}
+    {{- end }}
   namespaceSelector:
     matchNames:
       - {{ .Release.Namespace }}

charts/kubezero-logging/charts/fluent-bit/templates/vpa.yaml

@@ -0,0 +1,38 @@
+{{- if and (.Capabilities.APIVersions.Has "autoscaling.k8s.io/v1/VerticalPodAutoscaler") .Values.autoscaling.vpa.enabled }}
+apiVersion: autoscaling.k8s.io/v1
+kind: VerticalPodAutoscaler
+metadata:
+  name: {{ include "fluent-bit.fullname" . }}
+  labels:
+    {{- include "fluent-bit.labels" . | nindent 4 }}
+  {{- with .Values.autoscaling.vpa.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  resourcePolicy:
+    containerPolicies:
+    - containerName: {{ .Chart.Name }}
+      {{- with .Values.autoscaling.vpa.controlledResources }}
+      controlledResources: 
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.autoscaling.vpa.maxAllowed }}
+      maxAllowed:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.autoscaling.vpa.minAllowed }}
+      minAllowed:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+  targetRef:
+    apiVersion: apps/v1
+    kind: {{ .Values.kind }}
+    name: {{ include "fluent-bit.fullname" . }}
+  {{- if .Values.autoscaling.vpa.updatePolicy }}
+  updatePolicy:
+    {{- with .Values.autoscaling.vpa.updatePolicy.updateMode }}
+    updateMode: {{ . }}
+    {{- end }}
+  {{- end }}
+{{- end }}

charts/kubezero-logging/charts/fluent-bit/values.yaml

@@ -13,7 +13,7 @@ image:
   pullPolicy: Always
 
 testFramework:
-  enabled: false
+  enabled: true
   image:
     repository: busybox
     pullPolicy: Always
@@ -32,6 +32,11 @@ rbac:
   create: true
   nodeAccess: false
 
+# Configure podsecuritypolicy
+# Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/
+# from Kubernetes 1.25, PSP is deprecated
+# See: https://kubernetes.io/blog/2022/08/23/kubernetes-v1-25-release/#pod-security-changes
+# We automatically disable PSP if Kubernetes version is 1.25 or higher
 podSecurityPolicy:
   create: false
   annotations: {}
@@ -78,6 +83,8 @@ securityContext: {}
 service:
   type: ClusterIP
   port: 2020
+  loadBalancerClass:
+  loadBalancerSourceRanges: []
   labels: {}
   # nodePort: 30020
   # clusterIP: 172.16.10.1
@@ -111,6 +118,35 @@ serviceMonitor:
 #      targetLabel: nodename
 #      replacement: $1
 #      action: replace
+#  scheme: ""
+#  tlsConfig: {}
+
+  ## Beare in mind if youn want to collec metrics from a different port
+  ## you will need to configure the new ports on the extraPorts property.
+  additionalEndpoints: []
+  # - port: metrics
+  #   path: /metrics
+  #   interval: 10s
+  #   scrapeTimeout: 10s
+  #   scheme: ""
+  #   tlsConfig: {}
+  #   # metric relabel configs to apply to samples before ingestion.
+  #   #
+  #   metricRelabelings:
+  #     - sourceLabels: [__meta_kubernetes_service_label_cluster]
+  #       targetLabel: cluster
+  #       regex: (.*)
+  #       replacement: ${1}
+  #       action: replace
+  #   # relabel configs to apply to samples after ingestion.
+  #   #
+  #   relabelings:
+  #     - sourceLabels: [__meta_kubernetes_pod_node_name]
+  #       separator: ;
+  #       regex: ^(.*)$
+  #       targetLabel: nodename
+  #       replacement: $1
+  #       action: replace
 
 prometheusRule:
   enabled: false
@@ -177,6 +213,28 @@ ingress:
 
 ## only available if kind is Deployment
 autoscaling:
+  vpa:
+    enabled: false
+
+    annotations: {}
+
+    # List of resources that the vertical pod autoscaler can control. Defaults to cpu and memory
+    controlledResources: []
+
+    # Define the max allowed resources for the pod
+    maxAllowed: {}
+    # cpu: 200m
+    # memory: 100Mi
+    # Define the min allowed resources for the pod
+    minAllowed: {}
+    # cpu: 200m
+    # memory: 100Mi
+
+    updatePolicy:
+      # Specifies whether recommended updates are applied when a Pod is started and whether recommended updates
+      # are applied during the life of a Pod. Possible values are "Off", "Initial", "Recreate", and "Auto".
+      updateMode: Auto
+
   enabled: false
   minReplicas: 1
   maxReplicas: 3
@@ -222,6 +280,14 @@ podAnnotations: {}
 
 podLabels: {}
 
+## How long (in seconds) a pods needs to be stable before progressing the deployment
+##
+minReadySeconds:
+
+## How long (in seconds) a pod may take to exit (useful with lifecycle hooks to ensure lb deregistration is done)
+##
+terminationGracePeriodSeconds:
+
 priorityClassName: ""
 
 env: []
@@ -274,7 +340,7 @@ networkPolicy:
 
 luaScripts: {}
 
-## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/configuration-file
+## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/configuration-file
 config:
   service: |
     [SERVICE]
@@ -332,15 +398,8 @@ config:
         Retry_Limit False
 
   ## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/upstream-servers
+  ## This configuration is deprecated, please use `extraFiles` instead.
   upstream: {}
-#      upstream.conf: |
-#        [UPSTREAM]
-#            upstream1
-#
-#        [NODE]
-#            name       node-1
-#            host       127.0.0.1
-#            port       43000
 
   ## https://docs.fluentbit.io/manual/pipeline/parsers
   customParsers: |
@@ -354,6 +413,14 @@ config:
   # This allows adding more files with arbitary filenames to /fluent-bit/etc by providing key/value pairs.
   # The key becomes the filename, the value becomes the file content.
   extraFiles: {}
+#     upstream.conf: |
+#       [UPSTREAM]
+#           upstream1
+#
+#       [NODE]
+#           name       node-1
+#           host       127.0.0.1
+#           port       43000
 #     example.conf: |
 #       [OUTPUT]
 #           Name example

charts/kubezero-logging/dashboards.yaml

@@ -3,7 +3,7 @@ gzip: true
 folder: Logging
 dashboards:
 - name: fluent-logging
-  url: https://grafana.com/api/dashboards/7752/revisions/4/download
+  url: https://grafana.com/api/dashboards/7752/revisions/6/download
   #url: https://grafana.com/api/dashboards/13042/revisions/2/download
   tags:
   - fluentd

charts/kubezero-logging/fluent-bit.patch

@@ -1,24 +0,0 @@
-diff -tubr charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent-bit.zdt/templates/tests/test-connection.yaml
---- charts/fluent-bit/templates/tests/test-connection.yaml	2021-06-22 12:25:10.000000000 +0200
-+++ charts/fluent-bit.zdt/templates/tests/test-connection.yaml	2021-05-17 12:09:02.724057438 +0200
-@@ -1,3 +1,4 @@
-+{{- if .Values.testFramework.enabled }}
- apiVersion: v1
- kind: Pod
- metadata:
-@@ -18,3 +19,4 @@
-     {{- toYaml . | nindent 4 }}
-   {{- end }}
-   restartPolicy: Never
-+{{- end }}
-diff -tubr charts/fluent-bit/values.yaml charts/fluent-bit.zdt/values.yaml
---- charts/fluent-bit/values.yaml	2021-06-22 12:25:10.000000000 +0200
-+++ charts/fluent-bit.zdt/values.yaml	2021-07-19 10:23:01.383398153 +0200
-@@ -12,6 +12,7 @@
-   # tag:
- 
- testFramework:
-+  enabled: false
-   image:
-     repository: busybox
-     pullPolicy: Always

charts/kubezero-logging/update.sh

@@ -19,7 +19,7 @@ yq eval -Mi 'del(.dependencies)' charts/eck-operator/Chart.yaml
 rm -rf charts/fluent-bit
 curl -L -s -o - https://github.com/fluent/helm-charts/releases/download/fluent-bit-${FLUENT_BIT_VERSION}/fluent-bit-${FLUENT_BIT_VERSION}.tgz | tar xfz - -C charts
 
-patch -i fluent-bit.patch -p0 --no-backup-if-mismatch
+# patch -i fluent-bit.patch -p0 --no-backup-if-mismatch
 
 
 # FluentD

charts/kubezero-logging/values.yaml

@@ -48,7 +48,7 @@ fluentd:
   enabled: false
   image:
     repository: public.ecr.aws/zero-downtime/fluentd-concenter
-    tag: v1.14.2
+    tag: v1.16.0
   istio:
     enabled: false
 
@@ -246,7 +246,10 @@ fluent-bit:
 
   image:
     #repository: public.ecr.aws/zero-downtime/fluent-bit
-    tag: 1.9.8
+    tag: 2.0.10
+
+  testFramework:
+    enabled: false
 
   serviceMonitor:
     enabled: false
@@ -276,8 +279,8 @@ fluent-bit:
       tls: false
 
     input:
-      memBufLimit: 4MB
+      memBufLimit: 16MB
-      refreshInterval: 10
+      refreshInterval: 5
 
     logLevel: info
     flushInterval: 5
@@ -300,16 +303,19 @@ fluent-bit:
       [INPUT]
           Name tail
           Path /var/log/containers/*.log
+          # Exclude ourselves to current error spam, https://github.com/fluent/fluent-bit/issues/5769
+          Exclude_Path *logging-fluent-bit*
           multiline.parser cri
           Tag cri.*
           Skip_Long_Lines On
+          Skip_Empty_Lines On
           DB /var/log/flb_kube.db
           DB.Sync Normal
           DB.locking true
           # Buffer_Max_Size 1M
           {{- with .Values.config.input }}
-          Mem_Buf_Limit {{ default "4MB" .memBufLimit }}
+          Mem_Buf_Limit {{ default "16MB" .memBufLimit }}
-          Refresh_Interval {{ default 10 .refreshInterval }}
+          Refresh_Interval {{ default 5 .refreshInterval }}
           {{- end }}
 
     filters: |

charts/kubezero-metrics/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-metrics
 description: KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all Kubernetes integrations.
 type: application
-version: 0.8.9
+version: 0.9.2
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -19,16 +19,16 @@ dependencies:
     repository: https://cdn.zero-downtime.net/charts/
     # https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack
   - name: kube-prometheus-stack
-    version: 43.2.0
+    version: 45.27.2
     # Switch back to upstream once all alerts are fixed eg. etcd gpcr
     # repository: https://prometheus-community.github.io/helm-charts
   - name: prometheus-adapter
-    version: 3.5.0
+    version: 4.1.1
     repository: https://prometheus-community.github.io/helm-charts
     condition: prometheus-adapter.enabled
   - name: prometheus-pushgateway
-    version: 2.0.2
+    version: 2.1.3
     # Switch back to upstream once namespaces are supported
     repository: https://prometheus-community.github.io/helm-charts
     condition: prometheus-pushgateway.enabled
-kubeVersion: ">= 1.24.0"
+kubeVersion: ">= 1.25.0"

charts/kubezero-metrics/README.md

@@ -1,6 +1,6 @@
 # kubezero-metrics
 
-![Version: 0.8.9](https://img.shields.io/badge/Version-0.8.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.9.2](https://img.shields.io/badge/Version-0.9.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all Kubernetes integrations.
 
@@ -14,14 +14,14 @@ KubeZero Umbrella Chart for Prometheus, Grafana and Alertmanager as well as all
 
 ## Requirements
 
-Kubernetes: `>= 1.24.0`
+Kubernetes: `>= 1.25.0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-|  | kube-prometheus-stack | 43.2.0 |
+|  | kube-prometheus-stack | 45.27.2 |
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
-| https://prometheus-community.github.io/helm-charts | prometheus-adapter | 3.5.0 |
+| https://prometheus-community.github.io/helm-charts | prometheus-adapter | 4.1.1 |
-| https://prometheus-community.github.io/helm-charts | prometheus-pushgateway | 2.0.2 |
+| https://prometheus-community.github.io/helm-charts | prometheus-pushgateway | 2.1.3 |
 
 ## Values
 
@@ -155,7 +155,7 @@ Kubernetes: `>= 1.24.0`
 | kube-prometheus-stack.prometheus-node-exporter.prometheus.monitor.relabelings[0].replacement | string | `"$1"` |  |
 | kube-prometheus-stack.prometheus-node-exporter.prometheus.monitor.relabelings[0].separator | string | `";"` |  |
 | kube-prometheus-stack.prometheus-node-exporter.prometheus.monitor.relabelings[0].sourceLabels[0] | string | `"__meta_kubernetes_pod_node_name"` |  |
-| kube-prometheus-stack.prometheus-node-exporter.prometheus.monitor.relabelings[0].targetLabel | string | `"node"` |  |
+| kube-prometheus-stack.prometheus-node-exporter.prometheus.monitor.relabelings[0].targetLabel | string | `"instance"` |  |
 | kube-prometheus-stack.prometheus-node-exporter.resources.requests.cpu | string | `"20m"` |  |
 | kube-prometheus-stack.prometheus-node-exporter.resources.requests.memory | string | `"16Mi"` |  |
 | kube-prometheus-stack.prometheus.enabled | bool | `true` |  |
@@ -218,3 +218,5 @@ Kubernetes: `>= 1.24.0`
 - https://grafana.com/api/dashboards/3662/revisions/2/download
 ## AlertManager SNS Forwarder
 - https://github.com/DataReply/alertmanager-sns-forwarder
+## docker-registry
+- https://github.com/lstn/misc-grafana-dashboards/blob/master/dashboards/docker-registry.json

charts/kubezero-metrics/README.md.gotmpl

@@ -23,3 +23,5 @@
 - https://grafana.com/api/dashboards/3662/revisions/2/download
 ## AlertManager SNS Forwarder
 - https://github.com/DataReply/alertmanager-sns-forwarder
+## docker-registry
+- https://github.com/lstn/misc-grafana-dashboards/blob/master/dashboards/docker-registry.json

charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml

@@ -7,20 +7,20 @@ annotations:
       url: https://github.com/prometheus-operator/kube-prometheus
   artifacthub.io/operator: "true"
 apiVersion: v2
-appVersion: 0.61.1
+appVersion: v0.65.1
 dependencies:
 - condition: kubeStateMetrics.enabled
   name: kube-state-metrics
   repository: https://prometheus-community.github.io/helm-charts
-  version: 4.24.*
+  version: 5.5.*
 - condition: nodeExporter.enabled
   name: prometheus-node-exporter
   repository: https://prometheus-community.github.io/helm-charts
-  version: 4.8.*
+  version: 4.16.*
 - condition: grafana.enabled
   name: grafana
   repository: https://grafana.github.io/helm-charts
-  version: 6.48.*
+  version: 6.56.*
 description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards,
   and Prometheus rules combined with documentation and scripts to provide easy to
   operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus
@@ -52,4 +52,4 @@ sources:
 - https://github.com/prometheus-community/helm-charts
 - https://github.com/prometheus-operator/kube-prometheus
 type: application
-version: 43.2.0
+version: 45.27.2

charts/kubezero-metrics/charts/kube-prometheus-stack/README.md

@@ -80,6 +80,44 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen
 
 A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions.
 
+### From 44.x to 45.x
+
+This version upgrades Prometheus-Operator to v0.63.0, Prometheus to v2.43.0 and Thanos to v0.30.2.
+
+Run these commands to update the CRDs before applying the upgrade.
+
+```console
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.63.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
+```
+
+### From 43.x to 44.x
+
+This version upgrades Prometheus-Operator to v0.62.0, Prometheus to v2.41.0 and Thanos to v0.30.1.
+
+Run these commands to update the CRDs before applying the upgrade.
+
+```console
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.62.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
+```
+
+If you have explicitly set `prometheusOperator.admissionWebhooks.failurePolicy`, this value is now always used even when `.prometheusOperator.admissionWebhooks.patch.enabled` is `true` (the default).
+
+The values for `prometheusOperator.image.tag` & `prometheusOperator.prometheusConfigReloader.image.tag` are now empty by default and the Chart.yaml `appVersion` field is used instead.
+
 ### From 42.x to 43.x
 
 This version upgrades Prometheus-Operator to v0.61.1, Prometheus to v2.40.5 and Thanos to v0.29.0.

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 9.3.1
+appVersion: 9.5.1
 description: The leading tool for querying and visualizing time series and metrics.
 home: https://grafana.net
 icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png
@@ -19,4 +19,4 @@ name: grafana
 sources:
 - https://github.com/grafana/grafana
 type: application
-version: 6.48.0
+version: 6.56.2

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md

@@ -87,6 +87,7 @@ This version requires Helm >= 3.1.0.
 | `ingress.hosts`                           | Ingress accepted hostnames                    | `["chart-example.local"]`                                                    |
 | `ingress.extraPaths`                      | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-alb-ingress-controller/guide/ingress/annotation/#actions). Requires `ingress.hosts` to have one or more host entries. | `[]`                                                    |
 | `ingress.tls`                             | Ingress TLS configuration                     | `[]`                                                    |
+| `ingress.ingressClassName`                | Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 | `""`                              |
 | `resources`                               | CPU/Memory resource requests/limits           | `{}`                                                    |
 | `nodeSelector`                            | Node labels for pod assignment                | `{}`                                                    |
 | `tolerations`                             | Toleration labels for pod assignment          | `[]`                                                    |
@@ -146,7 +147,7 @@ This version requires Helm >= 3.1.0.
 | `podPortName`                             | Name of the grafana port on the pod           | `grafana`                                               |
 | `lifecycleHooks`                          | Lifecycle hooks for podStart and preStop [Example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/#define-poststart-and-prestop-handlers)     | `{}`                                                    |
 | `sidecar.image.repository`                | Sidecar image repository                      | `quay.io/kiwigrid/k8s-sidecar`                          |
-| `sidecar.image.tag`                       | Sidecar image tag                             | `1.19.2`                                                |
+| `sidecar.image.tag`                       | Sidecar image tag                             | `1.22.0`                                                |
 | `sidecar.image.sha`                       | Sidecar image sha (optional)                  | `""`                                                    |
 | `sidecar.imagePullPolicy`                 | Sidecar image pull policy                     | `IfNotPresent`                                          |
 | `sidecar.resources`                       | Sidecar resources                             | `{}`                                                    |
@@ -216,11 +217,12 @@ This version requires Helm >= 3.1.0.
 | `rbac.create`                             | Create and use RBAC resources                 | `true`                                                  |
 | `rbac.namespaced`                         | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance  | `false` |
 | `rbac.useExistingRole`                    | Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to the rolename set here. | `nil` |
-| `rbac.pspEnabled`                         | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `true`                 |
+| `rbac.pspEnabled`                         | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `false`                |
-| `rbac.pspUseAppArmor`                     | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`)  | `true`                    |
+| `rbac.pspUseAppArmor`                     | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`)  | `false`                   |
 | `rbac.extraRoleRules`                     | Additional rules to add to the Role           | []                                                      |
 | `rbac.extraClusterRoleRules`              | Additional rules to add to the ClusterRole    | []                                                      |
-| `command`                     | Define command to be executed by grafana container at startup  | `nil`                                              |
+| `command`                                 | Define command to be executed by grafana container at startup | `nil`                                   |
+| `args`                                    | Define additional args if command is used     | `nil`                                                   |
 | `testFramework.enabled`                   | Whether to create test-related resources      | `true`                                                  |
 | `testFramework.image`                     | `test-framework` image repository.            | `bats/bats`                                             |
 | `testFramework.tag`                       | `test-framework` image tag.                   | `v1.4.1`                                                |
@@ -250,6 +252,7 @@ This version requires Helm >= 3.1.0.
 | `imageRenderer.image.sha`                  | image-renderer Image sha (optional)                                                | `""`                             |
 | `imageRenderer.image.pullPolicy`           | image-renderer ImagePullPolicy                                                     | `Always`                         |
 | `imageRenderer.env`                        | extra env-vars for image-renderer                                                  | `{}`                             |
+| `imageRenderer.envValueFrom`               | Environment variables for image-renderer from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. Can be templated | `{}` |
 | `imageRenderer.serviceAccountName`         | image-renderer deployment serviceAccountName                                       | `""`                             |
 | `imageRenderer.securityContext`            | image-renderer deployment securityContext                                          | `{}`                             |
 | `imageRenderer.hostAliases`                | image-renderer deployment Host Aliases                                             | `[]`                             |
@@ -276,11 +279,10 @@ This version requires Helm >= 3.1.0.
 | `networkPolicy.egress.ports`               | An array of ports to allow for the egress                    | `[]`    |
 | `enableKubeBackwardCompatibility`          | Enable backward compatibility of kubernetes where pod's defintion version below 1.13 doesn't have the enableServiceLinks option  | `false`     |
 
-
-
 ### Example ingress with path
 
 With grafana 6.3 and above
+
 ```yaml
 grafana.ini:
   server:
@@ -397,9 +399,41 @@ filters out the ones with a label as defined in `sidecar.datasources.label`. The
 those secrets are written to a folder and accessed by grafana on startup. Using these yaml files,
 the data sources in grafana can be imported.
 
+Should you aim for reloading datasources in Grafana each time the config is changed, set `sidecar.datasources.skipReload: false` and adjust `sidecar.datasources.reloadURL` to `http://<svc-name>.<namespace>.svc.cluster.local/api/admin/provisioning/datasources/reload`.
+
 Secrets are recommended over configmaps for this usecase because datasources usually contain private
 data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those.
 
+Example values to add a postgres datasource as a kubernetes secret:
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: grafana-datasources
+  labels:
+    grafana_datasource: 'true' # default value for: sidecar.datasources.label
+stringData:
+  pg-db.yaml: |-
+    apiVersion: 1
+    datasources:
+      - name: My pg db datasource
+        type: postgres
+        url: my-postgresql-db:5432
+        user: db-readonly-user
+        secureJsonData:
+          password: 'SUperSEcretPa$$word'
+        jsonData:
+          database: my_datase
+          sslmode: 'disable' # disable/require/verify-ca/verify-full
+          maxOpenConns: 0 # Grafana v5.4+
+          maxIdleConns: 2 # Grafana v5.4+
+          connMaxLifetime: 14400 # Grafana v5.4+
+          postgresVersion: 1000 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10
+          timescaledb: false
+        # <bool> allow users to edit datasources from the UI.
+        editable: false
+```
+
 Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file):
 
 ```yaml
@@ -491,6 +525,51 @@ delete_notifiers:
     # default org_id: 1
 ```
 
+## Provision alert rules, contact points, notification policies and notification templates
+
+There are two methods to provision alerting configuration in Grafana. Below are some examples and explanations as to how to use each method:
+
+```yaml
+alerting:
+  team1-alert-rules.yaml:
+    file: alerting/team1/rules.yaml
+  team2-alert-rules.yaml:
+    file: alerting/team2/rules.yaml
+  team3-alert-rules.yaml:
+    file: alerting/team3/rules.yaml
+  notification-policies.yaml:
+    file: alerting/shared/notification-policies.yaml
+  notification-templates.yaml:
+    file: alerting/shared/notification-templates.yaml
+  contactpoints.yaml:
+    apiVersion: 1
+    contactPoints:
+      - orgId: 1
+        name: Slack channel
+        receivers:
+          - uid: default-receiver
+            type: slack
+            settings:
+              # Webhook URL to be filled in
+              url: ""
+              # We need to escape double curly braces for the tpl function.
+              text: '{{ `{{ template "default.message" . }}` }}'
+              title: '{{ `{{ template "default.title" . }}` }}'
+```
+
+There are two possibilities:
+
+* Inlining the file contents as described in the example `values.yaml` and the official [Grafana documentation](https://grafana.com/docs/grafana/next/alerting/set-up/provision-alerting-resources/file-provisioning/).
+* Importing a file using a relative path starting from the chart root directory.
+
+### Important notes on file provisioning
+
+* The chart supports importing YAML and JSON files.
+* The filename must be unique, otherwise one volume mount will overwrite the other.
+* In case of inlining, double curly braces that arise from the Grafana configuration format and are not intended as templates for the chart must be escaped.
+* The number of total files under `alerting:` is not limited. Each file will end up as a volume mount in the corresponding provisioning folder of the deployed Grafana instance.
+* The file size for each import is limited by what the function `.Files.Get` can handle, which suffices for most cases.
+
 ## How to serve Grafana with a path prefix (/grafana)
 
 In order to serve Grafana with a prefix (e.g., <http://example.com/grafana>), add the following to your values.yaml.
@@ -598,6 +677,9 @@ grafana.ini:
   unified_alerting:
     enabled: true
     ha_peers: {{ Name }}-headless:9094
+    ha_listen_address: ${POD_IP}:9094
+    ha_advertise_address: ${POD_IP}:9094
+
   alerting:
     enabled: false
 ```

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/NOTES.txt

@@ -1,6 +1,7 @@
 1. Get your '{{ .Values.adminUser }}' user password by running:
 
-   kubectl get secret --namespace {{ include "grafana.namespace" . }} {{ include "grafana.fullname" . }} -o jsonpath="{.data.admin-password}" | base64 --decode ; echo
+   kubectl get secret --namespace {{ include "grafana.namespace" . }} {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} -o jsonpath="{.data.{{ .Values.admin.passwordKey | default "admin-password" }}}" | base64 --decode ; echo
+
 
 2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster:
 

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl

@@ -68,7 +68,7 @@ Common labels
 helm.sh/chart: {{ include "grafana.chart" . }}
 {{ include "grafana.selectorLabels" . }}
 {{- if or .Chart.AppVersion .Values.image.tag }}
-app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }}
+app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }}
 {{- end }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- with .Values.extraLabels }}
@@ -91,7 +91,7 @@ Common labels
 helm.sh/chart: {{ include "grafana.chart" . }}
 {{ include "grafana.imageRenderer.selectorLabels" . }}
 {{- if or .Chart.AppVersion .Values.image.tag }}
-app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }}
+app.kubernetes.io/version: {{ mustRegexReplaceAllLiteral "@sha.*" .Values.image.tag "" | default .Chart.AppVersion | quote }}
 {{- end }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end }}
@@ -145,10 +145,12 @@ Return the appropriate apiVersion for ingress.
 Return the appropriate apiVersion for Horizontal Pod Autoscaler.
 */}}
 {{- define "grafana.hpa.apiVersion" -}}
-{{- if semverCompare "<1.23-0" .Capabilities.KubeVersion.Version }}
+{{- if $.Capabilities.APIVersions.Has "autoscaling/v2/HorizontalPodAutoscaler" }}
-{{- print "autoscaling/v2beta1" }}
-{{- else }}
 {{- print "autoscaling/v2" }}
+{{- else if $.Capabilities.APIVersions.Has "autoscaling/v2beta2/HorizontalPodAutoscaler" }}
+{{- print "autoscaling/v2beta2" }}
+{{- else }}
+{{- print "autoscaling/v2beta1" }}
 {{- end }}
 {{- end }}
 

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl

@@ -763,7 +763,13 @@ containers:
     {{- range .Values.command }}
       - {{ . | quote }}
     {{- end }}
-    {{- end}}
+    {{- end }}
+    {{- if .Values.args }}
+    args:
+    {{- range .Values.args }}
+      - {{ . | quote }}
+    {{- end }}
+    {{- end }}
     {{- with .Values.containerSecurityContext }}
     securityContext:
       {{- toYaml . | nindent 6 }}
@@ -780,7 +786,7 @@ containers:
       {{- range .Values.extraConfigmapMounts }}
       - name: {{ tpl .name $root }}
         mountPath: {{ tpl .mountPath $root }}
-        subPath: {{ (tpl .subPath $root) | default "" }}
+        subPath: {{ tpl (.subPath | default "") $root }}
         readOnly: {{ .readOnly }}
       {{- end }}
       - name: storage
@@ -878,7 +884,17 @@ containers:
       - name: {{ .Values.podPortName }}
         containerPort: {{ .Values.service.targetPort }}
         protocol: TCP
+      - name: {{ .Values.gossipPortName }}-tcp
+        containerPort: 9094
+        protocol: TCP
+      - name: {{ .Values.gossipPortName }}-udp
+        containerPort: 9094
+        protocol: UDP
     env:
+      - name: POD_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.podIP
       {{- if and (not .Values.env.GF_SECURITY_ADMIN_USER) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }}
       - name: GF_SECURITY_ADMIN_USER
         valueFrom:
@@ -1128,8 +1144,7 @@ volumes:
       path: {{ .hostPath }}
     {{- else if .csi }}
     csi:
-      data:
+      {{- toYaml .data | nindent 6 }}
-        {{- toYaml .data | nindent 8 }}
     {{- else }}
     emptyDir: {}
     {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/clusterrole.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.rbac.create (not .Values.rbac.namespaced) (not .Values.rbac.useExistingRole) }}
+{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) (not .Values.rbac.useExistingRole) }}
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -9,9 +9,9 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
   name: {{ include "grafana.fullname" . }}-clusterrole
-{{- if or .Values.sidecar.dashboards.enabled (or .Values.rbac.extraClusterRoleRules (or .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled)) }}
+{{- if or .Values.sidecar.dashboards.enabled .Values.rbac.extraClusterRoleRules .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }}
 rules:
-  {{- if or .Values.sidecar.dashboards.enabled (or .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled) }}
+  {{- if or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.sidecar.alerts.enabled }}
   - apiGroups: [""] # "" indicates the core API group
     resources: ["configmaps", "secrets"]
     verbs: ["get", "watch", "list"]

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/clusterrolebinding.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.rbac.create (not .Values.rbac.namespaced) }}
+{{- if and .Values.rbac.create (or (not .Values.rbac.namespaced) .Values.rbac.extraClusterRoleRules) }}
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.createConfigmap }}
+{{- $files := .Files }}
 {{- $root := . -}}
 apiVersion: v1
 kind: ConfigMap
@@ -53,9 +54,14 @@ data:
   {{- end }}
 
   {{- range $key, $value := .Values.alerting }}
+  {{- if (hasKey $value "file") }}
+  {{- $key | nindent 2 }}:
+  {{- toYaml ( $files.Get $value.file ) | nindent 4}}
+  {{- else }}
   {{- $key | nindent 2 }}: |
     {{- tpl (toYaml $value | nindent 4) $root }}
   {{- end }}
+  {{- end }}
 
   {{- range $key, $value := .Values.dashboardProviders }}
   {{- $key | nindent 2 }}: |
@@ -81,12 +87,19 @@ data:
     --connect-timeout 60 \
     --max-time 60 \
       {{- if not $value.b64content }}
+        {{- if not $value.acceptHeader }}
     -H "Accept: application/json" \
+        {{- else }}
+    -H "Accept: {{ $value.acceptHeader }}" \
+        {{- end }}
         {{- if $value.token }}
     -H "Authorization: token {{ $value.token }}" \
         {{- end }}
         {{- if $value.bearerToken }}
     -H "Authorization: Bearer {{ $value.bearerToken }}" \
+        {{- end }}
+        {{- if $value.basic }}
+    -H "Authorization: Basic {{ $value.basic }}" \
         {{- end }}
         {{- if $value.gitlabToken }}
     -H "PRIVATE-TOKEN: {{ $value.gitlabToken }}" \

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/deployment.yaml

@@ -42,6 +42,7 @@ spec:
         {{- if .Values.envRenderSecret }}
         checksum/secret-env: {{ include (print $.Template.BasePath "/secret-env.yaml") . | sha256sum }}
         {{- end }}
+        kubectl.kubernetes.io/default-container: {{ .Chart.Name }}
         {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/headless-service.yaml

@@ -17,7 +17,6 @@ spec:
     {{- include "grafana.selectorLabels" . | nindent 4 }}
   type: ClusterIP
   ports:
-  - protocol: TCP
+  - name: {{ .Values.gossipPortName }}-tcp
-    port: 3000
+    port: 9094
-    targetPort: {{ .Values.service.targetPort }}
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/hpa.yaml

@@ -26,7 +26,7 @@ spec:
     - type: Resource
       resource:
         name: memory
-        {{- if semverCompare "<1.23-0" .Capabilities.KubeVersion.Version }}
+        {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }}
         targetAverageUtilization: {{ .Values.autoscaling.targetMemory }}
         {{- else }}
         target:
@@ -38,7 +38,7 @@ spec:
     - type: Resource
       resource:
         name: cpu
-        {{- if semverCompare "<1.23-0" .Capabilities.KubeVersion.Version }}
+        {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }}
         targetAverageUtilization: {{ .Values.autoscaling.targetCPU }}
         {{- else }}
         target:
@@ -46,4 +46,7 @@ spec:
           averageUtilization: {{ .Values.autoscaling.targetCPU }}
         {{- end }}
     {{- end }}
+  {{- if .Values.autoscaling.behavior }}
+  behavior: {{ toYaml .Values.autoscaling.behavior | nindent 4 }}
+  {{- end }}
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml

@@ -15,7 +15,9 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
+  {{- if and (not .Values.imageRenderer.autoscaling.enabled) (.Values.imageRenderer.replicas) }}
   replicas: {{ .Values.imageRenderer.replicas }}
+  {{- end }}
   revisionHistoryLimit: {{ .Values.imageRenderer.revisionHistoryLimit }}
   selector:
     matchLabels:
@@ -86,6 +88,15 @@ spec:
           env:
             - name: HTTP_PORT
               value: {{ .Values.imageRenderer.service.targetPort | quote }}
+          {{- if .Values.imageRenderer.serviceMonitor.enabled }}
+            - name: ENABLE_METRICS
+              value: "true"
+          {{- end }}
+          {{- range $key, $value := .Values.imageRenderer.envValueFrom }}
+            - name: {{ $key | quote }}
+              valueFrom:
+                {{- tpl (toYaml $value) $ | nindent 16 }}
+          {{- end }}
           {{- range $key, $value := .Values.imageRenderer.env }}
             - name: {{ $key | quote }}
               value: {{ $value | quote }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-hpa.yaml

@@ -0,0 +1,47 @@
+{{- if and .Values.imageRenderer.enabled .Values.imageRenderer.autoscaling.enabled }}
+apiVersion: {{ include "grafana.hpa.apiVersion" . }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "grafana.fullname" . }}-image-renderer
+  namespace: {{ include "grafana.namespace" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer
+    helm.sh/chart: {{ include "grafana.chart" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "grafana.fullname" . }}-image-renderer
+  minReplicas: {{ .Values.imageRenderer.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.imageRenderer.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.imageRenderer.autoscaling.targetMemory }}
+    - type: Resource
+      resource:
+        name: memory
+        {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }}
+        targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }}
+        {{- else }}
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.imageRenderer.autoscaling.targetMemory }}
+        {{- end }}
+    {{- end }}
+    {{- if .Values.imageRenderer.autoscaling.targetCPU }}
+    - type: Resource
+      resource:
+        name: cpu
+        {{- if eq (include "grafana.hpa.apiVersion" .) "autoscaling/v2beta1" }}
+        targetAverageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }}
+        {{- else }}
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.imageRenderer.autoscaling.targetCPU }}
+        {{- end }}
+    {{- end }}
+  {{- if .Values.imageRenderer.autoscaling.behavior }}
+  behavior: {{ toYaml .Values.imageRenderer.autoscaling.behavior | nindent 4 }}
+  {{- end }}
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-network-policy.yaml

@@ -24,13 +24,16 @@ spec:
       from:
         - namespaceSelector:
             matchLabels:
-              name: {{ include "grafana.namespace" . }}
+              kubernetes.io/metadata.name: {{ include "grafana.namespace" . }}
-        - podSelector:
+          podSelector:
             matchLabels:
               {{- include "grafana.selectorLabels" . | nindent 14 }}
               {{- with .Values.podLabels }}
               {{- toYaml . | nindent 14 }}
               {{- end }}
+        {{- with .Values.imageRenderer.networkPolicy.extraIngressSelectors -}}
+        {{ toYaml . | nindent 8 }}
+        {{- end }}
 {{- end }}
 
 {{- if and .Values.imageRenderer.enabled .Values.imageRenderer.networkPolicy.limitEgress }}
@@ -61,10 +64,13 @@ spec:
           protocol: TCP
     # talk only to grafana
     - ports:
-        - port: {{ .Values.service.port }}
+        - port: {{ .Values.service.targetPort }}
           protocol: TCP
       to:
-        - podSelector:
+        - namespaceSelector:
+            matchLabels:
+              name: {{ include "grafana.namespace" . }}
+          podSelector:
             matchLabels:
               {{- include "grafana.selectorLabels" . | nindent 14 }}
               {{- with .Values.podLabels }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-servicemonitor.yaml

@@ -0,0 +1,48 @@
+{{- if .Values.imageRenderer.serviceMonitor.enabled }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "grafana.fullname" . }}-image-renderer
+  {{- if .Values.imageRenderer.serviceMonitor.namespace }}
+  namespace: {{ tpl .Values.imageRenderer.serviceMonitor.namespace . }}
+  {{- else }}
+  namespace: {{ include "grafana.namespace" . }}
+  {{- end }}
+  labels:
+    {{- include "grafana.imageRenderer.labels" . | nindent 4 }}
+    {{- with .Values.imageRenderer.serviceMonitor.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+  - port: {{ .Values.imageRenderer.service.portName }}
+    {{- with .Values.imageRenderer.serviceMonitor.interval }}
+    interval: {{ . }}
+    {{- end }}
+    {{- with .Values.imageRenderer.serviceMonitor.scrapeTimeout }}
+    scrapeTimeout: {{ . }}
+    {{- end }}
+    honorLabels: true
+    path: {{ .Values.imageRenderer.serviceMonitor.path }}
+    scheme: {{ .Values.imageRenderer.serviceMonitor.scheme }}
+    {{- with .Values.imageRenderer.serviceMonitor.tlsConfig }}
+    tlsConfig:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+    {{- with .Values.imageRenderer.serviceMonitor.relabelings }}
+    relabelings:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+  jobLabel: "{{ .Release.Name }}-image-renderer"
+  selector:
+    matchLabels:
+      {{- include "grafana.imageRenderer.selectorLabels" . | nindent 6 }}
+  namespaceSelector:
+    matchNames:
+      - {{ include "grafana.namespace" . }}
+  {{- with .Values.imageRenderer.serviceMonitor.targetLabels }}
+  targetLabels:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/role.yaml

@@ -12,7 +12,7 @@ metadata:
   {{- end }}
 {{- if or .Values.rbac.pspEnabled (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled .Values.sidecar.plugins.enabled .Values.rbac.extraRoleRules)) }}
 rules:
-  {{- if .Values.rbac.pspEnabled }}
+  {{- if and .Values.rbac.pspEnabled (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }}
   - apiGroups:      ['extensions']
     resources:      ['podsecuritypolicies']
     verbs:          ['use']

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/servicemonitor.yaml

@@ -41,4 +41,8 @@ spec:
   namespaceSelector:
     matchNames:
       - {{ include "grafana.namespace" . }}
+  {{- with .Values.serviceMonitor.targetLabels }}
+  targetLabels:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/statefulset.yaml

@@ -31,6 +31,7 @@ spec:
         {{- if and (or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret))) (not .Values.env.GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION) }}
         checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
         {{- end }}
+        kubectl.kubernetes.io/default-container: {{ .Chart.Name }}
         {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml

@@ -17,8 +17,8 @@ rbac:
   create: true
   ## Use an existing ClusterRole/Role (depending on rbac.namespaced false/true)
   # useExistingRole: name-of-some-(cluster)role
-  pspEnabled: true
+  pspEnabled: false
-  pspUseAppArmor: true
+  pspUseAppArmor: false
   namespaced: false
   extraRoleRules: []
   # - apiGroups: []
@@ -52,6 +52,7 @@ autoscaling:
   maxReplicas: 5
   targetCPU: "60"
   targetMemory: ""
+  behavior: {}
 
 ## See `kubectl explain poddisruptionbudget.spec` for more
 ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/
@@ -83,7 +84,7 @@ livenessProbe:
 # schedulerName: "default-scheduler"
 
 image:
-  repository: grafana/grafana
+  repository: docker.io/grafana/grafana
   # Overrides the Grafana image tag whose default is the chart appVersion
   tag: ""
   sha: ""
@@ -99,17 +100,23 @@ image:
 
 testFramework:
   enabled: true
-  image: "bats/bats"
+  image: docker.io/bats/bats
   tag: "v1.4.1"
   imagePullPolicy: IfNotPresent
   securityContext: {}
 
 securityContext:
+  runAsNonRoot: true
   runAsUser: 472
   runAsGroup: 472
   fsGroup: 472
 
-containerSecurityContext: {}
+containerSecurityContext:
+  capabilities:
+    drop:
+    - ALL
+  seccompProfile:
+    type: RuntimeDefault
 
 # Enable creating the grafana configmap
 createConfigmap: true
@@ -136,7 +143,7 @@ extraLabels: {}
 # priorityClassName:
 
 downloadDashboardsImage:
-  repository: curlimages/curl
+  repository: docker.io/curlimages/curl
   tag: 7.85.0
   sha: ""
   pullPolicy: IfNotPresent
@@ -145,7 +152,13 @@ downloadDashboards:
   env: {}
   envFromSecret: ""
   resources: {}
-  securityContext: {}
+  securityContext:
+    allowPrivilegeEscalation: false
+    capabilities:
+      drop:
+      - ALL
+    seccompProfile:
+      type: RuntimeDefault
   envValueFrom: {}
   #  ENV_NAME:
   #    configMapKeyRef:
@@ -159,7 +172,7 @@ downloadDashboards:
 # podLabels: {}
 
 podPortName: grafana
-
+gossipPortName: gossip
 ## Deployment annotations
 # annotations: {}
 
@@ -193,6 +206,7 @@ serviceMonitor:
   tlsConfig: {}
   scrapeTimeout: 30s
   relabelings: []
+  targetLabels: []
 
 extraExposePorts: []
  # - name: keycloak
@@ -344,7 +358,7 @@ initChownData:
   ## initChownData container image
   ##
   image:
-    repository: busybox
+    repository: docker.io/library/busybox
     tag: "1.31.1"
     sha: ""
     pullPolicy: IfNotPresent
@@ -362,7 +376,11 @@ initChownData:
   securityContext:
     runAsNonRoot: false
     runAsUser: 0
-
+    seccompProfile:
+      type: RuntimeDefault
+    capabilities:
+      add:
+        - CHOWN
 
 # Administrator credentials when not using an existing secret (see below)
 adminUser: admin
@@ -382,6 +400,14 @@ admin:
 # - "sh"
 # - "/run.sh"
 
+## Optionally define args if command is used
+## Needed if using `hashicorp/envconsul` to manage secrets
+## By default no arguments are set
+# args:
+# - "-secret"
+# - "secret/grafana"
+# - "./grafana"
+
 ## Extra environment variables that will be pass onto deployment pods
 ##
 ## to provide grafana with access to CloudWatch on AWS EKS:
@@ -510,6 +536,9 @@ lifecycleHooks: {}
 plugins: []
   # - digrich-bubblechart-panel
   # - grafana-clock-panel
+  ## You can also use other plugin download URL, as long as they are valid zip files,
+  ## and specify the name of the plugin after the semicolon. Like this:
+  # - https://grafana.com/api/plugins/marcusolsson-json-datasource/versions/1.3.2/download;marcusolsson-json-datasource
 
 ## Configure grafana datasources
 ## ref: http://docs.grafana.org/administration/provisioning/#datasources
@@ -663,6 +692,10 @@ dashboards: {}
   #   local-dashboard-bitbucket:
   #     url: https://example.com/repository/test-bitbucket.json
   #     bearerToken: ''
+  #   local-dashboard-azure:
+  #     url: https://example.com/repository/test-azure.json
+  #     basic: ''
+  #     acceptHeader: '*/*'
 
 ## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value.
 ## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both.
@@ -754,7 +787,7 @@ smtp:
 sidecar:
   image:
     repository: quay.io/kiwigrid/k8s-sidecar
-    tag: 1.21.0
+    tag: 1.22.0
     sha: ""
   imagePullPolicy: IfNotPresent
   resources: {}
@@ -764,7 +797,13 @@ sidecar:
 #   requests:
 #     cpu: 50m
 #     memory: 50Mi
-  securityContext: {}
+  securityContext:
+    allowPrivilegeEscalation: false
+    capabilities:
+      drop:
+      - ALL
+    seccompProfile:
+      type: RuntimeDefault
   # skipTlsVerify Set to true to skip tls verification for kube api calls
   # skipTlsVerify: true
   enableUniqueFilenames: false
@@ -1008,9 +1047,16 @@ imageRenderer:
   # Enable the image-renderer deployment & service
   enabled: false
   replicas: 1
+  autoscaling:
+    enabled: false
+    minReplicas: 1
+    maxReplicas: 5
+    targetCPU: "60"
+    targetMemory: ""
+    behavior: {}
   image:
     # image-renderer Image repository
-    repository: grafana/grafana-image-renderer
+    repository: docker.io/grafana/grafana-image-renderer
     # image-renderer Image tag
     tag: latest
     # image-renderer Image sha (optional)
@@ -1023,12 +1069,29 @@ imageRenderer:
     # RENDERING_ARGS: --no-sandbox,--disable-gpu,--window-size=1280x758
     # RENDERING_MODE: clustered
     # IGNORE_HTTPS_ERRORS: true
+
+  ## "valueFrom" environment variable references that will be added to deployment pods. Name is templated.
+  ## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#envvarsource-v1-core
+  ## Renders in container spec as:
+  ##   env:
+  ##     ...
+  ##     - name: <key>
+  ##       valueFrom:
+  ##         <value rendered as YAML>
+  envValueFrom: {}
+    #  ENV_NAME:
+    #    configMapKeyRef:
+    #      name: configmap-name
+    #      key: value_key
+
   # image-renderer deployment serviceAccount
   serviceAccountName: ""
   # image-renderer deployment securityContext
   securityContext: {}
   # image-renderer deployment container securityContext
   containerSecurityContext:
+    seccompProfile:
+      type: RuntimeDefault
     capabilities:
       drop: ['ALL']
     allowPrivilegeEscalation: false
@@ -1047,6 +1110,23 @@ imageRenderer:
     targetPort: 8081
     # Adds the appProtocol field to the image-renderer service. This allows to work with istio protocol selection. Ex: "http" or "tcp"
     appProtocol: ""
+  serviceMonitor:
+    ## If true, a ServiceMonitor CRD is created for a prometheus operator
+    ## https://github.com/coreos/prometheus-operator
+    ##
+    enabled: false
+    path: /metrics
+    #  namespace: monitoring  (defaults to use the namespace this chart is deployed to)
+    labels: {}
+    interval: 1m
+    scheme: http
+    tlsConfig: {}
+    scrapeTimeout: 30s
+    relabelings: []
+    # See: https://doc.crds.dev/github.com/prometheus-operator/kube-prometheus/monitoring.coreos.com/ServiceMonitor/v1@v0.11.0#spec-targetLabels
+    targetLabels: []
+      # - targetLabel1
+      # - targetLabel2
   # If https is enabled in Grafana, this needs to be set as 'https' to correctly configure the callback used in Grafana
   grafanaProtocol: http
   # In case a sub_path is used this needs to be added to the image renderer callback
@@ -1060,6 +1140,8 @@ imageRenderer:
     limitIngress: true
     # Enable a NetworkPolicy to limit outbound traffic to only the created grafana pods
     limitEgress: false
+    # Allow additional services to access image-renderer (eg. Prometheus operator when ServiceMonitor is enabled)
+    extraIngressSelectors: []
   resources: {}
 #   limits:
 #     cpu: 100m

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 2.7.0
+appVersion: 2.8.2
 description: Install kube-state-metrics to generate and expose cluster-level metrics
 home: https://github.com/kubernetes/kube-state-metrics/
 keywords:
@@ -18,4 +18,4 @@ name: kube-state-metrics
 sources:
 - https://github.com/kubernetes/kube-state-metrics/
 type: application
-version: 4.24.0
+version: 5.5.0

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md

@@ -2,14 +2,15 @@
 
 Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics).
 
-## Get Repo Info
+## Get Repository Info
-
+<!-- textlint-disable -->
 ```console
 helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 helm repo update
 ```
 
 _See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._
+<!-- textlint-enable -->
 
 ## Install Chart
 
@@ -43,20 +44,19 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen
 
 You can upgrade in-place:
 
-1. [get repo info](#get-repo-info)
+1. [get repository info](#get-repository-info)
-1. [upgrade](#upgrading-chart) your existing release name using the new chart repo
+1. [upgrade](#upgrading-chart) your existing release name using the new chart repository
-
 
 ## Upgrading to v3.0.0
 
 v3.0.0 includes kube-state-metrics v2.0, see the [changelog](https://github.com/kubernetes/kube-state-metrics/blob/release-2.0/CHANGELOG.md) for major changes on the application-side.
 
 The upgraded chart now the following changes:
+
 * Dropped support for helm v2 (helm v3 or later is required)
 * collectors key was renamed to resources
 * namespace key was renamed to namespaces
 
-
 ## Configuration
 
 See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments:
@@ -65,4 +65,21 @@ See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_h
 helm show values prometheus-community/kube-state-metrics
 ```
 
-You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options.
+### kube-rbac-proxy
+
+You can enable `kube-state-metrics` endpoint protection using `kube-rbac-proxy`. By setting `kubeRBACProxy.enabled: true`, this chart will deploy one RBAC proxy container per endpoint (metrics & telemetry).
+To authorize access, authenticate your requests (via a `ServiceAccount` for example) with a `ClusterRole` attached such as:
+
+```yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-state-metrics-read
+rules:
+  - apiGroups: [ "" ]
+    resources: ["services/kube-state-metrics"]
+    verbs:
+      - get
+```
+
+See [kube-rbac-proxy examples](https://github.com/brancz/kube-rbac-proxy/tree/master/examples/resource-attributes) for more details.

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/NOTES.txt

@@ -8,3 +8,16 @@ In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-s
 They are served either as plaintext or protobuf depending on the Accept header.
 They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint.
 
+{{- if .Values.kubeRBACProxy.enabled}}
+
+kube-rbac-proxy endpoint protections is enabled:
+- Metrics endpoints are now HTTPS
+- Ensure that the client authenticates the requests (e.g. via service account) with the following role permissions:
+```
+rules:
+  - apiGroups: [ "" ]
+    resources: ["services/{{ template "kube-state-metrics.fullname" . }}"]
+    verbs:
+      - get
+```
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/_helpers.tpl

@@ -77,9 +77,13 @@ release: {{ .Release.Name }}
 Selector labels
 */}}
 {{- define "kube-state-metrics.selectorLabels" }}
+{{- if .Values.selectorOverride }}
+{{ toYaml .Values.selectorOverride }}
+{{- else }}
 app.kubernetes.io/name: {{ include "kube-state-metrics.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
+{{- end }}
 
 {{/* Sets default scrape limits for servicemonitor */}}
 {{- define "servicemonitor.scrapeLimits" -}}
@@ -99,3 +103,54 @@ labelNameLengthLimit: {{ . }}
 labelValueLengthLimit: {{ . }}
 {{- end }}
 {{- end -}}
+
+{{/*
+Formats imagePullSecrets. Input is (dict "Values" .Values "imagePullSecrets" .{specific imagePullSecrets})
+*/}}
+{{- define "kube-state-metrics.imagePullSecrets" -}}
+{{- range (concat .Values.global.imagePullSecrets .imagePullSecrets) }}
+  {{- if eq (typeOf .) "map[string]interface {}" }}
+- {{ toYaml . | trim }}
+  {{- else }}
+- name: {{ . }}
+  {{- end }}
+{{- end }}
+{{- end -}}
+
+{{/*
+The image to use for kube-state-metrics
+*/}}
+{{- define "kube-state-metrics.image" -}}
+{{- if .Values.image.sha }}
+{{- if .Values.global.imageRegistry }}
+{{- printf "%s/%s:%s@%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }}
+{{- else }}
+{{- printf "%s/%s:%s@%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) .Values.image.sha }}
+{{- end }}
+{{- else }}
+{{- if .Values.global.imageRegistry }}
+{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }}
+{{- else }}
+{{- printf "%s/%s:%s" .Values.image.registry .Values.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.image.tag) }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+The image to use for kubeRBACProxy
+*/}}
+{{- define "kubeRBACProxy.image" -}}
+{{- if .Values.kubeRBACProxy.image.sha }}
+{{- if .Values.global.imageRegistry }}
+{{- printf "%s/%s:%s@%s" .Values.global.imageRegistry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }}
+{{- else }}
+{{- printf "%s/%s:%s@%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) .Values.kubeRBACProxy.image.sha }}
+{{- end }}
+{{- else }}
+{{- if .Values.global.imageRegistry }}
+{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }}
+{{- else }}
+{{- printf "%s/%s:%s" .Values.kubeRBACProxy.image.registry .Values.kubeRBACProxy.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.kubeRBACProxy.image.tag) }}
+{{- end }}
+{{- end }}
+{{- end }}

charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml

@@ -40,6 +40,8 @@ spec:
       priorityClassName: {{ .Values.priorityClassName }}
     {{- end }}
       containers:
+      {{- $httpPort := ternary 9090 (.Values.service.port | default 8080) .Values.kubeRBACProxy.enabled}}
+      {{- $telemetryPort := ternary 9091 (.Values.selfMonitor.telemetryPort | default 8081) .Values.kubeRBACProxy.enabled}}
       - name: {{ template "kube-state-metrics.name" . }}
         {{- if .Values.autosharding.enabled }}
         env:
@@ -56,9 +58,7 @@ spec:
         {{-  if .Values.extraArgs  }}
         {{- .Values.extraArgs | toYaml | nindent 8 }}
         {{-  end  }}
-        {{- if .Values.service.port }}
+        - --port={{ $httpPort }}
-        - --port={{ .Values.service.port | default 8080}}
-        {{-  end  }}
         {{-  if .Values.collectors  }}
         - --resources={{ .Values.collectors | join "," }}
         {{-  end  }}
@@ -96,11 +96,16 @@ spec:
         {{- if .Values.kubeconfig.enabled }}
         - --kubeconfig=/opt/k8s/.kube/config
         {{- end }}
+        {{- if .Values.kubeRBACProxy.enabled }}
+        - --telemetry-host=127.0.0.1
+        - --telemetry-port={{ $telemetryPort }}
+        {{- else }}
         {{- if .Values.selfMonitor.telemetryHost }}
         - --telemetry-host={{ .Values.selfMonitor.telemetryHost }}
         {{- end }}
         {{- if .Values.selfMonitor.telemetryPort }}
-        - --telemetry-port={{ .Values.selfMonitor.telemetryPort | default 8081 }}
+        - --telemetry-port={{ $telemetryPort }}
+        {{- end }}
         {{- end }}
         {{- if or (.Values.kubeconfig.enabled) (.Values.volumeMounts) }}
         volumeMounts:
@@ -114,28 +119,26 @@ spec:
         {{- end }}
         {{- end }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
-        {{- if .Values.image.sha }}
+        image: {{ include "kube-state-metrics.image" . }}
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}@sha256:{{ .Values.image.sha }}"
+        {{- if eq .Values.kubeRBACProxy.enabled false }}
-        {{- else }}
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-        {{- end }}
         ports:
         - containerPort: {{ .Values.service.port | default 8080}}
           name: "http"
         {{- if .Values.selfMonitor.enabled }}
-        - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }}
+        - containerPort: {{ $telemetryPort }}
           name: "metrics"
         {{- end }}
+        {{- end }}
         livenessProbe:
           httpGet:
             path: /healthz
-            port: {{ .Values.service.port | default 8080}}
+            port: {{ $httpPort }}
           initialDelaySeconds: 5
           timeoutSeconds: 5
         readinessProbe:
           httpGet:
             path: /
-            port: {{ .Values.service.port | default 8080}}
+            port: {{ $httpPort }}
           initialDelaySeconds: 5
           timeoutSeconds: 5
         {{- if .Values.resources }}
@@ -146,9 +149,87 @@ spec:
         securityContext:
 {{ toYaml .Values.containerSecurityContext | indent 10 }}
 {{- end }}
-{{- if .Values.imagePullSecrets }}
+        {{-  if .Values.kubeRBACProxy.enabled  }}
+      - name: kube-rbac-proxy-http
+        args:
+        {{-  if .Values.kubeRBACProxy.extraArgs  }}
+        {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }}
+        {{-  end  }}
+        - --secure-listen-address=:{{ .Values.service.port | default 8080}}
+        - --upstream=http://127.0.0.1:{{ $httpPort }}/
+        - --proxy-endpoints-port=8888
+        - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml
+        volumeMounts:
+          - name: kube-rbac-proxy-config
+            mountPath: /etc/kube-rbac-proxy-config
+          {{- with .Values.kubeRBACProxy.volumeMounts }}
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+        imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }}
+        image: {{ include "kubeRBACProxy.image" . }}
+        ports:
+          - containerPort: {{ .Values.service.port | default 8080}}
+            name: "http"
+          - containerPort: 8888
+            name: "http-healthz"
+        readinessProbe:
+          httpGet:
+            scheme: HTTPS
+            port: 8888
+            path: healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        {{- if .Values.kubeRBACProxy.resources }}
+        resources:
+{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }}
+{{- end }}
+{{- if .Values.kubeRBACProxy.containerSecurityContext }}
+        securityContext:
+{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }}
+{{- end }}
+      {{-  if .Values.selfMonitor.enabled  }}
+      - name: kube-rbac-proxy-telemetry
+        args:
+        {{-  if .Values.kubeRBACProxy.extraArgs  }}
+        {{- .Values.kubeRBACProxy.extraArgs | toYaml | nindent 8 }}
+        {{-  end  }}
+        - --secure-listen-address=:{{ .Values.selfMonitor.telemetryPort | default 8081 }}
+        - --upstream=http://127.0.0.1:{{ $telemetryPort }}/
+        - --proxy-endpoints-port=8889
+        - --config-file=/etc/kube-rbac-proxy-config/config-file.yaml
+        volumeMounts:
+          - name: kube-rbac-proxy-config
+            mountPath: /etc/kube-rbac-proxy-config
+          {{- with .Values.kubeRBACProxy.volumeMounts }}
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+        imagePullPolicy: {{ .Values.kubeRBACProxy.image.pullPolicy }}
+        image: {{ include "kubeRBACProxy.image" . }}
+        ports:
+          - containerPort: {{ .Values.selfMonitor.telemetryPort | default 8081 }}
+            name: "metrics"
+          - containerPort: 8889
+            name: "metrics-healthz"
+        readinessProbe:
+          httpGet:
+            scheme: HTTPS
+            port: 8889
+            path: healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        {{- if .Values.kubeRBACProxy.resources }}
+        resources:
+{{ toYaml .Values.kubeRBACProxy.resources | indent 10 }}
+{{- end }}
+{{- if .Values.kubeRBACProxy.containerSecurityContext }}
+        securityContext:
+{{ toYaml .Values.kubeRBACProxy.containerSecurityContext | indent 10 }}
+{{- end }}
+      {{- end }}
+      {{- end }}
+{{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }}
       imagePullSecrets:
-{{ toYaml .Values.imagePullSecrets | indent 8 }}
+        {{- include "kube-state-metrics.imagePullSecrets" (dict "Values" .Values "imagePullSecrets" .Values.imagePullSecrets) | indent 8 }}
       {{- end }}
       {{- if .Values.affinity }}
       affinity:
@@ -166,13 +247,18 @@ spec:
       topologySpreadConstraints:
 {{ toYaml .Values.topologySpreadConstraints | indent 8 }}
       {{- end }}
-      {{- if or (.Values.kubeconfig.enabled) (.Values.volumes) }}
+      {{- if or (.Values.kubeconfig.enabled) (.Values.volumes) (.Values.kubeRBACProxy.enabled) }}
       volumes:
       {{- if .Values.kubeconfig.enabled}}
         - name: kubeconfig
           secret:
             secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig
       {{- end }}
+      {{- if .Values.kubeRBACProxy.enabled}}
+        - name: kube-rbac-proxy-config
+          configMap:
+            name: {{ template "kube-state-metrics.fullname" . }}-rbac-config
+      {{- end }}
       {{- if .Values.volumes }}
 {{ toYaml .Values.volumes | indent 8 }}
       {{- end }}