charts/kubeadm/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubeadm
 description: KubeZero Kubeadm golden config
 type: application
-version: 1.19.8
+version: 1.19.9
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:

charts/kubeadm/templates/ClusterConfiguration.yaml

@@ -23,12 +23,14 @@ controllerManager:
     terminated-pod-gc-threshold: "300"
     leader-elect: {{ .Values.clusterHighAvailable | quote }}
     logging-format: json
+    feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" "platform" .Values.platform ) | trimSuffix "," | quote }}
 scheduler:
   extraArgs:
     profiling: "false"
     bind-address: {{ .Values.listenAddress }}
     leader-elect: {{ .Values.clusterHighAvailable | quote }}
     logging-format: json
+    feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" "platform" .Values.platform ) | trimSuffix "," | quote }}
 apiServer:
   certSANs:
   - {{ regexSplit ":" .Values.apiEndpoint -1 | first }}
@@ -44,13 +46,11 @@ apiServer:
     admission-control-config-file: /etc/kubernetes/apiserver/admission-configuration.yaml
     {{- if eq .Values.platform "aws" }}
     authentication-token-webhook-config-file: /etc/kubernetes/apiserver/aws-iam-authenticator.yaml
-    feature-gates: "CSIMigrationAWS=true,CSIMigrationAWSComplete=true,DefaultPodTopologySpread=true"
-    {{- else }}
-    feature-gates: "DefaultPodTopologySpread=true"
     {{- end }}
+    feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" "platform" .Values.platform ) | trimSuffix "," | quote }}
     enable-admission-plugins: NodeRestriction,EventRateLimit
     {{- if .Values.clusterHighAvailable }}
-    # goaway-chance: ".001"
+    goaway-chance: ".001"
     {{- end }}
     logging-format: json
     {{- with .Values.apiExtraArgs }}

charts/kubeadm/templates/KubeletConfiguration.yaml

@@ -17,21 +17,19 @@ eventRecordQPS: 0
 # tlsCertFile: /var/lib/kubelet/pki/kubelet.crt
 # tlsPrivateKeyFile: /var/lib/kubelet/pki/kubelet.key
 tlsCipherSuites: [TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256]
-featureGates:
+featureGates: {{ include "kubeadm.featuregates" ( dict "return" "map" "platform" .Values.platform ) }}
-  CustomCPUCFSQuotaPeriod: true
-{{- if eq .Values.platform "aws" }}
-  CSIMigrationAWS: true
-  CSIMigrationAWSComplete: true
-{{- end }}
 # Minimal unit is 50m per pod
 podsPerCore: 20
+# cpuCFSQuotaPeriod: 10ms
 # Basic OS on Ubuntu 20.04 incl. crio
-systemReserved:
+#systemReserved:
-  memory: 256Mi
+#  memory: 256Mi
 # This should be dynamic based on number of maxpods and available cores
 # https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture#memory_cpu
 # Below are default minimal for 2 cores and minimum kubelet 
 kubeReserved:
   cpu: 70m
-  memory: 128Mi
+#  memory: 128Mi
-# cpuCFSQuotaPeriod: 10ms
+# Lets use below to reserve memory for system processes as kubeReserved/sytemReserved doesnt go well with systemd it seems
+evictionHard:
+  memory.available: "484Mi"

charts/kubeadm/templates/_helpers.tpl

@@ -0,0 +1,18 @@
+{{- /*
+Feature gates for all control plane components
+*/ -}}
+{{- define "kubeadm.featuregates" -}}
+{{- $gates := dict "DefaultPodTopologySpread" "true" "CustomCPUCFSQuotaPeriod" "true" "GenericEphemeralVolume" "true" }}
+{{- if eq .platform "aws" }}
+{{- $gates = merge $gates ( dict "CSIMigrationAWS" "true" "CSIMigrationAWSComplete" "true") }}
+{{- end }}
+{{- if eq .return "csv" }}
+{{- range $key, $val := $gates }}
+{{- $key }}={{- $val }},
+{{- end }}
+{{- else }}
+{{- range $key, $val := $gates }}
+  {{ $key }}: {{ $val }}
+{{- end }}
+{{- end }}
+{{- end -}}

charts/kubezero-aws-ebs-csi-driver/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: kubezero-aws-ebs-csi-driver
 description: KubeZero Umbrella Chart for aws-ebs-csi-driver
 type: application
-version: 0.4.4
+version: 0.5.0
-appVersion: 0.9.0
+appVersion: 0.10.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 sources:
@@ -18,7 +18,7 @@ maintainers:
   - name: Quarky9
 dependencies:
   - name: aws-ebs-csi-driver
-    version: 0.9.14
+    version: 0.10.0
     repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver
   - name: kubezero-lib
     version: ">= 0.1.3"

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
-appVersion: 0.9.0
+appVersion: 0.10.0
 description: A Helm chart for AWS EBS CSI Driver
 home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver
 keywords:
@@ -15,4 +15,4 @@ maintainers:
 name: aws-ebs-csi-driver
 sources:
 - https://github.com/kubernetes-sigs/aws-ebs-csi-driver
-version: 0.9.14
+version: 0.10.0

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml

@@ -88,6 +88,12 @@ spec:
             - name: AWS_REGION
               value: {{ .Values.region }}
             {{- end }}
+            {{- if .Values.controller.extraVars }}
+            {{- range $key, $val :=  .Values.controller.extraVars }}
+            - name: {{ $key }}
+              value: "{{ $val }}"
+            {{- end }}
+            {{- end }}
 {{- if .Values.proxy.http_proxy }}
             - name: HTTP_PROXY
               value: {{ .Values.proxy.http_proxy | quote }}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml

@@ -39,6 +39,9 @@ spec:
       containers:
         - name: snapshot-controller
           image: {{ printf "%s:%s" .Values.snapshotController.repository .Values.snapshotController.tag }}
+          {{- with .Values.resources }}
+          resources: {{ toYaml . | nindent 12 }}
+          {{- end }}
           env:
 {{- if .Values.proxy.http_proxy }}
           - name: HTTP_PROXY

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml

@@ -6,7 +6,7 @@ replicaCount: 2
 
 image:
   repository: k8s.gcr.io/provider-aws/aws-ebs-csi-driver
-  tag: "v0.9.0"
+  tag: "v0.10.0"
   pullPolicy: IfNotPresent
 
 logLevel: 5
@@ -72,7 +72,7 @@ resources:
 
 priorityClassName: ""
 nodeSelector: {}
-tolerateAllTaints: true
+tolerateAllTaints: false
 tolerations: []
 affinity: {}
 
@@ -109,11 +109,15 @@ k8sTagClusterId: ""
 # region: us-east-1
 region: ""
 
+# Additonal environment variables for the controller
+controller:
+  extraVars: {}
+
 node:
   priorityClassName: ""
   nodeSelector: {}
   podAnnotations: {}
-  tolerateAllTaints: true
+  tolerateAllTaints: false
   tolerations: []
   resources: {}
 

charts/kubezero-aws-ebs-csi-driver/update.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-VERSION=0.9.14
+VERSION=0.10.0
 
 rm -rf charts/aws-ebs-csi-driver
 curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts

charts/kubezero-aws-efs-csi-driver/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: kubezero-aws-efs-csi-driver
 description: KubeZero Umbrella Chart for aws-efs-csi-driver
-version: 0.3.2
+version: 0.3.4
 appVersion: 1.2.0
 kubeVersion: ">=1.18.0-0"
 home: https://kubezero.com
@@ -21,5 +21,5 @@ dependencies:
     version: ">= 0.1.3"
     repository: https://zero-down-time.github.io/kubezero/
   - name: aws-efs-csi-driver
-    version: 1.2.1
+    version: 1.2.2
     # repository: https://kubernetes-sigs.github.io/aws-efs-csi-driver/

charts/kubezero-aws-efs-csi-driver/affinity_resources.patch

@@ -1,6 +1,6 @@
-diff -tubNr charts/aws-efs-csi-driver/templates/node-daemonset.yaml charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml
+diff -tubrN charts/aws-efs-csi-driver/templates/node-daemonset.yaml charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml
---- charts/aws-efs-csi-driver/templates/node-daemonset.yaml	2021-03-19 00:22:54.000000000 +0100
+--- charts/aws-efs-csi-driver/templates/node-daemonset.yaml	2021-03-23 14:34:03.000000000 +0100
-+++ charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml	2021-03-24 13:06:56.105762451 +0100
++++ charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml	2021-04-01 14:31:36.427375803 +0200
 @@ -40,15 +40,10 @@
          {{- with .Values.nodeSelector }}
          {{- . | toYaml | nindent 8 }}
@@ -31,9 +31,20 @@ diff -tubNr charts/aws-efs-csi-driver/templates/node-daemonset.yaml charts/aws-e
          - name: csi-driver-registrar
            image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }}
            args:
-diff -tubNr charts/aws-efs-csi-driver/values.yaml charts/aws-efs-csi-driver.zdt/values.yaml
+diff -tubrN charts/aws-efs-csi-driver/values.yaml charts/aws-efs-csi-driver.zdt/values.yaml
---- charts/aws-efs-csi-driver/values.yaml	2021-03-19 00:22:54.000000000 +0100
+--- charts/aws-efs-csi-driver/values.yaml	2021-03-23 14:34:03.000000000 +0100
-+++ charts/aws-efs-csi-driver.zdt/values.yaml	2021-03-24 13:06:56.105762451 +0100
++++ charts/aws-efs-csi-driver.zdt/values.yaml	2021-04-01 14:37:21.290724721 +0200
+@@ -11,8 +11,8 @@
+ 
+ sidecars:
+   livenessProbeImage:
+-    repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe
+-    tag: v2.1.0-eks-1-18-1
++    repository: k8s.gcr.io/sig-storage/livenessprobe
++    tag: "v2.2.0"
+   nodeDriverRegistrarImage:
+     repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar
+     tag: v2.0.1-eks-1-18-1
 @@ -43,7 +43,15 @@
  
  tolerations: []

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml

@@ -15,4 +15,4 @@ maintainers:
 name: aws-efs-csi-driver
 sources:
 - https://github.com/kubernetes-sigs/aws-efs-csi-driver
-version: 1.2.1
+version: 1.2.2

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl

@@ -48,7 +48,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 Create the name of the service account to use
 */}}
 {{- define "aws-efs-csi-driver.serviceAccountName" -}}
-{{- if .Values.serviceAccount.controller.create -}}
+{{- if .Values.controller.create -}}
     {{ default (include "aws-efs-csi-driver.fullname" .) .Values.serviceAccount.controller.name }}
 {{- else -}}
     {{ default "default" .Values.serviceAccount.controller.name }}

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml

@@ -34,9 +34,7 @@ spec:
         {{- with .Values.nodeSelector }}
         {{- . | toYaml | nindent 8 }}
         {{- end }}
-      {{- if .Values.serviceAccount.controller.create }}
       serviceAccountName: {{ include "aws-efs-csi-driver.serviceAccountName" . }}
-      {{- end }}
       priorityClassName: system-cluster-critical
       tolerations:
         - operator: Exists

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.serviceAccount.controller.create -}}
+{{- if .Values.serviceAccount.controller.create }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -9,6 +9,7 @@ metadata:
   annotations:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+{{- end }}
 ---
 
 kind: ClusterRole
@@ -56,4 +57,3 @@ roleRef:
   kind: ClusterRole
   name: efs-csi-external-provisioner-role
   apiGroup: rbac.authorization.k8s.io
-{{- end }}

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml

@@ -11,8 +11,8 @@ image:
 
 sidecars:
   livenessProbeImage:
-    repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe
+    repository: k8s.gcr.io/sig-storage/livenessprobe
-    tag: v2.1.0-eks-1-18-1
+    tag: "v2.2.0"
   nodeDriverRegistrarImage:
     repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar
     tag: v2.0.1-eks-1-18-1
@@ -98,7 +98,7 @@ storageClasses: []
 #   - tls
 #   parameters:
 #     provisioningMode: efs-ap
-#     fileSystemId: fs-92107410
+#     fileSystemId: fs-1122aabb
 #     directoryPerms: "700"
 #     gidRangeStart: "1000"
 #     gidRangeEnd: "2000"

charts/kubezero-aws-efs-csi-driver/update.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-VERSION=1.2.1
+VERSION=1.2.2
 
 rm -rf charts/aws-efs-csi-driver
 curl -L -s -o - https://github.com/kubernetes-sigs/aws-efs-csi-driver/releases/download/helm-chart-aws-efs-csi-driver-${VERSION}/aws-efs-csi-driver-${VERSION}.tgz | tar xfz - -C charts

charts/kubezero-istio-ingress/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: kubezero-istio-ingress
 description: KubeZero Umbrella Chart for Istio based Ingress
 type: application
-version: 0.5.0
+version: 0.5.1
-appVersion: 1.9.1
+appVersion: 1.9.2
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -16,9 +16,9 @@ dependencies:
     version: ">= 0.1.3"
     repository: https://zero-down-time.github.io/kubezero/
   - name: istio-ingress
-    version: 1.9.1
+    version: 1.9.2
     condition: istio-ingress.enabled
   - name: istio-private-ingress
-    version: 1.9.1
+    version: 1.9.2
     condition: istio-private-ingress.enabled
 kubeVersion: ">= 1.18.0"

charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: istio-ingress
-version: 1.9.1
+version: 1.9.2
 tillerVersion: ">=2.7.2"
 description: Helm chart for deploying Istio gateways
 keywords:

charts/kubezero-istio-ingress/charts/istio-ingress/templates/autoscale.yaml

@@ -3,7 +3,7 @@
 apiVersion: autoscaling/v2beta1
 kind: HorizontalPodAutoscaler
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}
+  name: {{ $gateway.name }}
   namespace: {{ .Release.Namespace }}
   labels:
 {{ $gateway.labels | toYaml | indent 4 }}
@@ -17,7 +17,7 @@ spec:
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
-    name: {{ $gateway.name | default "istio-ingressgateway" }}
+    name: {{ $gateway.name }}
   metrics:
     - type: Resource
       resource:

charts/kubezero-istio-ingress/charts/istio-ingress/templates/deployment.yaml

@@ -1,8 +1,10 @@
+
 {{- $gateway := index .Values "gateways" "istio-ingressgateway" }}
+{{- if eq $gateway.injectionTemplate "" }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}
+  name: {{ $gateway.name }}
   namespace: {{ .Release.Namespace }}
   labels:
 {{ $gateway.labels | toYaml | indent 4 }}
@@ -32,7 +34,7 @@ spec:
         release: istio
         chart: gateways
 {{- end }}
-        service.istio.io/canonical-name: {{ $gateway.name | default "istio-ingressgateway" }}
+        service.istio.io/canonical-name: {{ $gateway.name }}
         {{- if not (eq .Values.revision "") }}
         service.istio.io/canonical-revision: {{ .Values.revision }}
         {{- else}}
@@ -41,14 +43,17 @@ spec:
         istio.io/rev: {{ .Values.revision | default "default" }}
         install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }}
         operator.istio.io/component: "IngressGateways"
-        sidecar.istio.io/inject: "false"
+        sidecar.istio.io/inject: "{{- ne $gateway.injectionTemplate "" }}"
       annotations:
         {{- if .Values.meshConfig.enablePrometheusMerge }}
         prometheus.io/port: "15020"
         prometheus.io/scrape: "true"
         prometheus.io/path: "/stats/prometheus"
         {{- end }}
-        sidecar.istio.io/inject: "false"
+        sidecar.istio.io/inject: "{{- ne $gateway.injectionTemplate "" }}"
+        {{- if ne $gateway.injectionTemplate "" }}
+        inject.istio.io/templates: "{{ $gateway.injectionTemplate }}"
+        {{- end}}
 {{- if $gateway.podAnnotations }}
 {{ toYaml $gateway.podAnnotations | indent 8 }}
 {{ end }}
@@ -60,7 +65,7 @@ spec:
         runAsNonRoot: true
         fsGroup: 1337
 {{- end }}
-      serviceAccountName: {{ $gateway.name | default "istio-ingressgateway" }}-service-account
+      serviceAccountName: {{ $gateway.name }}-service-account
 {{- if .Values.global.priorityClassName }}
       priorityClassName: "{{ .Values.global.priorityClassName }}"
 {{- end }}
@@ -123,7 +128,7 @@ spec:
           - --log_as_json
         {{- end }}
           - --serviceCluster
-          - {{ $gateway.name | default "istio-ingressgateway" }}
+          - {{ $gateway.name }}
         {{- if .Values.global.sts.servicePort }}
           - --stsPort={{ .Values.global.sts.servicePort }}
         {{- end }}
@@ -201,9 +206,9 @@ spec:
               fieldRef:
                 fieldPath: metadata.labels['service.istio.io/canonical-revision']
           - name: ISTIO_META_WORKLOAD_NAME
-            value: {{ $gateway.name | default "istio-ingressgateway" }}
+            value: {{ $gateway.name }}
           - name: ISTIO_META_OWNER
-            value: kubernetes://apis/apps/v1/namespaces/{{ .Release.Namespace }}/deployments/{{ $gateway.name | default "istio-ingressgateway" }}
+            value: kubernetes://apis/apps/v1/namespaces/{{ .Release.Namespace }}/deployments/{{ $gateway.name }}
           {{- if $.Values.global.meshID }}
           - name: ISTIO_META_MESH_ID
             value: "{{ $.Values.global.meshID }}"
@@ -349,3 +354,4 @@ spec:
       tolerations:
 {{ toYaml .Values.global.defaultTolerations | indent 6 }}
 {{- end }}
+{{- end }}

charts/kubezero-istio-ingress/charts/istio-ingress/templates/injected-deployment.yaml

@@ -0,0 +1,143 @@
+{{- $gateway := index .Values "gateways" "istio-ingressgateway" }}
+{{- if ne $gateway.injectionTemplate "" }}
+{{/* This provides a minimal gateway, ready to be injected.
+     Any settings from values.gateways should be here - these are options specific to the gateway.
+     Global settings, like the image, various env vars and volumes, etc will be injected.
+     The normal Deployment is not suitable for this, as the original pod spec will override the injection template. */}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $gateway.name | default "istio-ingressgateway" }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+{{ $gateway.labels | toYaml | indent 4 }}
+    release: {{ .Release.Name }}
+    istio.io/rev: {{ .Values.revision | default "default" }}
+    install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }}
+    operator.istio.io/component: "IngressGateways"
+spec:
+{{- if not $gateway.autoscaleEnabled }}
+{{- if $gateway.replicaCount }}
+  replicas: {{ $gateway.replicaCount }}
+{{- end }}
+{{- end }}
+  selector:
+    matchLabels:
+{{ $gateway.labels | toYaml | indent 6 }}
+  strategy:
+    rollingUpdate:
+      maxSurge: {{ $gateway.rollingMaxSurge }}
+      maxUnavailable: {{ $gateway.rollingMaxUnavailable }}
+  template:
+    metadata:
+      labels:
+{{ $gateway.labels | toYaml | indent 8 }}
+{{- if eq .Release.Namespace "istio-system"}}
+        heritage: Tiller
+        release: istio
+        chart: gateways
+{{- end }}
+        install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }}
+        operator.istio.io/component: "IngressGateways"
+        sidecar.istio.io/inject: "true"
+        {{- with .Values.revision }}
+        istio.io/rev: {{ . }}
+        {{- end }}
+      annotations:
+        {{- if .Values.meshConfig.enablePrometheusMerge }}
+        prometheus.io/port: "15020"
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/stats/prometheus"
+        {{- end }}
+        sidecar.istio.io/inject: "true"
+        inject.istio.io/templates: "{{ $gateway.injectionTemplate }}"
+{{- if $gateway.podAnnotations }}
+{{ toYaml $gateway.podAnnotations | indent 8 }}
+{{ end }}
+    spec:
+{{- if not $gateway.runAsRoot }}
+      securityContext:
+        runAsUser: 1337
+        runAsGroup: 1337
+        runAsNonRoot: true
+        fsGroup: 1337
+{{- end }}
+      serviceAccountName: {{ $gateway.name | default "istio-ingressgateway" }}-service-account
+{{- if .Values.global.priorityClassName }}
+      priorityClassName: "{{ .Values.global.priorityClassName }}"
+{{- end }}
+      containers:
+        - name: istio-proxy
+          image: auto
+          ports:
+            {{- range $key, $val := $gateway.ports }}
+            - containerPort: {{ $val.targetPort | default $val.port }}
+              protocol: {{ $val.protocol | default "TCP" }}
+            {{- end }}
+            - containerPort: 15090
+              protocol: TCP
+              name: http-envoy-prom
+        {{- if not $gateway.runAsRoot }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+            privileged: false
+            readOnlyRootFilesystem: true
+        {{- end }}
+          resources:
+{{- if $gateway.resources }}
+{{ toYaml $gateway.resources | indent 12 }}
+{{- else }}
+{{ toYaml .Values.global.defaultResources | indent 12 }}
+{{- end }}
+          env:
+          {{- if not $gateway.runAsRoot }}
+          - name: ISTIO_META_UNPRIVILEGED_POD
+            value: "true"
+          {{- end }}
+          {{- range $key, $val := $gateway.env }}
+          - name: {{ $key }}
+            value: {{ $val }}
+          {{- end }}
+          volumeMounts:
+          {{- range $gateway.secretVolumes }}
+          - name: {{ .name }}
+            mountPath: {{ .mountPath | quote }}
+            readOnly: true
+          {{- end }}
+          {{- range $gateway.configVolumes }}
+          {{- if .mountPath }}
+          - name: {{ .name }}
+            mountPath: {{ .mountPath | quote }}
+            readOnly: true
+          {{- end }}
+          {{- end }}
+{{- if $gateway.additionalContainers }}
+{{ toYaml $gateway.additionalContainers | indent 8 }}
+{{- end }}
+      volumes:
+      {{- range $gateway.secretVolumes }}
+      - name: {{ .name }}
+        secret:
+          secretName: {{ .secretName | quote }}
+          optional: true
+      {{- end }}
+      {{- range $gateway.configVolumes }}
+      - name: {{ .name }}
+        configMap:
+          name: {{ .configMapName | quote }}
+          optional: true
+      {{- end }}
+      affinity:
+{{ include "nodeaffinity" (dict "global" .Values.global "nodeSelector" $gateway.nodeSelector) | trim | indent 8 }}
+      {{- include "podAntiAffinity" $gateway | indent 6 }}
+{{- if $gateway.tolerations }}
+      tolerations:
+{{ toYaml $gateway.tolerations | indent 6 }}
+{{- else if .Values.global.defaultTolerations }}
+      tolerations:
+{{ toYaml .Values.global.defaultTolerations | indent 6 }}
+{{- end }}
+{{- end }}

charts/kubezero-istio-ingress/charts/istio-ingress/templates/poddisruptionbudget.yaml

@@ -3,7 +3,7 @@
 apiVersion: policy/v1beta1
 kind: PodDisruptionBudget
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}
+  name: {{ $gateway.name }}
   namespace: {{ .Release.Namespace }}
   labels:
 {{ $gateway.labels | toYaml | trim | indent 4 }}

charts/kubezero-istio-ingress/charts/istio-ingress/templates/preconfigured.yaml

@@ -1,78 +0,0 @@
-{{ $gateway := index .Values "gateways" "istio-ingressgateway" }}
-{{- if .Values.global.multiCluster.enabled }}
-apiVersion: networking.istio.io/v1alpha3
-kind: Gateway
-metadata:
-  name: istio-multicluster-ingressgateway
-  namespace: {{ .Release.Namespace }}
-  labels:
-{{ $gateway.labels | toYaml | indent 4 }}
-    release: {{ .Release.Name }}
-    istio.io/rev: {{ .Values.revision | default "default" }}
-    install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }}
-    operator.istio.io/component: "IngressGateways"
-spec:
-  selector:
-    istio: ingressgateway
-  servers:
-  - hosts:
-    - "*.{{ .Values.global.multiCluster.globalDomainSuffix | trim }}"
-    port:
-      name: tls
-      number: 15443
-      protocol: TLS
-    tls:
-      mode: AUTO_PASSTHROUGH
----
-{{- if .Values.global.multiCluster.includeEnvoyFilter }}
-apiVersion: networking.istio.io/v1alpha3
-kind: EnvoyFilter
-metadata:
-  name: istio-multicluster-ingressgateway
-  namespace: {{ .Release.Namespace }}
-  labels:
-{{ $gateway.labels | toYaml | indent 4 }}
-    release: {{ .Release.Name }}
-spec:
-  workloadSelector:
-    labels:
-      istio: ingressgateway
-  configPatches:
-  - applyTo: NETWORK_FILTER
-    match:
-      context: GATEWAY
-      listener:
-        portNumber: 15443
-        filterChain:
-          filter:
-            name: "envoy.filters.network.sni_cluster"
-    patch:
-      operation: INSERT_AFTER
-      value:
-        name: "envoy.filters.network.tcp_cluster_rewrite"
-        typed_config:
-          "@type": "type.googleapis.com/istio.envoy.config.filter.network.tcp_cluster_rewrite.v2alpha1.TcpClusterRewrite"
-          cluster_pattern: "\\.{{ .Values.global.multiCluster.globalDomainSuffix | trim }}$"
-          cluster_replacement: ".svc.{{ .Values.global.proxy.clusterDomain }}"
----
-{{- end }}
-## To ensure all traffic to globalDomainSuffix is using mTLS
-apiVersion: networking.istio.io/v1alpha3
-kind: DestinationRule
-metadata:
-  name: istio-multicluster-ingressgateway
-  namespace: {{ .Release.Namespace }}
-  labels:
-{{ $gateway.labels | toYaml | indent 4 }}
-    release: {{ .Release.Name }}
-spec:
-  host: "*.{{ .Values.global.multiCluster.globalDomainSuffix | trim }}"
-  {{- if .Values.global.defaultConfigVisibilitySettings }}
-  exportTo:
-  - '*'
-  {{- end }}
-  trafficPolicy:
-    tls:
-      mode: ISTIO_MUTUAL
----
-{{- end }}

charts/kubezero-istio-ingress/charts/istio-ingress/templates/role.yaml

@@ -2,7 +2,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}-sds
+  name: {{ $gateway.name }}-sds
   namespace: {{ .Release.Namespace }}
   labels:
     release: {{ .Release.Name }}

charts/kubezero-istio-ingress/charts/istio-ingress/templates/rolebindings.yaml

@@ -2,7 +2,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}-sds
+  name: {{ $gateway.name }}-sds
   namespace: {{ .Release.Namespace }}
   labels:
     release: {{ .Release.Name }}
@@ -12,8 +12,8 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: {{ $gateway.name | default "istio-ingressgateway" }}-sds
+  name: {{ $gateway.name }}-sds
 subjects:
 - kind: ServiceAccount
-  name: {{ $gateway.name | default "istio-ingressgateway" }}-service-account
+  name: {{ $gateway.name }}-service-account
 ---

charts/kubezero-istio-ingress/charts/istio-ingress/templates/service.yaml

@@ -3,7 +3,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}
+  name: {{ $gateway.name }}
   namespace: {{ .Release.Namespace }}
   annotations:
     {{- range $key, $val := $gateway.serviceAnnotations }}

charts/kubezero-istio-ingress/charts/istio-ingress/templates/serviceaccount.yaml

@@ -8,7 +8,7 @@ imagePullSecrets:
 {{- end }}
 {{- end }}
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}-service-account
+  name: {{ $gateway.name }}-service-account
   namespace: {{ .Release.Namespace }}
   labels:
 {{ $gateway.labels | toYaml | trim | indent 4 }}

charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml

@@ -120,6 +120,9 @@ gateways:
     # whether to run the gateway in a privileged container
     runAsRoot: false
 
+    # The injection template to use for the gateway. If not set, no injection will be performed.
+    injectionTemplate: ""
+
 # Revision is set as 'version' label and part of the resource names when installing multiple control planes.
 revision: ""
 
@@ -171,7 +174,7 @@ global:
   hub: docker.io/istio
 
   # Default tag for Istio images.
-  tag: 1.9.1
+  tag: 1.9.2
 
   # Specify image pull policy if default behavior isn't desired.
   # Default behavior: latest images will be Always else IfNotPresent.

charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: istio-private-ingress
-version: 1.9.1
+version: 1.9.2
 tillerVersion: ">=2.7.2"
 description: Helm chart for deploying Istio gateways
 keywords:

charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/autoscale.yaml

@@ -3,7 +3,7 @@
 apiVersion: autoscaling/v2beta1
 kind: HorizontalPodAutoscaler
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}
+  name: {{ $gateway.name }}
   namespace: {{ .Release.Namespace }}
   labels:
 {{ $gateway.labels | toYaml | indent 4 }}
@@ -17,7 +17,7 @@ spec:
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
-    name: {{ $gateway.name | default "istio-ingressgateway" }}
+    name: {{ $gateway.name }}
   metrics:
     - type: Resource
       resource:

charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/deployment.yaml

@@ -1,8 +1,10 @@
+
 {{- $gateway := index .Values "gateways" "istio-ingressgateway" }}
+{{- if eq $gateway.injectionTemplate "" }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}
+  name: {{ $gateway.name }}
   namespace: {{ .Release.Namespace }}
   labels:
 {{ $gateway.labels | toYaml | indent 4 }}
@@ -32,7 +34,7 @@ spec:
         release: istio
         chart: gateways
 {{- end }}
-        service.istio.io/canonical-name: {{ $gateway.name | default "istio-ingressgateway" }}
+        service.istio.io/canonical-name: {{ $gateway.name }}
         {{- if not (eq .Values.revision "") }}
         service.istio.io/canonical-revision: {{ .Values.revision }}
         {{- else}}
@@ -41,14 +43,17 @@ spec:
         istio.io/rev: {{ .Values.revision | default "default" }}
         install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }}
         operator.istio.io/component: "IngressGateways"
-        sidecar.istio.io/inject: "false"
+        sidecar.istio.io/inject: "{{- ne $gateway.injectionTemplate "" }}"
       annotations:
         {{- if .Values.meshConfig.enablePrometheusMerge }}
         prometheus.io/port: "15020"
         prometheus.io/scrape: "true"
         prometheus.io/path: "/stats/prometheus"
         {{- end }}
-        sidecar.istio.io/inject: "false"
+        sidecar.istio.io/inject: "{{- ne $gateway.injectionTemplate "" }}"
+        {{- if ne $gateway.injectionTemplate "" }}
+        inject.istio.io/templates: "{{ $gateway.injectionTemplate }}"
+        {{- end}}
 {{- if $gateway.podAnnotations }}
 {{ toYaml $gateway.podAnnotations | indent 8 }}
 {{ end }}
@@ -60,7 +65,7 @@ spec:
         runAsNonRoot: true
         fsGroup: 1337
 {{- end }}
-      serviceAccountName: {{ $gateway.name | default "istio-ingressgateway" }}-service-account
+      serviceAccountName: {{ $gateway.name }}-service-account
 {{- if .Values.global.priorityClassName }}
       priorityClassName: "{{ .Values.global.priorityClassName }}"
 {{- end }}
@@ -123,7 +128,7 @@ spec:
           - --log_as_json
         {{- end }}
           - --serviceCluster
-          - {{ $gateway.name | default "istio-ingressgateway" }}
+          - {{ $gateway.name }}
         {{- if .Values.global.sts.servicePort }}
           - --stsPort={{ .Values.global.sts.servicePort }}
         {{- end }}
@@ -201,9 +206,9 @@ spec:
               fieldRef:
                 fieldPath: metadata.labels['service.istio.io/canonical-revision']
           - name: ISTIO_META_WORKLOAD_NAME
-            value: {{ $gateway.name | default "istio-ingressgateway" }}
+            value: {{ $gateway.name }}
           - name: ISTIO_META_OWNER
-            value: kubernetes://apis/apps/v1/namespaces/{{ .Release.Namespace }}/deployments/{{ $gateway.name | default "istio-ingressgateway" }}
+            value: kubernetes://apis/apps/v1/namespaces/{{ .Release.Namespace }}/deployments/{{ $gateway.name }}
           {{- if $.Values.global.meshID }}
           - name: ISTIO_META_MESH_ID
             value: "{{ $.Values.global.meshID }}"
@@ -349,3 +354,4 @@ spec:
       tolerations:
 {{ toYaml .Values.global.defaultTolerations | indent 6 }}
 {{- end }}
+{{- end }}

charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/injected-deployment.yaml

@@ -0,0 +1,143 @@
+{{- $gateway := index .Values "gateways" "istio-ingressgateway" }}
+{{- if ne $gateway.injectionTemplate "" }}
+{{/* This provides a minimal gateway, ready to be injected.
+     Any settings from values.gateways should be here - these are options specific to the gateway.
+     Global settings, like the image, various env vars and volumes, etc will be injected.
+     The normal Deployment is not suitable for this, as the original pod spec will override the injection template. */}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $gateway.name | default "istio-ingressgateway" }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+{{ $gateway.labels | toYaml | indent 4 }}
+    release: {{ .Release.Name }}
+    istio.io/rev: {{ .Values.revision | default "default" }}
+    install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }}
+    operator.istio.io/component: "IngressGateways"
+spec:
+{{- if not $gateway.autoscaleEnabled }}
+{{- if $gateway.replicaCount }}
+  replicas: {{ $gateway.replicaCount }}
+{{- end }}
+{{- end }}
+  selector:
+    matchLabels:
+{{ $gateway.labels | toYaml | indent 6 }}
+  strategy:
+    rollingUpdate:
+      maxSurge: {{ $gateway.rollingMaxSurge }}
+      maxUnavailable: {{ $gateway.rollingMaxUnavailable }}
+  template:
+    metadata:
+      labels:
+{{ $gateway.labels | toYaml | indent 8 }}
+{{- if eq .Release.Namespace "istio-system"}}
+        heritage: Tiller
+        release: istio
+        chart: gateways
+{{- end }}
+        install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }}
+        operator.istio.io/component: "IngressGateways"
+        sidecar.istio.io/inject: "true"
+        {{- with .Values.revision }}
+        istio.io/rev: {{ . }}
+        {{- end }}
+      annotations:
+        {{- if .Values.meshConfig.enablePrometheusMerge }}
+        prometheus.io/port: "15020"
+        prometheus.io/scrape: "true"
+        prometheus.io/path: "/stats/prometheus"
+        {{- end }}
+        sidecar.istio.io/inject: "true"
+        inject.istio.io/templates: "{{ $gateway.injectionTemplate }}"
+{{- if $gateway.podAnnotations }}
+{{ toYaml $gateway.podAnnotations | indent 8 }}
+{{ end }}
+    spec:
+{{- if not $gateway.runAsRoot }}
+      securityContext:
+        runAsUser: 1337
+        runAsGroup: 1337
+        runAsNonRoot: true
+        fsGroup: 1337
+{{- end }}
+      serviceAccountName: {{ $gateway.name | default "istio-ingressgateway" }}-service-account
+{{- if .Values.global.priorityClassName }}
+      priorityClassName: "{{ .Values.global.priorityClassName }}"
+{{- end }}
+      containers:
+        - name: istio-proxy
+          image: auto
+          ports:
+            {{- range $key, $val := $gateway.ports }}
+            - containerPort: {{ $val.targetPort | default $val.port }}
+              protocol: {{ $val.protocol | default "TCP" }}
+            {{- end }}
+            - containerPort: 15090
+              protocol: TCP
+              name: http-envoy-prom
+        {{- if not $gateway.runAsRoot }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+            privileged: false
+            readOnlyRootFilesystem: true
+        {{- end }}
+          resources:
+{{- if $gateway.resources }}
+{{ toYaml $gateway.resources | indent 12 }}
+{{- else }}
+{{ toYaml .Values.global.defaultResources | indent 12 }}
+{{- end }}
+          env:
+          {{- if not $gateway.runAsRoot }}
+          - name: ISTIO_META_UNPRIVILEGED_POD
+            value: "true"
+          {{- end }}
+          {{- range $key, $val := $gateway.env }}
+          - name: {{ $key }}
+            value: {{ $val }}
+          {{- end }}
+          volumeMounts:
+          {{- range $gateway.secretVolumes }}
+          - name: {{ .name }}
+            mountPath: {{ .mountPath | quote }}
+            readOnly: true
+          {{- end }}
+          {{- range $gateway.configVolumes }}
+          {{- if .mountPath }}
+          - name: {{ .name }}
+            mountPath: {{ .mountPath | quote }}
+            readOnly: true
+          {{- end }}
+          {{- end }}
+{{- if $gateway.additionalContainers }}
+{{ toYaml $gateway.additionalContainers | indent 8 }}
+{{- end }}
+      volumes:
+      {{- range $gateway.secretVolumes }}
+      - name: {{ .name }}
+        secret:
+          secretName: {{ .secretName | quote }}
+          optional: true
+      {{- end }}
+      {{- range $gateway.configVolumes }}
+      - name: {{ .name }}
+        configMap:
+          name: {{ .configMapName | quote }}
+          optional: true
+      {{- end }}
+      affinity:
+{{ include "nodeaffinity" (dict "global" .Values.global "nodeSelector" $gateway.nodeSelector) | trim | indent 8 }}
+      {{- include "podAntiAffinity" $gateway | indent 6 }}
+{{- if $gateway.tolerations }}
+      tolerations:
+{{ toYaml $gateway.tolerations | indent 6 }}
+{{- else if .Values.global.defaultTolerations }}
+      tolerations:
+{{ toYaml .Values.global.defaultTolerations | indent 6 }}
+{{- end }}
+{{- end }}

charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/poddisruptionbudget.yaml

@@ -3,7 +3,7 @@
 apiVersion: policy/v1beta1
 kind: PodDisruptionBudget
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}
+  name: {{ $gateway.name }}
   namespace: {{ .Release.Namespace }}
   labels:
 {{ $gateway.labels | toYaml | trim | indent 4 }}

charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/preconfigured.yaml

@@ -1,78 +0,0 @@
-{{ $gateway := index .Values "gateways" "istio-ingressgateway" }}
-{{- if .Values.global.multiCluster.enabled }}
-apiVersion: networking.istio.io/v1alpha3
-kind: Gateway
-metadata:
-  name: istio-multicluster-ingressgateway
-  namespace: {{ .Release.Namespace }}
-  labels:
-{{ $gateway.labels | toYaml | indent 4 }}
-    release: {{ .Release.Name }}
-    istio.io/rev: {{ .Values.revision | default "default" }}
-    install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }}
-    operator.istio.io/component: "IngressGateways"
-spec:
-  selector:
-    istio: ingressgateway
-  servers:
-  - hosts:
-    - "*.{{ .Values.global.multiCluster.globalDomainSuffix | trim }}"
-    port:
-      name: tls
-      number: 15443
-      protocol: TLS
-    tls:
-      mode: AUTO_PASSTHROUGH
----
-{{- if .Values.global.multiCluster.includeEnvoyFilter }}
-apiVersion: networking.istio.io/v1alpha3
-kind: EnvoyFilter
-metadata:
-  name: istio-multicluster-ingressgateway
-  namespace: {{ .Release.Namespace }}
-  labels:
-{{ $gateway.labels | toYaml | indent 4 }}
-    release: {{ .Release.Name }}
-spec:
-  workloadSelector:
-    labels:
-      istio: ingressgateway
-  configPatches:
-  - applyTo: NETWORK_FILTER
-    match:
-      context: GATEWAY
-      listener:
-        portNumber: 15443
-        filterChain:
-          filter:
-            name: "envoy.filters.network.sni_cluster"
-    patch:
-      operation: INSERT_AFTER
-      value:
-        name: "envoy.filters.network.tcp_cluster_rewrite"
-        typed_config:
-          "@type": "type.googleapis.com/istio.envoy.config.filter.network.tcp_cluster_rewrite.v2alpha1.TcpClusterRewrite"
-          cluster_pattern: "\\.{{ .Values.global.multiCluster.globalDomainSuffix | trim }}$"
-          cluster_replacement: ".svc.{{ .Values.global.proxy.clusterDomain }}"
----
-{{- end }}
-## To ensure all traffic to globalDomainSuffix is using mTLS
-apiVersion: networking.istio.io/v1alpha3
-kind: DestinationRule
-metadata:
-  name: istio-multicluster-ingressgateway
-  namespace: {{ .Release.Namespace }}
-  labels:
-{{ $gateway.labels | toYaml | indent 4 }}
-    release: {{ .Release.Name }}
-spec:
-  host: "*.{{ .Values.global.multiCluster.globalDomainSuffix | trim }}"
-  {{- if .Values.global.defaultConfigVisibilitySettings }}
-  exportTo:
-  - '*'
-  {{- end }}
-  trafficPolicy:
-    tls:
-      mode: ISTIO_MUTUAL
----
-{{- end }}

charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/role.yaml

@@ -2,7 +2,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}-sds
+  name: {{ $gateway.name }}-sds
   namespace: {{ .Release.Namespace }}
   labels:
     release: {{ .Release.Name }}

charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/rolebindings.yaml

@@ -2,7 +2,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}-sds
+  name: {{ $gateway.name }}-sds
   namespace: {{ .Release.Namespace }}
   labels:
     release: {{ .Release.Name }}
@@ -12,8 +12,8 @@ metadata:
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
-  name: {{ $gateway.name | default "istio-ingressgateway" }}-sds
+  name: {{ $gateway.name }}-sds
 subjects:
 - kind: ServiceAccount
-  name: {{ $gateway.name | default "istio-ingressgateway" }}-service-account
+  name: {{ $gateway.name }}-service-account
 ---

charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/service.yaml

@@ -3,7 +3,7 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}
+  name: {{ $gateway.name }}
   namespace: {{ .Release.Namespace }}
   annotations:
     {{- range $key, $val := $gateway.serviceAnnotations }}

charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/serviceaccount.yaml

@@ -8,7 +8,7 @@ imagePullSecrets:
 {{- end }}
 {{- end }}
 metadata:
-  name: {{ $gateway.name | default "istio-ingressgateway" }}-service-account
+  name: {{ $gateway.name }}-service-account
   namespace: {{ .Release.Namespace }}
   labels:
 {{ $gateway.labels | toYaml | trim | indent 4 }}

charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml

@@ -120,6 +120,9 @@ gateways:
     # whether to run the gateway in a privileged container
     runAsRoot: false
 
+    # The injection template to use for the gateway. If not set, no injection will be performed.
+    injectionTemplate: ""
+
 # Revision is set as 'version' label and part of the resource names when installing multiple control planes.
 revision: ""
 
@@ -171,7 +174,7 @@ global:
   hub: docker.io/istio
 
   # Default tag for Istio images.
-  tag: 1.9.1
+  tag: 1.9.2
 
   # Specify image pull policy if default behavior isn't desired.
   # Default behavior: latest images will be Always else IfNotPresent.

charts/kubezero-istio/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: kubezero-istio
 description: KubeZero Umbrella Chart for Istio
 type: application
-version: 0.5.0
+version: 0.5.1
-appVersion: 1.9.1
+appVersion: 1.9.2
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -16,7 +16,7 @@ dependencies:
     version: ">= 0.1.3"
     repository: https://zero-down-time.github.io/kubezero/
   - name: base
-    version: 1.9.1
+    version: 1.9.2
   - name: istio-discovery
-    version: 1.9.1
+    version: 1.9.2
 kubeVersion: ">= 1.18.0"

charts/kubezero-istio/charts/base/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: base
-version: 1.9.1
+version: 1.9.2
 tillerVersion: ">=2.7.2"
 description: Helm chart for deploying Istio cluster resources and CRDs
 keywords:

charts/kubezero-istio/charts/base/crds/crd-all.gen.yaml

@@ -2473,7 +2473,8 @@ spec:
                         format: int32
                         type: integer
                       perTryTimeout:
-                        description: Timeout per retry attempt for a given request.
+                        description: Timeout per attempt for a given request, including
+                          the initial call and any retries.
                         type: string
                       retryOn:
                         description: Specifies the conditions under which retry takes

charts/kubezero-istio/charts/base/files/gen-istio-cluster.yaml

@@ -2475,7 +2475,8 @@ spec:
                         format: int32
                         type: integer
                       perTryTimeout:
-                        description: Timeout per retry attempt for a given request.
+                        description: Timeout per attempt for a given request, including
+                          the initial call and any retries.
                         type: string
                       retryOn:
                         description: Specifies the conditions under which retry takes

charts/kubezero-istio/charts/istio-discovery/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: istio-discovery
-version: 1.9.1
+version: 1.9.2
 tillerVersion: ">=2.7.2"
 description: Helm chart for istio control plane
 keywords:

charts/kubezero-istio/charts/istio-discovery/files/gateway-injection-template.yaml

@@ -0,0 +1,233 @@
+{{- $containers := list }}
+{{- range $index, $container := .Spec.Containers }}{{ if not (eq $container.Name "istio-proxy") }}{{ $containers = append $containers $container.Name }}{{end}}{{- end}}
+metadata:
+  labels:
+    service.istio.io/canonical-name: {{ index .ObjectMeta.Labels `service.istio.io/canonical-name` | default (index .ObjectMeta.Labels `app.kubernetes.io/name`) | default (index .ObjectMeta.Labels `app`) | default .DeploymentMeta.Name  | quote }}
+    service.istio.io/canonical-revision: {{ index .ObjectMeta.Labels `service.istio.io/canonical-revision` | default (index .ObjectMeta.Labels `app.kubernetes.io/version`) | default (index .ObjectMeta.Labels `version`) | default "latest"  | quote }}
+    istio.io/rev: {{ .Revision | default "default" | quote }}
+  annotations: {
+    {{- if eq (len $containers) 1 }}
+    kubectl.kubernetes.io/default-logs-container: "{{ index $containers 0 }}",
+    {{ end }}
+  }
+spec:
+  containers:
+  - name: istio-proxy
+  {{- if contains "/" .Values.global.proxy.image }}
+    image: "{{ annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy.image }}"
+  {{- else }}
+    image: "{{ .Values.global.hub }}/{{ .Values.global.proxy.image }}:{{ .Values.global.tag }}"
+  {{- end }}
+    ports:
+    - containerPort: 15090
+      protocol: TCP
+      name: http-envoy-prom
+    args:
+    - proxy
+    - router
+    - --domain
+    - $(POD_NAMESPACE).svc.{{ .Values.global.proxy.clusterDomain }}
+    - --serviceCluster
+    {{ if ne "" (index .ObjectMeta.Labels "app") -}}
+    - "{{ index .ObjectMeta.Labels `app` }}.$(POD_NAMESPACE)"
+    {{ else -}}
+    - "{{ valueOrDefault .DeploymentMeta.Name `istio-proxy` }}.{{ valueOrDefault .DeploymentMeta.Namespace `default` }}"
+    {{ end -}}
+    - --proxyLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/logLevel` .Values.global.proxy.logLevel }}
+    - --proxyComponentLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/componentLogLevel` .Values.global.proxy.componentLogLevel }}
+    - --log_output_level={{ annotation .ObjectMeta `sidecar.istio.io/agentLogLevel` .Values.global.logging.level }}
+  {{- if .Values.global.sts.servicePort }}
+    - --stsPort={{ .Values.global.sts.servicePort }}
+  {{- end }}
+  {{- if .Values.global.logAsJson }}
+    - --log_as_json
+  {{- end }}
+  {{- if .Values.global.proxy.lifecycle }}
+    lifecycle:
+      {{ toYaml .Values.global.proxy.lifecycle | indent 6 }}
+  {{- end }}
+    env:
+    - name: JWT_POLICY
+      value: {{ .Values.global.jwtPolicy }}
+    - name: PILOT_CERT_PROVIDER
+      value: {{ .Values.global.pilotCertProvider }}
+    - name: CA_ADDR
+    {{- if .Values.global.caAddress }}
+      value: {{ .Values.global.caAddress }}
+    {{- else }}
+      value: istiod{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }}.{{ .Values.global.istioNamespace }}.svc:15012
+    {{- end }}
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+    - name: INSTANCE_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    - name: SERVICE_ACCOUNT
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.serviceAccountName
+    - name: HOST_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.hostIP
+    - name: CANONICAL_SERVICE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.labels['service.istio.io/canonical-name']
+    - name: CANONICAL_REVISION
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.labels['service.istio.io/canonical-revision']
+    - name: PROXY_CONFIG
+      value: |
+             {{ protoToJSON .ProxyConfig }}
+    - name: ISTIO_META_POD_PORTS
+      value: |-
+        [
+        {{- $first := true }}
+        {{- range $index1, $c := .Spec.Containers }}
+          {{- range $index2, $p := $c.Ports }}
+            {{- if (structToJSON $p) }}
+            {{if not $first}},{{end}}{{ structToJSON $p }}
+            {{- $first = false }}
+            {{- end }}
+          {{- end}}
+        {{- end}}
+        ]
+    - name: ISTIO_META_APP_CONTAINERS
+      value: "{{ $containers | join "," }}"
+    - name: ISTIO_META_CLUSTER_ID
+      value: "{{ valueOrDefault .Values.global.multiCluster.clusterName `Kubernetes` }}"
+    - name: ISTIO_META_INTERCEPTION_MODE
+      value: "{{ .ProxyConfig.InterceptionMode.String }}"
+    {{- if .Values.global.network }}
+    - name: ISTIO_META_NETWORK
+      value: "{{ .Values.global.network }}"
+    {{- end }}
+    {{ if .ObjectMeta.Annotations }}
+    - name: ISTIO_METAJSON_ANNOTATIONS
+      value: |
+             {{ toJSON .ObjectMeta.Annotations }}
+    {{ end }}
+    {{- if .DeploymentMeta.Name }}
+    - name: ISTIO_META_WORKLOAD_NAME
+      value: "{{ .DeploymentMeta.Name }}"
+    {{ end }}
+    {{- if and .TypeMeta.APIVersion .DeploymentMeta.Name }}
+    - name: ISTIO_META_OWNER
+      value: kubernetes://apis/{{ .TypeMeta.APIVersion }}/namespaces/{{ valueOrDefault .DeploymentMeta.Namespace `default` }}/{{ toLower .TypeMeta.Kind}}s/{{ .DeploymentMeta.Name }}
+    {{- end}}
+    {{- if .Values.global.meshID }}
+    - name: ISTIO_META_MESH_ID
+      value: "{{ .Values.global.meshID }}"
+    {{- else if (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain) }}
+    - name: ISTIO_META_MESH_ID
+      value: "{{ (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain) }}"
+    {{- end }}
+    {{- with (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain)  }}
+    - name: TRUST_DOMAIN
+      value: "{{ . }}"
+    {{- end }}
+    {{- range $key, $value := .ProxyConfig.ProxyMetadata }}
+    - name: {{ $key }}
+      value: "{{ $value }}"
+    {{- end }}
+    {{with .Values.global.imagePullPolicy }}imagePullPolicy: "{{.}}"{{end}}
+    readinessProbe:
+      httpGet:
+        path: /healthz/ready
+        port: 15021
+      initialDelaySeconds: {{.Values.global.proxy.readinessInitialDelaySeconds }}
+      periodSeconds: {{ .Values.global.proxy.readinessPeriodSeconds }}
+      timeoutSeconds: 3
+      failureThreshold: {{ .Values.global.proxy.readinessFailureThreshold }}
+    volumeMounts:
+    {{- if eq .Values.global.pilotCertProvider "istiod" }}
+    - mountPath: /var/run/secrets/istio
+      name: istiod-ca-cert
+    {{- end }}
+    - mountPath: /var/lib/istio/data
+      name: istio-data
+    # SDS channel between istioagent and Envoy
+    - mountPath: /etc/istio/proxy
+      name: istio-envoy
+    {{- if eq .Values.global.jwtPolicy "third-party-jwt" }}
+    - mountPath: /var/run/secrets/tokens
+      name: istio-token
+    {{- end }}
+    {{- if .Values.global.mountMtlsCerts }}
+    # Use the key and cert mounted to /etc/certs/ for the in-cluster mTLS communications.
+    - mountPath: /etc/certs/
+      name: istio-certs
+      readOnly: true
+    {{- end }}
+    - name: istio-podinfo
+      mountPath: /etc/istio/pod
+  volumes:
+  # SDS channel between istioagent and Envoy
+  - emptyDir:
+      medium: Memory
+    name: istio-envoy
+  - name: istio-data
+    emptyDir: {}
+  - name: istio-podinfo
+    downwardAPI:
+      items:
+        - path: "labels"
+          fieldRef:
+            fieldPath: metadata.labels
+        - path: "annotations"
+          fieldRef:
+            fieldPath: metadata.annotations
+        - path: "cpu-limit"
+          resourceFieldRef:
+            containerName: istio-proxy
+            resource: limits.cpu
+            divisor: 1m
+        - path: "cpu-request"
+          resourceFieldRef:
+            containerName: istio-proxy
+            resource: requests.cpu
+            divisor: 1m
+  {{- if eq .Values.global.jwtPolicy "third-party-jwt" }}
+  - name: istio-token
+    projected:
+      sources:
+      - serviceAccountToken:
+          path: istio-token
+          expirationSeconds: 43200
+          audience: {{ .Values.global.sds.token.aud }}
+  {{- end }}
+  {{- if eq .Values.global.pilotCertProvider "istiod" }}
+  - name: istiod-ca-cert
+    configMap:
+      name: istio-ca-root-cert
+  {{- end }}
+  {{- if .Values.global.mountMtlsCerts }}
+  # Use the key and cert mounted to /etc/certs/ for the in-cluster mTLS communications.
+  - name: istio-certs
+    secret:
+      optional: true
+      {{ if eq .Spec.ServiceAccountName "" }}
+      secretName: istio.default
+      {{ else -}}
+      secretName: {{  printf "istio.%s" .Spec.ServiceAccountName }}
+      {{  end -}}
+  {{- end }}
+  {{- if .Values.global.imagePullSecrets }}
+  imagePullSecrets:
+    {{- range .Values.global.imagePullSecrets }}
+    - name: {{ . }}
+    {{- end }}
+  {{- end }}
+  {{- if eq (env "ENABLE_LEGACY_FSGROUP_INJECTION" "true") "true" }}
+  securityContext:
+    fsGroup: 1337
+  {{- end }}

charts/kubezero-istio/charts/istio-discovery/files/gen-istio.yaml

@@ -695,6 +695,240 @@ data:
           securityContext:
             fsGroup: 1337
           {{- end }}
+      gateway: |
+        {{- $containers := list }}
+        {{- range $index, $container := .Spec.Containers }}{{ if not (eq $container.Name "istio-proxy") }}{{ $containers = append $containers $container.Name }}{{end}}{{- end}}
+        metadata:
+          labels:
+            service.istio.io/canonical-name: {{ index .ObjectMeta.Labels `service.istio.io/canonical-name` | default (index .ObjectMeta.Labels `app.kubernetes.io/name`) | default (index .ObjectMeta.Labels `app`) | default .DeploymentMeta.Name  | quote }}
+            service.istio.io/canonical-revision: {{ index .ObjectMeta.Labels `service.istio.io/canonical-revision` | default (index .ObjectMeta.Labels `app.kubernetes.io/version`) | default (index .ObjectMeta.Labels `version`) | default "latest"  | quote }}
+            istio.io/rev: {{ .Revision | default "default" | quote }}
+          annotations: {
+            {{- if eq (len $containers) 1 }}
+            kubectl.kubernetes.io/default-logs-container: "{{ index $containers 0 }}",
+            {{ end }}
+          }
+        spec:
+          containers:
+          - name: istio-proxy
+          {{- if contains "/" .Values.global.proxy.image }}
+            image: "{{ annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy.image }}"
+          {{- else }}
+            image: "{{ .Values.global.hub }}/{{ .Values.global.proxy.image }}:{{ .Values.global.tag }}"
+          {{- end }}
+            ports:
+            - containerPort: 15090
+              protocol: TCP
+              name: http-envoy-prom
+            args:
+            - proxy
+            - router
+            - --domain
+            - $(POD_NAMESPACE).svc.{{ .Values.global.proxy.clusterDomain }}
+            - --serviceCluster
+            {{ if ne "" (index .ObjectMeta.Labels "app") -}}
+            - "{{ index .ObjectMeta.Labels `app` }}.$(POD_NAMESPACE)"
+            {{ else -}}
+            - "{{ valueOrDefault .DeploymentMeta.Name `istio-proxy` }}.{{ valueOrDefault .DeploymentMeta.Namespace `default` }}"
+            {{ end -}}
+            - --proxyLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/logLevel` .Values.global.proxy.logLevel }}
+            - --proxyComponentLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/componentLogLevel` .Values.global.proxy.componentLogLevel }}
+            - --log_output_level={{ annotation .ObjectMeta `sidecar.istio.io/agentLogLevel` .Values.global.logging.level }}
+          {{- if .Values.global.sts.servicePort }}
+            - --stsPort={{ .Values.global.sts.servicePort }}
+          {{- end }}
+          {{- if .Values.global.logAsJson }}
+            - --log_as_json
+          {{- end }}
+          {{- if .Values.global.proxy.lifecycle }}
+            lifecycle:
+              {{ toYaml .Values.global.proxy.lifecycle | indent 6 }}
+          {{- end }}
+            env:
+            - name: JWT_POLICY
+              value: {{ .Values.global.jwtPolicy }}
+            - name: PILOT_CERT_PROVIDER
+              value: {{ .Values.global.pilotCertProvider }}
+            - name: CA_ADDR
+            {{- if .Values.global.caAddress }}
+              value: {{ .Values.global.caAddress }}
+            {{- else }}
+              value: istiod{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }}.{{ .Values.global.istioNamespace }}.svc:15012
+            {{- end }}
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: INSTANCE_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: SERVICE_ACCOUNT
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.serviceAccountName
+            - name: HOST_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+            - name: CANONICAL_SERVICE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.labels['service.istio.io/canonical-name']
+            - name: CANONICAL_REVISION
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.labels['service.istio.io/canonical-revision']
+            - name: PROXY_CONFIG
+              value: |
+                     {{ protoToJSON .ProxyConfig }}
+            - name: ISTIO_META_POD_PORTS
+              value: |-
+                [
+                {{- $first := true }}
+                {{- range $index1, $c := .Spec.Containers }}
+                  {{- range $index2, $p := $c.Ports }}
+                    {{- if (structToJSON $p) }}
+                    {{if not $first}},{{end}}{{ structToJSON $p }}
+                    {{- $first = false }}
+                    {{- end }}
+                  {{- end}}
+                {{- end}}
+                ]
+            - name: ISTIO_META_APP_CONTAINERS
+              value: "{{ $containers | join "," }}"
+            - name: ISTIO_META_CLUSTER_ID
+              value: "{{ valueOrDefault .Values.global.multiCluster.clusterName `Kubernetes` }}"
+            - name: ISTIO_META_INTERCEPTION_MODE
+              value: "{{ .ProxyConfig.InterceptionMode.String }}"
+            {{- if .Values.global.network }}
+            - name: ISTIO_META_NETWORK
+              value: "{{ .Values.global.network }}"
+            {{- end }}
+            {{ if .ObjectMeta.Annotations }}
+            - name: ISTIO_METAJSON_ANNOTATIONS
+              value: |
+                     {{ toJSON .ObjectMeta.Annotations }}
+            {{ end }}
+            {{- if .DeploymentMeta.Name }}
+            - name: ISTIO_META_WORKLOAD_NAME
+              value: "{{ .DeploymentMeta.Name }}"
+            {{ end }}
+            {{- if and .TypeMeta.APIVersion .DeploymentMeta.Name }}
+            - name: ISTIO_META_OWNER
+              value: kubernetes://apis/{{ .TypeMeta.APIVersion }}/namespaces/{{ valueOrDefault .DeploymentMeta.Namespace `default` }}/{{ toLower .TypeMeta.Kind}}s/{{ .DeploymentMeta.Name }}
+            {{- end}}
+            {{- if .Values.global.meshID }}
+            - name: ISTIO_META_MESH_ID
+              value: "{{ .Values.global.meshID }}"
+            {{- else if (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain) }}
+            - name: ISTIO_META_MESH_ID
+              value: "{{ (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain) }}"
+            {{- end }}
+            {{- with (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain)  }}
+            - name: TRUST_DOMAIN
+              value: "{{ . }}"
+            {{- end }}
+            {{- range $key, $value := .ProxyConfig.ProxyMetadata }}
+            - name: {{ $key }}
+              value: "{{ $value }}"
+            {{- end }}
+            {{with .Values.global.imagePullPolicy }}imagePullPolicy: "{{.}}"{{end}}
+            readinessProbe:
+              httpGet:
+                path: /healthz/ready
+                port: 15021
+              initialDelaySeconds: {{.Values.global.proxy.readinessInitialDelaySeconds }}
+              periodSeconds: {{ .Values.global.proxy.readinessPeriodSeconds }}
+              timeoutSeconds: 3
+              failureThreshold: {{ .Values.global.proxy.readinessFailureThreshold }}
+            volumeMounts:
+            {{- if eq .Values.global.pilotCertProvider "istiod" }}
+            - mountPath: /var/run/secrets/istio
+              name: istiod-ca-cert
+            {{- end }}
+            - mountPath: /var/lib/istio/data
+              name: istio-data
+            # SDS channel between istioagent and Envoy
+            - mountPath: /etc/istio/proxy
+              name: istio-envoy
+            {{- if eq .Values.global.jwtPolicy "third-party-jwt" }}
+            - mountPath: /var/run/secrets/tokens
+              name: istio-token
+            {{- end }}
+            {{- if .Values.global.mountMtlsCerts }}
+            # Use the key and cert mounted to /etc/certs/ for the in-cluster mTLS communications.
+            - mountPath: /etc/certs/
+              name: istio-certs
+              readOnly: true
+            {{- end }}
+            - name: istio-podinfo
+              mountPath: /etc/istio/pod
+          volumes:
+          # SDS channel between istioagent and Envoy
+          - emptyDir:
+              medium: Memory
+            name: istio-envoy
+          - name: istio-data
+            emptyDir: {}
+          - name: istio-podinfo
+            downwardAPI:
+              items:
+                - path: "labels"
+                  fieldRef:
+                    fieldPath: metadata.labels
+                - path: "annotations"
+                  fieldRef:
+                    fieldPath: metadata.annotations
+                - path: "cpu-limit"
+                  resourceFieldRef:
+                    containerName: istio-proxy
+                    resource: limits.cpu
+                    divisor: 1m
+                - path: "cpu-request"
+                  resourceFieldRef:
+                    containerName: istio-proxy
+                    resource: requests.cpu
+                    divisor: 1m
+          {{- if eq .Values.global.jwtPolicy "third-party-jwt" }}
+          - name: istio-token
+            projected:
+              sources:
+              - serviceAccountToken:
+                  path: istio-token
+                  expirationSeconds: 43200
+                  audience: {{ .Values.global.sds.token.aud }}
+          {{- end }}
+          {{- if eq .Values.global.pilotCertProvider "istiod" }}
+          - name: istiod-ca-cert
+            configMap:
+              name: istio-ca-root-cert
+          {{- end }}
+          {{- if .Values.global.mountMtlsCerts }}
+          # Use the key and cert mounted to /etc/certs/ for the in-cluster mTLS communications.
+          - name: istio-certs
+            secret:
+              optional: true
+              {{ if eq .Spec.ServiceAccountName "" }}
+              secretName: istio.default
+              {{ else -}}
+              secretName: {{  printf "istio.%s" .Spec.ServiceAccountName }}
+              {{  end -}}
+          {{- end }}
+          {{- if .Values.global.imagePullSecrets }}
+          imagePullSecrets:
+            {{- range .Values.global.imagePullSecrets }}
+            - name: {{ . }}
+            {{- end }}
+          {{- end }}
+          {{- if eq (env "ENABLE_LEGACY_FSGROUP_INJECTION" "true") "true" }}
+          securityContext:
+            fsGroup: 1337
+          {{- end }}
 ---
 # Source: istio-discovery/templates/service.yaml
 apiVersion: v1

charts/kubezero-istio/charts/istio-discovery/files/injection-template.yaml

@@ -1,4 +1,4 @@
- {{- $containers := list }}
+{{- $containers := list }}
 {{- range $index, $container := .Spec.Containers }}{{ if not (eq $container.Name "istio-proxy") }}{{ $containers = append $containers $container.Name }}{{end}}{{- end}}
 metadata:
   labels:

charts/kubezero-istio/charts/istio-discovery/templates/istiod-injector-configmap.yaml

@@ -48,6 +48,10 @@ data:
       sidecar: |
 {{ .Files.Get "files/injection-template.yaml" | trim | indent 8 }}
 {{- end }}
+{{- if not (hasKey .Values.sidecarInjectorWebhook.templates "gateway") }}
+      gateway: |
+{{ .Files.Get "files/gateway-injection-template.yaml" | trim | indent 8 }}
+{{- end }}
 {{- with .Values.sidecarInjectorWebhook.templates }}
 {{ toYaml . | trim | indent 6 }}
 {{- end }}

charts/kubezero-istio/charts/istio-discovery/values.yaml

@@ -232,7 +232,7 @@ global:
   # Dev builds from prow are on gcr.io
   hub: docker.io/istio
   # Default tag for Istio images.
-  tag: 1.9.1
+  tag: 1.9.2
 
   # Specify image pull policy if default behavior isn't desired.
   # Default behavior: latest images will be Always else IfNotPresent.

charts/kubezero-istio/update.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -ex
 
-export ISTIO_VERSION=1.9.1
+export ISTIO_VERSION=1.9.2
 
 if [ ! -d istio-$ISTIO_VERSION ]; then
   NAME="istio-$ISTIO_VERSION"

charts/kubezero-logging/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-logging
 description: KubeZero Umbrella Chart for complete EFK stack
 type: application
-version: 0.6.2
+version: 0.6.4
 appVersion: 1.4.1
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@@ -26,6 +26,6 @@ dependencies:
     version: 0.2.2
     condition: fluentd.enabled
   - name: fluent-bit
-    version: 0.12.3
+    version: 0.15.4
     condition: fluent-bit.enabled
-kubeVersion: ">= 1.16.0"
+kubeVersion: ">= 1.18.0"

charts/kubezero-logging/charts/fluent-bit/Chart.yaml

@@ -1,8 +1,8 @@
 annotations:
   artifacthub.io/changes: |
-    - add custom annotations on dashboards configmap
+    - Upgrade fluent-bit image to v1.7.3
 apiVersion: v2
-appVersion: 1.7.2
+appVersion: 1.7.3
 description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
 home: https://fluentbit.io/
 icon: https://fluentbit.io/assets/img/logo1-default.png
@@ -20,4 +20,4 @@ maintainers:
 name: fluent-bit
 sources:
 - https://github.com/fluent/fluent-bit/
-version: 0.12.3
+version: 0.15.4

charts/kubezero-logging/charts/fluent-bit/dashboards/fluent-bit.json

@@ -122,7 +122,7 @@
       "pluginVersion": "7.2.1",
       "targets": [
         {
-          "expr": "sum(kube_pod_info{pod=~\".*fluent-bit.*\"})",
+          "expr": "sum(kube_pod_info{pod=~\".*{{ include "fluent-bit.fullname" . }}.*\"})",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -221,7 +221,7 @@
           "hide": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{ pod }}/{{name}}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}/{{"{{"}}name{{"}}"}}",
           "refId": "A"
         }
       ],
@@ -325,7 +325,7 @@
           "hide": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{ pod }}/{{name}}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}/{{"{{"}}name{{"}}"}}",
           "refId": "A"
         }
       ],
@@ -429,7 +429,7 @@
           "hide": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{ pod }}/{{name}}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}/{{"{{"}}name{{"}}"}}",
           "refId": "A"
         }
       ],
@@ -535,7 +535,7 @@
           "hide": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{ pod }}/{{name}}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}/{{"{{"}}name{{"}}"}}",
           "refId": "A"
         }
       ],
@@ -641,7 +641,7 @@
           "hide": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{pod}} Retries to {{name}}",
+          "legendFormat": "{{"{{"}}pod{{"}}"}} Retries to {{"{{"}}name{{"}}"}}",
           "refId": "A"
         },
         {
@@ -649,7 +649,7 @@
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{pod}} Failed Retries to {{ name }}",
+          "legendFormat": "{{"{{"}}pod{{"}}"}} Failed Retries to {{"{{"}} name {{"}}"}}",
           "refId": "B"
         }
       ],
@@ -756,7 +756,7 @@
           "hide": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{ pod }}/{{ name }}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}/{{"{{"}} name {{"}}"}}",
           "refId": "A"
         }
       ],
@@ -851,7 +851,7 @@
         {
           "expr": "sum(rate(fluentbit_filter_drop_records_total{pod=~\"$pod\"}[5m])) by (pod, instance, name)",
           "interval": "",
-          "legendFormat": "{{ pod }} / {{ name }}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}} / {{"{{"}} name {{"}}"}}",
           "refId": "A"
         }
       ],
@@ -946,7 +946,7 @@
         {
           "expr": "sum(rate(fluentbit_filter_add_records_total{pod=~\"$pod\"}[5m])) by (pod, instance, name)",
           "interval": "",
-          "legendFormat": "{{ pod }} / {{ name }}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}} / {{"{{"}} name {{"}}"}}",
           "refId": "A"
         }
       ],
@@ -1059,15 +1059,15 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "container_memory_working_set_bytes{pod=~\".*fluent-bit.*\",pod=~\"$pod\", image!=\"\", container!=\"POD\"}\n",
+          "expr": "container_memory_working_set_bytes{pod=~\".*{{ include "fluent-bit.fullname" . }}.*\",pod=~\"$pod\", image!=\"\", container!=\"POD\"}\n",
           "interval": "",
-          "legendFormat": "{{ pod }}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}",
           "refId": "A"
         },
         {
-          "expr": "avg(kube_pod_container_resource_requests_memory_bytes{pod=~\".*fluent-bit.*\",pod=~\"$pod\"}) by (pod)",
+          "expr": "avg(kube_pod_container_resource_requests_memory_bytes{pod=~\".*{{ include "fluent-bit.fullname" . }}.*\",pod=~\"$pod\"}) by (pod)",
           "interval": "",
-          "legendFormat": "{{ pod }} request",
+          "legendFormat": "{{"{{"}} pod {{"}}"}} request",
           "refId": "B"
         }
       ],
@@ -1075,7 +1075,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Memroy Usage",
+      "title": "Memory Usage",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -1166,15 +1166,15 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "rate(container_cpu_usage_seconds_total{pod=~\".*fluent-bit.*\",pod=~\"$pod\",image!=\"\",container!=\"POD\"}[5m])",
+          "expr": "rate(container_cpu_usage_seconds_total{pod=~\".*{{ include "fluent-bit.fullname" . }}.*\",pod=~\"$pod\",image!=\"\",container!=\"POD\"}[5m])",
           "interval": "",
-          "legendFormat": "{{ pod }}",
+          "legendFormat": "{{"{{"}} pod {{"}}"}}",
           "refId": "A"
         },
         {
           "expr": "avg(kube_pod_container_resource_requests_cpu_cores{pod=~\"$pod\"}) by (pod)",
           "interval": "",
-          "legendFormat": "{{ pod }} request",
+          "legendFormat": "{{"{{"}} pod {{"}}"}} request",
           "refId": "B"
         }
       ],
@@ -1249,14 +1249,14 @@
         "allValue": null,
         "current": {},
         "datasource": "$DS_PROMETHEUS",
-        "definition": "label_values(kube_pod_info{pod=~\".*fluent-bit.*\"}, pod)",
+        "definition": "label_values(kube_pod_info{pod=~\".*{{ include "fluent-bit.fullname" . }}.*\"}, pod)",
         "hide": 0,
         "includeAll": true,
         "label": "pod",
         "multi": false,
         "name": "pod",
         "options": [],
-        "query": "label_values(kube_pod_info{pod=~\".*fluent-bit.*\"}, pod)",
+        "query": "label_values(kube_pod_info{pod=~\".*{{ include "fluent-bit.fullname" . }}.*\"}, pod)",
         "refresh": 2,
         "regex": "",
         "skipUrlSync": false,
@@ -1299,7 +1299,7 @@
     ]
   },
   "timezone": "",
-  "title": "Fluent Bit",
+  "title": "{{ include "fluent-bit.fullname" . }}",
-  "uid": "fluentbit",
+  "uid": "{{ include "fluent-bit.fullname" . }}",
   "version": 2
-}
+}

charts/kubezero-logging/charts/fluent-bit/templates/_pod.tpl

@@ -28,6 +28,14 @@ containers:
   {{- if .Values.envFrom }}
     envFrom:
     {{- toYaml .Values.envFrom | nindent 4 }}
+  {{- end }}
+  {{- if .Values.args }}
+    args:
+    {{- toYaml .Values.args | nindent 6 }}
+  {{- end}}
+  {{- if .Values.command }}
+    command:
+    {{- toYaml .Values.command | nindent 6 }}
   {{- end }}
     ports:
       - name: http
@@ -43,35 +51,39 @@ containers:
     {{- if .Values.livenessProbe }}
     livenessProbe:
       {{- toYaml .Values.livenessProbe | nindent 6 }}
+    {{- else }}
+    livenessProbe:
+      httpGet:
+        path: /
+        port: http
     {{- end }}
     {{- if .Values.readinessProbe }}
     readinessProbe:
       {{- toYaml .Values.readinessProbe | nindent 6 }}
+    {{- else }}
+    readinessProbe:
+      httpGet:
+        path: /
+        port: http
     {{- end }}
     resources:
       {{- toYaml .Values.resources | nindent 6 }}
     volumeMounts:
-      - name: config
+      {{- toYaml .Values.volumeMounts | nindent 6 }}
-        mountPath: /fluent-bit/etc/fluent-bit.conf
-        subPath: fluent-bit.conf
-      - name: config
-        mountPath: /fluent-bit/etc/custom_parsers.conf
-        subPath: custom_parsers.conf
     {{- range $key, $value := .Values.luaScripts }}
       - name: luascripts
         mountPath: /fluent-bit/scripts/{{ $key }}
         subPath: {{ $key }}
     {{- end }}
     {{- if eq .Values.kind "DaemonSet" }}
-      - name: varlog
+      {{- toYaml .Values.daemonSetVolumeMounts | nindent 6 }}
-        mountPath: /var/log
-      - name: etcmachineid
-        mountPath: /etc/machine-id
-        readOnly: true
     {{- end }}
     {{- if .Values.extraVolumeMounts }}
       {{- toYaml .Values.extraVolumeMounts | nindent 6 }}
     {{- end }}
+  {{- if .Values.extraContainers }}
+    {{- toYaml .Values.extraContainers | nindent 2 }}
+  {{- end }}
 volumes:
   - name: config
     configMap:
@@ -82,13 +94,7 @@ volumes:
       name: {{ include "fluent-bit.fullname" . }}-luascripts
 {{- end }}
 {{- if eq .Values.kind "DaemonSet" }}
-  - name: varlog
+  {{- toYaml .Values.daemonSetVolumes | nindent 2 }}
-    hostPath:
-      path: /var/log
-  - name: etcmachineid
-    hostPath:
-      path: /etc/machine-id
-      type: File
 {{- end }}
 {{- if .Values.extraVolumes }}
   {{- toYaml .Values.extraVolumes | nindent 2 }}

charts/kubezero-logging/charts/fluent-bit/templates/clusterrole.yaml

@@ -11,8 +11,8 @@ rules:
     resources:
       - pods
       - namespaces
-      - nodes
+      #- nodes
-      - nodes/proxy
+      #- nodes/proxy
     verbs:
       - get
       - list

charts/kubezero-logging/charts/fluent-bit/templates/configmap-dashboards.yaml

@@ -14,7 +14,7 @@ metadata:
     {{ $.Values.dashboards.labelKey }}: "1"
 data:
   {{ base $path }}: |
-    {{- $.Files.Get $path | nindent 4 }}
+    {{- tpl ($.Files.Get $path) $ | nindent 4 }}
 ---
 {{- end }}
 {{- end -}}

charts/kubezero-logging/charts/fluent-bit/templates/psp.yaml

@@ -3,6 +3,10 @@ apiVersion: policy/v1beta1
 kind: PodSecurityPolicy
 metadata:
   name: {{ include "fluent-bit.fullname" . }}
+{{- if .Values.podSecurityPolicy.annotations }}
+  annotations:
+    {{- toYaml .Values.podSecurityPolicy.annotations | nindent 4 }}
+{{- end }}
 spec:
   privileged: false
   # Required to prevent escalations to root.

charts/kubezero-logging/charts/fluent-bit/values.yaml

@@ -32,6 +32,7 @@ rbac:
 
 podSecurityPolicy:
   create: false
+  annotations: {}
 
 podSecurityContext:
   {}
@@ -96,15 +97,15 @@ dashboards:
   annotations: {}
 
 
-livenessProbe:
+livenessProbe: {}
-  httpGet:
+  # httpGet:
-    path: /
+  #   path: /
-    port: http
+  #   port: http
 
 readinessProbe:
-  httpGet:
+  # httpGet:
-    path: /
+  #   path: /
-    port: http
+  #   port: http
 
 resources:
   {}
@@ -131,6 +132,11 @@ env: []
 
 envFrom: []
 
+extraContainers: []
+  # - name: do-something
+  #   image: busybox
+  #   command: ['do', 'something']
+
 extraPorts: []
 #   - port: 5170
 #     containerPort: 5170
@@ -220,3 +226,38 @@ config:
         Time_Keep Off
         Time_Key time
         Time_Format %Y-%m-%dT%H:%M:%S.%L
+
+# The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname"
+volumeMounts:
+  - name: config
+    mountPath: /fluent-bit/etc/fluent-bit.conf
+    subPath: fluent-bit.conf
+  - name: config
+    mountPath: /fluent-bit/etc/custom_parsers.conf
+    subPath: custom_parsers.conf
+
+daemonSetVolumes:
+  - name: varlog
+    hostPath:
+      path: /var/log
+  - name: varlibdockercontainers
+    hostPath:
+      path: /var/lib/docker/containers
+  - name: etcmachineid
+    hostPath:
+      path: /etc/machine-id
+      type: File
+
+daemonSetVolumeMounts:
+  - name: varlog
+    mountPath: /var/log
+  - name: varlibdockercontainers
+    mountPath: /var/lib/docker/containers
+    readOnly: true
+  - name: etcmachineid
+    mountPath: /etc/machine-id
+    readOnly: true
+
+args: []
+
+command: []

charts/kubezero-logging/fluent-bit.patch

@@ -1,20 +1,18 @@
-diff -tubNr charts/fluent-bit/Chart.yaml charts/fluent-bit.zdt/Chart.yaml
+diff -tubrN charts/fluent-bit/Chart.yaml charts/fluent-bit.zdt/Chart.yaml
---- charts/fluent-bit/Chart.yaml	2021-02-23 14:58:39.000000000 +0100
+--- charts/fluent-bit/Chart.yaml	2021-04-06 13:55:48.000000000 +0200
-+++ charts/fluent-bit.zdt/Chart.yaml	2021-03-07 12:32:34.360084252 +0100
++++ charts/fluent-bit.zdt/Chart.yaml	2021-04-07 11:49:25.955685229 +0200
-@@ -1,8 +1,8 @@
+@@ -1,7 +1,7 @@
  annotations:
    artifacthub.io/changes: |
-     - add custom annotations on dashboards configmap
+     - Upgrade fluent-bit image to v1.7.3
 -apiVersion: v1
--appVersion: 1.7.1
 +apiVersion: v2
-+appVersion: 1.7.2
+ appVersion: 1.7.3
  description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems.
  home: https://fluentbit.io/
- icon: https://fluentbit.io/assets/img/logo1-default.png
+diff -tubrN charts/fluent-bit/templates/_pod.tpl charts/fluent-bit.zdt/templates/_pod.tpl
-diff -tubNr charts/fluent-bit/templates/_pod.tpl charts/fluent-bit.zdt/templates/_pod.tpl
+--- charts/fluent-bit/templates/_pod.tpl	2021-04-06 13:55:48.000000000 +0200
---- charts/fluent-bit/templates/_pod.tpl	2021-02-23 14:58:39.000000000 +0100
++++ charts/fluent-bit.zdt/templates/_pod.tpl	2021-04-07 11:53:45.609030378 +0200
-+++ charts/fluent-bit.zdt/templates/_pod.tpl	2021-03-07 12:45:11.766785251 +0100
 @@ -13,6 +13,8 @@
  dnsConfig:
    {{- toYaml . | nindent 2 }}
@@ -24,41 +22,21 @@ diff -tubNr charts/fluent-bit/templates/_pod.tpl charts/fluent-bit.zdt/templates
  containers:
    - name: {{ .Chart.Name }}
      securityContext:
-@@ -63,9 +65,6 @@
+diff -tubrN charts/fluent-bit/templates/clusterrole.yaml charts/fluent-bit.zdt/templates/clusterrole.yaml
-     {{- if eq .Values.kind "DaemonSet" }}
+--- charts/fluent-bit/templates/clusterrole.yaml	2021-04-06 13:55:48.000000000 +0200
-       - name: varlog
++++ charts/fluent-bit.zdt/templates/clusterrole.yaml	2021-04-07 11:53:26.049029491 +0200
-         mountPath: /var/log
--      - name: varlibdockercontainers
--        mountPath: /var/lib/docker/containers
--        readOnly: true
-       - name: etcmachineid
-         mountPath: /etc/machine-id
-         readOnly: true
-@@ -86,9 +85,6 @@
-   - name: varlog
-     hostPath:
-       path: /var/log
--  - name: varlibdockercontainers
--    hostPath:
--      path: /var/lib/docker/containers
-   - name: etcmachineid
-     hostPath:
-       path: /etc/machine-id
-diff -tubNr charts/fluent-bit/templates/clusterrole.yaml charts/fluent-bit.zdt/templates/clusterrole.yaml
---- charts/fluent-bit/templates/clusterrole.yaml	2021-02-23 14:58:39.000000000 +0100
-+++ charts/fluent-bit.zdt/templates/clusterrole.yaml	2021-03-07 12:32:34.363417586 +0100
 @@ -11,6 +11,8 @@
      resources:
        - pods
        - namespaces
-+      - nodes
++      #- nodes
-+      - nodes/proxy
++      #- nodes/proxy
      verbs:
        - get
        - list
-diff -tubNr charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent-bit.zdt/templates/tests/test-connection.yaml
+diff -tubrN charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent-bit.zdt/templates/tests/test-connection.yaml
---- charts/fluent-bit/templates/tests/test-connection.yaml	2021-02-23 14:58:39.000000000 +0100
+--- charts/fluent-bit/templates/tests/test-connection.yaml	2021-04-06 13:55:48.000000000 +0200
-+++ charts/fluent-bit.zdt/templates/tests/test-connection.yaml	2021-03-07 12:32:34.363417586 +0100
++++ charts/fluent-bit.zdt/templates/tests/test-connection.yaml	2021-04-07 11:52:34.199027132 +0200
 @@ -1,3 +1,4 @@
 +{{- if .Values.testFramework.enabled }}
  apiVersion: v1
@@ -69,9 +47,9 @@ diff -tubNr charts/fluent-bit/templates/tests/test-connection.yaml charts/fluent
    {{- end }}
    restartPolicy: Never
 +{{- end }}
-diff -tubNr charts/fluent-bit/values.yaml charts/fluent-bit.zdt/values.yaml
+diff -tubrN charts/fluent-bit/values.yaml charts/fluent-bit.zdt/values.yaml
---- charts/fluent-bit/values.yaml	2021-02-23 14:58:39.000000000 +0100
+--- charts/fluent-bit/values.yaml	2021-04-06 13:55:48.000000000 +0200
-+++ charts/fluent-bit.zdt/values.yaml	2021-03-07 12:32:34.363417586 +0100
++++ charts/fluent-bit.zdt/values.yaml	2021-04-07 11:52:54.742361399 +0200
 @@ -12,6 +12,7 @@
    # tag:
  

charts/kubezero-logging/update.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-FLUENT_BIT_VERSION=0.12.3
+FLUENT_BIT_VERSION=0.15.4
 FLUENTD_VERSION=0.2.2
 
 # Fluent Bit

charts/kubezero-logging/values.yaml

@@ -227,7 +227,7 @@ fluent-bit:
       tls: false
 
     input:
-      memBufLimit: 16MB
+      memBufLimit: 4MB
       refreshInterval: 10
 
     logLevel: warn
@@ -256,7 +256,7 @@ fluent-bit:
           DB /var/log/flb_kube.db
           DB.Sync Normal
           {{- with .Values.config.input }}
-          Mem_Buf_Limit {{ default "16MB" .memBufLimit }}
+          Mem_Buf_Limit {{ default "4MB" .memBufLimit }}
           Refresh_Interval {{ default 10 .refreshInterval }}
           {{- end }}
 
@@ -350,3 +350,19 @@ fluent-bit:
           end
           return 1, timestamp, new_record
       end
+
+  daemonSetVolumes:
+  - name: varlog
+    hostPath:
+      path: /var/log
+  - name: etcmachineid
+    hostPath:
+      path: /etc/machine-id
+      type: File
+
+  daemonSetVolumeMounts:
+  - name: varlog
+    mountPath: /var/log
+  - name: etcmachineid
+    mountPath: /etc/machine-id
+    readOnly: true

charts/kubezero-redis/templates/envoyfilter-redis-proxy.yaml

@@ -31,5 +31,5 @@ spec:
             op_timeout: 5s
             enable_redirection: true
             enable_command_stats: true
-            read_policy: PREFER_REPLICA
+            read_policy: PREFER_MASTER
 {{- end }}

docs/Upgrade-2.19.md

@@ -31,21 +31,24 @@ This change was required to enable node restrictions via the upstream aws-iam-au
   - Fluentd replicaCount default from 2 to 1
 
 ## Upgrade - Without ArgoCD
-### CRDs:
+1. Update CRDs of all enabled components:  
-( commands assume latest kubezero repository being checkout next to this git repository to deploy master / non-released version )
+  `./bootstrap.sh crds all clusters/$CLUSTER`
-  
-  `./bootstrap.sh crds all clusters/$CLUSTER ../../../kubezero/charts`
 
-### Components
+2. Prepare upgrade
-`./bootstrap.sh deploy all clusters/$CLUSTER ../../../kubezero/charts`
+- delete old fluentd deployement because labels are immutable and they changed due to the migration to new upstream helm chart  
+`kubectl delete deployment logging-fluentd -n logging`
+
+3. Upgrade all components  
+`./bootstrap.sh deploy all clusters/$CLUSTER`
 
 ## Upgrade - ArgoCD
-- push latest config to git repo
+- ArgoCD needs to be upgraded first to support latest Helm chart requirements: `./bootstrap.sh deploy argocd clusters/$CLUSTER`
-- verify correct branch etc. ( argoless branch is retired ! )
+- push latest cluster config to your git repo
+- verify correct branch etc. ( eg. argoless branch has been retired ! )
 - trigger sync in ArgoCD starting with the KubeZero root app  
 ( only if auto-sync is not enabled )
 
 ## Verification / Tests
 - check if all pods are RUNNING
 - check any Ingress services
-- ...
+- ...

kubezero-repo.GFV/repo

@@ -0,0 +1 @@
+Subproject commit 25b8ebe3cc2af88643a4674a63e651e9c31245cb

scripts/git-pre-receive-hook.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Reject pushes that contain commits with messages that do not adhere
+# to the defined regex.
+
+# [1] https://www.conventionalcommits.org/en/v1.0.0/#specification
+
+set -e
+
+zero_commit='0000000000000000000000000000000000000000'
+msg_regex='^((fixup! |squash! )?(fix|feat|build|chore|ci|docs|style|refactor|perf|test)(?:\(([^\)\s]+)\))?: (.+))(?:\n|$){0,2}?((?:^.+(\n|$))+(?:\n|$){0,2}?)*((?:^.+(\n|$))+)*'
+
+while read -r oldrev newrev refname; do
+  # Branch or tag got deleted, ignore the push
+  [ "$newrev" = "$zero_commit" ] && continue
+
+  # Calculate range for new branch/updated branch
+  [ "$oldrev" = "$zero_commit" ] && range="$newrev" || range="$oldrev..$newrev"
+
+  for commit in $(git rev-list "$range" --not --all); do
+    if ! git log --max-count=1 --format=%B $commit | grep -iqE "$msg_regex"; then
+      echo "ERROR:"
+      echo "ERROR: Your push was rejected because the commit"
+      echo "ERROR: $commit in ${refname#refs/heads/}"
+      echo "ERROR: is not adhering to convential commit format."
+      echo "ERROR:"
+      echo "ERROR: Please fix the commit message and push again."
+      echo "ERROR: https://www.conventionalcommits.org/en/v1.0.0/#specification"
+      echo "ERROR"
+      exit 1
+    fi
+  done
+done

scripts/publish.sh

@@ -11,7 +11,7 @@ GIT_PUSH=${GIT_PUSH:-true}
 [[ "$(git branch --show-current)" == "stable" || -n "$FORCE" ]] || { echo "Helm packages should only be built from stable branch !"; exit 1; }
 
 TMPDIR=$(mktemp -d kubezero-repo.XXX)
-mkdir -p $TMPDIR/stage
+mkdir -p $TMPDIR/stage && trap 'rm -rf $TMPDIR' ERR EXIT
 
 git clone -b gh-pages ssh://git@git.zero-downtime.net:22000/ZeroDownTime/KubeZero.git $TMPDIR/repo
 # Reset all
@@ -52,7 +52,7 @@ git status
 
 if [ "$GIT_PUSH" == "true" ]
 then
-    git add . && git commit -m "Publish charts" && git push ssh://git@git.zero-downtime.net:22000/ZeroDownTime/KubeZero.git gh-pages
+    git add . && git commit -m "ci: Publish charts" && git push ssh://git@git.zero-downtime.net:22000/ZeroDownTime/KubeZero.git gh-pages
 fi
 
 cd -