chore(deps): update helm release nats to v1.3.7

admin/hooks-1.32.sh

@@ -2,13 +2,7 @@
 
 # All things BEFORE the first controller / control plane upgrade
 pre_control_plane_upgrade_cluster() {
-  if [ "$PLATFORM" != "gke" ];then
-    # patch multus DS to ONLY run pods on 1.31 controllers
-    kubectl patch ds kube-multus-ds -n kube-system -p '{"spec": {"template": {"spec": {"nodeSelector": {"node.kubernetes.io/kubezero.version": "v1.31.6"}}}}}' || true
-
-    # patch kube-proxy DS to ONLY run pods on 1.31 controllers
-    kubectl patch ds kube-proxy -n kube-system -p '{"spec": {"template": {"spec": {"nodeSelector": {"node.kubernetes.io/kubezero.version": "v1.31.6"}}}}}' || true
-  fi
+  echo
 }
 
 
@@ -22,20 +16,7 @@ post_control_plane_upgrade_cluster() {
 pre_cluster_upgrade_final() {
   set +e
 
-  if [ "$PLATFORM" != "gke" ];then
-    # cleanup multus
-    kubectl delete clusterrolebinding multus
-    kubectl delete clusterrole multus
-    kubectl delete serviceaccount multus -n kube-system
-    kubectl delete cm multus-cni-config -n kube-system
-    kubectl delete ds kube-multus-ds -n kube-system
-    kubectl delete NetworkAttachmentDefinition cilium
-    kubectl delete crd network-attachment-definitions.k8s.cni.cncf.io
-
-    # remove kube-proxy
-    kubectl -n kube-system delete ds kube-proxy
-    kubectl -n kube-system delete cm kube-proxy
-  fi
+  echo
 
   set -e
 }

admin/kubezero.sh

@@ -63,7 +63,7 @@ render_kubeadm() {
 
   # Assemble kubeadm config
   cat /dev/null > ${HOSTFS}/etc/kubernetes/kubeadm.yaml
-  for f in Cluster Kubelet; do
+  for f in Cluster KubeProxy Kubelet; do
     # echo "---" >> /etc/kubernetes/kubeadm.yaml
     cat ${WORKDIR}/kubeadm/templates/${f}Configuration.yaml >> ${HOSTFS}/etc/kubernetes/kubeadm.yaml
   done
@@ -169,7 +169,7 @@ kubeadm_upgrade() {
   else
     pre_cluster_upgrade_final
 
-    _kubeadm upgrade apply phase addon coredns $KUBE_VERSION
+    _kubeadm upgrade apply phase addon all $KUBE_VERSION
 
     post_cluster_upgrade_final
 
@@ -239,7 +239,7 @@ control_plane_node() {
   if [[ "$CMD" =~ ^(join)$ ]]; then
     # Delete any former self in case forseti did not delete yet
     kubectl delete node ${NODENAME} --wait=true || true
-    # Wait for all pods to be deleted otherwise we end up with stale pods
+    # Wait for all pods to be deleted otherwise we end up with stale pods eg. kube-proxy and all goes to ....
     kubectl delete pods -n kube-system --field-selector spec.nodeName=${NODENAME}
 
     # get current running etcd pods for etcdctl commands
@@ -309,9 +309,8 @@ control_plane_node() {
   _kubeadm init phase mark-control-plane
   _kubeadm init phase kubelet-finalize all
 
-  # we skip kube-proxy
   if [[ "$CMD" =~ ^(bootstrap|restore)$ ]]; then
-    _kubeadm init phase addon coredns
+    _kubeadm init phase addon all
   fi
 
   post_kubeadm

admin/libhelm.sh

@@ -3,7 +3,6 @@
 # Simulate well-known CRDs being available
 API_VERSIONS="-a monitoring.coreos.com/v1 -a snapshot.storage.k8s.io/v1 -a policy/v1/PodDisruptionBudget -a apiregistration.k8s.io/v1"
 LOCAL_DEV=${LOCAL_DEV:-""}
-ENV_VALUES=""
 
 export HELM_SECRETS_BACKEND="vals"
 
@@ -81,19 +80,15 @@ function get_kubezero_secret() {
   get_secret_val kubezero kubezero-secrets "$1"
 }
 
-
 function ensure_kubezero_secret_key() {
-  local ns=$1
-  local secret=$2
+  local secret="$(kubectl get secret -n kubezero kubezero-secrets -o yaml)"
+  local key=""
+  local val=""
 
-  local secret="$(kubectl get secret -n $ns $secret -o yaml)"
-  local key
-  local val
-
-  for key in $1; do
-    val=$(echo $secret | yq ".data.\"$key\"")
+  for key in $@; do
+    val=$(echo "$secret" | yq ".data.\"$key\"")
     if [ "$val" == "null" ]; then
-      set_kubezero_secret $key ""
+      kubectl patch secret -n kubezero kubezero-secrets --patch="{\"data\": { \"$key\": \"\" }}"
     fi
   done
 }
@@ -267,11 +262,6 @@ function _helm() {
 
     crds
 
-  elif [ $action == "dryrun" ]; then
-    cat $WORKDIR/values.yaml
-    render
-    cat $WORKDIR/helm.yaml
-
   elif [ $action == "apply" -o $action == "replace" ]; then
     echo "using values to $action of module $module: "
     cat $WORKDIR/values.yaml

admin/migrate_argo_values.py

@@ -10,14 +10,7 @@ def migrate(values):
 
     # 1.32
     try:
-        values["istio-ingress"]["gateway"]["service"]["extraPorts"] = values["istio-ingress"]["gateway"]["service"]["ports"]
-        values["istio-ingress"]["gateway"]["service"].pop("ports")
-    except KeyError:
         pass
-
-    try:
-        values["istio-private-ingress"]["gateway"]["service"]["extraPorts"] = values["istio-private-ingress"]["gateway"]["service"]["ports"]
-        values["istio-private-ingress"]["gateway"]["service"].pop("ports")
     except KeyError:
         pass
 

charts/kubeadm/README.md

@@ -47,6 +47,7 @@ Kubernetes: `>= 1.32.0-0`
 - https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3
 - https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go
 - https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/
+- https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration
 
 - https://github.com/awslabs/amazon-eks-ami
 

charts/kubeadm/README.md.gotmpl

@@ -22,6 +22,7 @@
 - https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3
 - https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go
 - https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/
+- https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration
 
 - https://github.com/awslabs/amazon-eks-ami
 

charts/kubeadm/create_audit_policy.sh

@@ -29,6 +29,12 @@ kind: Policy
 rules:
   # The following requests were manually identified as high-volume and low-risk,
   # so drop them.
+  - level: None
+    users: ["system:kube-proxy"]
+    verbs: ["watch"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints", "services", "services/status"]
   - level: None
     # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
     # TODO(#46983): Change this to the ingress controller service account.

charts/kubeadm/templates/ClusterConfiguration.yaml

@@ -6,8 +6,6 @@ featureGates:
   ControlPlaneKubeletLocalMode: true
   NodeLocalCRISocket: true
 controlPlaneEndpoint: {{ .Values.api.endpoint }}
-proxy:
-  disabled: true
 networking:
   podSubnet: 10.244.0.0/16
 etcd:

charts/kubeadm/templates/InitConfiguration.yaml

@@ -7,8 +7,6 @@ localAPIEndpoint:
 patches:
   directory: {{ . }}
 {{- end }}
-skipPhases:
-  - addon/kube-proxy
 nodeRegistration:
   criSocket: "unix:///run/containerd/containerd.sock"
   ignorePreflightErrors:

charts/kubeadm/templates/KubeProxyConfiguration.yaml

@@ -0,0 +1,10 @@
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+# kube-proxy doesnt really support setting dynamic bind-address via config, replaced by cilium long-term anyways
+metricsBindAddress: "0.0.0.0:10249"
+mode: "iptables"
+logging:
+  format: json
+iptables:
+  localhostNodePorts: false
+#nodePortAddresses: primary

charts/kubeadm/templates/apiserver/audit-policy.yaml

@@ -3,6 +3,12 @@ kind: Policy
 rules:
   # The following requests were manually identified as high-volume and low-risk,
   # so drop them.
+  - level: None
+    users: ["system:kube-proxy"]
+    verbs: ["watch"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints", "services", "services/status"]
   - level: None
     # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
     # TODO(#46983): Change this to the ingress controller service account.
@@ -108,7 +114,7 @@ rules:
   # Get responses can be large; skip them.
   - level: Request
     verbs: ["get", "list", "watch"]
-    resources:
+    resources: 
       - group: "" # core
       - group: "admissionregistration.k8s.io"
       - group: "apiextensions.k8s.io"
@@ -131,7 +137,7 @@ rules:
       - "RequestReceived"
   # Default level for known APIs
   - level: RequestResponse
-    resources:
+    resources: 
       - group: "" # core
       - group: "admissionregistration.k8s.io"
       - group: "apiextensions.k8s.io"

charts/kubezero-ci/values.yaml

@@ -289,7 +289,7 @@ trivy:
   #tag: 0.57.0
   persistence:
     enabled: true
-    size: 2Gi
+    size: 1Gi
   rbac:
     create: false
 

charts/kubezero-lib/templates/_helpers.tpl

@@ -42,3 +42,17 @@ helm.sh/chart: {{ include "kubezero-lib.chart" . }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 app.kubernetes.io/part-of: kubezero
 {{- end -}}
+
+{{- /*
+kubezero-lib.util.merge will merge two YAML templates and output the result.
+This takes an array of three values:
+- the top context
+- the template name of the overrides (destination)
+- the template name of the base (source)
+*/ -}}
+{{- define "kubezero-lib.util.merge" -}}
+{{- $top := first . -}}
+{{- $overrides := fromYaml (include (index . 1) $top) | default (dict ) -}}
+{{- $tpl := fromYaml (include (index . 2) $top) | default (dict ) -}}
+{{- toYaml (merge $overrides $tpl) -}}
+{{- end -}}

charts/kubezero-metrics/values.yaml

@@ -12,7 +12,7 @@ kube-prometheus-stack:
   kubeStateMetrics:
     enabled: true
   kubeProxy:
-    enabled: false
+    enabled: true
 
   kubeEtcd:
     enabled: true

charts/kubezero-mq/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-mq
 description: KubeZero umbrella chart for MQ systems like NATS, RabbitMQ
 type: application
-version: 0.3.11
+version: 0.3.12
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -17,7 +17,7 @@ dependencies:
     version: 0.2.1
     repository: https://cdn.zero-downtime.net/charts/
   - name: nats
-    version: 1.3.3
+    version: 1.3.7
     repository: https://nats-io.github.io/k8s/helm/charts/
     condition: nats.enabled
   - name: rabbitmq

charts/kubezero-network/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-network
 description: KubeZero umbrella chart for all things network
 type: application
-version: 0.5.9
+version: 0.5.8
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -19,7 +19,7 @@ dependencies:
     version: 0.2.1
     repository: https://cdn.zero-downtime.net/charts/
   - name: cilium
-    version: 1.17.4
+    version: 1.17.3
     repository: https://helm.cilium.io/
     condition: cilium.enabled
   - name: metallb

charts/kubezero-network/README.md

@@ -1,6 +1,6 @@
 # kubezero-network
 
-![Version: 0.5.9](https://img.shields.io/badge/Version-0.5.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.5.8](https://img.shields.io/badge/Version-0.5.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero umbrella chart for all things network
 
@@ -20,7 +20,7 @@ Kubernetes: `>= 1.30.0-0`
 |------------|------|---------|
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | 0.2.1 |
 | https://haproxytech.github.io/helm-charts | haproxy | 1.24.0 |
-| https://helm.cilium.io/ | cilium | 1.17.4 |
+| https://helm.cilium.io/ | cilium | 1.17.3 |
 | https://metallb.github.io/metallb | metallb | 0.14.9 |
 
 ## Values
@@ -45,8 +45,6 @@ Kubernetes: `>= 1.30.0-0`
 | cilium.hubble.ui.enabled | bool | `false` |  |
 | cilium.image.useDigest | bool | `false` |  |
 | cilium.ipam.operator.clusterPoolIPv4PodCIDRList[0] | string | `"10.240.0.0/16"` |  |
-| cilium.k8s.apiServerURLs | string | `""` |  |
-| cilium.kubeProxyReplacement | bool | `true` |  |
 | cilium.l7Proxy | bool | `false` |  |
 | cilium.operator.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` |  |
 | cilium.operator.prometheus.enabled | bool | `false` |  |

charts/kubezero-network/values.yaml

@@ -33,18 +33,17 @@ cilium:
 
   resources:
     requests:
-      cpu: 50m
-      memory: 256Mi
-#   limits:
-#     memory: 1Gi
-#     cpu: 4000m
+      cpu: 10m
+      memory: 160Mi
+    limits:
+      memory: 1Gi
+      # cpu: 4000m
 
   cni:
     binPath: "/usr/libexec/cni"
     logFile: /var/log/cilium-cni.log
     #-- Ensure this is false if multus is enabled
-    exclusive: true
-    iptablesRemoveAWSRules: false
+    exclusive: false
 
   cluster:
     # This should match the second octet of clusterPoolIPv4PodCIDRList
@@ -64,32 +63,13 @@ cilium:
     enabled: false
   #rollOutCiliumPods: true
 
-  kubeProxyReplacement: true
-  dnsProxy:
-    enableTransparentMode: true
-
-# For LB support via L2announcement or BGP - on-prem only
-# l2announcements:
-#   enabled: true
-# Not needed normally
-# externalIPs:
-#   enabled: true
-
-  k8sServiceHost: ""
-  k8sServicePort: 6443
-# k8s:
-#   # This has to be set to the DNS name of all API servers
-#   # For example "https://192.168.0.1:6443 https://192.168.0.2:6443"
-#   apiServerURLs: ""
-
   cgroup:
     autoMount:
       enabled: false
     hostRoot: "/sys/fs/cgroup"
 
-  bpf:
-    preallocateMaps: true
   # we need biDirectional so use helm init-container
+  #bpf:
   # autoMount:
   #   enabled: false
 
@@ -111,11 +91,9 @@ cilium:
     - key: node-role.kubernetes.io/control-plane
       effect: NoSchedule
     # the operator removes the taints,
-    # so we need to break chicken egg
+    # so we need to break chicken egg on single controller
     - key: node.cilium.io/agent-not-ready
       effect: NoSchedule
-    - key: node.kubernetes.io/not-ready
-      effect: NoSchedule
 
     nodeSelector:
       node-role.kubernetes.io/control-plane: ""

charts/kubezero/templates/_app.tpl

@@ -28,7 +28,7 @@ spec:
     helm:
       skipTests: true
       valuesObject:
-        {{- toYaml (merge (omit (index .Values $name) "enabled" "namespace" "retain" "targetRevision") (fromYaml (include (print $name "-values") $ ))) | nindent 8 }}
+        {{- include (print $name "-values") $ | nindent 8 }}
 
   destination:
     server: "https://kubernetes.default.svc"

charts/kubezero/templates/istio-ingress.yaml

@@ -67,7 +67,7 @@ gateway:
       gatewayProtocol: HTTPS
       tls:
         mode: SIMPLE
-    {{- with index .Values "istio-ingress" "gateway" "service" "extraPorts" }}
+    {{- with index .Values "istio-ingress" "gateway" "service" "ports" }}
     {{- toYaml . | nindent 4 }}
     {{- end }}
 
@@ -93,6 +93,7 @@ certificates:
   {{- toYaml $cert.dnsNames | nindent 4 }}
 {{- end }}
 {{- end }}
+proxyProtocol: {{ default true (index .Values "istio-ingress" "proxyProtocol") }}
 {{- with (index .Values "istio-ingress" "hardening") }}
 hardening:
   {{- toYaml . | nindent 2 }}

charts/kubezero/templates/istio-private-ingress.yaml

@@ -64,7 +64,7 @@ gateway:
       gatewayProtocol: HTTPS
       tls:
         mode: SIMPLE
-  {{- with index .Values "istio-private-ingress" "gateway" "service" "extraPorts" }}
+  {{- with index .Values "istio-private-ingress" "gateway" "service" "ports" }}
   {{- toYaml . | nindent 4 }}
   {{- end }}
 
@@ -88,6 +88,7 @@ certificates:
   dnsNames:
   {{- toYaml $cert.dnsNames | nindent 4 }}
 {{- end }}
+proxyProtocol: {{ default true (index .Values "istio-private-ingress" "proxyProtocol") }}
 {{- with (index .Values "istio-private-ingress" "hardening") }}
 hardening:
   {{- toYaml . | nindent 2 }}

charts/kubezero/templates/network.yaml

@@ -1,6 +1,6 @@
 {{- define "network-values" }}
 multus:
-  enabled: false
+  enabled: true
   clusterNetwork: "cilium"
 
 # {{- if eq .Values.global.platform "aws" }}
@@ -9,23 +9,22 @@ multus:
 # {{- end }}
 
 cilium:
-  k8sServiceHost: {{ .Values.global.apiServerUrl }}
+  enabled: true
 
 # {{- if eq .Values.global.platform "aws" }}
 # image:
 #   pullPolicy: Never
 # {{- end }}
-# k8s:
-#   apiServerURLs: "https://{{ .Values.global.apiServerUrl }}"
 
   cluster:
     name: {{ .Values.global.clusterName }}
-  {{- with .Values.network.cilium.cluster.id }}
+    {{- with .Values.network.cilium.cluster.id }}
+    id: {{ . }}
   ipam:
     operator:
       clusterPoolIPv4PodCIDRList:
         - 10.{{ . }}.0.0/16
-  {{- end }}
+    {{- end }}
 
   prometheus:
     enabled: {{ .Values.metrics.enabled }}

charts/kubezero/templates/telemetry.yaml

@@ -61,10 +61,8 @@ opensearch:
   {{- toYaml . | nindent 2 }}
   {{- end }}
 
-# Disabled until upstream made up their mind
-# https://github.com/opensearch-project/technical-steering/issues/35
-# serviceMonitor:
-#   enabled: {{ .Values.metrics.enabled }}
+  serviceMonitor:
+    enabled: {{ .Values.metrics.enabled }}
 {{- end }}
 
 {{- if index .Values "telemetry" "opensearch-dashboards" }}
@@ -73,10 +71,8 @@ opensearch-dashboards:
   {{- toYaml . | nindent 2 }}
   {{- end }}
 
-# Disabled until upstream made up their mind
-# https://github.com/opensearch-project/technical-steering/issues/35
-# serviceMonitor:
-#   enabled: {{ .Values.metrics.enabled }}
+  serviceMonitor:
+    enabled: {{ .Values.metrics.enabled }}
 {{- end }}
 
 {{- end }}

charts/kubezero/values.yaml

@@ -1,6 +1,5 @@
 global:
   clusterName: zdt-trial-cluster
-  apiServerUrl: localhost:6443
 
   # platform: aws (kubeadm, default), gke, or nocloud
   platform: "aws"
@@ -33,7 +32,7 @@ addons:
 network:
   enabled: true
   retain: true
-  targetRevision: 0.5.9
+  targetRevision: 0.5.8
   cilium:
     cluster: {}
 
@@ -61,13 +60,13 @@ storage:
 istio:
   enabled: false
   namespace: istio-system
-  targetRevision: 0.24.6
+  targetRevision: 0.24.3
 
 istio-ingress:
   enabled: false
   chart: kubezero-istio-gateway
   namespace: istio-ingress
-  targetRevision: 0.24.6
+  targetRevision: 0.24.3
   gateway:
     service: {}
 
@@ -75,7 +74,7 @@ istio-private-ingress:
   enabled: false
   chart: kubezero-istio-gateway
   namespace: istio-ingress
-  targetRevision: 0.24.6
+  targetRevision: 0.24.3
   gateway:
     service: {}