chore(deps): update kubezero-ci-dependencies

Reviewed-on: #118

admin/hooks-1.32.sh

@@ -2,7 +2,13 @@
 
 # All things BEFORE the first controller / control plane upgrade
 pre_control_plane_upgrade_cluster() {
-  echo
+  if [ "$PLATFORM" != "gke" ];then
+    # patch multus DS to ONLY run pods on 1.31 controllers
+    kubectl patch ds kube-multus-ds -n kube-system -p '{"spec": {"template": {"spec": {"nodeSelector": {"node.kubernetes.io/kubezero.version": "v1.31.6"}}}}}' || true
+
+    # patch kube-proxy DS to ONLY run pods on 1.31 controllers
+    kubectl patch ds kube-proxy -n kube-system -p '{"spec": {"template": {"spec": {"nodeSelector": {"node.kubernetes.io/kubezero.version": "v1.31.6"}}}}}' || true
+  fi
 }
 
 
@@ -16,7 +22,20 @@ post_control_plane_upgrade_cluster() {
 pre_cluster_upgrade_final() {
   set +e
 
-  echo
+  if [ "$PLATFORM" != "gke" ];then
+    # cleanup multus
+    kubectl delete clusterrolebinding multus
+    kubectl delete clusterrole multus
+    kubectl delete serviceaccount multus -n kube-system
+    kubectl delete cm multus-cni-config -n kube-system
+    kubectl delete ds kube-multus-ds -n kube-system
+    kubectl delete NetworkAttachmentDefinition cilium
+    kubectl delete crd network-attachment-definitions.k8s.cni.cncf.io
+
+    # remove kube-proxy
+    kubectl -n kube-system delete ds kube-proxy
+    kubectl -n kube-system delete cm kube-proxy
+  fi
 
   set -e
 }

admin/kubezero.sh

@@ -63,7 +63,7 @@ render_kubeadm() {
 
   # Assemble kubeadm config
   cat /dev/null > ${HOSTFS}/etc/kubernetes/kubeadm.yaml
-  for f in Cluster KubeProxy Kubelet; do
+  for f in Cluster Kubelet; do
     # echo "---" >> /etc/kubernetes/kubeadm.yaml
     cat ${WORKDIR}/kubeadm/templates/${f}Configuration.yaml >> ${HOSTFS}/etc/kubernetes/kubeadm.yaml
   done
@@ -169,7 +169,7 @@ kubeadm_upgrade() {
   else
     pre_cluster_upgrade_final
 
-    _kubeadm upgrade apply phase addon all $KUBE_VERSION
+    _kubeadm upgrade apply phase addon coredns $KUBE_VERSION
 
     post_cluster_upgrade_final
 
@@ -239,7 +239,7 @@ control_plane_node() {
   if [[ "$CMD" =~ ^(join)$ ]]; then
     # Delete any former self in case forseti did not delete yet
     kubectl delete node ${NODENAME} --wait=true || true
-    # Wait for all pods to be deleted otherwise we end up with stale pods eg. kube-proxy and all goes to ....
+    # Wait for all pods to be deleted otherwise we end up with stale pods
     kubectl delete pods -n kube-system --field-selector spec.nodeName=${NODENAME}
 
     # get current running etcd pods for etcdctl commands
@@ -309,8 +309,9 @@ control_plane_node() {
   _kubeadm init phase mark-control-plane
   _kubeadm init phase kubelet-finalize all
 
+  # we skip kube-proxy
   if [[ "$CMD" =~ ^(bootstrap|restore)$ ]]; then
-    _kubeadm init phase addon all
+    _kubeadm init phase addon coredns
   fi
 
   post_kubeadm

admin/libhelm.sh

@@ -81,15 +81,16 @@ function get_kubezero_secret() {
   get_secret_val kubezero kubezero-secrets "$1"
 }
 
-function ensure_kubezero_secret_key() {
-  local secret="$(kubectl get secret -n kubezero kubezero-secrets -o yaml)"
-  local key=""
-  local val=""
 
-  for key in $@; do
+function ensure_kubezero_secret_key() {
-    val=$(echo "$secret" | yq ".data.\"$key\"")
+  local secret="$(kubectl get secret -n $ns $secret -o yaml)"
+  local key
+  local val
+
+  for key in $1; do
+    val=$(echo $secret | yq ".data.\"$key\"")
     if [ "$val" == "null" ]; then
-      kubectl patch secret -n kubezero kubezero-secrets --patch="{\"data\": { \"$key\": \"\" }}"
+      set_kubezero_secret $key ""
     fi
   done
 }

charts/kubeadm/README.md

@@ -47,7 +47,6 @@ Kubernetes: `>= 1.32.0-0`
 - https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3
 - https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go
 - https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/
-- https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration
 
 - https://github.com/awslabs/amazon-eks-ami
 

charts/kubeadm/README.md.gotmpl

@@ -22,7 +22,6 @@
 - https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3
 - https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go
 - https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/
-- https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration
 
 - https://github.com/awslabs/amazon-eks-ami
 

charts/kubeadm/create_audit_policy.sh

@@ -29,12 +29,6 @@ kind: Policy
 rules:
   # The following requests were manually identified as high-volume and low-risk,
   # so drop them.
-  - level: None
-    users: ["system:kube-proxy"]
-    verbs: ["watch"]
-    resources:
-      - group: "" # core
-        resources: ["endpoints", "services", "services/status"]
   - level: None
     # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
     # TODO(#46983): Change this to the ingress controller service account.

charts/kubeadm/templates/ClusterConfiguration.yaml

@@ -6,6 +6,8 @@ featureGates:
   ControlPlaneKubeletLocalMode: true
   NodeLocalCRISocket: true
 controlPlaneEndpoint: {{ .Values.api.endpoint }}
+proxy:
+  disabled: true
 networking:
   podSubnet: 10.244.0.0/16
 etcd:

charts/kubeadm/templates/InitConfiguration.yaml

@@ -7,6 +7,8 @@ localAPIEndpoint:
 patches:
   directory: {{ . }}
 {{- end }}
+skipPhases:
+  - addon/kube-proxy
 nodeRegistration:
   criSocket: "unix:///run/containerd/containerd.sock"
   ignorePreflightErrors:

charts/kubeadm/templates/KubeProxyConfiguration.yaml

@@ -1,10 +0,0 @@
-apiVersion: kubeproxy.config.k8s.io/v1alpha1
-kind: KubeProxyConfiguration
-# kube-proxy doesnt really support setting dynamic bind-address via config, replaced by cilium long-term anyways
-metricsBindAddress: "0.0.0.0:10249"
-mode: "iptables"
-logging:
-  format: json
-iptables:
-  localhostNodePorts: false
-#nodePortAddresses: primary

charts/kubeadm/templates/apiserver/audit-policy.yaml

@@ -3,12 +3,6 @@ kind: Policy
 rules:
   # The following requests were manually identified as high-volume and low-risk,
   # so drop them.
-  - level: None
-    users: ["system:kube-proxy"]
-    verbs: ["watch"]
-    resources:
-      - group: "" # core
-        resources: ["endpoints", "services", "services/status"]
   - level: None
     # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
     # TODO(#46983): Change this to the ingress controller service account.

charts/kubezero-ci/Chart.yaml

@@ -30,7 +30,7 @@ dependencies:
     repository: https://aquasecurity.github.io/helm-charts/
     condition: trivy.enabled
   - name: renovate
-    version: 40.35.3
+    version: 40.36.8
     repository: https://docs.renovatebot.com/helm-charts
     condition: renovate.enabled
 kubeVersion: ">= 1.25.0"

charts/kubezero-network/README.md

@@ -1,6 +1,6 @@
 # kubezero-network
 
-![Version: 0.5.8](https://img.shields.io/badge/Version-0.5.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.5.9](https://img.shields.io/badge/Version-0.5.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero umbrella chart for all things network
 
@@ -20,7 +20,7 @@ Kubernetes: `>= 1.30.0-0`
 |------------|------|---------|
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | 0.2.1 |
 | https://haproxytech.github.io/helm-charts | haproxy | 1.24.0 |
-| https://helm.cilium.io/ | cilium | 1.17.3 |
+| https://helm.cilium.io/ | cilium | 1.17.4 |
 | https://metallb.github.io/metallb | metallb | 0.14.9 |
 
 ## Values
@@ -45,6 +45,8 @@ Kubernetes: `>= 1.30.0-0`
 | cilium.hubble.ui.enabled | bool | `false` |  |
 | cilium.image.useDigest | bool | `false` |  |
 | cilium.ipam.operator.clusterPoolIPv4PodCIDRList[0] | string | `"10.240.0.0/16"` |  |
+| cilium.k8s.apiServerURLs | string | `""` |  |
+| cilium.kubeProxyReplacement | bool | `true` |  |
 | cilium.l7Proxy | bool | `false` |  |
 | cilium.operator.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` |  |
 | cilium.operator.prometheus.enabled | bool | `false` |  |

charts/kubezero-network/values.yaml

@@ -43,7 +43,8 @@ cilium:
     binPath: "/usr/libexec/cni"
     logFile: /var/log/cilium-cni.log
     #-- Ensure this is false if multus is enabled
-    exclusive: false
+    exclusive: true
+    iptablesRemoveAWSRules: false
 
   cluster:
     # This should match the second octet of clusterPoolIPv4PodCIDRList
@@ -63,6 +64,17 @@ cilium:
     enabled: false
   #rollOutCiliumPods: true
 
+  kubeProxyReplacement: true
+  dnsProxy:
+    enableTransparentMode: true
+
+  k8sServiceHost: ""
+  k8sServicePort: 6443
+# k8s:
+#   # This has to be set to the DNS name of all API servers
+#   # For example "https://192.168.0.1:6443 https://192.168.0.2:6443"
+#   apiServerURLs: ""
+
   cgroup:
     autoMount:
       enabled: false
@@ -91,9 +103,11 @@ cilium:
     - key: node-role.kubernetes.io/control-plane
       effect: NoSchedule
     # the operator removes the taints,
-    # so we need to break chicken egg on single controller
+    # so we need to break chicken egg
     - key: node.cilium.io/agent-not-ready
       effect: NoSchedule
+    - key: node.kubernetes.io/not-ready
+      effect: NoSchedule
 
     nodeSelector:
       node-role.kubernetes.io/control-plane: ""

charts/kubezero/templates/network.yaml

@@ -1,6 +1,6 @@
 {{- define "network-values" }}
 multus:
-  enabled: true
+  enabled: false
   clusterNetwork: "cilium"
 
 # {{- if eq .Values.global.platform "aws" }}
@@ -15,6 +15,9 @@ cilium:
 # image:
 #   pullPolicy: Never
 # {{- end }}
+  k8sServiceHost: {{ .Values.global.apiServerUrl }}
+# k8s:
+#   apiServerURLs: "https://{{ .Values.global.apiServerUrl }}"
 
   cluster:
     name: {{ .Values.global.clusterName }}

charts/kubezero/values.yaml

@@ -1,5 +1,6 @@
 global:
   clusterName: zdt-trial-cluster
+  apiServerUrl: localhost:6443
 
   # platform: aws (kubeadm, default), gke, or nocloud
   platform: "aws"
@@ -32,7 +33,7 @@ addons:
 network:
   enabled: true
   retain: true
-  targetRevision: 0.5.8
+  targetRevision: 0.5.9
   cilium:
     cluster: {}