From 7527e085ea9fa0a307d03a6f219767640d15524d Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 29 May 2025 20:24:40 +0000 Subject: [PATCH 1/3] feat: remove kube-proxy, enable cilium --- admin/kubezero.sh | 7 ++++--- charts/kubeadm/README.md | 1 - charts/kubeadm/README.md.gotmpl | 1 - charts/kubeadm/create_audit_policy.sh | 6 ------ charts/kubeadm/templates/ClusterConfiguration.yaml | 2 ++ charts/kubeadm/templates/InitConfiguration.yaml | 2 ++ charts/kubeadm/templates/KubeProxyConfiguration.yaml | 10 ---------- charts/kubeadm/templates/apiserver/audit-policy.yaml | 10 ++-------- charts/kubezero-network/README.md | 6 ++++-- charts/kubezero-network/values.yaml | 8 ++++++++ charts/kubezero/templates/network.yaml | 3 +++ charts/kubezero/values.yaml | 1 + 12 files changed, 26 insertions(+), 31 deletions(-) delete mode 100644 charts/kubeadm/templates/KubeProxyConfiguration.yaml diff --git a/admin/kubezero.sh b/admin/kubezero.sh index 5891334c..f65445e3 100755 --- a/admin/kubezero.sh +++ b/admin/kubezero.sh @@ -63,7 +63,7 @@ render_kubeadm() { # Assemble kubeadm config cat /dev/null > ${HOSTFS}/etc/kubernetes/kubeadm.yaml - for f in Cluster KubeProxy Kubelet; do + for f in Cluster Kubelet; do # echo "---" >> /etc/kubernetes/kubeadm.yaml cat ${WORKDIR}/kubeadm/templates/${f}Configuration.yaml >> ${HOSTFS}/etc/kubernetes/kubeadm.yaml done @@ -239,7 +239,7 @@ control_plane_node() { if [[ "$CMD" =~ ^(join)$ ]]; then # Delete any former self in case forseti did not delete yet kubectl delete node ${NODENAME} --wait=true || true - # Wait for all pods to be deleted otherwise we end up with stale pods eg. kube-proxy and all goes to .... + # Wait for all pods to be deleted otherwise we end up with stale pods kubectl delete pods -n kube-system --field-selector spec.nodeName=${NODENAME} # get current running etcd pods for etcdctl commands @@ -309,8 +309,9 @@ control_plane_node() { _kubeadm init phase mark-control-plane _kubeadm init phase kubelet-finalize all + # we skip kube-proxy if [[ "$CMD" =~ ^(bootstrap|restore)$ ]]; then - _kubeadm init phase addon all + _kubeadm init phase addon coredns fi post_kubeadm diff --git a/charts/kubeadm/README.md b/charts/kubeadm/README.md index 44cff2ff..d22caf94 100644 --- a/charts/kubeadm/README.md +++ b/charts/kubeadm/README.md @@ -47,7 +47,6 @@ Kubernetes: `>= 1.32.0-0` - https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3 - https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go - https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/ -- https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration - https://github.com/awslabs/amazon-eks-ami diff --git a/charts/kubeadm/README.md.gotmpl b/charts/kubeadm/README.md.gotmpl index 43d20a82..bfeeeda9 100644 --- a/charts/kubeadm/README.md.gotmpl +++ b/charts/kubeadm/README.md.gotmpl @@ -22,7 +22,6 @@ - https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3 - https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go - https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/ -- https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration - https://github.com/awslabs/amazon-eks-ami diff --git a/charts/kubeadm/create_audit_policy.sh b/charts/kubeadm/create_audit_policy.sh index a4d24701..870501db 100755 --- a/charts/kubeadm/create_audit_policy.sh +++ b/charts/kubeadm/create_audit_policy.sh @@ -29,12 +29,6 @@ kind: Policy rules: # The following requests were manually identified as high-volume and low-risk, # so drop them. - - level: None - users: ["system:kube-proxy"] - verbs: ["watch"] - resources: - - group: "" # core - resources: ["endpoints", "services", "services/status"] - level: None # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port. # TODO(#46983): Change this to the ingress controller service account. diff --git a/charts/kubeadm/templates/ClusterConfiguration.yaml b/charts/kubeadm/templates/ClusterConfiguration.yaml index c247f9e1..64e7bc93 100644 --- a/charts/kubeadm/templates/ClusterConfiguration.yaml +++ b/charts/kubeadm/templates/ClusterConfiguration.yaml @@ -6,6 +6,8 @@ featureGates: ControlPlaneKubeletLocalMode: true NodeLocalCRISocket: true controlPlaneEndpoint: {{ .Values.api.endpoint }} +proxy: + disabled: true networking: podSubnet: 10.244.0.0/16 etcd: diff --git a/charts/kubeadm/templates/InitConfiguration.yaml b/charts/kubeadm/templates/InitConfiguration.yaml index 5eab7ba2..3511554f 100644 --- a/charts/kubeadm/templates/InitConfiguration.yaml +++ b/charts/kubeadm/templates/InitConfiguration.yaml @@ -7,6 +7,8 @@ localAPIEndpoint: patches: directory: {{ . }} {{- end }} +skipPhases: + - addon/kube-proxy nodeRegistration: criSocket: "unix:///run/containerd/containerd.sock" ignorePreflightErrors: diff --git a/charts/kubeadm/templates/KubeProxyConfiguration.yaml b/charts/kubeadm/templates/KubeProxyConfiguration.yaml deleted file mode 100644 index 61ca7926..00000000 --- a/charts/kubeadm/templates/KubeProxyConfiguration.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: kubeproxy.config.k8s.io/v1alpha1 -kind: KubeProxyConfiguration -# kube-proxy doesnt really support setting dynamic bind-address via config, replaced by cilium long-term anyways -metricsBindAddress: "0.0.0.0:10249" -mode: "iptables" -logging: - format: json -iptables: - localhostNodePorts: false -#nodePortAddresses: primary diff --git a/charts/kubeadm/templates/apiserver/audit-policy.yaml b/charts/kubeadm/templates/apiserver/audit-policy.yaml index 9285f6e5..a77e7c2b 100644 --- a/charts/kubeadm/templates/apiserver/audit-policy.yaml +++ b/charts/kubeadm/templates/apiserver/audit-policy.yaml @@ -3,12 +3,6 @@ kind: Policy rules: # The following requests were manually identified as high-volume and low-risk, # so drop them. - - level: None - users: ["system:kube-proxy"] - verbs: ["watch"] - resources: - - group: "" # core - resources: ["endpoints", "services", "services/status"] - level: None # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port. # TODO(#46983): Change this to the ingress controller service account. @@ -114,7 +108,7 @@ rules: # Get responses can be large; skip them. - level: Request verbs: ["get", "list", "watch"] - resources: + resources: - group: "" # core - group: "admissionregistration.k8s.io" - group: "apiextensions.k8s.io" @@ -137,7 +131,7 @@ rules: - "RequestReceived" # Default level for known APIs - level: RequestResponse - resources: + resources: - group: "" # core - group: "admissionregistration.k8s.io" - group: "apiextensions.k8s.io" diff --git a/charts/kubezero-network/README.md b/charts/kubezero-network/README.md index 776bcb10..7dae8001 100644 --- a/charts/kubezero-network/README.md +++ b/charts/kubezero-network/README.md @@ -1,6 +1,6 @@ # kubezero-network -![Version: 0.5.8](https://img.shields.io/badge/Version-0.5.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.5.9](https://img.shields.io/badge/Version-0.5.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero umbrella chart for all things network @@ -20,7 +20,7 @@ Kubernetes: `>= 1.30.0-0` |------------|------|---------| | https://cdn.zero-downtime.net/charts/ | kubezero-lib | 0.2.1 | | https://haproxytech.github.io/helm-charts | haproxy | 1.24.0 | -| https://helm.cilium.io/ | cilium | 1.17.3 | +| https://helm.cilium.io/ | cilium | 1.17.4 | | https://metallb.github.io/metallb | metallb | 0.14.9 | ## Values @@ -45,6 +45,8 @@ Kubernetes: `>= 1.30.0-0` | cilium.hubble.ui.enabled | bool | `false` | | | cilium.image.useDigest | bool | `false` | | | cilium.ipam.operator.clusterPoolIPv4PodCIDRList[0] | string | `"10.240.0.0/16"` | | +| cilium.k8s.apiServerURLs | string | `""` | | +| cilium.kubeProxyReplacement | bool | `true` | | | cilium.l7Proxy | bool | `false` | | | cilium.operator.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | | | cilium.operator.prometheus.enabled | bool | `false` | | diff --git a/charts/kubezero-network/values.yaml b/charts/kubezero-network/values.yaml index 64f7a681..8ee9a7fa 100644 --- a/charts/kubezero-network/values.yaml +++ b/charts/kubezero-network/values.yaml @@ -64,6 +64,14 @@ cilium: enabled: false #rollOutCiliumPods: true + k8sServiceHost: localhost + k8sServicePort: 6443 + kubeProxyReplacement: true + k8s: + # This has to be set to the DNS name of all API servers + # For example "https://192.168.0.1:6443 https://192.168.0.2:6443" + apiServerURLs: "" + cgroup: autoMount: enabled: false diff --git a/charts/kubezero/templates/network.yaml b/charts/kubezero/templates/network.yaml index df6a3a02..6ec13a71 100644 --- a/charts/kubezero/templates/network.yaml +++ b/charts/kubezero/templates/network.yaml @@ -15,6 +15,9 @@ cilium: # image: # pullPolicy: Never # {{- end }} + k8sServiceHost: {{ .Values.global.apiServerUrl }} + k8s: + apiServerURLs: "https://{{ .Values.global.apiServerUrl }}" cluster: name: {{ .Values.global.clusterName }} diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 36b70606..809b820c 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -1,5 +1,6 @@ global: clusterName: zdt-trial-cluster + apiServerUrl: localhost:6443 # platform: aws (kubeadm, default), gke, or nocloud platform: "aws" From 87e7f5fe20e07a7ccd1c06e7eae794338d03d101 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 30 May 2025 00:06:57 +0000 Subject: [PATCH 2/3] fix: ensure dnsproxy is set, clean default values --- charts/kubezero-network/values.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/charts/kubezero-network/values.yaml b/charts/kubezero-network/values.yaml index 8ee9a7fa..5214a38a 100644 --- a/charts/kubezero-network/values.yaml +++ b/charts/kubezero-network/values.yaml @@ -64,13 +64,16 @@ cilium: enabled: false #rollOutCiliumPods: true - k8sServiceHost: localhost - k8sServicePort: 6443 kubeProxyReplacement: true - k8s: - # This has to be set to the DNS name of all API servers - # For example "https://192.168.0.1:6443 https://192.168.0.2:6443" - apiServerURLs: "" + k8sServiceHost: auto + k8sServicePort: auto + dnsProxy: + enableTransparentMode: true + +# k8s: +# # This has to be set to the DNS name of all API servers +# # For example "https://192.168.0.1:6443 https://192.168.0.2:6443" +# apiServerURLs: "" cgroup: autoMount: From f72ef007f2ce2880f9c9ed5ce31a7b1a6d8088a9 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 30 May 2025 12:34:17 +0000 Subject: [PATCH 3/3] fix: more upgrade fixes --- admin/kubezero.sh | 2 +- charts/kubezero-network/values.yaml | 4 ++-- charts/kubezero/templates/network.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/admin/kubezero.sh b/admin/kubezero.sh index f65445e3..58be1ac3 100755 --- a/admin/kubezero.sh +++ b/admin/kubezero.sh @@ -169,7 +169,7 @@ kubeadm_upgrade() { else pre_cluster_upgrade_final - _kubeadm upgrade apply phase addon all $KUBE_VERSION + _kubeadm upgrade apply phase addon coredns $KUBE_VERSION post_cluster_upgrade_final diff --git a/charts/kubezero-network/values.yaml b/charts/kubezero-network/values.yaml index 5214a38a..985d946c 100644 --- a/charts/kubezero-network/values.yaml +++ b/charts/kubezero-network/values.yaml @@ -65,11 +65,11 @@ cilium: #rollOutCiliumPods: true kubeProxyReplacement: true - k8sServiceHost: auto - k8sServicePort: auto dnsProxy: enableTransparentMode: true + k8sServiceHost: "" + k8sServicePort: 6443 # k8s: # # This has to be set to the DNS name of all API servers # # For example "https://192.168.0.1:6443 https://192.168.0.2:6443" diff --git a/charts/kubezero/templates/network.yaml b/charts/kubezero/templates/network.yaml index 6ec13a71..f40fb6b0 100644 --- a/charts/kubezero/templates/network.yaml +++ b/charts/kubezero/templates/network.yaml @@ -16,8 +16,8 @@ cilium: # pullPolicy: Never # {{- end }} k8sServiceHost: {{ .Values.global.apiServerUrl }} - k8s: - apiServerURLs: "https://{{ .Values.global.apiServerUrl }}" +# k8s: +# apiServerURLs: "https://{{ .Values.global.apiServerUrl }}" cluster: name: {{ .Values.global.clusterName }}