Merge pull request 'remove-kube-proxy' (#118) from remove-kube-proxy into main

Reviewed-on: #118

admin/kubezero.sh

@@ -63,7 +63,7 @@ render_kubeadm() {
 
   # Assemble kubeadm config
   cat /dev/null > ${HOSTFS}/etc/kubernetes/kubeadm.yaml
-  for f in Cluster KubeProxy Kubelet; do
+  for f in Cluster Kubelet; do
     # echo "---" >> /etc/kubernetes/kubeadm.yaml
     cat ${WORKDIR}/kubeadm/templates/${f}Configuration.yaml >> ${HOSTFS}/etc/kubernetes/kubeadm.yaml
   done
@@ -169,7 +169,7 @@ kubeadm_upgrade() {
   else
     pre_cluster_upgrade_final
 
-    _kubeadm upgrade apply phase addon all $KUBE_VERSION
+    _kubeadm upgrade apply phase addon coredns $KUBE_VERSION
 
     post_cluster_upgrade_final
 
@@ -239,7 +239,7 @@ control_plane_node() {
   if [[ "$CMD" =~ ^(join)$ ]]; then
     # Delete any former self in case forseti did not delete yet
     kubectl delete node ${NODENAME} --wait=true || true
-    # Wait for all pods to be deleted otherwise we end up with stale pods eg. kube-proxy and all goes to ....
+    # Wait for all pods to be deleted otherwise we end up with stale pods
     kubectl delete pods -n kube-system --field-selector spec.nodeName=${NODENAME}
 
     # get current running etcd pods for etcdctl commands
@@ -309,8 +309,9 @@ control_plane_node() {
   _kubeadm init phase mark-control-plane
   _kubeadm init phase kubelet-finalize all
 
+  # we skip kube-proxy
   if [[ "$CMD" =~ ^(bootstrap|restore)$ ]]; then
-    _kubeadm init phase addon all
+    _kubeadm init phase addon coredns
   fi
 
   post_kubeadm

charts/kubeadm/README.md

@@ -47,7 +47,6 @@ Kubernetes: `>= 1.32.0-0`
 - https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3
 - https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go
 - https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/
-- https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration
 
 - https://github.com/awslabs/amazon-eks-ami
 

charts/kubeadm/README.md.gotmpl

@@ -22,7 +22,6 @@
 - https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3
 - https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go
 - https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/
-- https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration
 
 - https://github.com/awslabs/amazon-eks-ami
 

charts/kubeadm/create_audit_policy.sh

@@ -29,12 +29,6 @@ kind: Policy
 rules:
   # The following requests were manually identified as high-volume and low-risk,
   # so drop them.
-  - level: None
-    users: ["system:kube-proxy"]
-    verbs: ["watch"]
-    resources:
-      - group: "" # core
-        resources: ["endpoints", "services", "services/status"]
   - level: None
     # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
     # TODO(#46983): Change this to the ingress controller service account.

charts/kubeadm/templates/ClusterConfiguration.yaml

@@ -6,6 +6,8 @@ featureGates:
   ControlPlaneKubeletLocalMode: true
   NodeLocalCRISocket: true
 controlPlaneEndpoint: {{ .Values.api.endpoint }}
+proxy:
+  disabled: true
 networking:
   podSubnet: 10.244.0.0/16
 etcd:

charts/kubeadm/templates/InitConfiguration.yaml

@@ -7,6 +7,8 @@ localAPIEndpoint:
 patches:
   directory: {{ . }}
 {{- end }}
+skipPhases:
+  - addon/kube-proxy
 nodeRegistration:
   criSocket: "unix:///run/containerd/containerd.sock"
   ignorePreflightErrors:

charts/kubeadm/templates/KubeProxyConfiguration.yaml

@@ -1,10 +0,0 @@
-apiVersion: kubeproxy.config.k8s.io/v1alpha1
-kind: KubeProxyConfiguration
-# kube-proxy doesnt really support setting dynamic bind-address via config, replaced by cilium long-term anyways
-metricsBindAddress: "0.0.0.0:10249"
-mode: "iptables"
-logging:
-  format: json
-iptables:
-  localhostNodePorts: false
-#nodePortAddresses: primary

charts/kubeadm/templates/apiserver/audit-policy.yaml

@@ -3,12 +3,6 @@ kind: Policy
 rules:
   # The following requests were manually identified as high-volume and low-risk,
   # so drop them.
-  - level: None
-    users: ["system:kube-proxy"]
-    verbs: ["watch"]
-    resources:
-      - group: "" # core
-        resources: ["endpoints", "services", "services/status"]
   - level: None
     # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
     # TODO(#46983): Change this to the ingress controller service account.
@@ -114,7 +108,7 @@ rules:
   # Get responses can be large; skip them.
   - level: Request
     verbs: ["get", "list", "watch"]
-    resources: 
+    resources:
       - group: "" # core
       - group: "admissionregistration.k8s.io"
       - group: "apiextensions.k8s.io"
@@ -137,7 +131,7 @@ rules:
       - "RequestReceived"
   # Default level for known APIs
   - level: RequestResponse
-    resources: 
+    resources:
       - group: "" # core
       - group: "admissionregistration.k8s.io"
       - group: "apiextensions.k8s.io"

charts/kubezero-network/README.md

@@ -1,6 +1,6 @@
 # kubezero-network
 
-![Version: 0.5.8](https://img.shields.io/badge/Version-0.5.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.5.9](https://img.shields.io/badge/Version-0.5.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero umbrella chart for all things network
 
@@ -20,7 +20,7 @@ Kubernetes: `>= 1.30.0-0`
 |------------|------|---------|
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | 0.2.1 |
 | https://haproxytech.github.io/helm-charts | haproxy | 1.24.0 |
-| https://helm.cilium.io/ | cilium | 1.17.3 |
+| https://helm.cilium.io/ | cilium | 1.17.4 |
 | https://metallb.github.io/metallb | metallb | 0.14.9 |
 
 ## Values
@@ -45,6 +45,8 @@ Kubernetes: `>= 1.30.0-0`
 | cilium.hubble.ui.enabled | bool | `false` |  |
 | cilium.image.useDigest | bool | `false` |  |
 | cilium.ipam.operator.clusterPoolIPv4PodCIDRList[0] | string | `"10.240.0.0/16"` |  |
+| cilium.k8s.apiServerURLs | string | `""` |  |
+| cilium.kubeProxyReplacement | bool | `true` |  |
 | cilium.l7Proxy | bool | `false` |  |
 | cilium.operator.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` |  |
 | cilium.operator.prometheus.enabled | bool | `false` |  |

charts/kubezero-network/values.yaml

@@ -64,6 +64,17 @@ cilium:
     enabled: false
   #rollOutCiliumPods: true
 
+  kubeProxyReplacement: true
+  dnsProxy:
+    enableTransparentMode: true
+
+  k8sServiceHost: ""
+  k8sServicePort: 6443
+# k8s:
+#   # This has to be set to the DNS name of all API servers
+#   # For example "https://192.168.0.1:6443 https://192.168.0.2:6443"
+#   apiServerURLs: ""
+
   cgroup:
     autoMount:
       enabled: false

charts/kubezero/templates/network.yaml

@@ -15,6 +15,9 @@ cilium:
 # image:
 #   pullPolicy: Never
 # {{- end }}
+  k8sServiceHost: {{ .Values.global.apiServerUrl }}
+# k8s:
+#   apiServerURLs: "https://{{ .Values.global.apiServerUrl }}"
 
   cluster:
     name: {{ .Values.global.clusterName }}

charts/kubezero/values.yaml

@@ -1,5 +1,6 @@
 global:
   clusterName: zdt-trial-cluster
+  apiServerUrl: localhost:6443
 
   # platform: aws (kubeadm, default), gke, or nocloud
   platform: "aws"