diff --git a/admin/kubezero.sh b/admin/kubezero.sh index 5891334c..58be1ac3 100755 --- a/admin/kubezero.sh +++ b/admin/kubezero.sh @@ -63,7 +63,7 @@ render_kubeadm() { # Assemble kubeadm config cat /dev/null > ${HOSTFS}/etc/kubernetes/kubeadm.yaml - for f in Cluster KubeProxy Kubelet; do + for f in Cluster Kubelet; do # echo "---" >> /etc/kubernetes/kubeadm.yaml cat ${WORKDIR}/kubeadm/templates/${f}Configuration.yaml >> ${HOSTFS}/etc/kubernetes/kubeadm.yaml done @@ -169,7 +169,7 @@ kubeadm_upgrade() { else pre_cluster_upgrade_final - _kubeadm upgrade apply phase addon all $KUBE_VERSION + _kubeadm upgrade apply phase addon coredns $KUBE_VERSION post_cluster_upgrade_final @@ -239,7 +239,7 @@ control_plane_node() { if [[ "$CMD" =~ ^(join)$ ]]; then # Delete any former self in case forseti did not delete yet kubectl delete node ${NODENAME} --wait=true || true - # Wait for all pods to be deleted otherwise we end up with stale pods eg. kube-proxy and all goes to .... + # Wait for all pods to be deleted otherwise we end up with stale pods kubectl delete pods -n kube-system --field-selector spec.nodeName=${NODENAME} # get current running etcd pods for etcdctl commands @@ -309,8 +309,9 @@ control_plane_node() { _kubeadm init phase mark-control-plane _kubeadm init phase kubelet-finalize all + # we skip kube-proxy if [[ "$CMD" =~ ^(bootstrap|restore)$ ]]; then - _kubeadm init phase addon all + _kubeadm init phase addon coredns fi post_kubeadm diff --git a/charts/kubeadm/README.md b/charts/kubeadm/README.md index 44cff2ff..d22caf94 100644 --- a/charts/kubeadm/README.md +++ b/charts/kubeadm/README.md @@ -47,7 +47,6 @@ Kubernetes: `>= 1.32.0-0` - https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3 - https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go - https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/ -- https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration - https://github.com/awslabs/amazon-eks-ami diff --git a/charts/kubeadm/README.md.gotmpl b/charts/kubeadm/README.md.gotmpl index 43d20a82..bfeeeda9 100644 --- a/charts/kubeadm/README.md.gotmpl +++ b/charts/kubeadm/README.md.gotmpl @@ -22,7 +22,6 @@ - https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3 - https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go - https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/ -- https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration - https://github.com/awslabs/amazon-eks-ami diff --git a/charts/kubeadm/create_audit_policy.sh b/charts/kubeadm/create_audit_policy.sh index a4d24701..870501db 100755 --- a/charts/kubeadm/create_audit_policy.sh +++ b/charts/kubeadm/create_audit_policy.sh @@ -29,12 +29,6 @@ kind: Policy rules: # The following requests were manually identified as high-volume and low-risk, # so drop them. - - level: None - users: ["system:kube-proxy"] - verbs: ["watch"] - resources: - - group: "" # core - resources: ["endpoints", "services", "services/status"] - level: None # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port. # TODO(#46983): Change this to the ingress controller service account. diff --git a/charts/kubeadm/templates/ClusterConfiguration.yaml b/charts/kubeadm/templates/ClusterConfiguration.yaml index c247f9e1..64e7bc93 100644 --- a/charts/kubeadm/templates/ClusterConfiguration.yaml +++ b/charts/kubeadm/templates/ClusterConfiguration.yaml @@ -6,6 +6,8 @@ featureGates: ControlPlaneKubeletLocalMode: true NodeLocalCRISocket: true controlPlaneEndpoint: {{ .Values.api.endpoint }} +proxy: + disabled: true networking: podSubnet: 10.244.0.0/16 etcd: diff --git a/charts/kubeadm/templates/InitConfiguration.yaml b/charts/kubeadm/templates/InitConfiguration.yaml index 5eab7ba2..3511554f 100644 --- a/charts/kubeadm/templates/InitConfiguration.yaml +++ b/charts/kubeadm/templates/InitConfiguration.yaml @@ -7,6 +7,8 @@ localAPIEndpoint: patches: directory: {{ . }} {{- end }} +skipPhases: + - addon/kube-proxy nodeRegistration: criSocket: "unix:///run/containerd/containerd.sock" ignorePreflightErrors: diff --git a/charts/kubeadm/templates/KubeProxyConfiguration.yaml b/charts/kubeadm/templates/KubeProxyConfiguration.yaml deleted file mode 100644 index 61ca7926..00000000 --- a/charts/kubeadm/templates/KubeProxyConfiguration.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: kubeproxy.config.k8s.io/v1alpha1 -kind: KubeProxyConfiguration -# kube-proxy doesnt really support setting dynamic bind-address via config, replaced by cilium long-term anyways -metricsBindAddress: "0.0.0.0:10249" -mode: "iptables" -logging: - format: json -iptables: - localhostNodePorts: false -#nodePortAddresses: primary diff --git a/charts/kubeadm/templates/apiserver/audit-policy.yaml b/charts/kubeadm/templates/apiserver/audit-policy.yaml index 9285f6e5..a77e7c2b 100644 --- a/charts/kubeadm/templates/apiserver/audit-policy.yaml +++ b/charts/kubeadm/templates/apiserver/audit-policy.yaml @@ -3,12 +3,6 @@ kind: Policy rules: # The following requests were manually identified as high-volume and low-risk, # so drop them. - - level: None - users: ["system:kube-proxy"] - verbs: ["watch"] - resources: - - group: "" # core - resources: ["endpoints", "services", "services/status"] - level: None # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port. # TODO(#46983): Change this to the ingress controller service account. @@ -114,7 +108,7 @@ rules: # Get responses can be large; skip them. - level: Request verbs: ["get", "list", "watch"] - resources: + resources: - group: "" # core - group: "admissionregistration.k8s.io" - group: "apiextensions.k8s.io" @@ -137,7 +131,7 @@ rules: - "RequestReceived" # Default level for known APIs - level: RequestResponse - resources: + resources: - group: "" # core - group: "admissionregistration.k8s.io" - group: "apiextensions.k8s.io" diff --git a/charts/kubezero-network/README.md b/charts/kubezero-network/README.md index 776bcb10..7dae8001 100644 --- a/charts/kubezero-network/README.md +++ b/charts/kubezero-network/README.md @@ -1,6 +1,6 @@ # kubezero-network -![Version: 0.5.8](https://img.shields.io/badge/Version-0.5.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.5.9](https://img.shields.io/badge/Version-0.5.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero umbrella chart for all things network @@ -20,7 +20,7 @@ Kubernetes: `>= 1.30.0-0` |------------|------|---------| | https://cdn.zero-downtime.net/charts/ | kubezero-lib | 0.2.1 | | https://haproxytech.github.io/helm-charts | haproxy | 1.24.0 | -| https://helm.cilium.io/ | cilium | 1.17.3 | +| https://helm.cilium.io/ | cilium | 1.17.4 | | https://metallb.github.io/metallb | metallb | 0.14.9 | ## Values @@ -45,6 +45,8 @@ Kubernetes: `>= 1.30.0-0` | cilium.hubble.ui.enabled | bool | `false` | | | cilium.image.useDigest | bool | `false` | | | cilium.ipam.operator.clusterPoolIPv4PodCIDRList[0] | string | `"10.240.0.0/16"` | | +| cilium.k8s.apiServerURLs | string | `""` | | +| cilium.kubeProxyReplacement | bool | `true` | | | cilium.l7Proxy | bool | `false` | | | cilium.operator.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | | | cilium.operator.prometheus.enabled | bool | `false` | | diff --git a/charts/kubezero-network/values.yaml b/charts/kubezero-network/values.yaml index bad4f6c2..b8863cf4 100644 --- a/charts/kubezero-network/values.yaml +++ b/charts/kubezero-network/values.yaml @@ -63,6 +63,17 @@ cilium: enabled: false #rollOutCiliumPods: true + kubeProxyReplacement: true + dnsProxy: + enableTransparentMode: true + + k8sServiceHost: "" + k8sServicePort: 6443 +# k8s: +# # This has to be set to the DNS name of all API servers +# # For example "https://192.168.0.1:6443 https://192.168.0.2:6443" +# apiServerURLs: "" + cgroup: autoMount: enabled: false diff --git a/charts/kubezero/templates/network.yaml b/charts/kubezero/templates/network.yaml index 61dfddf5..60c2ed4b 100644 --- a/charts/kubezero/templates/network.yaml +++ b/charts/kubezero/templates/network.yaml @@ -15,6 +15,9 @@ cilium: # image: # pullPolicy: Never # {{- end }} + k8sServiceHost: {{ .Values.global.apiServerUrl }} +# k8s: +# apiServerURLs: "https://{{ .Values.global.apiServerUrl }}" cluster: name: {{ .Values.global.clusterName }} diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 8b6e3b98..809b820c 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -1,5 +1,6 @@ global: clusterName: zdt-trial-cluster + apiServerUrl: localhost:6443 # platform: aws (kubeadm, default), gke, or nocloud platform: "aws" @@ -32,7 +33,7 @@ addons: network: enabled: true retain: true - targetRevision: 0.5.8 + targetRevision: 0.5.9 cilium: cluster: {}