feat: new network module incl. optional cilium as second CNI support, multus update and fixes

admin/kubezero.sh

@@ -319,6 +319,11 @@ elif [[ "$1" =~ "^(bootstrap|restore|join)$" ]]; then
   if [[ "$1" =~ "^(bootstrap|join)$" ]]; then
     # network
     yq eval '.network // ""' ${HOSTFS}/etc/kubernetes/kubezero.yaml > _values.yaml
+
+    # Ensure multus is first
+    helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \
+      --set multus.enabled=true --kube-version $KUBE_VERSION | kubectl apply -f - $LOG
+
     helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \
       -f _values.yaml --kube-version $KUBE_VERSION | kubectl apply --namespace kube-system -f - $LOG
 

charts/kubeadm/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubeadm
 description: KubeZero Kubeadm cluster config
 type: application
-version: 1.22.8
+version: 1.23.8
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:

charts/kubeadm/templates/ClusterConfiguration.yaml

@@ -2,13 +2,14 @@ apiVersion: kubeadm.k8s.io/v1beta3
 kind: ClusterConfiguration
 kubernetesVersion: {{ .Chart.Version }}
 clusterName: {{ .Values.clusterName }}
+featureGates:
+  UnversionedKubeletConfigMap: true
 controlPlaneEndpoint: {{ .Values.api.endpoint }}
 networking:
   podSubnet: 10.244.0.0/16
 etcd:
   local:
-    # As 3.5 is not recommended stick with 3.4.13 till 1.23
-    imageTag: 3.4.13-0
+    imageTag: 3.5.4-0
     extraArgs:
       ### DNS discovery
       #discovery-srv: {{ .Values.domain }}

charts/kubeadm/templates/KubeProxyConfiguration.yaml

@@ -2,4 +2,6 @@ apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
 # kube-proxy doesnt really support setting dynamic bind-address via config, replaced by cilium long-term anyways
 metricsBindAddress: "0.0.0.0:10249"
-mode: "ipvs"
+# calico < 3.22.1 breaks starting with 1.23, see https://github.com/projectcalico/calico/issues/5011
+# we go Cilium anyways
+mode: "iptables"

charts/kubeadm/templates/_helpers.tpl

@@ -1,6 +1,6 @@
 {{- /* Feature gates for all control plane components */ -}}
 {{- define "kubeadm.featuregates" }}
-{{- $gates := list "CustomCPUCFSQuotaPeriod" "GenericEphemeralVolume" "KubeletCredentialProviders"}}
+{{- $gates := list "CustomCPUCFSQuotaPeriod" "KubeletCredentialProviders"}}
 {{- if eq .return "csv" }}
 {{- range $key := $gates }}
 {{- $key }}=true,

charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml

@@ -95,11 +95,11 @@ spec:
     type: RollingUpdate
   template:
     metadata:
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ""
       labels:
         k8s-app: aws-iam-authenticator
     spec:
+      priorityClassName: system-cluster-critical
+
       # use service account with access to
       serviceAccountName: aws-iam-authenticator
 

charts/kubezero-network/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-network
 description: KubeZero umbrella chart for all things network
 type: application
-version: 0.2.1
+version: 0.3.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -15,15 +15,17 @@ maintainers:
   - name: Stefan Reimer
     email: stefan@zero-downtime.net
 dependencies:
+  - name: kubezero-lib
+    version: ">= 0.1.5"
+    repository: https://cdn.zero-downtime.net/charts/
   - name: cilium
-    version: 1.11.3
+    version: 1.11.6
     repository: https://helm.cilium.io/
     condition: cilium.enabled
   - name: metallb
-    version: 0.10.2
+    version: 0.13.3
     repository: https://metallb.github.io/metallb
     condition: metallb.enabled
-  # Legact / Testing support
   - name: calico
     version: 0.2.2
     condition: calico.enabled

charts/kubezero-network/crds/multus.yaml

@@ -1,4 +1,3 @@
-{{- if .Values.multus.enabled }}
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
@@ -43,4 +42,3 @@ spec:
                 config:
                   description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
                   type: string
-{{- end }}

charts/kubezero-network/templates/metallb/config.yaml

@@ -0,0 +1,27 @@
+{{- if .Values.metallb.enabled }}
+apiVersion: metallb.io/v1beta1
+kind: L2Advertisement
+metadata:
+  name: l2advertisement1
+  namespace: kube-system
+spec:
+  ipAddressPools:
+    {{- range $key, $val := .Values.metallb.ipAddressPools }}
+    {{- if eq $val.protocol "layer2" }}
+    - {{ $val.name }}
+    {{- end }}
+    {{- end }}
+---
+
+{{- range $key, $val := .Values.metallb.ipAddressPools }}
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: {{ $val.name }}
+  namespace: kube-system
+spec:
+  addresses:
+  {{- $val.addresses | toYaml | nindent 4 }}
+{{- end }}
+---
+{{- end }}

charts/kubezero-network/templates/multus/calico-network.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.calico.enabled }}
+apiVersion: k8s.cni.cncf.io/v1
+kind: NetworkAttachmentDefinition
+metadata:
+  name: calico
+  namespace: kube-system
+spec:
+  config: '{
+      "type": "calico",
+      "cniVersion": "0.3.1",
+      "log_level": "info",
+      "log_file_path": "/var/log/calico/cni/cni.log",
+      "datastore_type": "kubernetes",
+      "mtu": 8941,
+      "ipam": {
+          "type": "calico-ipam"
+      },
+      "policy": {
+          "type": "k8s"
+      },
+      "kubernetes": {
+          "kubeconfig": "/etc/cni/net.d/calico-kubeconfig"
+      }
+    }'
+{{- end }}

charts/kubezero-network/templates/multus/cilium-network.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.cilium.enabled }}
+apiVersion: k8s.cni.cncf.io/v1
+kind: NetworkAttachmentDefinition
+metadata:
+  name: cilium
+  namespace: kube-system
+spec:
+  config: '{
+    "cniVersion": "0.3.1",
+    "name": "cilium",
+    "type": "cilium-cni",
+    "enable-debug": false
+  }'
+{{- end }}

charts/kubezero-network/templates/multus/config.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.multus.enabled }}
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: multus-cni-config
+  namespace: kube-system
+  labels:
+    {{- include "kubezero-lib.labels" . | nindent 4 }}
+data:
+  cni-conf.json: |
+    {
+      "name": "multus-cni-network",
+      "type": "multus",
+      "cniVersion": "0.3.1",
+      "capabilities": {
+        "portMappings": true
+      },
+      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig",
+      "clusterNetwork": {{ .Values.multus.clusterNetwork | quote }},
+      "defaultNetworks": {{ .Values.multus.defaultNetworks | toJson }},
+      "systemNamespaces": [""],
+      "readinessindicatorfile": {{ .Values.multus.readinessindicatorfile | quote }}
+    }
+{{- end }}

charts/kubezero-network/templates/multus/daemonset.yaml

@@ -1,105 +1,21 @@
 {{- if .Values.multus.enabled }}
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus
-rules:
-  - apiGroups: ["k8s.cni.cncf.io"]
-    resources:
-      - '*'
-    verbs:
-      - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-      - pods/status
-    verbs:
-      - get
-      - update
-  - apiGroups:
-      - ""
-      - events.k8s.io
-    resources:
-      - events
-    verbs:
-      - create
-      - patch
-      - update
----
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
-  name: multus
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: multus
-subjects:
-- kind: ServiceAccount
-  name: multus
-  namespace: kube-system
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: multus
-  namespace: kube-system
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: multus-cni-config
-  namespace: kube-system
-  labels:
-    tier: node
-    app: multus
-data:
-  # NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
-  # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
-  # change the "args" line below from
-  # - "--multus-conf-file=auto"
-  # to:
-  # "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
-  # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
-  # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
-  cni-conf.json: |
-    {
-      "cniVersion": "0.3.1",
-      "name": "multus-cni-network",
-      "type": "multus",
-      "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig",
-      "delegates": [
-        {
-          "cniVersion": "0.3.1",
-          "name": "cilium",
-          "type": "cilium-cni",
-          "enable-debug": false
-        }
-      ]
-    }
----
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: kube-multus-ds
   namespace: kube-system
   labels:
-    tier: node
-    app: multus
-    name: multus
+    {{- include "kubezero-lib.labels" . | nindent 4 }}
 spec:
   selector:
     matchLabels:
-      name: multus
+      {{- include "kubezero-lib.selectorLabels" . | nindent 6 }}
   updateStrategy:
     type: RollingUpdate
   template:
     metadata:
       labels:
-        tier: node
-        app: multus
-        name: multus
+        {{- include "kubezero-lib.labels" . | nindent 8 }}
     spec:
       hostNetwork: true
       tolerations:
@@ -113,8 +29,8 @@ spec:
         image: ghcr.io/k8snetworkplumbingwg/multus-cni:{{ .Values.multus.tag }}
         command: ["/entrypoint.sh"]
         args:
-        - "--multus-conf-file=auto"
-        - "--rename-conf-file=true"
+        - "--multus-conf-file=/tmp/multus-conf/00-multus.conf"
+        - "--rename-conf-file=false"
         - "--cni-bin-dir=/host/usr/libexec/cni"
         - "--cni-version=0.3.1"
         resources:
@@ -153,5 +69,5 @@ spec:
             name: multus-cni-config
             items:
             - key: cni-conf.json
-              path: 70-multus.conf
+              path: 00-multus.conf
 {{- end }}

charts/kubezero-network/templates/multus/rbac.yaml

@@ -0,0 +1,48 @@
+{{- if .Values.multus.enabled }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - update
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: kube-system
+{{- end }}

charts/kubezero-network/values.yaml

@@ -10,8 +10,7 @@ metallb:
     nodeSelector:
       node-role.kubernetes.io/control-plane: ""
 
-  configInline: {}
-#   address-pools:
+  ipAddressPools: []
   #- name: my-ip-space
   #  protocol: layer2
   #  addresses:
@@ -19,14 +18,25 @@ metallb:
 
 multus:
   enabled: false
-  tag: "v3.8.1"
+  tag: "v3.9"
+
+  clusterNetwork: "calico"
+  defaultNetworks: []
+   # - "cilium"
+  readinessindicatorfile: "/etc/cni/net.d/10-calico.conflist"
 
 cilium:
   enabled: false
 
   cni:
+    binPath: "/usr/libexec/cni"
     #-- Ensure this is false if multus is enabled
-    exclusive: true
+    exclusive: false
+
+  cgroup:
+    autoMount:
+      enabled: false
+    hostRoot: "/sys/fs/cgroup"
 
   tunnel: geneve
 
@@ -36,6 +46,11 @@ cilium:
 
   operator:
     replicas: 1
+    tolerations:
+    - key: node-role.kubernetes.io/master
+      effect: NoSchedule
+    nodeSelector:
+      node-role.kubernetes.io/control-plane: ""
 
   hubble:
     enabled: false