diff --git a/admin/kubezero.sh b/admin/kubezero.sh index 31359fe..b43d78a 100755 --- a/admin/kubezero.sh +++ b/admin/kubezero.sh @@ -319,6 +319,11 @@ elif [[ "$1" =~ "^(bootstrap|restore|join)$" ]]; then if [[ "$1" =~ "^(bootstrap|join)$" ]]; then # network yq eval '.network // ""' ${HOSTFS}/etc/kubernetes/kubezero.yaml > _values.yaml + + # Ensure multus is first + helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \ + --set multus.enabled=true --kube-version $KUBE_VERSION | kubectl apply -f - $LOG + helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \ -f _values.yaml --kube-version $KUBE_VERSION | kubectl apply --namespace kube-system -f - $LOG diff --git a/charts/kubeadm/Chart.yaml b/charts/kubeadm/Chart.yaml index 087112d..934be1d 100644 --- a/charts/kubeadm/Chart.yaml +++ b/charts/kubeadm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubeadm description: KubeZero Kubeadm cluster config type: application -version: 1.22.8 +version: 1.23.8 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubeadm/templates/ClusterConfiguration.yaml b/charts/kubeadm/templates/ClusterConfiguration.yaml index f64e200..7a73e22 100644 --- a/charts/kubeadm/templates/ClusterConfiguration.yaml +++ b/charts/kubeadm/templates/ClusterConfiguration.yaml @@ -2,13 +2,14 @@ apiVersion: kubeadm.k8s.io/v1beta3 kind: ClusterConfiguration kubernetesVersion: {{ .Chart.Version }} clusterName: {{ .Values.clusterName }} +featureGates: + UnversionedKubeletConfigMap: true controlPlaneEndpoint: {{ .Values.api.endpoint }} networking: podSubnet: 10.244.0.0/16 etcd: local: - # As 3.5 is not recommended stick with 3.4.13 till 1.23 - imageTag: 3.4.13-0 + imageTag: 3.5.4-0 extraArgs: ### DNS discovery #discovery-srv: {{ .Values.domain }} diff --git a/charts/kubeadm/templates/KubeProxyConfiguration.yaml b/charts/kubeadm/templates/KubeProxyConfiguration.yaml index fa741b0..26da352 100644 --- a/charts/kubeadm/templates/KubeProxyConfiguration.yaml +++ b/charts/kubeadm/templates/KubeProxyConfiguration.yaml @@ -2,4 +2,6 @@ apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration # kube-proxy doesnt really support setting dynamic bind-address via config, replaced by cilium long-term anyways metricsBindAddress: "0.0.0.0:10249" -mode: "ipvs" +# calico < 3.22.1 breaks starting with 1.23, see https://github.com/projectcalico/calico/issues/5011 +# we go Cilium anyways +mode: "iptables" diff --git a/charts/kubeadm/templates/_helpers.tpl b/charts/kubeadm/templates/_helpers.tpl index 3b8976b..1d193f4 100644 --- a/charts/kubeadm/templates/_helpers.tpl +++ b/charts/kubeadm/templates/_helpers.tpl @@ -1,6 +1,6 @@ {{- /* Feature gates for all control plane components */ -}} {{- define "kubeadm.featuregates" }} -{{- $gates := list "CustomCPUCFSQuotaPeriod" "GenericEphemeralVolume" "KubeletCredentialProviders"}} +{{- $gates := list "CustomCPUCFSQuotaPeriod" "KubeletCredentialProviders"}} {{- if eq .return "csv" }} {{- range $key := $gates }} {{- $key }}=true, diff --git a/charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml b/charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml index d14a109..607bfb6 100644 --- a/charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml +++ b/charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml @@ -95,11 +95,11 @@ spec: type: RollingUpdate template: metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" labels: k8s-app: aws-iam-authenticator spec: + priorityClassName: system-cluster-critical + # use service account with access to serviceAccountName: aws-iam-authenticator diff --git a/charts/kubezero-network/Chart.yaml b/charts/kubezero-network/Chart.yaml index 613784d..5ae0c26 100644 --- a/charts/kubezero-network/Chart.yaml +++ b/charts/kubezero-network/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-network description: KubeZero umbrella chart for all things network type: application -version: 0.2.1 +version: 0.3.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -15,15 +15,17 @@ maintainers: - name: Stefan Reimer email: stefan@zero-downtime.net dependencies: + - name: kubezero-lib + version: ">= 0.1.5" + repository: https://cdn.zero-downtime.net/charts/ - name: cilium - version: 1.11.3 + version: 1.11.6 repository: https://helm.cilium.io/ condition: cilium.enabled - name: metallb - version: 0.10.2 + version: 0.13.3 repository: https://metallb.github.io/metallb condition: metallb.enabled - # Legact / Testing support - name: calico version: 0.2.2 condition: calico.enabled diff --git a/charts/kubezero-network/templates/multus/crds.yaml b/charts/kubezero-network/crds/multus.yaml similarity index 97% rename from charts/kubezero-network/templates/multus/crds.yaml rename to charts/kubezero-network/crds/multus.yaml index ff4bcf1..8aa15e6 100644 --- a/charts/kubezero-network/templates/multus/crds.yaml +++ b/charts/kubezero-network/crds/multus.yaml @@ -1,4 +1,3 @@ -{{- if .Values.multus.enabled }} apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -43,4 +42,3 @@ spec: config: description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration' type: string -{{- end }} diff --git a/charts/kubezero-network/templates/metallb/config.yaml b/charts/kubezero-network/templates/metallb/config.yaml new file mode 100644 index 0000000..bdadb1f --- /dev/null +++ b/charts/kubezero-network/templates/metallb/config.yaml @@ -0,0 +1,27 @@ +{{- if .Values.metallb.enabled }} +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: l2advertisement1 + namespace: kube-system +spec: + ipAddressPools: + {{- range $key, $val := .Values.metallb.ipAddressPools }} + {{- if eq $val.protocol "layer2" }} + - {{ $val.name }} + {{- end }} + {{- end }} +--- + +{{- range $key, $val := .Values.metallb.ipAddressPools }} +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: {{ $val.name }} + namespace: kube-system +spec: + addresses: + {{- $val.addresses | toYaml | nindent 4 }} +{{- end }} +--- +{{- end }} diff --git a/charts/kubezero-network/templates/multus/calico-network.yaml b/charts/kubezero-network/templates/multus/calico-network.yaml new file mode 100644 index 0000000..a0c41b4 --- /dev/null +++ b/charts/kubezero-network/templates/multus/calico-network.yaml @@ -0,0 +1,25 @@ +{{- if .Values.calico.enabled }} +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: calico + namespace: kube-system +spec: + config: '{ + "type": "calico", + "cniVersion": "0.3.1", + "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", + "datastore_type": "kubernetes", + "mtu": 8941, + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "/etc/cni/net.d/calico-kubeconfig" + } + }' +{{- end }} diff --git a/charts/kubezero-network/templates/multus/cilium-network.yaml b/charts/kubezero-network/templates/multus/cilium-network.yaml new file mode 100644 index 0000000..4f05e67 --- /dev/null +++ b/charts/kubezero-network/templates/multus/cilium-network.yaml @@ -0,0 +1,14 @@ +{{- if .Values.cilium.enabled }} +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: cilium + namespace: kube-system +spec: + config: '{ + "cniVersion": "0.3.1", + "name": "cilium", + "type": "cilium-cni", + "enable-debug": false + }' +{{- end }} diff --git a/charts/kubezero-network/templates/multus/config.yaml b/charts/kubezero-network/templates/multus/config.yaml new file mode 100644 index 0000000..2bc4851 --- /dev/null +++ b/charts/kubezero-network/templates/multus/config.yaml @@ -0,0 +1,24 @@ +{{- if .Values.multus.enabled }} +kind: ConfigMap +apiVersion: v1 +metadata: + name: multus-cni-config + namespace: kube-system + labels: + {{- include "kubezero-lib.labels" . | nindent 4 }} +data: + cni-conf.json: | + { + "name": "multus-cni-network", + "type": "multus", + "cniVersion": "0.3.1", + "capabilities": { + "portMappings": true + }, + "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig", + "clusterNetwork": {{ .Values.multus.clusterNetwork | quote }}, + "defaultNetworks": {{ .Values.multus.defaultNetworks | toJson }}, + "systemNamespaces": [""], + "readinessindicatorfile": {{ .Values.multus.readinessindicatorfile | quote }} + } +{{- end }} diff --git a/charts/kubezero-network/templates/multus/daemonset.yaml b/charts/kubezero-network/templates/multus/daemonset.yaml index 5fc6269..c18b92f 100644 --- a/charts/kubezero-network/templates/multus/daemonset.yaml +++ b/charts/kubezero-network/templates/multus/daemonset.yaml @@ -1,105 +1,21 @@ {{- if .Values.multus.enabled }} -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: multus -rules: - - apiGroups: ["k8s.cni.cncf.io"] - resources: - - '*' - verbs: - - '*' - - apiGroups: - - "" - resources: - - pods - - pods/status - verbs: - - get - - update - - apiGroups: - - "" - - events.k8s.io - resources: - - events - verbs: - - create - - patch - - update ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: multus -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: multus -subjects: -- kind: ServiceAccount - name: multus - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: multus - namespace: kube-system ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: multus-cni-config - namespace: kube-system - labels: - tier: node - app: multus -data: - # NOTE: If you'd prefer to manually apply a configuration file, you may create one here. - # In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod - # change the "args" line below from - # - "--multus-conf-file=auto" - # to: - # "--multus-conf-file=/tmp/multus-conf/70-multus.conf" - # Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the - # /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet. - cni-conf.json: | - { - "cniVersion": "0.3.1", - "name": "multus-cni-network", - "type": "multus", - "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig", - "delegates": [ - { - "cniVersion": "0.3.1", - "name": "cilium", - "type": "cilium-cni", - "enable-debug": false - } - ] - } ---- apiVersion: apps/v1 kind: DaemonSet metadata: name: kube-multus-ds namespace: kube-system labels: - tier: node - app: multus - name: multus + {{- include "kubezero-lib.labels" . | nindent 4 }} spec: selector: matchLabels: - name: multus + {{- include "kubezero-lib.selectorLabels" . | nindent 6 }} updateStrategy: type: RollingUpdate template: metadata: labels: - tier: node - app: multus - name: multus + {{- include "kubezero-lib.labels" . | nindent 8 }} spec: hostNetwork: true tolerations: @@ -113,8 +29,8 @@ spec: image: ghcr.io/k8snetworkplumbingwg/multus-cni:{{ .Values.multus.tag }} command: ["/entrypoint.sh"] args: - - "--multus-conf-file=auto" - - "--rename-conf-file=true" + - "--multus-conf-file=/tmp/multus-conf/00-multus.conf" + - "--rename-conf-file=false" - "--cni-bin-dir=/host/usr/libexec/cni" - "--cni-version=0.3.1" resources: @@ -153,5 +69,5 @@ spec: name: multus-cni-config items: - key: cni-conf.json - path: 70-multus.conf + path: 00-multus.conf {{- end }} diff --git a/charts/kubezero-network/templates/multus/rbac.yaml b/charts/kubezero-network/templates/multus/rbac.yaml new file mode 100644 index 0000000..1f07d26 --- /dev/null +++ b/charts/kubezero-network/templates/multus/rbac.yaml @@ -0,0 +1,48 @@ +{{- if .Values.multus.enabled }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: multus +rules: + - apiGroups: ["k8s.cni.cncf.io"] + resources: + - '*' + verbs: + - '*' + - apiGroups: + - "" + resources: + - pods + - pods/status + verbs: + - get + - update + - apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - patch + - update +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: multus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: multus +subjects: +- kind: ServiceAccount + name: multus + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: multus + namespace: kube-system +{{- end }} diff --git a/charts/kubezero-network/values.yaml b/charts/kubezero-network/values.yaml index d54acaa..6ba7b1c 100644 --- a/charts/kubezero-network/values.yaml +++ b/charts/kubezero-network/values.yaml @@ -10,23 +10,33 @@ metallb: nodeSelector: node-role.kubernetes.io/control-plane: "" - configInline: {} -# address-pools: -# - name: my-ip-space -# protocol: layer2 -# addresses: -# - 192.168.42.0/24 + ipAddressPools: [] + #- name: my-ip-space + # protocol: layer2 + # addresses: + # - 192.168.42.0/24 multus: enabled: false - tag: "v3.8.1" + tag: "v3.9" + + clusterNetwork: "calico" + defaultNetworks: [] + # - "cilium" + readinessindicatorfile: "/etc/cni/net.d/10-calico.conflist" cilium: enabled: false cni: + binPath: "/usr/libexec/cni" #-- Ensure this is false if multus is enabled - exclusive: true + exclusive: false + + cgroup: + autoMount: + enabled: false + hostRoot: "/sys/fs/cgroup" tunnel: geneve @@ -36,6 +46,11 @@ cilium: operator: replicas: 1 + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + nodeSelector: + node-role.kubernetes.io/control-plane: "" hubble: enabled: false