feat: new network module incl. optional cilium as second CNI support, multus update and fixes

This commit is contained in:
Stefan Reimer 2022-07-14 14:59:06 +00:00
parent 23d87073bb
commit 9fe008efcb
15 changed files with 188 additions and 111 deletions

View File

@ -319,6 +319,11 @@ elif [[ "$1" =~ "^(bootstrap|restore|join)$" ]]; then
if [[ "$1" =~ "^(bootstrap|join)$" ]]; then if [[ "$1" =~ "^(bootstrap|join)$" ]]; then
# network # network
yq eval '.network // ""' ${HOSTFS}/etc/kubernetes/kubezero.yaml > _values.yaml yq eval '.network // ""' ${HOSTFS}/etc/kubernetes/kubezero.yaml > _values.yaml
# Ensure multus is first
helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \
--set multus.enabled=true --kube-version $KUBE_VERSION | kubectl apply -f - $LOG
helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \ helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \
-f _values.yaml --kube-version $KUBE_VERSION | kubectl apply --namespace kube-system -f - $LOG -f _values.yaml --kube-version $KUBE_VERSION | kubectl apply --namespace kube-system -f - $LOG

View File

@ -2,7 +2,7 @@ apiVersion: v2
name: kubeadm name: kubeadm
description: KubeZero Kubeadm cluster config description: KubeZero Kubeadm cluster config
type: application type: application
version: 1.22.8 version: 1.23.8
home: https://kubezero.com home: https://kubezero.com
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
keywords: keywords:

View File

@ -2,13 +2,14 @@ apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration kind: ClusterConfiguration
kubernetesVersion: {{ .Chart.Version }} kubernetesVersion: {{ .Chart.Version }}
clusterName: {{ .Values.clusterName }} clusterName: {{ .Values.clusterName }}
featureGates:
UnversionedKubeletConfigMap: true
controlPlaneEndpoint: {{ .Values.api.endpoint }} controlPlaneEndpoint: {{ .Values.api.endpoint }}
networking: networking:
podSubnet: 10.244.0.0/16 podSubnet: 10.244.0.0/16
etcd: etcd:
local: local:
# As 3.5 is not recommended stick with 3.4.13 till 1.23 imageTag: 3.5.4-0
imageTag: 3.4.13-0
extraArgs: extraArgs:
### DNS discovery ### DNS discovery
#discovery-srv: {{ .Values.domain }} #discovery-srv: {{ .Values.domain }}

View File

@ -2,4 +2,6 @@ apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration kind: KubeProxyConfiguration
# kube-proxy doesnt really support setting dynamic bind-address via config, replaced by cilium long-term anyways # kube-proxy doesnt really support setting dynamic bind-address via config, replaced by cilium long-term anyways
metricsBindAddress: "0.0.0.0:10249" metricsBindAddress: "0.0.0.0:10249"
mode: "ipvs" # calico < 3.22.1 breaks starting with 1.23, see https://github.com/projectcalico/calico/issues/5011
# we go Cilium anyways
mode: "iptables"

View File

@ -1,6 +1,6 @@
{{- /* Feature gates for all control plane components */ -}} {{- /* Feature gates for all control plane components */ -}}
{{- define "kubeadm.featuregates" }} {{- define "kubeadm.featuregates" }}
{{- $gates := list "CustomCPUCFSQuotaPeriod" "GenericEphemeralVolume" "KubeletCredentialProviders"}} {{- $gates := list "CustomCPUCFSQuotaPeriod" "KubeletCredentialProviders"}}
{{- if eq .return "csv" }} {{- if eq .return "csv" }}
{{- range $key := $gates }} {{- range $key := $gates }}
{{- $key }}=true, {{- $key }}=true,

View File

@ -95,11 +95,11 @@ spec:
type: RollingUpdate type: RollingUpdate
template: template:
metadata: metadata:
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ""
labels: labels:
k8s-app: aws-iam-authenticator k8s-app: aws-iam-authenticator
spec: spec:
priorityClassName: system-cluster-critical
# use service account with access to # use service account with access to
serviceAccountName: aws-iam-authenticator serviceAccountName: aws-iam-authenticator

View File

@ -2,7 +2,7 @@ apiVersion: v2
name: kubezero-network name: kubezero-network
description: KubeZero umbrella chart for all things network description: KubeZero umbrella chart for all things network
type: application type: application
version: 0.2.1 version: 0.3.0
home: https://kubezero.com home: https://kubezero.com
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
keywords: keywords:
@ -15,15 +15,17 @@ maintainers:
- name: Stefan Reimer - name: Stefan Reimer
email: stefan@zero-downtime.net email: stefan@zero-downtime.net
dependencies: dependencies:
- name: kubezero-lib
version: ">= 0.1.5"
repository: https://cdn.zero-downtime.net/charts/
- name: cilium - name: cilium
version: 1.11.3 version: 1.11.6
repository: https://helm.cilium.io/ repository: https://helm.cilium.io/
condition: cilium.enabled condition: cilium.enabled
- name: metallb - name: metallb
version: 0.10.2 version: 0.13.3
repository: https://metallb.github.io/metallb repository: https://metallb.github.io/metallb
condition: metallb.enabled condition: metallb.enabled
# Legact / Testing support
- name: calico - name: calico
version: 0.2.2 version: 0.2.2
condition: calico.enabled condition: calico.enabled

View File

@ -1,4 +1,3 @@
{{- if .Values.multus.enabled }}
apiVersion: apiextensions.k8s.io/v1 apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition kind: CustomResourceDefinition
metadata: metadata:
@ -43,4 +42,3 @@ spec:
config: config:
description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration' description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
type: string type: string
{{- end }}

View File

@ -0,0 +1,27 @@
{{- if .Values.metallb.enabled }}
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
name: l2advertisement1
namespace: kube-system
spec:
ipAddressPools:
{{- range $key, $val := .Values.metallb.ipAddressPools }}
{{- if eq $val.protocol "layer2" }}
- {{ $val.name }}
{{- end }}
{{- end }}
---
{{- range $key, $val := .Values.metallb.ipAddressPools }}
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
name: {{ $val.name }}
namespace: kube-system
spec:
addresses:
{{- $val.addresses | toYaml | nindent 4 }}
{{- end }}
---
{{- end }}

View File

@ -0,0 +1,25 @@
{{- if .Values.calico.enabled }}
apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
name: calico
namespace: kube-system
spec:
config: '{
"type": "calico",
"cniVersion": "0.3.1",
"log_level": "info",
"log_file_path": "/var/log/calico/cni/cni.log",
"datastore_type": "kubernetes",
"mtu": 8941,
"ipam": {
"type": "calico-ipam"
},
"policy": {
"type": "k8s"
},
"kubernetes": {
"kubeconfig": "/etc/cni/net.d/calico-kubeconfig"
}
}'
{{- end }}

View File

@ -0,0 +1,14 @@
{{- if .Values.cilium.enabled }}
apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
name: cilium
namespace: kube-system
spec:
config: '{
"cniVersion": "0.3.1",
"name": "cilium",
"type": "cilium-cni",
"enable-debug": false
}'
{{- end }}

View File

@ -0,0 +1,24 @@
{{- if .Values.multus.enabled }}
kind: ConfigMap
apiVersion: v1
metadata:
name: multus-cni-config
namespace: kube-system
labels:
{{- include "kubezero-lib.labels" . | nindent 4 }}
data:
cni-conf.json: |
{
"name": "multus-cni-network",
"type": "multus",
"cniVersion": "0.3.1",
"capabilities": {
"portMappings": true
},
"kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig",
"clusterNetwork": {{ .Values.multus.clusterNetwork | quote }},
"defaultNetworks": {{ .Values.multus.defaultNetworks | toJson }},
"systemNamespaces": [""],
"readinessindicatorfile": {{ .Values.multus.readinessindicatorfile | quote }}
}
{{- end }}

View File

@ -1,105 +1,21 @@
{{- if .Values.multus.enabled }} {{- if .Values.multus.enabled }}
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: multus
rules:
- apiGroups: ["k8s.cni.cncf.io"]
resources:
- '*'
verbs:
- '*'
- apiGroups:
- ""
resources:
- pods
- pods/status
verbs:
- get
- update
- apiGroups:
- ""
- events.k8s.io
resources:
- events
verbs:
- create
- patch
- update
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: multus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: multus
subjects:
- kind: ServiceAccount
name: multus
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: multus
namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
name: multus-cni-config
namespace: kube-system
labels:
tier: node
app: multus
data:
# NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
# In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
# change the "args" line below from
# - "--multus-conf-file=auto"
# to:
# "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
# Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
# /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
cni-conf.json: |
{
"cniVersion": "0.3.1",
"name": "multus-cni-network",
"type": "multus",
"kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig",
"delegates": [
{
"cniVersion": "0.3.1",
"name": "cilium",
"type": "cilium-cni",
"enable-debug": false
}
]
}
---
apiVersion: apps/v1 apiVersion: apps/v1
kind: DaemonSet kind: DaemonSet
metadata: metadata:
name: kube-multus-ds name: kube-multus-ds
namespace: kube-system namespace: kube-system
labels: labels:
tier: node {{- include "kubezero-lib.labels" . | nindent 4 }}
app: multus
name: multus
spec: spec:
selector: selector:
matchLabels: matchLabels:
name: multus {{- include "kubezero-lib.selectorLabels" . | nindent 6 }}
updateStrategy: updateStrategy:
type: RollingUpdate type: RollingUpdate
template: template:
metadata: metadata:
labels: labels:
tier: node {{- include "kubezero-lib.labels" . | nindent 8 }}
app: multus
name: multus
spec: spec:
hostNetwork: true hostNetwork: true
tolerations: tolerations:
@ -113,8 +29,8 @@ spec:
image: ghcr.io/k8snetworkplumbingwg/multus-cni:{{ .Values.multus.tag }} image: ghcr.io/k8snetworkplumbingwg/multus-cni:{{ .Values.multus.tag }}
command: ["/entrypoint.sh"] command: ["/entrypoint.sh"]
args: args:
- "--multus-conf-file=auto" - "--multus-conf-file=/tmp/multus-conf/00-multus.conf"
- "--rename-conf-file=true" - "--rename-conf-file=false"
- "--cni-bin-dir=/host/usr/libexec/cni" - "--cni-bin-dir=/host/usr/libexec/cni"
- "--cni-version=0.3.1" - "--cni-version=0.3.1"
resources: resources:
@ -153,5 +69,5 @@ spec:
name: multus-cni-config name: multus-cni-config
items: items:
- key: cni-conf.json - key: cni-conf.json
path: 70-multus.conf path: 00-multus.conf
{{- end }} {{- end }}

View File

@ -0,0 +1,48 @@
{{- if .Values.multus.enabled }}
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: multus
rules:
- apiGroups: ["k8s.cni.cncf.io"]
resources:
- '*'
verbs:
- '*'
- apiGroups:
- ""
resources:
- pods
- pods/status
verbs:
- get
- update
- apiGroups:
- ""
- events.k8s.io
resources:
- events
verbs:
- create
- patch
- update
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: multus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: multus
subjects:
- kind: ServiceAccount
name: multus
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: multus
namespace: kube-system
{{- end }}

View File

@ -10,8 +10,7 @@ metallb:
nodeSelector: nodeSelector:
node-role.kubernetes.io/control-plane: "" node-role.kubernetes.io/control-plane: ""
configInline: {} ipAddressPools: []
# address-pools:
#- name: my-ip-space #- name: my-ip-space
# protocol: layer2 # protocol: layer2
# addresses: # addresses:
@ -19,14 +18,25 @@ metallb:
multus: multus:
enabled: false enabled: false
tag: "v3.8.1" tag: "v3.9"
clusterNetwork: "calico"
defaultNetworks: []
# - "cilium"
readinessindicatorfile: "/etc/cni/net.d/10-calico.conflist"
cilium: cilium:
enabled: false enabled: false
cni: cni:
binPath: "/usr/libexec/cni"
#-- Ensure this is false if multus is enabled #-- Ensure this is false if multus is enabled
exclusive: true exclusive: false
cgroup:
autoMount:
enabled: false
hostRoot: "/sys/fs/cgroup"
tunnel: geneve tunnel: geneve
@ -36,6 +46,11 @@ cilium:
operator: operator:
replicas: 1 replicas: 1
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
nodeSelector:
node-role.kubernetes.io/control-plane: ""
hubble: hubble:
enabled: false enabled: false