feat: new network module incl. optional cilium as second CNI support, multus update and fixes

This commit is contained in:
Stefan Reimer 2022-07-14 14:59:06 +00:00
parent 915857dbbc
commit dfbb4d48e6
15 changed files with 188 additions and 111 deletions

View File

@ -319,6 +319,11 @@ elif [[ "$1" =~ "^(bootstrap|restore|join)$" ]]; then
if [[ "$1" =~ "^(bootstrap|join)$" ]]; then
# network
yq eval '.network // ""' ${HOSTFS}/etc/kubernetes/kubezero.yaml > _values.yaml
# Ensure multus is first
helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \
--set multus.enabled=true --kube-version $KUBE_VERSION | kubectl apply -f - $LOG
helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \
-f _values.yaml --kube-version $KUBE_VERSION | kubectl apply --namespace kube-system -f - $LOG

View File

@ -2,7 +2,7 @@ apiVersion: v2
name: kubeadm
description: KubeZero Kubeadm cluster config
type: application
version: 1.22.8
version: 1.23.8
home: https://kubezero.com
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
keywords:

View File

@ -2,13 +2,14 @@ apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: {{ .Chart.Version }}
clusterName: {{ .Values.clusterName }}
featureGates:
UnversionedKubeletConfigMap: true
controlPlaneEndpoint: {{ .Values.api.endpoint }}
networking:
podSubnet: 10.244.0.0/16
etcd:
local:
# As 3.5 is not recommended stick with 3.4.13 till 1.23
imageTag: 3.4.13-0
imageTag: 3.5.4-0
extraArgs:
### DNS discovery
#discovery-srv: {{ .Values.domain }}

View File

@ -2,4 +2,6 @@ apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
# kube-proxy doesnt really support setting dynamic bind-address via config, replaced by cilium long-term anyways
metricsBindAddress: "0.0.0.0:10249"
mode: "ipvs"
# calico < 3.22.1 breaks starting with 1.23, see https://github.com/projectcalico/calico/issues/5011
# we go Cilium anyways
mode: "iptables"

View File

@ -1,6 +1,6 @@
{{- /* Feature gates for all control plane components */ -}}
{{- define "kubeadm.featuregates" }}
{{- $gates := list "CustomCPUCFSQuotaPeriod" "GenericEphemeralVolume" "KubeletCredentialProviders"}}
{{- $gates := list "CustomCPUCFSQuotaPeriod" "KubeletCredentialProviders"}}
{{- if eq .return "csv" }}
{{- range $key := $gates }}
{{- $key }}=true,

View File

@ -95,11 +95,11 @@ spec:
type: RollingUpdate
template:
metadata:
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ""
labels:
k8s-app: aws-iam-authenticator
spec:
priorityClassName: system-cluster-critical
# use service account with access to
serviceAccountName: aws-iam-authenticator

View File

@ -2,7 +2,7 @@ apiVersion: v2
name: kubezero-network
description: KubeZero umbrella chart for all things network
type: application
version: 0.2.1
version: 0.3.0
home: https://kubezero.com
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
keywords:
@ -15,15 +15,17 @@ maintainers:
- name: Stefan Reimer
email: stefan@zero-downtime.net
dependencies:
- name: kubezero-lib
version: ">= 0.1.5"
repository: https://cdn.zero-downtime.net/charts/
- name: cilium
version: 1.11.3
version: 1.11.6
repository: https://helm.cilium.io/
condition: cilium.enabled
- name: metallb
version: 0.10.2
version: 0.13.3
repository: https://metallb.github.io/metallb
condition: metallb.enabled
# Legact / Testing support
- name: calico
version: 0.2.2
condition: calico.enabled

View File

@ -1,4 +1,3 @@
{{- if .Values.multus.enabled }}
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
@ -43,4 +42,3 @@ spec:
config:
description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
type: string
{{- end }}

View File

@ -0,0 +1,27 @@
{{- if .Values.metallb.enabled }}
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
name: l2advertisement1
namespace: kube-system
spec:
ipAddressPools:
{{- range $key, $val := .Values.metallb.ipAddressPools }}
{{- if eq $val.protocol "layer2" }}
- {{ $val.name }}
{{- end }}
{{- end }}
---
{{- range $key, $val := .Values.metallb.ipAddressPools }}
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
name: {{ $val.name }}
namespace: kube-system
spec:
addresses:
{{- $val.addresses | toYaml | nindent 4 }}
{{- end }}
---
{{- end }}

View File

@ -0,0 +1,25 @@
{{- if .Values.calico.enabled }}
apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
name: calico
namespace: kube-system
spec:
config: '{
"type": "calico",
"cniVersion": "0.3.1",
"log_level": "info",
"log_file_path": "/var/log/calico/cni/cni.log",
"datastore_type": "kubernetes",
"mtu": 8941,
"ipam": {
"type": "calico-ipam"
},
"policy": {
"type": "k8s"
},
"kubernetes": {
"kubeconfig": "/etc/cni/net.d/calico-kubeconfig"
}
}'
{{- end }}

View File

@ -0,0 +1,14 @@
{{- if .Values.cilium.enabled }}
apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
name: cilium
namespace: kube-system
spec:
config: '{
"cniVersion": "0.3.1",
"name": "cilium",
"type": "cilium-cni",
"enable-debug": false
}'
{{- end }}

View File

@ -0,0 +1,24 @@
{{- if .Values.multus.enabled }}
kind: ConfigMap
apiVersion: v1
metadata:
name: multus-cni-config
namespace: kube-system
labels:
{{- include "kubezero-lib.labels" . | nindent 4 }}
data:
cni-conf.json: |
{
"name": "multus-cni-network",
"type": "multus",
"cniVersion": "0.3.1",
"capabilities": {
"portMappings": true
},
"kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig",
"clusterNetwork": {{ .Values.multus.clusterNetwork | quote }},
"defaultNetworks": {{ .Values.multus.defaultNetworks | toJson }},
"systemNamespaces": [""],
"readinessindicatorfile": {{ .Values.multus.readinessindicatorfile | quote }}
}
{{- end }}

View File

@ -1,105 +1,21 @@
{{- if .Values.multus.enabled }}
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: multus
rules:
- apiGroups: ["k8s.cni.cncf.io"]
resources:
- '*'
verbs:
- '*'
- apiGroups:
- ""
resources:
- pods
- pods/status
verbs:
- get
- update
- apiGroups:
- ""
- events.k8s.io
resources:
- events
verbs:
- create
- patch
- update
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: multus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: multus
subjects:
- kind: ServiceAccount
name: multus
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: multus
namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
name: multus-cni-config
namespace: kube-system
labels:
tier: node
app: multus
data:
# NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
# In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
# change the "args" line below from
# - "--multus-conf-file=auto"
# to:
# "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
# Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
# /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
cni-conf.json: |
{
"cniVersion": "0.3.1",
"name": "multus-cni-network",
"type": "multus",
"kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig",
"delegates": [
{
"cniVersion": "0.3.1",
"name": "cilium",
"type": "cilium-cni",
"enable-debug": false
}
]
}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-multus-ds
namespace: kube-system
labels:
tier: node
app: multus
name: multus
{{- include "kubezero-lib.labels" . | nindent 4 }}
spec:
selector:
matchLabels:
name: multus
{{- include "kubezero-lib.selectorLabels" . | nindent 6 }}
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
tier: node
app: multus
name: multus
{{- include "kubezero-lib.labels" . | nindent 8 }}
spec:
hostNetwork: true
tolerations:
@ -113,8 +29,8 @@ spec:
image: ghcr.io/k8snetworkplumbingwg/multus-cni:{{ .Values.multus.tag }}
command: ["/entrypoint.sh"]
args:
- "--multus-conf-file=auto"
- "--rename-conf-file=true"
- "--multus-conf-file=/tmp/multus-conf/00-multus.conf"
- "--rename-conf-file=false"
- "--cni-bin-dir=/host/usr/libexec/cni"
- "--cni-version=0.3.1"
resources:
@ -153,5 +69,5 @@ spec:
name: multus-cni-config
items:
- key: cni-conf.json
path: 70-multus.conf
path: 00-multus.conf
{{- end }}

View File

@ -0,0 +1,48 @@
{{- if .Values.multus.enabled }}
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: multus
rules:
- apiGroups: ["k8s.cni.cncf.io"]
resources:
- '*'
verbs:
- '*'
- apiGroups:
- ""
resources:
- pods
- pods/status
verbs:
- get
- update
- apiGroups:
- ""
- events.k8s.io
resources:
- events
verbs:
- create
- patch
- update
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: multus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: multus
subjects:
- kind: ServiceAccount
name: multus
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: multus
namespace: kube-system
{{- end }}

View File

@ -10,23 +10,33 @@ metallb:
nodeSelector:
node-role.kubernetes.io/control-plane: ""
configInline: {}
# address-pools:
# - name: my-ip-space
# protocol: layer2
# addresses:
# - 192.168.42.0/24
ipAddressPools: []
#- name: my-ip-space
# protocol: layer2
# addresses:
# - 192.168.42.0/24
multus:
enabled: false
tag: "v3.8.1"
tag: "v3.9"
clusterNetwork: "calico"
defaultNetworks: []
# - "cilium"
readinessindicatorfile: "/etc/cni/net.d/10-calico.conflist"
cilium:
enabled: false
cni:
binPath: "/usr/libexec/cni"
#-- Ensure this is false if multus is enabled
exclusive: true
exclusive: false
cgroup:
autoMount:
enabled: false
hostRoot: "/sys/fs/cgroup"
tunnel: geneve
@ -36,6 +46,11 @@ cilium:
operator:
replicas: 1
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
nodeSelector:
node-role.kubernetes.io/control-plane: ""
hubble:
enabled: false