feat: new network module incl. optional cilium as second CNI support, multus update and fixes
This commit is contained in:
parent
23d87073bb
commit
9fe008efcb
@ -319,6 +319,11 @@ elif [[ "$1" =~ "^(bootstrap|restore|join)$" ]]; then
|
|||||||
if [[ "$1" =~ "^(bootstrap|join)$" ]]; then
|
if [[ "$1" =~ "^(bootstrap|join)$" ]]; then
|
||||||
# network
|
# network
|
||||||
yq eval '.network // ""' ${HOSTFS}/etc/kubernetes/kubezero.yaml > _values.yaml
|
yq eval '.network // ""' ${HOSTFS}/etc/kubernetes/kubezero.yaml > _values.yaml
|
||||||
|
|
||||||
|
# Ensure multus is first
|
||||||
|
helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \
|
||||||
|
--set multus.enabled=true --kube-version $KUBE_VERSION | kubectl apply -f - $LOG
|
||||||
|
|
||||||
helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \
|
helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \
|
||||||
-f _values.yaml --kube-version $KUBE_VERSION | kubectl apply --namespace kube-system -f - $LOG
|
-f _values.yaml --kube-version $KUBE_VERSION | kubectl apply --namespace kube-system -f - $LOG
|
||||||
|
|
||||||
|
@ -2,7 +2,7 @@ apiVersion: v2
|
|||||||
name: kubeadm
|
name: kubeadm
|
||||||
description: KubeZero Kubeadm cluster config
|
description: KubeZero Kubeadm cluster config
|
||||||
type: application
|
type: application
|
||||||
version: 1.22.8
|
version: 1.23.8
|
||||||
home: https://kubezero.com
|
home: https://kubezero.com
|
||||||
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
|
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
|
||||||
keywords:
|
keywords:
|
||||||
|
@ -2,13 +2,14 @@ apiVersion: kubeadm.k8s.io/v1beta3
|
|||||||
kind: ClusterConfiguration
|
kind: ClusterConfiguration
|
||||||
kubernetesVersion: {{ .Chart.Version }}
|
kubernetesVersion: {{ .Chart.Version }}
|
||||||
clusterName: {{ .Values.clusterName }}
|
clusterName: {{ .Values.clusterName }}
|
||||||
|
featureGates:
|
||||||
|
UnversionedKubeletConfigMap: true
|
||||||
controlPlaneEndpoint: {{ .Values.api.endpoint }}
|
controlPlaneEndpoint: {{ .Values.api.endpoint }}
|
||||||
networking:
|
networking:
|
||||||
podSubnet: 10.244.0.0/16
|
podSubnet: 10.244.0.0/16
|
||||||
etcd:
|
etcd:
|
||||||
local:
|
local:
|
||||||
# As 3.5 is not recommended stick with 3.4.13 till 1.23
|
imageTag: 3.5.4-0
|
||||||
imageTag: 3.4.13-0
|
|
||||||
extraArgs:
|
extraArgs:
|
||||||
### DNS discovery
|
### DNS discovery
|
||||||
#discovery-srv: {{ .Values.domain }}
|
#discovery-srv: {{ .Values.domain }}
|
||||||
|
@ -2,4 +2,6 @@ apiVersion: kubeproxy.config.k8s.io/v1alpha1
|
|||||||
kind: KubeProxyConfiguration
|
kind: KubeProxyConfiguration
|
||||||
# kube-proxy doesnt really support setting dynamic bind-address via config, replaced by cilium long-term anyways
|
# kube-proxy doesnt really support setting dynamic bind-address via config, replaced by cilium long-term anyways
|
||||||
metricsBindAddress: "0.0.0.0:10249"
|
metricsBindAddress: "0.0.0.0:10249"
|
||||||
mode: "ipvs"
|
# calico < 3.22.1 breaks starting with 1.23, see https://github.com/projectcalico/calico/issues/5011
|
||||||
|
# we go Cilium anyways
|
||||||
|
mode: "iptables"
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
{{- /* Feature gates for all control plane components */ -}}
|
{{- /* Feature gates for all control plane components */ -}}
|
||||||
{{- define "kubeadm.featuregates" }}
|
{{- define "kubeadm.featuregates" }}
|
||||||
{{- $gates := list "CustomCPUCFSQuotaPeriod" "GenericEphemeralVolume" "KubeletCredentialProviders"}}
|
{{- $gates := list "CustomCPUCFSQuotaPeriod" "KubeletCredentialProviders"}}
|
||||||
{{- if eq .return "csv" }}
|
{{- if eq .return "csv" }}
|
||||||
{{- range $key := $gates }}
|
{{- range $key := $gates }}
|
||||||
{{- $key }}=true,
|
{{- $key }}=true,
|
||||||
|
@ -95,11 +95,11 @@ spec:
|
|||||||
type: RollingUpdate
|
type: RollingUpdate
|
||||||
template:
|
template:
|
||||||
metadata:
|
metadata:
|
||||||
annotations:
|
|
||||||
scheduler.alpha.kubernetes.io/critical-pod: ""
|
|
||||||
labels:
|
labels:
|
||||||
k8s-app: aws-iam-authenticator
|
k8s-app: aws-iam-authenticator
|
||||||
spec:
|
spec:
|
||||||
|
priorityClassName: system-cluster-critical
|
||||||
|
|
||||||
# use service account with access to
|
# use service account with access to
|
||||||
serviceAccountName: aws-iam-authenticator
|
serviceAccountName: aws-iam-authenticator
|
||||||
|
|
||||||
|
@ -2,7 +2,7 @@ apiVersion: v2
|
|||||||
name: kubezero-network
|
name: kubezero-network
|
||||||
description: KubeZero umbrella chart for all things network
|
description: KubeZero umbrella chart for all things network
|
||||||
type: application
|
type: application
|
||||||
version: 0.2.1
|
version: 0.3.0
|
||||||
home: https://kubezero.com
|
home: https://kubezero.com
|
||||||
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
|
icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
|
||||||
keywords:
|
keywords:
|
||||||
@ -15,15 +15,17 @@ maintainers:
|
|||||||
- name: Stefan Reimer
|
- name: Stefan Reimer
|
||||||
email: stefan@zero-downtime.net
|
email: stefan@zero-downtime.net
|
||||||
dependencies:
|
dependencies:
|
||||||
|
- name: kubezero-lib
|
||||||
|
version: ">= 0.1.5"
|
||||||
|
repository: https://cdn.zero-downtime.net/charts/
|
||||||
- name: cilium
|
- name: cilium
|
||||||
version: 1.11.3
|
version: 1.11.6
|
||||||
repository: https://helm.cilium.io/
|
repository: https://helm.cilium.io/
|
||||||
condition: cilium.enabled
|
condition: cilium.enabled
|
||||||
- name: metallb
|
- name: metallb
|
||||||
version: 0.10.2
|
version: 0.13.3
|
||||||
repository: https://metallb.github.io/metallb
|
repository: https://metallb.github.io/metallb
|
||||||
condition: metallb.enabled
|
condition: metallb.enabled
|
||||||
# Legact / Testing support
|
|
||||||
- name: calico
|
- name: calico
|
||||||
version: 0.2.2
|
version: 0.2.2
|
||||||
condition: calico.enabled
|
condition: calico.enabled
|
||||||
|
@ -1,4 +1,3 @@
|
|||||||
{{- if .Values.multus.enabled }}
|
|
||||||
apiVersion: apiextensions.k8s.io/v1
|
apiVersion: apiextensions.k8s.io/v1
|
||||||
kind: CustomResourceDefinition
|
kind: CustomResourceDefinition
|
||||||
metadata:
|
metadata:
|
||||||
@ -43,4 +42,3 @@ spec:
|
|||||||
config:
|
config:
|
||||||
description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
|
description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
|
||||||
type: string
|
type: string
|
||||||
{{- end }}
|
|
27
charts/kubezero-network/templates/metallb/config.yaml
Normal file
27
charts/kubezero-network/templates/metallb/config.yaml
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
{{- if .Values.metallb.enabled }}
|
||||||
|
apiVersion: metallb.io/v1beta1
|
||||||
|
kind: L2Advertisement
|
||||||
|
metadata:
|
||||||
|
name: l2advertisement1
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
ipAddressPools:
|
||||||
|
{{- range $key, $val := .Values.metallb.ipAddressPools }}
|
||||||
|
{{- if eq $val.protocol "layer2" }}
|
||||||
|
- {{ $val.name }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
|
---
|
||||||
|
|
||||||
|
{{- range $key, $val := .Values.metallb.ipAddressPools }}
|
||||||
|
apiVersion: metallb.io/v1beta1
|
||||||
|
kind: IPAddressPool
|
||||||
|
metadata:
|
||||||
|
name: {{ $val.name }}
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
addresses:
|
||||||
|
{{- $val.addresses | toYaml | nindent 4 }}
|
||||||
|
{{- end }}
|
||||||
|
---
|
||||||
|
{{- end }}
|
25
charts/kubezero-network/templates/multus/calico-network.yaml
Normal file
25
charts/kubezero-network/templates/multus/calico-network.yaml
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
{{- if .Values.calico.enabled }}
|
||||||
|
apiVersion: k8s.cni.cncf.io/v1
|
||||||
|
kind: NetworkAttachmentDefinition
|
||||||
|
metadata:
|
||||||
|
name: calico
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
config: '{
|
||||||
|
"type": "calico",
|
||||||
|
"cniVersion": "0.3.1",
|
||||||
|
"log_level": "info",
|
||||||
|
"log_file_path": "/var/log/calico/cni/cni.log",
|
||||||
|
"datastore_type": "kubernetes",
|
||||||
|
"mtu": 8941,
|
||||||
|
"ipam": {
|
||||||
|
"type": "calico-ipam"
|
||||||
|
},
|
||||||
|
"policy": {
|
||||||
|
"type": "k8s"
|
||||||
|
},
|
||||||
|
"kubernetes": {
|
||||||
|
"kubeconfig": "/etc/cni/net.d/calico-kubeconfig"
|
||||||
|
}
|
||||||
|
}'
|
||||||
|
{{- end }}
|
14
charts/kubezero-network/templates/multus/cilium-network.yaml
Normal file
14
charts/kubezero-network/templates/multus/cilium-network.yaml
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
{{- if .Values.cilium.enabled }}
|
||||||
|
apiVersion: k8s.cni.cncf.io/v1
|
||||||
|
kind: NetworkAttachmentDefinition
|
||||||
|
metadata:
|
||||||
|
name: cilium
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
config: '{
|
||||||
|
"cniVersion": "0.3.1",
|
||||||
|
"name": "cilium",
|
||||||
|
"type": "cilium-cni",
|
||||||
|
"enable-debug": false
|
||||||
|
}'
|
||||||
|
{{- end }}
|
24
charts/kubezero-network/templates/multus/config.yaml
Normal file
24
charts/kubezero-network/templates/multus/config.yaml
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
{{- if .Values.multus.enabled }}
|
||||||
|
kind: ConfigMap
|
||||||
|
apiVersion: v1
|
||||||
|
metadata:
|
||||||
|
name: multus-cni-config
|
||||||
|
namespace: kube-system
|
||||||
|
labels:
|
||||||
|
{{- include "kubezero-lib.labels" . | nindent 4 }}
|
||||||
|
data:
|
||||||
|
cni-conf.json: |
|
||||||
|
{
|
||||||
|
"name": "multus-cni-network",
|
||||||
|
"type": "multus",
|
||||||
|
"cniVersion": "0.3.1",
|
||||||
|
"capabilities": {
|
||||||
|
"portMappings": true
|
||||||
|
},
|
||||||
|
"kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig",
|
||||||
|
"clusterNetwork": {{ .Values.multus.clusterNetwork | quote }},
|
||||||
|
"defaultNetworks": {{ .Values.multus.defaultNetworks | toJson }},
|
||||||
|
"systemNamespaces": [""],
|
||||||
|
"readinessindicatorfile": {{ .Values.multus.readinessindicatorfile | quote }}
|
||||||
|
}
|
||||||
|
{{- end }}
|
@ -1,105 +1,21 @@
|
|||||||
{{- if .Values.multus.enabled }}
|
{{- if .Values.multus.enabled }}
|
||||||
kind: ClusterRole
|
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
|
||||||
metadata:
|
|
||||||
name: multus
|
|
||||||
rules:
|
|
||||||
- apiGroups: ["k8s.cni.cncf.io"]
|
|
||||||
resources:
|
|
||||||
- '*'
|
|
||||||
verbs:
|
|
||||||
- '*'
|
|
||||||
- apiGroups:
|
|
||||||
- ""
|
|
||||||
resources:
|
|
||||||
- pods
|
|
||||||
- pods/status
|
|
||||||
verbs:
|
|
||||||
- get
|
|
||||||
- update
|
|
||||||
- apiGroups:
|
|
||||||
- ""
|
|
||||||
- events.k8s.io
|
|
||||||
resources:
|
|
||||||
- events
|
|
||||||
verbs:
|
|
||||||
- create
|
|
||||||
- patch
|
|
||||||
- update
|
|
||||||
---
|
|
||||||
kind: ClusterRoleBinding
|
|
||||||
apiVersion: rbac.authorization.k8s.io/v1
|
|
||||||
metadata:
|
|
||||||
name: multus
|
|
||||||
roleRef:
|
|
||||||
apiGroup: rbac.authorization.k8s.io
|
|
||||||
kind: ClusterRole
|
|
||||||
name: multus
|
|
||||||
subjects:
|
|
||||||
- kind: ServiceAccount
|
|
||||||
name: multus
|
|
||||||
namespace: kube-system
|
|
||||||
---
|
|
||||||
apiVersion: v1
|
|
||||||
kind: ServiceAccount
|
|
||||||
metadata:
|
|
||||||
name: multus
|
|
||||||
namespace: kube-system
|
|
||||||
---
|
|
||||||
kind: ConfigMap
|
|
||||||
apiVersion: v1
|
|
||||||
metadata:
|
|
||||||
name: multus-cni-config
|
|
||||||
namespace: kube-system
|
|
||||||
labels:
|
|
||||||
tier: node
|
|
||||||
app: multus
|
|
||||||
data:
|
|
||||||
# NOTE: If you'd prefer to manually apply a configuration file, you may create one here.
|
|
||||||
# In the case you'd like to customize the Multus installation, you should change the arguments to the Multus pod
|
|
||||||
# change the "args" line below from
|
|
||||||
# - "--multus-conf-file=auto"
|
|
||||||
# to:
|
|
||||||
# "--multus-conf-file=/tmp/multus-conf/70-multus.conf"
|
|
||||||
# Additionally -- you should ensure that the name "70-multus.conf" is the alphabetically first name in the
|
|
||||||
# /etc/cni/net.d/ directory on each node, otherwise, it will not be used by the Kubelet.
|
|
||||||
cni-conf.json: |
|
|
||||||
{
|
|
||||||
"cniVersion": "0.3.1",
|
|
||||||
"name": "multus-cni-network",
|
|
||||||
"type": "multus",
|
|
||||||
"kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig",
|
|
||||||
"delegates": [
|
|
||||||
{
|
|
||||||
"cniVersion": "0.3.1",
|
|
||||||
"name": "cilium",
|
|
||||||
"type": "cilium-cni",
|
|
||||||
"enable-debug": false
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
---
|
|
||||||
apiVersion: apps/v1
|
apiVersion: apps/v1
|
||||||
kind: DaemonSet
|
kind: DaemonSet
|
||||||
metadata:
|
metadata:
|
||||||
name: kube-multus-ds
|
name: kube-multus-ds
|
||||||
namespace: kube-system
|
namespace: kube-system
|
||||||
labels:
|
labels:
|
||||||
tier: node
|
{{- include "kubezero-lib.labels" . | nindent 4 }}
|
||||||
app: multus
|
|
||||||
name: multus
|
|
||||||
spec:
|
spec:
|
||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
name: multus
|
{{- include "kubezero-lib.selectorLabels" . | nindent 6 }}
|
||||||
updateStrategy:
|
updateStrategy:
|
||||||
type: RollingUpdate
|
type: RollingUpdate
|
||||||
template:
|
template:
|
||||||
metadata:
|
metadata:
|
||||||
labels:
|
labels:
|
||||||
tier: node
|
{{- include "kubezero-lib.labels" . | nindent 8 }}
|
||||||
app: multus
|
|
||||||
name: multus
|
|
||||||
spec:
|
spec:
|
||||||
hostNetwork: true
|
hostNetwork: true
|
||||||
tolerations:
|
tolerations:
|
||||||
@ -113,8 +29,8 @@ spec:
|
|||||||
image: ghcr.io/k8snetworkplumbingwg/multus-cni:{{ .Values.multus.tag }}
|
image: ghcr.io/k8snetworkplumbingwg/multus-cni:{{ .Values.multus.tag }}
|
||||||
command: ["/entrypoint.sh"]
|
command: ["/entrypoint.sh"]
|
||||||
args:
|
args:
|
||||||
- "--multus-conf-file=auto"
|
- "--multus-conf-file=/tmp/multus-conf/00-multus.conf"
|
||||||
- "--rename-conf-file=true"
|
- "--rename-conf-file=false"
|
||||||
- "--cni-bin-dir=/host/usr/libexec/cni"
|
- "--cni-bin-dir=/host/usr/libexec/cni"
|
||||||
- "--cni-version=0.3.1"
|
- "--cni-version=0.3.1"
|
||||||
resources:
|
resources:
|
||||||
@ -153,5 +69,5 @@ spec:
|
|||||||
name: multus-cni-config
|
name: multus-cni-config
|
||||||
items:
|
items:
|
||||||
- key: cni-conf.json
|
- key: cni-conf.json
|
||||||
path: 70-multus.conf
|
path: 00-multus.conf
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
48
charts/kubezero-network/templates/multus/rbac.yaml
Normal file
48
charts/kubezero-network/templates/multus/rbac.yaml
Normal file
@ -0,0 +1,48 @@
|
|||||||
|
{{- if .Values.multus.enabled }}
|
||||||
|
kind: ClusterRole
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: multus
|
||||||
|
rules:
|
||||||
|
- apiGroups: ["k8s.cni.cncf.io"]
|
||||||
|
resources:
|
||||||
|
- '*'
|
||||||
|
verbs:
|
||||||
|
- '*'
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- pods
|
||||||
|
- pods/status
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- update
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
- events.k8s.io
|
||||||
|
resources:
|
||||||
|
- events
|
||||||
|
verbs:
|
||||||
|
- create
|
||||||
|
- patch
|
||||||
|
- update
|
||||||
|
---
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: multus
|
||||||
|
roleRef:
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
kind: ClusterRole
|
||||||
|
name: multus
|
||||||
|
subjects:
|
||||||
|
- kind: ServiceAccount
|
||||||
|
name: multus
|
||||||
|
namespace: kube-system
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ServiceAccount
|
||||||
|
metadata:
|
||||||
|
name: multus
|
||||||
|
namespace: kube-system
|
||||||
|
{{- end }}
|
@ -10,8 +10,7 @@ metallb:
|
|||||||
nodeSelector:
|
nodeSelector:
|
||||||
node-role.kubernetes.io/control-plane: ""
|
node-role.kubernetes.io/control-plane: ""
|
||||||
|
|
||||||
configInline: {}
|
ipAddressPools: []
|
||||||
# address-pools:
|
|
||||||
#- name: my-ip-space
|
#- name: my-ip-space
|
||||||
# protocol: layer2
|
# protocol: layer2
|
||||||
# addresses:
|
# addresses:
|
||||||
@ -19,14 +18,25 @@ metallb:
|
|||||||
|
|
||||||
multus:
|
multus:
|
||||||
enabled: false
|
enabled: false
|
||||||
tag: "v3.8.1"
|
tag: "v3.9"
|
||||||
|
|
||||||
|
clusterNetwork: "calico"
|
||||||
|
defaultNetworks: []
|
||||||
|
# - "cilium"
|
||||||
|
readinessindicatorfile: "/etc/cni/net.d/10-calico.conflist"
|
||||||
|
|
||||||
cilium:
|
cilium:
|
||||||
enabled: false
|
enabled: false
|
||||||
|
|
||||||
cni:
|
cni:
|
||||||
|
binPath: "/usr/libexec/cni"
|
||||||
#-- Ensure this is false if multus is enabled
|
#-- Ensure this is false if multus is enabled
|
||||||
exclusive: true
|
exclusive: false
|
||||||
|
|
||||||
|
cgroup:
|
||||||
|
autoMount:
|
||||||
|
enabled: false
|
||||||
|
hostRoot: "/sys/fs/cgroup"
|
||||||
|
|
||||||
tunnel: geneve
|
tunnel: geneve
|
||||||
|
|
||||||
@ -36,6 +46,11 @@ cilium:
|
|||||||
|
|
||||||
operator:
|
operator:
|
||||||
replicas: 1
|
replicas: 1
|
||||||
|
tolerations:
|
||||||
|
- key: node-role.kubernetes.io/master
|
||||||
|
effect: NoSchedule
|
||||||
|
nodeSelector:
|
||||||
|
node-role.kubernetes.io/control-plane: ""
|
||||||
|
|
||||||
hubble:
|
hubble:
|
||||||
enabled: false
|
enabled: false
|
||||||
|
Loading…
Reference in New Issue
Block a user