From 26b66016ad8308774cbf20bd456211c5c7a7339a Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 1 Dec 2021 13:33:11 +0100 Subject: [PATCH] feat: various re-org for 1.21 --- containers/admin/upgrade-cluster.yaml | 2 +- containers/admin/v1.21/kubeadm/README.md | 9 ++-- .../templates/ClusterConfiguration.yaml | 8 +-- .../templates/KubeletConfiguration.yaml | 2 +- .../kubeadm/templates/admin-aws-iam.yaml | 4 +- .../apiserver/aws-iam-authenticator.yaml | 2 +- ...aml => 50-aws-iam-authenticator-crds.yaml} | 2 +- ... 51-aws-iam-authenticator-deployment.yaml} | 6 +-- ...=> 52-aws-iam-authenticator-mappings.yaml} | 6 +-- .../templates/resources/90-backup-secret.yaml | 9 ---- .../resources/91-backup-cluster.yaml | 53 ------------------- containers/admin/v1.21/kubeadm/values.yaml | 44 +++++++++------ containers/admin/v1.21/kubezero.sh | 49 ++++++++++++++--- 13 files changed, 92 insertions(+), 104 deletions(-) rename containers/admin/v1.21/kubeadm/templates/resources/{00-aws-iam-authenticator-crds.yaml => 50-aws-iam-authenticator-crds.yaml} (94%) rename containers/admin/v1.21/kubeadm/templates/resources/{01-aws-iam-authenticator-deployment.yaml => 51-aws-iam-authenticator-deployment.yaml} (96%) rename containers/admin/v1.21/kubeadm/templates/resources/{02-aws-iam-authenticator-mappings.yaml => 52-aws-iam-authenticator-mappings.yaml} (80%) delete mode 100644 containers/admin/v1.21/kubeadm/templates/resources/90-backup-secret.yaml delete mode 100644 containers/admin/v1.21/kubeadm/templates/resources/91-backup-cluster.yaml diff --git a/containers/admin/upgrade-cluster.yaml b/containers/admin/upgrade-cluster.yaml index 0411a7fd..9af82232 100644 --- a/containers/admin/upgrade-cluster.yaml +++ b/containers/admin/upgrade-cluster.yaml @@ -1,7 +1,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: kubezero-upgrade + name: kubezero-upgrade-v1.21.7 namespace: kube-system spec: backoffLimit: 1 diff --git a/containers/admin/v1.21/kubeadm/README.md b/containers/admin/v1.21/kubeadm/README.md index 8c7ac28a..8f1953a7 100644 --- a/containers/admin/v1.21/kubeadm/README.md +++ b/containers/admin/v1.21/kubeadm/README.md @@ -22,12 +22,14 @@ Kubernetes: `>= 1.20.0` |-----|------|---------|-------------| | api.allEtcdEndpoints | string | `""` | | | api.apiAudiences | string | `"istio-ca"` | | -| api.awsIamAuth | string | `"false"` | | | api.endpoint | string | `"kube-api.changeme.org:6443"` | | | api.extraArgs | object | `{}` | | | api.listenPort | int | `6443` | | | api.oidcEndpoint | string | `""` | s3://${CFN[ConfigBucket]}/k8s/$CLUSTERNAME | | api.serviceAccountIssuer | string | `""` | https://s3.${REGION}.amazonaws.com/${CFN[ConfigBucket]}/k8s/$CLUSTERNAME | +| awsIamAuth.enabled | bool | `false` | | +| awsIamAuth.kubeAdminRole | string | `"arn:aws:iam::000000000000:role/KubernetesNode"` | | +| awsIamAuth.workerNodeRole | string | `"arn:aws:iam::000000000000:role/KubernetesNode"` | | | backup.passwordFile | string | `""` | /etc/cloudbender/clusterBackup.passphrase | | backup.repository | string | `""` | s3:https://s3.amazonaws.com/${CFN[ConfigBucket]}/k8s/${CLUSTERNAME}/clusterBackup | | clusterName | string | `"pleasechangeme"` | | @@ -35,13 +37,12 @@ Kubernetes: `>= 1.20.0` | etcd.extraArgs | object | `{}` | | | etcd.nodeName | string | `"set_via_cmdline"` | | | highAvailable | bool | `false` | | -| kubeAdminRole | string | `"arn:aws:iam::000000000000:role/KubernetesNode"` | | | listenAddress | string | `"0.0.0.0"` | Needs to be set to primary node IP | +| network.multus.enabled | bool | `true` | | +| network.multus.tag | string | `"v3.8"` | | | nodeName | string | `"localhost"` | set to $HOSTNAME | -| platform | string | `"aws"` | supported values aws,bare-metal | | protectKernelDefaults | bool | `true` | | | systemd | bool | `true` | Set to false for openrc, eg. on Gentoo or Alpine | -| workerNodeRole | string | `"arn:aws:iam::000000000000:role/KubernetesNode"` | | ## Resources diff --git a/containers/admin/v1.21/kubeadm/templates/ClusterConfiguration.yaml b/containers/admin/v1.21/kubeadm/templates/ClusterConfiguration.yaml index 92b73bdb..a7c5fc13 100644 --- a/containers/admin/v1.21/kubeadm/templates/ClusterConfiguration.yaml +++ b/containers/admin/v1.21/kubeadm/templates/ClusterConfiguration.yaml @@ -36,13 +36,13 @@ controllerManager: terminated-pod-gc-threshold: "300" # leader-elect: {{ .Values.highAvailable | quote }} logging-format: json - feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" "platform" .Values.platform ) | trimSuffix "," | quote }} + feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }} scheduler: extraArgs: profiling: "false" # leader-elect: {{ .Values.highAvailable | quote }} logging-format: json - feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" "platform" .Values.platform ) | trimSuffix "," | quote }} + feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }} apiServer: certSANs: - {{ regexSplit ":" .Values.api.endpoint -1 | first }} @@ -62,10 +62,10 @@ apiServer: service-account-issuer: "{{ .Values.api.serviceAccountIssuer }}" service-account-jwks-uri: "{{ .Values.api.serviceAccountIssuer }}/openid/v1/jwks" {{- end }} - {{- if eq .Values.platform "aws" }} + {{- if .Values.api.awsIamAuth.enabled }} authentication-token-webhook-config-file: /etc/kubernetes/apiserver/aws-iam-authenticator.yaml {{- end }} - feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" "platform" .Values.platform ) | trimSuffix "," | quote }} + feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }} enable-admission-plugins: DenyServiceExternalIPs,NodeRestriction,EventRateLimit # {{- if .Values.highAvailable }} # goaway-chance: ".001" diff --git a/containers/admin/v1.21/kubeadm/templates/KubeletConfiguration.yaml b/containers/admin/v1.21/kubeadm/templates/KubeletConfiguration.yaml index d7a814aa..e5622e51 100644 --- a/containers/admin/v1.21/kubeadm/templates/KubeletConfiguration.yaml +++ b/containers/admin/v1.21/kubeadm/templates/KubeletConfiguration.yaml @@ -16,7 +16,7 @@ eventRecordQPS: 0 # tlsCertFile: /var/lib/kubelet/pki/kubelet.crt # tlsPrivateKeyFile: /var/lib/kubelet/pki/kubelet.key tlsCipherSuites: [TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256] -featureGates: {{ include "kubeadm.featuregates" ( dict "return" "map" "platform" .Values.platform ) }} +featureGates: {{ include "kubeadm.featuregates" ( dict "return" "map" ) }} # Minimal unit is 50m per pod podsPerCore: 20 # cpuCFSQuotaPeriod: 10ms diff --git a/containers/admin/v1.21/kubeadm/templates/admin-aws-iam.yaml b/containers/admin/v1.21/kubeadm/templates/admin-aws-iam.yaml index 62ad49f6..99e9c7a3 100644 --- a/containers/admin/v1.21/kubeadm/templates/admin-aws-iam.yaml +++ b/containers/admin/v1.21/kubeadm/templates/admin-aws-iam.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.platform "aws" }} +{{- if .Values.api.awsIamAuth.enabled }} apiVersion: v1 kind: Config clusters: @@ -23,5 +23,5 @@ users: - "-i" - "{{ .Values.clusterName }}" - "-r" - - "{{ .Values.kubeAdminRole }}" + - "{{ .Values.api.awsIamAuth.kubeAdminRole }}" {{- end }} diff --git a/containers/admin/v1.21/kubeadm/templates/apiserver/aws-iam-authenticator.yaml b/containers/admin/v1.21/kubeadm/templates/apiserver/aws-iam-authenticator.yaml index f869ec54..1a6818a2 100644 --- a/containers/admin/v1.21/kubeadm/templates/apiserver/aws-iam-authenticator.yaml +++ b/containers/admin/v1.21/kubeadm/templates/apiserver/aws-iam-authenticator.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.platform "aws" }} +{{- if .Values.api.awsIamAuth.enabled }} # clusters refers to the remote service. clusters: - name: aws-iam-authenticator diff --git a/containers/admin/v1.21/kubeadm/templates/resources/00-aws-iam-authenticator-crds.yaml b/containers/admin/v1.21/kubeadm/templates/resources/50-aws-iam-authenticator-crds.yaml similarity index 94% rename from containers/admin/v1.21/kubeadm/templates/resources/00-aws-iam-authenticator-crds.yaml rename to containers/admin/v1.21/kubeadm/templates/resources/50-aws-iam-authenticator-crds.yaml index c1977a82..f17f214f 100644 --- a/containers/admin/v1.21/kubeadm/templates/resources/00-aws-iam-authenticator-crds.yaml +++ b/containers/admin/v1.21/kubeadm/templates/resources/50-aws-iam-authenticator-crds.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.platform "aws" }} +{{- if .Values.api.awsIamAuth.enabled }} apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: diff --git a/containers/admin/v1.21/kubeadm/templates/resources/01-aws-iam-authenticator-deployment.yaml b/containers/admin/v1.21/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml similarity index 96% rename from containers/admin/v1.21/kubeadm/templates/resources/01-aws-iam-authenticator-deployment.yaml rename to containers/admin/v1.21/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml index aa78a07b..be5741f4 100644 --- a/containers/admin/v1.21/kubeadm/templates/resources/01-aws-iam-authenticator-deployment.yaml +++ b/containers/admin/v1.21/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.platform "aws" }} +{{- if .Values.api.awsIamAuth.enabled }} kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -106,9 +106,9 @@ spec: # run on the host network (don't depend on CNI) hostNetwork: true - # run on each master node + # run on each controller nodeSelector: - node-role.kubernetes.io/master: "" + node-role.kubernetes.io/control-plane: "" tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master diff --git a/containers/admin/v1.21/kubeadm/templates/resources/02-aws-iam-authenticator-mappings.yaml b/containers/admin/v1.21/kubeadm/templates/resources/52-aws-iam-authenticator-mappings.yaml similarity index 80% rename from containers/admin/v1.21/kubeadm/templates/resources/02-aws-iam-authenticator-mappings.yaml rename to containers/admin/v1.21/kubeadm/templates/resources/52-aws-iam-authenticator-mappings.yaml index 7310ef0f..0ff2b1a9 100644 --- a/containers/admin/v1.21/kubeadm/templates/resources/02-aws-iam-authenticator-mappings.yaml +++ b/containers/admin/v1.21/kubeadm/templates/resources/52-aws-iam-authenticator-mappings.yaml @@ -1,11 +1,11 @@ -{{- if eq .Values.platform "aws" }} +{{- if .Values.api.awsIamAuth.enabled }} # Controller role for consistency, similar to kubeadm admin.conf apiVersion: iamauthenticator.k8s.aws/v1alpha1 kind: IAMIdentityMapping metadata: name: kubezero-worker-nodes spec: - arn: {{ .Values.workerNodeRole }} + arn: {{ .Values.api.awsIamAuth.workerNodeRole }} username: system:node:{{ "{{" }}EC2PrivateDNSName{{ "}}" }} groups: # For now use masters, define properly with 1.20 @@ -19,7 +19,7 @@ kind: IAMIdentityMapping metadata: name: kubernetes-admin spec: - arn: {{ .Values.kubeAdminRole }} + arn: {{ .Values.api.awsIamAuth.kubeAdminRole }} username: kubernetes-admin groups: - system:masters diff --git a/containers/admin/v1.21/kubeadm/templates/resources/90-backup-secret.yaml b/containers/admin/v1.21/kubeadm/templates/resources/90-backup-secret.yaml deleted file mode 100644 index 7ed653cb..00000000 --- a/containers/admin/v1.21/kubeadm/templates/resources/90-backup-secret.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: kubezero-backup-restic - namespace: kube-system -type: Opaque -data: - repository: {{ default "" .Values.backup.repository | b64enc }} - password: {{ default "" .Values.backup.password | b64enc }} diff --git a/containers/admin/v1.21/kubeadm/templates/resources/91-backup-cluster.yaml b/containers/admin/v1.21/kubeadm/templates/resources/91-backup-cluster.yaml deleted file mode 100644 index 670e3cad..00000000 --- a/containers/admin/v1.21/kubeadm/templates/resources/91-backup-cluster.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: batch/v1 -kind: CronJob -metadata: - name: kubezero-backup - namespace: kube-system -spec: - schedule: "0 * * * *" - jobTemplate: - spec: - backoffLimit: 1 - template: - spec: - containers: - - name: kubezero-admin - image: public.ecr.aws/zero-downtime/kubezero-admin:v{{ .Chart.Version }} - imagePullPolicy: Always - command: ["kubezero.sh"] - args: - - backup - volumeMounts: - - name: host - mountPath: /host - - name: workdir - mountPath: /tmp - env: - - name: DEBUG - value: "1" - - name: RESTIC_REPOSITORY - valueFrom: - secretKeyRef: - name: kubezero-backup-restic - key: repository - - name: RESTIC_PASSWORD - valueFrom: - secretKeyRef: - name: kubezero-backup-restic - key: password - #securityContext: - # readOnlyRootFilesystem: true - hostNetwork: true - volumes: - - name: host - hostPath: - path: / - type: Directory - - name: workdir - emptyDir: {} - nodeSelector: - node-role.kubernetes.io/master: "" - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - restartPolicy: Never diff --git a/containers/admin/v1.21/kubeadm/values.yaml b/containers/admin/v1.21/kubeadm/values.yaml index faed292a..107e34da 100644 --- a/containers/admin/v1.21/kubeadm/values.yaml +++ b/containers/admin/v1.21/kubeadm/values.yaml @@ -16,26 +16,40 @@ api: # -- s3://${CFN[ConfigBucket]}/k8s/$CLUSTERNAME oidcEndpoint: "" apiAudiences: "istio-ca" - awsIamAuth: "false" + + awsIamAuth: + enabled: false + workerNodeRole: "arn:aws:iam::000000000000:role/KubernetesNode" + kubeAdminRole: "arn:aws:iam::000000000000:role/KubernetesNode" + +addons: + aws-node-termination-handler: + enabled: false + # -- arn:aws:sqs:${REGION}:${AWS_ACCOUNT_ID}:${CLUSTERNAME}_Nth + queueURL: "" + + clusterBackup: + enabled: false + # -- s3:https://s3.amazonaws.com/${CFN[ConfigBucket]}/k8s/${CLUSTERNAME}/clusterBackup + repository: "" + # -- /etc/cloudbender/clusterBackup.passphrase + passwordFile: "" + +network: + multus: + enabled: false + tag: "v3.8" + cilium: + enabled: false + calico: + enabled: false + +highAvailable: false etcd: nodeName: set_via_cmdline extraArgs: {} -backup: - # -- s3:https://s3.amazonaws.com/${CFN[ConfigBucket]}/k8s/${CLUSTERNAME}/clusterBackup - repository: "" - # -- /etc/cloudbender/clusterBackup.passphrase - passwordFile: "" - -highAvailable: false - -# -- supported values aws,bare-metal -platform: "aws" - # -- Set to false for openrc, eg. on Gentoo or Alpine systemd: true protectKernelDefaults: true - -workerNodeRole: "arn:aws:iam::000000000000:role/KubernetesNode" -kubeAdminRole: "arn:aws:iam::000000000000:role/KubernetesNode" diff --git a/containers/admin/v1.21/kubezero.sh b/containers/admin/v1.21/kubezero.sh index d50b80d9..f12b2d8c 100755 --- a/containers/admin/v1.21/kubezero.sh +++ b/containers/admin/v1.21/kubezero.sh @@ -41,7 +41,7 @@ render_kubeadm() { yq eval 'del(.etcd.local.serverCertSANs) | del(.etcd.local.peerCertSANs)' \ ${HOSTFS}/etc/kubernetes/kubeadm-etcd.yaml > ${HOSTFS}/etc/kubernetes/kubeadm.yaml - # Copy JoinConfig + # Copy JoinConfig cp ${WORKDIR}/kubeadm/templates/JoinConfiguration.yaml ${HOSTFS}/etc/kubernetes # hack to "uncloack" the json patches after they go processed by helm @@ -55,10 +55,12 @@ render_kubeadm() { parse_kubezero() { [ -f ${HOSTFS}/etc/kubernetes/kubezero.yaml ] || { echo "Missing /etc/kubernetes/kubezero.yaml!"; exit 1; } + KUBE_VERSION=$(kubeadm version -o yaml | yq eval .clientVersion.gitVersion -) CLUSTERNAME=$(yq eval '.clusterName' ${HOSTFS}/etc/kubernetes/kubezero.yaml) NODENAME=$(yq eval '.nodeName' ${HOSTFS}/etc/kubernetes/kubezero.yaml) - AWS_IAM_AUTH=$(yq eval '.api.awsIamAuth // "true"' ${HOSTFS}/etc/kubernetes/kubezero.yaml) + AWS_IAM_AUTH=$(yq eval '.api.awsIamAuth.enabled' ${HOSTFS}/etc/kubernetes/kubezero.yaml) + AWS_NTH=$(yq eval '.addons.aws-node-termination-handler.enabled' ${HOSTFS}/etc/kubernetes/kubezero.yaml) } @@ -66,7 +68,7 @@ parse_kubezero() { pre_kubeadm() { # update all apiserver addons first cp -r ${WORKDIR}/kubeadm/templates/apiserver ${HOSTFS}/etc/kubernetes - + # aws-iam-authenticator enabled ? if [ "$AWS_IAM_AUTH" == "true" ]; then @@ -89,7 +91,9 @@ pre_kubeadm() { # Shared steps after calling kubeadm post_kubeadm() { # KubeZero resources - cat ${WORKDIR}/kubeadm/templates/resources/*.yaml | kubectl apply -f - $LOG + for f in ${WORKDIR}/kubeadm/templates/resources/*.yaml; do + kubectl apply -f $f $LOG + done # Patch coreDNS addon, ideally we prevent kubeadm to reset coreDNS to its defaults kubectl patch deployment coredns -n kube-system --patch-file ${WORKDIR}/kubeadm/templates/patches/coredns0.yaml $LOG @@ -132,6 +136,23 @@ if [ "$1" == 'upgrade' ]; then ### POST 1.21 specific ###################### + helm repo add kubezero https://cdn.zero-downtime.net/charts/ + + # if Calico, install multus to prepare migration + kubectl get ds calico-node -n kube-system && \ + helm template kubezero/kubezero-network --version 0.1.0 --include-crds --namespace kube-system --kube-version $KUBE_VERSION --name-template network \ + --set multus.enabled=true \ + | kubectl apply -f - $LOG + + # migrate backup + if [ -f ${HOSTFS}/usr/local/sbin/backup_control_plane.sh ]; then + _repo=$(grep "export RESTIC_REPOSITORY" ${HOSTFS}/usr/local/sbin/backup_control_plane.sh) + helm template kubezero/kubezero-addons --version 0.2.0 --include-crds --namespace kube-system --kube-version $KUBE_VERSION --name-template addons \ + --set clusterBackup.enabled=true \ + --set clusterBackup.repository="${_repo##*=}" \ + --set clusterBackup.password="$(cat ${HOSTFS}/etc/kubernetes/clusterBackup.passphrase)" \ + | kubectl apply -f - $LOG + fi ###################### @@ -147,7 +168,6 @@ if [ "$1" == 'upgrade' ]; then # Removed: # - update oidc do we need that ? - # - backup right after upgrade ... not so sure about that one elif [[ "$1" =~ "^(bootstrap|recover|join)$" ]]; then @@ -203,8 +223,23 @@ elif [[ "$1" =~ "^(bootstrap|recover|join)$" ]]; then yq eval -M ".clusters[0].cluster.certificate-authority-data = \"$(cat ${HOSTFS}/etc/kubernetes/pki/ca.crt | base64 -w0)\"" ${WORKDIR}/kubeadm/templates/admin-aws-iam.yaml > ${HOSTFS}/etc/kubernetes/admin-aws-iam.yaml fi + # Install some basics on bootstrap + if [[ "$1" =~ "^(bootstrap)$" ]]; then + helm repo add kubezero https://cdn.zero-downtime.net/charts/ + + # network + yq eval '.network // ""' ${HOSTFS}/etc/kubernetes/kubezero.yaml > _values.yaml + helm template kubezero/kubezero-network --version 0.1.0 --include-crds --namespace kube-system --name-template network \ + -f _values.yaml --kube-version $KUBE_VERSION | kubectl apply -f - $LOG + + # addons + yq eval '.addons // ""' ${HOSTFS}/etc/kubernetes/kubezero.yaml > _values.yaml + helm template kubezero/kubezero-addons --version 0.2.0 --include-crds --namespace kube-system --name-template addons \ + -f _values.yaml --kube-version $KUBE_VERSION | kubectl apply -f - $LOG + fi + post_kubeadm - + echo "${1} cluster $CLUSTERNAME successfull." @@ -225,7 +260,7 @@ elif [ "$1" == 'backup' ]; then # pki & cluster-admin access cp -r ${HOSTFS}/etc/kubernetes/pki ${WORKDIR} cp -r ${HOSTFS}/etc/kubernetes/admin.conf ${WORKDIR} - + # Backup via restic restic snapshots || restic init restic backup ${WORKDIR} -H $CLUSTERNAME