diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8e2e6e2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +ARG ALPINE_VERSION + +FROM alpine:${ALPINE_VERSION} + +ARG ALPINE_VERSION +ARG KUBE_VERSION + +RUN cd /etc/apk/keys && \ + wget "https://cdn.zero-downtime.net/alpine/stefan@zero-downtime.net-61bb6bfb.rsa.pub" && \ + echo "@kubezero https://cdn.zero-downtime.net/alpine/v${ALPINE_VERSION}/kubezero" >> /etc/apk/repositories && \ + echo "@testing http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \ + apk upgrade -U -a --no-cache && \ + apk --no-cache add \ + jq \ + yq \ + cri-tools@kubezero \ + kubeadm@kubezero~=${KUBE_VERSION} \ + kubectl@kubezero~=${KUBE_VERSION} \ + etcd-ctl@testing \ + restic@testing \ + helm@testing + +ADD releases/v${KUBE_VERSION}/kubezero.sh /usr/bin +ADD charts/kubeadm /charts/kubeadm +ADD charts/kubezero-addons /charts/kubezero-addons +ADD charts/kubezero-network /charts/kubezero-network + +ENTRYPOINT ["kubezero.sh"] diff --git a/Makefile b/Makefile index ceadf7e..7c84e43 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,39 @@ -BUCKET ?= zero-downtime -BUCKET_PREFIX ?= /cloudbender/distfiles -FILES ?= distfiles.txt +VERSION ?= 1.22.8 +ALPINE_VERSION ?= 3.15 +REGISTRY := public.ecr.aws/zero-downtime +REPOSITORY := kubezero-admin +TAG := $(REPOSITORY):v$(VERSION) +KUBE_VERSION := $(shell echo $(VERSION) | sed -e 's/\.[[:digit:]]*$$//') -.PHONY: update docs +.PHONY: build push clean scan -all: update +all: build push -update: +build: + podman build --rm --build-arg KUBE_VERSION=$(KUBE_VERSION) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) -t $(TAG) . + +push: + aws ecr-public get-login-password --region us-east-1 | podman login --username AWS --password-stdin $(REGISTRY) + podman tag $(TAG) $(REGISTRY)/$(TAG) + podman push $(REGISTRY)/$(TAG) + +clean: + podman image prune -f + +scan: + podman system service& + sleep 5; trivy $(TAG) + +update-charts: ./scripts/update_helm.sh -docs: +update-chart-docs: for c in charts/*; do \ [[ $$c =~ "kubezero-lib" ]] && continue ; \ [[ $$c =~ "kubeadm" ]] && continue ; \ helm-docs -c $$c ; \ done -publish: +publish-charts: ./scripts/publish.sh + diff --git a/containers/admin/v1.21/kubeadm/.helmignore b/charts/kubeadm/.helmignore similarity index 100% rename from containers/admin/v1.21/kubeadm/.helmignore rename to charts/kubeadm/.helmignore diff --git a/containers/admin/v1.21/kubeadm/Chart.yaml b/charts/kubeadm/Chart.yaml similarity index 95% rename from containers/admin/v1.21/kubeadm/Chart.yaml rename to charts/kubeadm/Chart.yaml index 61624b1..087112d 100644 --- a/containers/admin/v1.21/kubeadm/Chart.yaml +++ b/charts/kubeadm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubeadm description: KubeZero Kubeadm cluster config type: application -version: 1.21.9 +version: 1.22.8 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/containers/admin/v1.21/kubeadm/README.md b/charts/kubeadm/README.md similarity index 92% rename from containers/admin/v1.21/kubeadm/README.md rename to charts/kubeadm/README.md index 72baa7a..d461e46 100644 --- a/containers/admin/v1.21/kubeadm/README.md +++ b/charts/kubeadm/README.md @@ -1,6 +1,6 @@ # kubeadm -![Version: 1.21.9](https://img.shields.io/badge/Version-1.21.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 1.22.8](https://img.shields.io/badge/Version-1.22.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero Kubeadm cluster config @@ -14,7 +14,7 @@ KubeZero Kubeadm cluster config ## Requirements -Kubernetes: `>= 1.20.0` +Kubernetes: `>= 1.22.0` ## Values @@ -47,8 +47,8 @@ Kubernetes: `>= 1.20.0` | network.multus.enabled | bool | `false` | | | network.multus.tag | string | `"v3.8"` | | | nodeName | string | `"kubezero-node"` | set to $HOSTNAME | -| protectKernelDefaults | bool | `true` | | -| systemd | bool | `true` | Set to false for openrc, eg. on Gentoo or Alpine | +| protectKernelDefaults | bool | `false` | | +| systemd | bool | `false` | Set to false for openrc, eg. on Gentoo or Alpine | ## Resources diff --git a/containers/admin/v1.21/kubeadm/README.md.gotmpl b/charts/kubeadm/README.md.gotmpl similarity index 100% rename from containers/admin/v1.21/kubeadm/README.md.gotmpl rename to charts/kubeadm/README.md.gotmpl diff --git a/containers/admin/v1.21/kubeadm/templates/ClusterConfiguration.yaml b/charts/kubeadm/templates/ClusterConfiguration.yaml similarity index 96% rename from containers/admin/v1.21/kubeadm/templates/ClusterConfiguration.yaml rename to charts/kubeadm/templates/ClusterConfiguration.yaml index 78ca579..f64e200 100644 --- a/containers/admin/v1.21/kubeadm/templates/ClusterConfiguration.yaml +++ b/charts/kubeadm/templates/ClusterConfiguration.yaml @@ -1,4 +1,4 @@ -apiVersion: kubeadm.k8s.io/v1beta2 +apiVersion: kubeadm.k8s.io/v1beta3 kind: ClusterConfiguration kubernetesVersion: {{ .Chart.Version }} clusterName: {{ .Values.clusterName }} @@ -7,6 +7,8 @@ networking: podSubnet: 10.244.0.0/16 etcd: local: + # As 3.5 is not recommended stick with 3.4.13 till 1.23 + imageTag: 3.4.13-0 extraArgs: ### DNS discovery #discovery-srv: {{ .Values.domain }} diff --git a/containers/admin/v1.21/kubeadm/templates/InitConfiguration.yaml b/charts/kubeadm/templates/InitConfiguration.yaml similarity index 89% rename from containers/admin/v1.21/kubeadm/templates/InitConfiguration.yaml rename to charts/kubeadm/templates/InitConfiguration.yaml index 908f86f..2ef6ee0 100644 --- a/containers/admin/v1.21/kubeadm/templates/InitConfiguration.yaml +++ b/charts/kubeadm/templates/InitConfiguration.yaml @@ -1,8 +1,10 @@ -apiVersion: kubeadm.k8s.io/v1beta2 +apiVersion: kubeadm.k8s.io/v1beta3 kind: InitConfiguration localAPIEndpoint: advertiseAddress: {{ .Values.listenAddress }} bindPort: {{ .Values.api.listenPort }} +patches: + directory: /tmp/patches nodeRegistration: ignorePreflightErrors: - DirAvailable--var-lib-etcd diff --git a/charts/kubeadm/templates/JoinConfiguration.yaml b/charts/kubeadm/templates/JoinConfiguration.yaml new file mode 100644 index 0000000..9fdbf1f --- /dev/null +++ b/charts/kubeadm/templates/JoinConfiguration.yaml @@ -0,0 +1,4 @@ +apiVersion: kubeadm.k8s.io/v1beta3 +kind: JoinConfiguration +patches: + directory: /tmp/patches diff --git a/containers/admin/v1.21/kubeadm/templates/KubeProxyConfiguration.yaml b/charts/kubeadm/templates/KubeProxyConfiguration.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/KubeProxyConfiguration.yaml rename to charts/kubeadm/templates/KubeProxyConfiguration.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/KubeletConfiguration.yaml b/charts/kubeadm/templates/KubeletConfiguration.yaml similarity index 91% rename from containers/admin/v1.21/kubeadm/templates/KubeletConfiguration.yaml rename to charts/kubeadm/templates/KubeletConfiguration.yaml index 2e5418b..3faadf6 100644 --- a/containers/admin/v1.21/kubeadm/templates/KubeletConfiguration.yaml +++ b/charts/kubeadm/templates/KubeletConfiguration.yaml @@ -20,16 +20,16 @@ featureGates: {{ include "kubeadm.featuregates" ( dict "return" "map" ) }} # Minimal unit is 50m per pod podsPerCore: 20 # cpuCFSQuotaPeriod: 10ms -# Basic OS on Ubuntu 20.04 incl. crio +# Basic OS incl. crio systemReserved: - memory: 256Mi - ephemeral-storage: "2Gi" + memory: 96Mi + #ephemeral-storage: "1Gi" # kubelet memory should be static as runc,conmon are added to each pod's cgroup kubeReserved: cpu: 70m - memory: 128Mi + memory: 96Mi # Lets use below to reserve memory for system processes as kubeReserved/sytemReserved doesnt go well with systemd it seems #evictionHard: # memory.available: "484Mi" imageGCLowThresholdPercent: 70 -kernelMemcgNotification: true +# kernelMemcgNotification: true diff --git a/containers/admin/v1.21/kubeadm/templates/README.md b/charts/kubeadm/templates/README.md similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/README.md rename to charts/kubeadm/templates/README.md diff --git a/containers/admin/v1.21/kubeadm/templates/_helpers.tpl b/charts/kubeadm/templates/_helpers.tpl similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/_helpers.tpl rename to charts/kubeadm/templates/_helpers.tpl diff --git a/containers/admin/v1.21/kubeadm/templates/admin-aws-iam.yaml b/charts/kubeadm/templates/admin-aws-iam.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/admin-aws-iam.yaml rename to charts/kubeadm/templates/admin-aws-iam.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/apiserver/admission-configuration.yaml b/charts/kubeadm/templates/apiserver/admission-configuration.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/apiserver/admission-configuration.yaml rename to charts/kubeadm/templates/apiserver/admission-configuration.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/apiserver/audit-policy.yaml b/charts/kubeadm/templates/apiserver/audit-policy.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/apiserver/audit-policy.yaml rename to charts/kubeadm/templates/apiserver/audit-policy.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/apiserver/aws-iam-authenticator.yaml b/charts/kubeadm/templates/apiserver/aws-iam-authenticator.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/apiserver/aws-iam-authenticator.yaml rename to charts/kubeadm/templates/apiserver/aws-iam-authenticator.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/apiserver/event-config.yaml b/charts/kubeadm/templates/apiserver/event-config.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/apiserver/event-config.yaml rename to charts/kubeadm/templates/apiserver/event-config.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/patches/coredns0.yaml b/charts/kubeadm/templates/patches/coredns0.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/patches/coredns0.yaml rename to charts/kubeadm/templates/patches/coredns0.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/patches/etcd0.yaml b/charts/kubeadm/templates/patches/etcd0.yaml similarity index 76% rename from containers/admin/v1.21/kubeadm/templates/patches/etcd0.yaml rename to charts/kubeadm/templates/patches/etcd0.yaml index b33390b..8bebff4 100644 --- a/containers/admin/v1.21/kubeadm/templates/patches/etcd0.yaml +++ b/charts/kubeadm/templates/patches/etcd0.yaml @@ -5,4 +5,4 @@ spec: requests: cpu: 200m memory: 192Mi - ephemeral-storage: 1Gi + #ephemeral-storage: 1Gi diff --git a/containers/admin/v1.21/kubeadm/templates/patches/kube-apiserver0.yaml b/charts/kubeadm/templates/patches/kube-apiserver0.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/patches/kube-apiserver0.yaml rename to charts/kubeadm/templates/patches/kube-apiserver0.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/patches/kube-apiserver1+json.yaml b/charts/kubeadm/templates/patches/kube-apiserver1+json.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/patches/kube-apiserver1+json.yaml rename to charts/kubeadm/templates/patches/kube-apiserver1+json.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/patches/kube-controller-manager0.yaml b/charts/kubeadm/templates/patches/kube-controller-manager0.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/patches/kube-controller-manager0.yaml rename to charts/kubeadm/templates/patches/kube-controller-manager0.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/patches/kube-controller-manager1+json.yaml b/charts/kubeadm/templates/patches/kube-controller-manager1+json.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/patches/kube-controller-manager1+json.yaml rename to charts/kubeadm/templates/patches/kube-controller-manager1+json.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/patches/kube-scheduler0.yaml b/charts/kubeadm/templates/patches/kube-scheduler0.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/patches/kube-scheduler0.yaml rename to charts/kubeadm/templates/patches/kube-scheduler0.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/patches/kube-scheduler1+json.yaml b/charts/kubeadm/templates/patches/kube-scheduler1+json.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/patches/kube-scheduler1+json.yaml rename to charts/kubeadm/templates/patches/kube-scheduler1+json.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/resources/10-runtimeClass.yaml b/charts/kubeadm/templates/resources/10-runtimeClass.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/resources/10-runtimeClass.yaml rename to charts/kubeadm/templates/resources/10-runtimeClass.yaml diff --git a/containers/admin/v1.21/kubeadm/templates/resources/20-oicd-public-rbac.yaml b/charts/kubeadm/templates/resources/20-oicd-public-rbac.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/resources/20-oicd-public-rbac.yaml rename to charts/kubeadm/templates/resources/20-oicd-public-rbac.yaml diff --git a/charts/kubeadm/templates/resources/50-aws-iam-authenticator-crds.yaml b/charts/kubeadm/templates/resources/50-aws-iam-authenticator-crds.yaml new file mode 100644 index 0000000..be77195 --- /dev/null +++ b/charts/kubeadm/templates/resources/50-aws-iam-authenticator-crds.yaml @@ -0,0 +1,46 @@ +{{- if .Values.api.awsIamAuth.enabled }} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: iamidentitymappings.iamauthenticator.k8s.aws +spec: + group: iamauthenticator.k8s.aws + scope: Cluster + names: + plural: iamidentitymappings + singular: iamidentitymapping + kind: IAMIdentityMapping + categories: + - all + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + required: + - arn + - username + properties: + arn: + type: string + username: + type: string + groups: + type: array + items: + type: string + status: + type: object + properties: + canonicalARN: + type: string + userID: + type: string + subresources: + status: {} +{{- end }} diff --git a/containers/admin/v1.21/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml b/charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml similarity index 99% rename from containers/admin/v1.21/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml rename to charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml index be5741f..ba375df 100644 --- a/containers/admin/v1.21/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml +++ b/charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml @@ -115,7 +115,7 @@ spec: containers: - name: aws-iam-authenticator - image: public.ecr.aws/zero-downtime/aws-iam-authenticator:v0.5.3 + image: public.ecr.aws/zero-downtime/aws-iam-authenticator:v0.5.4 args: - server - --backend-mode=CRD,MountedFile diff --git a/containers/admin/v1.21/kubeadm/templates/resources/52-aws-iam-authenticator-mappings.yaml b/charts/kubeadm/templates/resources/52-aws-iam-authenticator-mappings.yaml similarity index 100% rename from containers/admin/v1.21/kubeadm/templates/resources/52-aws-iam-authenticator-mappings.yaml rename to charts/kubeadm/templates/resources/52-aws-iam-authenticator-mappings.yaml diff --git a/containers/admin/v1.21/kubeadm/values.yaml b/charts/kubeadm/values.yaml similarity index 96% rename from containers/admin/v1.21/kubeadm/values.yaml rename to charts/kubeadm/values.yaml index 40718de..6950ee9 100644 --- a/containers/admin/v1.21/kubeadm/values.yaml +++ b/charts/kubeadm/values.yaml @@ -52,5 +52,5 @@ etcd: extraArgs: {} # -- Set to false for openrc, eg. on Gentoo or Alpine -systemd: true -protectKernelDefaults: true +systemd: false +protectKernelDefaults: false diff --git a/containers/admin/Dockerfile b/containers/admin/Dockerfile deleted file mode 100644 index 248c106..0000000 --- a/containers/admin/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -ARG ALPINE_VERSION - -FROM alpine:${ALPINE_VERSION} - -ARG KUBE_VERSION - -RUN cd /etc/apk/keys && \ - wget "https://cdn.zero-downtime.net/alpine/stefan@zero-downtime.net-61bb6bfb.rsa.pub" && \ - echo "@kubezero https://cdn.zero-downtime.net/alpine/v${KUBE_VERSION}/kubezero" >> /etc/apk/repositories && \ - echo "@testing http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \ - apk upgrade -U -a && \ - apk --no-cache add \ - yq \ - cri-tools@kubezero \ - kubeadm@kubezero \ - kubectl@kubezero \ - etcd-ctl@testing \ - restic@testing \ - helm@testing - -ADD v${KUBE_VERSION}/kubezero.sh /usr/bin -ADD v${KUBE_VERSION}/kubeadm /opt/kubeadm - -ENTRYPOINT ["kubezero.sh"] diff --git a/containers/admin/Makefile b/containers/admin/Makefile deleted file mode 100644 index 5acb8b2..0000000 --- a/containers/admin/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -VERSION ?= 1.21.9 -ALPINE_VERSION ?= 3.15 -REGISTRY := public.ecr.aws/zero-downtime -REPOSITORY := kubezero-admin -TAG := $(REPOSITORY):v$(VERSION) -KUBE_VERSION := $(shell echo $(VERSION) | sed -e 's/\.[[:digit:]]*$$//') - -.PHONY: build push clean scan - -all: build push - -build: - podman build --rm --squash-all --build-arg KUBE_VERSION=$(KUBE_VERSION) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) -t $(TAG) . - -push: - aws ecr-public get-login-password --region us-east-1 | podman login --username AWS --password-stdin $(REGISTRY) - podman tag $(TAG) $(REGISTRY)/$(TAG) - podman push $(REGISTRY)/$(TAG) - -clean: - podman image prune -f - -scan: - podman system service& - sleep 5; trivy $(TAG) diff --git a/containers/admin/v1.21/kubeadm/templates/resources/50-aws-iam-authenticator-crds.yaml b/containers/admin/v1.21/kubeadm/templates/resources/50-aws-iam-authenticator-crds.yaml deleted file mode 100644 index f17f214..0000000 --- a/containers/admin/v1.21/kubeadm/templates/resources/50-aws-iam-authenticator-crds.yaml +++ /dev/null @@ -1,34 +0,0 @@ -{{- if .Values.api.awsIamAuth.enabled }} -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: iamidentitymappings.iamauthenticator.k8s.aws -spec: - group: iamauthenticator.k8s.aws - version: v1alpha1 - scope: Cluster - names: - plural: iamidentitymappings - singular: iamidentitymapping - kind: IAMIdentityMapping - categories: - - all - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - required: - - arn - - username - properties: - arn: - type: string - username: - type: string - groups: - type: array - items: - type: string -{{- end }} diff --git a/containers/admin/v1.21/README.md b/releases/v1.22/README.md similarity index 100% rename from containers/admin/v1.21/README.md rename to releases/v1.22/README.md diff --git a/containers/admin/v1.21/kubezero.sh b/releases/v1.22/kubezero.sh similarity index 58% rename from containers/admin/v1.21/kubezero.sh rename to releases/v1.22/kubezero.sh index 38061ef..e377393 100755 --- a/containers/admin/v1.21/kubezero.sh +++ b/releases/v1.22/kubezero.sh @@ -1,5 +1,4 @@ #!/bin/sh -set -e if [ -n "$DEBUG" ]; then set -x @@ -9,9 +8,8 @@ fi # Export vars to ease use in debug_shell etc export WORKDIR=/tmp/kubezero export HOSTFS=/host -export VERSION=v1.21 -export NETWORK_VERSION=0.1.7 -export ADDONS_VERSION=0.4.3 +export CHARTS=/charts +export VERSION=v1.22 export KUBECONFIG="${HOSTFS}/root/.kube/config" @@ -21,6 +19,8 @@ export ETCDCTL_CACERT=${HOSTFS}/etc/kubernetes/pki/etcd/ca.crt export ETCDCTL_CERT=${HOSTFS}/etc/kubernetes/pki/apiserver-etcd-client.crt export ETCDCTL_KEY=${HOSTFS}/etc/kubernetes/pki/apiserver-etcd-client.key +mkdir -p ${WORKDIR} + # Generic retry utility retry() { local tries=$1 @@ -43,11 +43,11 @@ _kubeadm() { # Render cluster config render_kubeadm() { - helm template /opt/kubeadm --output-dir ${WORKDIR} -f ${HOSTFS}/etc/kubernetes/kubezero.yaml + helm template $CHARTS/kubeadm --output-dir ${WORKDIR} -f ${HOSTFS}/etc/kubernetes/kubezero.yaml # Assemble kubeadm config cat /dev/null > ${HOSTFS}/etc/kubernetes/kubeadm.yaml - for f in Cluster Init KubeProxy Kubelet; do + for f in Cluster Init Join KubeProxy Kubelet; do # echo "---" >> /etc/kubernetes/kubeadm.yaml cat ${WORKDIR}/kubeadm/templates/${f}Configuration.yaml >> ${HOSTFS}/etc/kubernetes/kubeadm.yaml done @@ -61,14 +61,18 @@ render_kubeadm() { parse_kubezero() { - [ -f ${HOSTFS}/etc/kubernetes/kubezero.yaml ] || { echo "Missing /etc/kubernetes/kubezero.yaml!"; exit 1; } + [ -f ${HOSTFS}/etc/kubernetes/kubezero.yaml ] || { echo "Missing /etc/kubernetes/kubezero.yaml!"; return 1; } export KUBE_VERSION=$(kubeadm version -o yaml | yq eval .clientVersion.gitVersion -) export CLUSTERNAME=$(yq eval '.clusterName' ${HOSTFS}/etc/kubernetes/kubezero.yaml) export ETCD_NODENAME=$(yq eval '.etcd.nodeName' ${HOSTFS}/etc/kubernetes/kubezero.yaml) + export NODENAME=$(yq eval '.nodeName' ${HOSTFS}/etc/kubernetes/kubezero.yaml) export AWS_IAM_AUTH=$(yq eval '.api.awsIamAuth.enabled' ${HOSTFS}/etc/kubernetes/kubezero.yaml) export AWS_NTH=$(yq eval '.addons.aws-node-termination-handler.enabled' ${HOSTFS}/etc/kubernetes/kubezero.yaml) + + # From here on bail out, allows debug_shell even in error cases + set -e } @@ -114,17 +118,9 @@ post_kubeadm() { parse_kubezero if [ "$1" == 'upgrade' ]; then - ### PRE 1.21 specific + ### PRE 1.22 specific ##################### - # Migrate aws-iam-authenticator from file certs to secret - if [ "$AWS_IAM_AUTH" == "true" ]; then - kubectl get secrets -n kube-system aws-iam-certs || \ - kubectl create secret generic aws-iam-certs -n kube-system \ - --from-file=key.pem=${HOSTFS}/etc/kubernetes/pki/aws-iam-authenticator.key \ - --from-file=cert.pem=${HOSTFS}/etc/kubernetes/pki/aws-iam-authenticator.crt - fi - ##################### render_kubeadm @@ -132,7 +128,7 @@ if [ "$1" == 'upgrade' ]; then pre_kubeadm # Upgrade - _kubeadm upgrade apply -y --experimental-patches /tmp/patches + _kubeadm upgrade apply -y post_kubeadm @@ -142,20 +138,16 @@ if [ "$1" == 'upgrade' ]; then fi ### POST 1.21 specific - # Delete any previous aws-node-term config as they changed the labels ;-( - kubectl delete deployment aws-node-termination-handler -n kube-system || true ###################### - helm repo add kubezero https://cdn.zero-downtime.net/charts/ - # network yq eval '.network // ""' ${HOSTFS}/etc/kubernetes/kubezero.yaml > _values.yaml - helm template kubezero/kubezero-network --version $NETWORK_VERSION --namespace kube-system --include-crds --name-template network \ + helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \ -f _values.yaml --kube-version $KUBE_VERSION | kubectl apply --namespace kube-system -f - $LOG # addons yq eval '.addons // ""' ${HOSTFS}/etc/kubernetes/kubezero.yaml > _values.yaml - helm template kubezero/kubezero-addons --version $ADDONS_VERSION --namespace kube-system --include-crds --name-template addons \ + helm template $CHARTS/kubezero-addons --namespace kube-system --include-crds --name-template addons \ -f _values.yaml --kube-version $KUBE_VERSION | kubectl apply --namespace kube-system -f - $LOG ###################### @@ -181,67 +173,58 @@ if [ "$1" == 'upgrade' ]; then elif [[ "$1" == 'node-upgrade' ]]; then echo "Starting node upgrade ..." - echo "Migrating kubezero.yaml" - yq -i eval '.api.etcdServers = .api.allEtcdEndpoints | .network.multus.enabled = "true"' ${HOSTFS}/etc/kubernetes/kubezero.yaml - - # remove old aws-node-termination-handler config, first new controller will do the right thing - yq -i eval 'del(.addons.aws-node-termination-handler)' ${HOSTFS}/etc/kubernetes/kubezero.yaml - - # AWS - if [ -f ${HOSTFS}/etc/cloudbender/clusterBackup.passphrase ]; then - if [ -f ${HOSTFS}/usr/local/sbin/backup_control_plane.sh ]; then - mv ${HOSTFS}/usr/local/sbin/backup_control_plane.sh ${HOSTFS}/usr/local/sbin/backup_control_plane.disabled - echo "Disabled old cluster backup OS cronjob" - fi - - # enable backup and awsIamAuth & multus - yq -i eval ' - .api.awsIamAuth.enabled = "true" | .api.awsIamAuth.workerNodeRole = .workerNodeRole | .api.awsIamAuth.kubeAdminRole = .kubeAdminRole - | .api.serviceAccountIssuer = .serviceAccountIssuer | .api.apiAudiences = "istio-ca,sts.amazonaws.com" - ' ${HOSTFS}/etc/kubernetes/kubezero.yaml - - export restic_repo=$(grep "export RESTIC_REPOSITORY" ${HOSTFS}/usr/local/sbin/backup_control_plane.disabled | sed -e 's/.*=//' | sed -e 's/"//g') - export restic_pw="$(cat ${HOSTFS}/etc/cloudbender/clusterBackup.passphrase)" - export REGION=$(kubectl get node $NODE_NAME -o yaml | yq eval '.metadata.labels."topology.kubernetes.io/region"' -) - - if [ -n "$restic_repo" ]; then - yq -i eval ' - .addons.clusterBackup.enabled = "true" | .addons.clusterBackup.repository = strenv(restic_repo) | .addons.clusterBackup.password = strenv(restic_pw) - | .addons.clusterBackup.image.tag = strenv(KUBE_VERSION) - | .addons.clusterBackup.extraEnv[0].name = "AWS_DEFAULT_REGION" | .addons.clusterBackup.extraEnv[0].value = strenv(REGION) - ' ${HOSTFS}/etc/kubernetes/kubezero.yaml - fi - fi - echo "All done." -elif [[ "$1" =~ "^(bootstrap|recover|join)$" ]]; then +elif [[ "$1" =~ "^(bootstrap|restore|join)$" ]]; then render_kubeadm + # Esnure clean slate if bootstrap, restore PKI otherwise if [[ "$1" =~ "^(bootstrap)$" ]]; then - # Create all certs during bootstrap - _kubeadm init phase certs all + rm -rf ${HOSTFS}/var/lib/etcd/member else - # Recert certificates for THIS node - rm -f ${HOSTFS}/etc/kubernetes/pki/etcd/peer.* ${HOSTFS}/etc/kubernetes/pki/etcd/server.* ${HOSTFS}/etc/kubernetes/pki/apiserver.* - _kubeadm init phase certs etcd-server - _kubeadm init phase certs etcd-peer - _kubeadm init phase certs apiserver + restic restore latest --no-lock -t / --tag $VERSION + + # Make last etcd snapshot available + cp ${WORKDIR}/etcd_snapshot ${HOSTFS}/etc/kubernetes + + # Put PKI in place + cp -r ${WORKDIR}/pki ${HOSTFS}/etc/kubernetes + + # Always use kubeadm kubectl config to never run into chicken egg with custom auth hooks + cp ${WORKDIR}/admin.conf ${HOSTFS}/root/.kube/config + + # etcd needs to resync during join + if [[ "$1" =~ "^(restore)$" ]]; then + # Only restore etcd data set if none exists already + if [ ! -d ${HOSTFS}/var/lib/etcd/member ]; then + etcdctl snapshot restore ${HOSTFS}/etc/kubernetes/etcd_snapshot \ + --name $ETCD_NODENAME \ + --data-dir="${HOSTFS}/var/lib/etcd" \ + --initial-cluster-token etcd-${CLUSTERNAME} \ + --initial-advertise-peer-urls https://${ETCD_NODENAME}:2380 \ + --initial-cluster $ETCD_NODENAME=https://${ETCD_NODENAME}:2380 + fi + fi fi + # Delete old node certs in case they are around + rm -f ${HOSTFS}/etc/kubernetes/pki/etcd/peer.* ${HOSTFS}/etc/kubernetes/pki/etcd/server.* ${HOSTFS}/etc/kubernetes/pki/etcd/healthcheck-client.* \ + ${HOSTFS}/etc/kubernetes/pki/apiserver* ${HOSTFS}/etc/kubernetes/pki/front-proxy-client.* + + # Issue all certs first, needed for eg. aws-iam-authenticator setup + _kubeadm init phase certs all + pre_kubeadm + # Pull all images + _kubeadm config images pull + + _kubeadm init phase preflight + _kubeadm init phase kubeconfig all + if [[ "$1" =~ "^(join)$" ]]; then - - _kubeadm init phase preflight - _kubeadm init phase kubeconfig all - _kubeadm init phase kubelet-start - - # flush etcd data directory from restore - rm -rf ${HOSTFS}/var/lib/etcd/member - # get current running etcd pods for etcdctl commands while true; do etcd_endpoints=$(kubectl get pods -n kube-system -l component=etcd -o yaml | \ @@ -250,14 +233,30 @@ elif [[ "$1" =~ "^(bootstrap|recover|join)$" ]]; then sleep 3 done - # is our $ETCD_NODENAME already in the etcd cluster ? - # Remove former self first + # if we are NOT member already, flush etcd to be able to join MY_ID=$(etcdctl member list --endpoints=$etcd_endpoints | grep $ETCD_NODENAME | awk '{print $1}' | sed -e 's/,$//') - [ -n "$MY_ID" ] && retry 12 5 5 etcdctl member remove $MY_ID --endpoints=$etcd_endpoints - # Announce new etcd member and capture ETCD_INITIAL_CLUSTER, retry needed in case another node joining causes temp quorum loss - ETCD_ENVS=$(retry 12 5 5 etcdctl member add $ETCD_NODENAME --peer-urls="https://${ETCD_NODENAME}:2380" --endpoints=$etcd_endpoints) - export $(echo "$ETCD_ENVS" | grep ETCD_INITIAL_CLUSTER= | sed -e 's/"//g') + # Failsafe / etcd on ephmeral: we were a member but our dataset is missing + # -> remove former self so we can re-join + if [ -n "$MY_ID" -a ! -d ${HOSTFS}/var/lib/etcd/member ]; then + # Remove former self first + [ -n "$MY_ID" ] && retry 12 5 5 etcdctl member remove $MY_ID --endpoints=$etcd_endpoints + MY_ID="" + fi + + + if [ -z "$MY_ID" ]; then + # flush etcd data directory from restore + rm -rf ${HOSTFS}/var/lib/etcd/member + + # Announce new etcd member and capture ETCD_INITIAL_CLUSTER, retry needed in case another node joining causes temp quorum loss + ETCD_ENVS=$(retry 12 5 5 etcdctl member add $ETCD_NODENAME --peer-urls="https://${ETCD_NODENAME}:2380" --endpoints=$etcd_endpoints) + export $(echo "$ETCD_ENVS" | grep ETCD_INITIAL_CLUSTER= | sed -e 's/"//g') + else + # build initial_cluster string from running cluster + _cluster=$(etcdctl member list --endpoints=$etcd_endpoints -w json | jq -r '.members[] | "\(.name)=\(.peerURLs[]),"') + export ETCD_INITIAL_CLUSTER=$(echo ${_cluster%%,} | sed -e 's/ //g') + fi # Patch kubezero.yaml and re-render to get etcd manifest patched yq eval -i '.etcd.state = "existing" @@ -265,21 +264,36 @@ elif [[ "$1" =~ "^(bootstrap|recover|join)$" ]]; then ' ${HOSTFS}/etc/kubernetes/kubezero.yaml render_kubeadm - # Generate our advanced etcd yaml - _kubeadm init phase etcd local --experimental-patches /tmp/patches - - _kubeadm init phase control-plane all --experimental-patches /tmp/patches - _kubeadm init phase mark-control-plane - _kubeadm init phase kubelet-finalize all - - else - _kubeadm init --experimental-patches /tmp/patches --skip-token-print + # Delete any former self in case forseti did not delete yet + kubectl delete node ${NODENAME} --wait=true || true fi + # Generate our custom etcd yaml + _kubeadm init phase etcd local + _kubeadm init phase control-plane all + + _kubeadm init phase kubelet-start + cp ${HOSTFS}/etc/kubernetes/admin.conf ${HOSTFS}/root/.kube/config # Wait for api to be online - retry 0 10 30 kubectl cluster-info --request-timeout 3 + echo "Waiting for Kubernetes API to be online ..." + retry 0 5 30 kubectl cluster-info --request-timeout 3 >/dev/null + + if [[ ! "$1" =~ "^(join)$" ]]; then + _kubeadm init phase upload-config all + _kubeadm init phase upload-certs --skip-certificate-key-print + + # This sets up the ClusterRoleBindings to allow bootstrap nodes to create CSRs etc. + _kubeadm init phase bootstrap-token --skip-token-print + fi + + _kubeadm init phase mark-control-plane + _kubeadm init phase kubelet-finalize all + + if [[ ! "$1" =~ "^(join)$" ]]; then + _kubeadm init phase addon all + fi # Ensure aws-iam-authenticator secret is in place if [ "$AWS_IAM_AUTH" == "true" ]; then @@ -293,17 +307,15 @@ elif [[ "$1" =~ "^(bootstrap|recover|join)$" ]]; then fi # Install some basics on bootstrap and join for 1.21 to get new modules in place - if [[ "$1" =~ "^(bootstrap|join|recover)$" ]]; then - helm repo add kubezero https://cdn.zero-downtime.net/charts/ - + if [[ "$1" =~ "^(bootstrap|join|restore)$" ]]; then # network yq eval '.network // ""' ${HOSTFS}/etc/kubernetes/kubezero.yaml > _values.yaml - helm template kubezero/kubezero-network --version $NETWORK_VERSION --namespace kube-system --include-crds --name-template network \ + helm template $CHARTS/kubezero-network --namespace kube-system --include-crds --name-template network \ -f _values.yaml --kube-version $KUBE_VERSION | kubectl apply --namespace kube-system -f - $LOG # addons yq eval '.addons // ""' ${HOSTFS}/etc/kubernetes/kubezero.yaml > _values.yaml - helm template kubezero/kubezero-addons --version $ADDONS_VERSION --namespace kube-system --include-crds --name-template addons \ + helm template $CHARTS/kubezero-addons --namespace kube-system --include-crds --name-template addons \ -f _values.yaml --kube-version $KUBE_VERSION | kubectl apply --namespace kube-system -f - $LOG fi @@ -314,8 +326,6 @@ elif [[ "$1" =~ "^(bootstrap|recover|join)$" ]]; then # Since 1.21 we only need to backup etcd + /etc/kubernetes/pki ! elif [ "$1" == 'backup' ]; then - mkdir -p ${WORKDIR} - restic snapshots || restic init || exit 1 etcdctl --endpoints=https://${ETCD_NODENAME}:2379 snapshot save ${WORKDIR}/etcd_snapshot @@ -340,32 +350,11 @@ elif [ "$1" == 'backup' ]; then etcdctl --endpoints=https://${ETCD_NODENAME}:2379 defrag -elif [ "$1" == 'restore' ]; then - mkdir -p ${WORKDIR} - - restic restore latest --no-lock -t / --tag $VERSION - - # Make last etcd snapshot available - cp ${WORKDIR}/etcd_snapshot ${HOSTFS}/etc/kubernetes - - # Put PKI in place - cp -r ${WORKDIR}/pki ${HOSTFS}/etc/kubernetes - - # Always use kubeadm kubectl config to never run into chicken egg with custom auth hooks - cp ${WORKDIR}/admin.conf ${HOSTFS}/root/.kube/config - - etcdctl snapshot restore ${HOSTFS}/etc/kubernetes/etcd_snapshot \ - --name $ETCD_NODENAME \ - --data-dir="${HOSTFS}/var/lib/etcd" \ - --initial-cluster-token etcd-${CLUSTERNAME} \ - --initial-advertise-peer-urls https://${ETCD_NODENAME}:2380 \ - --initial-cluster $ETCD_NODENAME=https://${ETCD_NODENAME}:2380 - - echo "Backup restored." - - elif [ "$1" == 'debug_shell' ]; then echo "Entering debug shell" + + printf "For manual etcdctl commands use:\n # export ETCDCTL_ENDPOINTS=$ETCD_NODENAME:2379\n" + /bin/sh else diff --git a/containers/admin/v1.21/kubezero_121.sh b/releases/v1.22/kubezero_121.sh similarity index 100% rename from containers/admin/v1.21/kubezero_121.sh rename to releases/v1.22/kubezero_121.sh diff --git a/containers/admin/v1.21/migrate_argo.sh b/releases/v1.22/migrate_argo.sh similarity index 100% rename from containers/admin/v1.21/migrate_argo.sh rename to releases/v1.22/migrate_argo.sh diff --git a/containers/admin/v1.21/upgrade_121.sh b/releases/v1.22/upgrade_121.sh similarity index 99% rename from containers/admin/v1.21/upgrade_121.sh rename to releases/v1.22/upgrade_121.sh index c53f5e9..0ca3835 100755 --- a/containers/admin/v1.21/upgrade_121.sh +++ b/releases/v1.22/upgrade_121.sh @@ -8,6 +8,7 @@ VERSION="v1.21.9" unset AWS_DEFAULT_PROFILE nodes=$(kubectl get nodes -l node-role.kubernetes.io/control-plane -o json | jq .items[].metadata.name -r) +nodes="" for node in $nodes; do echo "Deploying node upgrade job on $node..."