From 9f389d5c13d7a111e98484ef07c570940e56d591 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sun, 11 Sep 2022 13:54:56 +0200 Subject: [PATCH] feat: bootstrap / upgrade reorg as part of 1.23 --- Dockerfile | 10 +- admin/kubezero.sh | 84 ++++++++----- admin/libhelm.sh | 110 ++++++++++++++++++ admin/migrate_argo.py | 90 ++++++-------- admin/upgrade_cluster.sh | 56 ++++++--- charts/kubeadm/README.md | 4 +- .../templates/ClusterConfiguration.yaml | 12 +- charts/kubeadm/templates/admin-aws-iam.yaml | 10 +- .../kubeadm/templates/patches/coredns0.yaml | 2 +- .../51-aws-iam-authenticator-deployment.yaml | 2 +- charts/kubeadm/values.yaml | 7 +- charts/kubezero-addons/Chart.yaml | 6 +- charts/kubezero-addons/README.md | 15 ++- charts/kubezero-addons/values.yaml | 48 +++++++- charts/kubezero-auth/README.md | 7 +- charts/kubezero-logging/README.md | 2 +- charts/kubezero-logging/values.yaml | 2 +- charts/kubezero-network/README.md | 9 +- .../charts/calico/templates/calico.yaml | 2 + .../templates/multus/calico-network.yaml | 4 +- .../templates/multus/cilium-network.yaml | 2 +- charts/kubezero-network/values.yaml | 30 +++-- charts/kubezero/Chart.yaml | 2 +- charts/kubezero/README.md | 8 +- charts/kubezero/clusters/README.md | 0 charts/kubezero/templates/addons.yaml | 42 ++++++- charts/kubezero/templates/istio.yaml | 9 -- charts/kubezero/templates/storage.yaml | 2 +- charts/kubezero/values.yaml | 6 +- scripts/publish.sh | 4 + 30 files changed, 428 insertions(+), 159 deletions(-) create mode 100755 admin/libhelm.sh delete mode 100644 charts/kubezero/clusters/README.md diff --git a/Dockerfile b/Dockerfile index 0c8c556..fcb87f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,9 @@ RUN cd /etc/apk/keys && \ jq \ yq \ diffutils \ + bash \ + python3 \ + py3-yaml \ cri-tools@kubezero \ kubeadm@kubezero~=${KUBE_VERSION} \ kubectl@kubezero~=${KUBE_VERSION} \ @@ -22,9 +25,10 @@ RUN cd /etc/apk/keys && \ restic@testing \ helm@testing -ADD admin/kubezero.sh /usr/bin +RUN helm repo add kubezero https://cdn.zero-downtime.net/charts + +ADD admin/kubezero.sh admin/libhelm.sh /usr/bin ADD charts/kubeadm /charts/kubeadm -ADD charts/kubezero-addons /charts/kubezero-addons -ADD charts/kubezero-network /charts/kubezero-network +ADD charts/kubezero /charts/kubezero ENTRYPOINT ["kubezero.sh"] diff --git a/admin/kubezero.sh b/admin/kubezero.sh index 2eb8213..3cc1258 100755 --- a/admin/kubezero.sh +++ b/admin/kubezero.sh @@ -1,10 +1,13 @@ -#!/bin/sh +#!/bin/bash if [ -n "$DEBUG" ]; then set -x LOG="--v=5" fi +# include helm lib +. libhelm.sh + # Export vars to ease use in debug_shell etc export WORKDIR=/tmp/kubezero export HOSTFS=/host @@ -44,7 +47,7 @@ _kubeadm() { # Render cluster config render_kubeadm() { - helm template $CHARTS/kubeadm --output-dir ${WORKDIR} -f ${HOSTFS}/etc/kubernetes/kubezero.yaml + helm template $CHARTS/kubeadm --output-dir ${WORKDIR} -f ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml # Assemble kubeadm config cat /dev/null > ${HOSTFS}/etc/kubernetes/kubeadm.yaml @@ -62,13 +65,17 @@ render_kubeadm() { parse_kubezero() { - [ -f ${HOSTFS}/etc/kubernetes/kubezero.yaml ] || { echo "Missing /etc/kubernetes/kubezero.yaml!"; return 1; } + # remove with 1.24 + if [ ! -f ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml ]; then + [ -f ${HOSTFS}/etc/kubernetes/kubezero.yaml ] && cp ${HOSTFS}/etc/kubernetes/kubezero.yaml ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml + fi - export CLUSTERNAME=$(yq eval '.clusterName' ${HOSTFS}/etc/kubernetes/kubezero.yaml) - export ETCD_NODENAME=$(yq eval '.etcd.nodeName' ${HOSTFS}/etc/kubernetes/kubezero.yaml) - export NODENAME=$(yq eval '.nodeName' ${HOSTFS}/etc/kubernetes/kubezero.yaml) - export PROVIDER_ID=$(yq eval '.providerID' ${HOSTFS}/etc/kubernetes/kubezero.yaml) - export AWS_IAM_AUTH=$(yq eval '.api.awsIamAuth.enabled' ${HOSTFS}/etc/kubernetes/kubezero.yaml) + export CLUSTERNAME=$(yq eval '.global.clusterName' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml) + export HIGHAVAILABLE=$(yq eval '.global.highAvailable // "false"' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml) + export ETCD_NODENAME=$(yq eval '.etcd.nodeName' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml) + export NODENAME=$(yq eval '.nodeName' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml) + export PROVIDER_ID=$(yq eval '.providerID' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml) + export AWS_IAM_AUTH=$(yq eval '.api.awsIamAuth.enabled // "false"' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml) # From here on bail out, allows debug_shell even in error cases set -e @@ -117,11 +124,27 @@ cluster_upgrade() { ### PRE 1.23 specific ##################### - # Migrate addons and network values into CM from kubezero.yaml + # Migrate addons and network values from local kubeadm-values.yaml on controllers into CM + # - remove secrets from addons + # - enable cilium + + if [[ $PROVIDER_ID =~ ^aws ]]; then + REGION=$(echo $PROVIDER_ID | sed -e 's,aws:///,,' -e 's,/.*,,' -e 's/\w$//') + fi + kubectl get cm -n kube-system kubezero-values || \ kubectl create configmap -n kube-system kubezero-values \ - --from-literal addons="$(yq e '.addons | del .clusterBackup.repository | del .clusterBackup.password' ${HOSTFS}/etc/kubernetes/kubezero.yaml)" \ - --from-literal network="$(yq e .network ${HOSTFS}/etc/kubernetes/kubezero.yaml)" + --from-literal values.yaml="$(yq e 'del .addons.clusterBackup.repository | del .addons.clusterBackup.password | \ + .addons.clusterBackup.image.tag =strenv(KUBE_VERSION) | \ + .network.cilium.enabled = true | .network.multus.defaultNetworks = ["cilium"] | \ + .network.cilium.cluster.name = strenv(CLUSTERNAME) | \ + .global.clusterName = strenv(CLUSTERNAME) | \ + .global.highAvailable = strenv(HIGHAVAILABLE) | \ + .global.aws.region = strenv(REGION)' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml)" + + # Create kubeadm-values CM if not available + kubectl get cm -n kube-system kubeadm-values || \ + kubectl create configmap -n kube-system kubeadm-values ##################### @@ -254,10 +277,10 @@ control_plane_node() { export ETCD_INITIAL_CLUSTER=$(echo ${_cluster%%,} | sed -e 's/ //g') fi - # Patch kubezero.yaml and re-render to get etcd manifest patched + # Patch kubeadm-values.yaml and re-render to get etcd manifest patched yq eval -i '.etcd.state = "existing" | .etcd.initialCluster = strenv(ETCD_INITIAL_CLUSTER) - ' ${HOSTFS}/etc/kubernetes/kubezero.yaml + ' ${HOSTFS}/etc/kubernetes/kubeadm-values.yaml render_kubeadm fi @@ -318,30 +341,33 @@ control_plane_node() { apply_module() { - MODULE=$1 + MODULES=$1 - # network - kubectl get configmap -n kube-system kubezero-values -o custom-columns=NAME:".data.$MODULE" --no-headers=true > _values.yaml + kubectl get configmap -n kube-system kubezero-values -o yaml | yq '.data."values.yaml"' > $WORKDIR/_values.yaml - helm template $CHARTS/kubezero-$MODULE --namespace kube-system --name-template $MODULE --skip-crds --set installCRDs=false -f _values.yaml --kube-version $KUBE_VERSION > helm-no-crds.yaml - helm template $CHARTS/kubezero-$MODULE --namespace kube-system --name-template $MODULE --include-crds --set installCRDs=true -f _values.yaml --kube-version $KUBE_VERSION > helm-crds.yaml - diff -e helm-no-crds.yaml helm-crds.yaml | head -n-1 | tail -n+2 > crds.yaml + # Always use embedded kubezero chart + helm template $CHARTS/kubezero -f $WORKDIR/_values.yaml --version ~$KUBE_VERSION --devel --output-dir $WORKDIR - # Only apply if there are actually any crds - if [ -s crds.yaml ]; then - kubectl apply -f crds.yaml --server-side $LOG - fi + # Resolve all the all enabled modules - helm template $CHARTS/kubezero-$MODULE --namespace kube-system --include-crds --name-template $MODULE \ - -f _values.yaml --kube-version $KUBE_VERSION | kubectl apply --namespace kube-system -f - $LOG + [ -z "$MODULES" ] && MODULES="$(ls ${WORKDIR}/kubezero/templates | sed -e 's/.yaml//g')" - echo "Applied KubeZero module: $MODULE" + # CRDs first + for t in $MODULES; do + _helm crds $t + done + + for t in $MODULES; do + _helm apply $t + done + + echo "Applied KubeZero modules: $MODULES" } # backup etcd + /etc/kubernetes/pki backup() { - # Display all ENVs, careful this exposes the password ! + # Display all ENVs, careful this exposes the password ! [ -n "$DEBUG" ] && env restic snapshots || restic init || exit 1 @@ -380,10 +406,10 @@ debug_shell() { printf "For manual etcdctl commands use:\n # export ETCDCTL_ENDPOINTS=$ETCD_NODENAME:2379\n" - /bin/sh + /bin/bash } -# First parse kubezero.yaml +# First parse kubeadm-values.yaml parse_kubezero # Execute tasks diff --git a/admin/libhelm.sh b/admin/libhelm.sh new file mode 100755 index 0000000..b572e85 --- /dev/null +++ b/admin/libhelm.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +# Simulate well-known CRDs being available +API_VERSIONS="-a monitoring.coreos.com/v1 -a snapshot.storage.k8s.io/v1" + +# Waits for max 300s and retries +function wait_for() { + local TRIES=0 + while true; do + eval " $@" && break + [ $TRIES -eq 100 ] && return 1 + let TRIES=$TRIES+1 + sleep 3 + done +} + + +function chart_location() { + echo "$1 --repo https://cdn.zero-downtime.net/charts" +} + + +# make sure namespace exists prior to calling helm as the create-namespace options doesn't work +function create_ns() { + local namespace=$1 + if [ "$namespace" != "kube-system" ]; then + kubectl get ns $namespace || kubectl create ns $namespace + fi +} + + +# delete non kube-system ns +function delete_ns() { + local namespace=$1 + [ "$namespace" != "kube-system" ] && kubectl delete ns $namespace +} + + +# Extract crds via helm calls and apply delta=crds only +function _crds() { + helm template $(chart_location $chart) -n $namespace --name-template $module $targetRevision --skip-crds --set ${module}.installCRDs=false -f $WORKDIR/values.yaml $API_VERSIONS --kube-version $KUBE_VERSION > $WORKDIR/helm-no-crds.yaml + helm template $(chart_location $chart) -n $namespace --name-template $module $targetRevision --include-crds --set ${module}.installCRDs=true -f $WORKDIR/values.yaml $API_VERSIONS --kube-version $KUBE_VERSION > $WORKDIR/helm-crds.yaml + diff -e $WORKDIR/helm-no-crds.yaml $WORKDIR/helm-crds.yaml | head -n-1 | tail -n+2 > $WORKDIR/crds.yaml + + # Only apply if there are actually any crds + if [ -s $WORKDIR/crds.yaml ]; then + kubectl apply -f $WORKDIR/crds.yaml --server-side + fi +} + + +# helm template | kubectl apply -f - +# confine to one namespace if possible +function apply() { + helm template $(chart_location $chart) -n $namespace --name-template $module $targetRevision --skip-crds -f $WORKDIR/values.yaml $API_VERSIONS --kube-version $KUBE_VERSION $@ \ + | python3 -c ' +#!/usr/bin/python3 +import yaml +import sys + +for manifest in yaml.safe_load_all(sys.stdin): + if manifest: + if "metadata" in manifest and "namespace" not in manifest["metadata"]: + manifest["metadata"]["namespace"] = sys.argv[1] + print("---") + print(yaml.dump(manifest))' $namespace > $WORKDIR/helm.yaml + + kubectl $action -f $WORKDIR/helm.yaml && rc=$? || rc=$? +} + + +function _helm() { + local action=$1 + local module=$2 + + local chart="$(yq eval '.spec.source.chart' $WORKDIR/kubezero/templates/${module}.yaml)" + local namespace="$(yq eval '.spec.destination.namespace' $WORKDIR/kubezero/templates/${module}.yaml)" + + targetRevision="" + _version="$(yq eval '.spec.source.targetRevision' $WORKDIR/kubezero/templates/${module}.yaml)" + + [ -n "$_version" ] && targetRevision="--version $_version" + + yq eval '.spec.source.helm.values' $WORKDIR/kubezero/templates/${module}.yaml > $WORKDIR/values.yaml + + if [ $action == "crds" ]; then + # Allow custom CRD handling + declare -F ${module}-crds && ${module}-crds || _crds + + elif [ $action == "apply" ]; then + # namespace must exist prior to apply + create_ns $namespace + + # Optional pre hook + declare -F ${module}-pre && ${module}-pre + + apply + + # Optional post hook + declare -F ${module}-post && ${module}-post + + elif [ $action == "delete" ]; then + apply + + # Delete dedicated namespace if not kube-system + [ -n "$DELETE_NS" ] && delete_ns $namespace + fi + + return 0 +} diff --git a/admin/migrate_argo.py b/admin/migrate_argo.py index c60f6d7..ee98add 100755 --- a/admin/migrate_argo.py +++ b/admin/migrate_argo.py @@ -11,11 +11,34 @@ yaml.explicit_start = True yaml.indent(mapping=2, sequence=4, offset=2) -parser = argparse.ArgumentParser(description="Update Route53 entries") +def rec_sort(d): + if isinstance(d, dict): + res = dict() + + # Always have "enabled" first if present + if "enabled" in d.keys(): + res["enabled"] = rec_sort(d["enabled"]) + d.pop("enabled") + + # next is "name" if present + if "name" in d.keys(): + res["name"] = rec_sort(d["name"]) + d.pop("name") + + for k in sorted(d.keys()): + res[k] = rec_sort(d[k]) + return res + if isinstance(d, list): + for idx, elem in enumerate(d): + d[idx] = rec_sort(elem) + return d + + +parser = argparse.ArgumentParser(description="Migrate ArgoCD Kubezero values to new cluster config") parser.add_argument( "--version", dest="version", - default="1.22.8-10", + default="1.23.10", action="store", required=False, help="Update KubeZero version", @@ -34,62 +57,25 @@ values = yaml.load(application["spec"]["source"]["helm"]["values"]) ### Do your thing -# New Istio Gateway charts -if "private" in values["istio-ingress"]: - values["istio-private-ingress"] = { - "enabled": True, - "certificates": values["istio-ingress"]["private"]["certificates"].copy() - } +# migrate ClusterName to clusterName +if "ClusterName" in values: + values["clusterName"] = values["ClusterName"] + values.pop("ClusterName") - if "gateway" in values["istio-ingress"]["private"]: - values["istio-private-ingress"]["gateway"] = {} - - try: - values["istio-private-ingress"]["gateway"]["replicaCount"] = values["istio-ingress"]["private"]["gateway"]["replicaCount"] - except KeyError: - pass - - if "ports" in values["istio-ingress"]["private"]["gateway"]: - values["istio-private-ingress"]["gateway"]["service"] = {} - values["istio-private-ingress"]["gateway"]["service"]["ports"] = [] - for port in values["istio-ingress"]["private"]["gateway"]["ports"]: - if port["name"] not in ["status-port", "http2", "https"]: - values["istio-private-ingress"]["gateway"]["service"]["ports"].append(port) - - values["istio-ingress"].pop("private") - -if "public" in values["istio-ingress"]: - values["istio-ingress"]["certificates"] = values["istio-ingress"]["public"]["certificates"].copy() - - if "gateway" in values["istio-ingress"]["public"]: - values["istio-ingress"]["gateway"] = {} - - try: - values["istio-ingress"]["gateway"]["replicaCount"] = values["istio-ingress"]["public"]["gateway"]["replicaCount"] - except KeyError: - pass - - if "ports" in values["istio-ingress"]["public"]["gateway"]: - values["istio-ingress"]["gateway"]["service"] = {} - values["istio-ingress"]["gateway"]["service"]["ports"] = [] - for port in values["istio-ingress"]["public"]["gateway"]["ports"]: - if port["name"] not in ["status-port", "http2", "https"]: - values["istio-ingress"]["gateway"]["service"]["ports"].append(port) - - values["istio-ingress"].pop("public") - -if "global" in values["istio-ingress"]: - values["istio-ingress"].pop("global") - -# Remove Kiam -if "kiam" in values: - values.pop("kiam") +# Create new clusterwide cloudprovider data if possible +try: + if values["cert-manager"]["clusterIssuer"]["solvers"][0]["dns01"]["route53"]["regions"]: + if "aws" not in values: + values["aws"] = {} + values["aws"]["region"] = values["cert-manager"]["clusterIssuer"]["solvers"][0]["dns01"]["route53"]["region"] +except KeyError: + pass ### End # Merge new values buffer = io.StringIO() -yaml.dump(values, buffer) +yaml.dump(rec_sort(values), buffer) application["spec"]["source"]["helm"]["values"] = buffer.getvalue() # Dump final yaml diff --git a/admin/upgrade_cluster.sh b/admin/upgrade_cluster.sh index 8b9cc6c..67faa8e 100755 --- a/admin/upgrade_cluster.sh +++ b/admin/upgrade_cluster.sh @@ -1,13 +1,16 @@ #!/bin/bash -e -VERSION="v1.23" +VERSION="v1.23.10-1" [ -n "$DEBUG" ] && set -x # unset any AWS_DEFAULT_PROFILE as it will break aws-iam-auth unset AWS_DEFAULT_PROFILE -controller_nodes_upgrade() { + +all_nodes_upgrade() { + CMD="$1" + echo "Deploying node upgrade daemonSet..." cat </dev/null -while true; do - kubectl logs kubezero-upgrade-${VERSION//.} -n kube-system -f 2>/dev/null && break - sleep 3 -done -kubectl delete pod kubezero-upgrade-${VERSION//.} -n kube-system + kubectl wait pod kubezero-upgrade-${VERSION//.} -n kube-system --timeout 120s --for=condition=initialized 2>/dev/null + while true; do + kubectl logs kubezero-upgrade-${VERSION//.} -n kube-system -f 2>/dev/null && break + sleep 3 + done + kubectl delete pod kubezero-upgrade-${VERSION//.} -n kube-system +} + +all_nodes_upgrade "mount --make-shared /host/sys/fs/cgroup; mount --make-shared /host/sys;" + +control_plane_upgrade cluster_upgrade + +echo "Adjust kubezero-values CM !!" +read + +#kubectl delete ds kube-multus-ds -n kube-system + +control_plane_upgrade "apply_network, apply_addons" +exit 0 + +kubectl rollout restart daemonset/calico-node -n kube-system +kubectl rollout restart daemonset/cilium -n kube-system + +kubectl rollout restart daemonset/kube-multus-ds -n kube-system diff --git a/charts/kubeadm/README.md b/charts/kubeadm/README.md index 20a2273..6b7c5aa 100644 --- a/charts/kubeadm/README.md +++ b/charts/kubeadm/README.md @@ -30,12 +30,12 @@ Kubernetes: `>= 1.20.0` | api.listenPort | int | `6443` | | | api.oidcEndpoint | string | `""` | s3://${CFN[ConfigBucket]}/k8s/$CLUSTERNAME | | api.serviceAccountIssuer | string | `""` | https://s3.${REGION}.amazonaws.com/${CFN[ConfigBucket]}/k8s/$CLUSTERNAME | -| clusterName | string | `"pleasechangeme"` | | | domain | string | `"changeme.org"` | | | etcd.extraArgs | object | `{}` | | | etcd.nodeName | string | `"etcd"` | | | etcd.state | string | `"new"` | | -| highAvailable | bool | `false` | | +| global.clusterName | string | `"pleasechangeme"` | | +| global.highAvailable | bool | `false` | | | listenAddress | string | `"0.0.0.0"` | Needs to be set to primary node IP | | nodeName | string | `"kubezero-node"` | set to $HOSTNAME | | protectKernelDefaults | bool | `false` | | diff --git a/charts/kubeadm/templates/ClusterConfiguration.yaml b/charts/kubeadm/templates/ClusterConfiguration.yaml index 7a73e22..44f374b 100644 --- a/charts/kubeadm/templates/ClusterConfiguration.yaml +++ b/charts/kubeadm/templates/ClusterConfiguration.yaml @@ -1,7 +1,7 @@ apiVersion: kubeadm.k8s.io/v1beta3 kind: ClusterConfiguration kubernetesVersion: {{ .Chart.Version }} -clusterName: {{ .Values.clusterName }} +clusterName: {{ .Values.global.clusterName }} featureGates: UnversionedKubeletConfigMap: true controlPlaneEndpoint: {{ .Values.api.endpoint }} @@ -13,12 +13,12 @@ etcd: extraArgs: ### DNS discovery #discovery-srv: {{ .Values.domain }} - #discovery-srv-name: {{ .Values.clusterName }} + #discovery-srv-name: {{ .Values.global.clusterName }} advertise-client-urls: https://{{ .Values.etcd.nodeName }}:2379 initial-advertise-peer-urls: https://{{ .Values.etcd.nodeName }}:2380 initial-cluster: {{ include "kubeadm.etcd.initialCluster" .Values.etcd | quote }} initial-cluster-state: {{ .Values.etcd.state }} - initial-cluster-token: etcd-{{ .Values.clusterName }} + initial-cluster-token: etcd-{{ .Values.global.clusterName }} name: {{ .Values.etcd.nodeName }} listen-peer-urls: https://{{ .Values.listenAddress }}:2380 listen-client-urls: https://{{ .Values.listenAddress }}:2379 @@ -40,13 +40,13 @@ controllerManager: extraArgs: profiling: "false" terminated-pod-gc-threshold: "300" - leader-elect: {{ .Values.highAvailable | quote }} + leader-elect: {{ .Values.global.highAvailable | quote }} logging-format: json feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }} scheduler: extraArgs: profiling: "false" - leader-elect: {{ .Values.highAvailable | quote }} + leader-elect: {{ .Values.global.highAvailable | quote }} logging-format: json feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }} apiServer: @@ -73,7 +73,7 @@ apiServer: {{- end }} feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }} enable-admission-plugins: DenyServiceExternalIPs,NodeRestriction,EventRateLimit - # {{- if .Values.highAvailable }} + # {{- if .Values.global.highAvailable }} # goaway-chance: ".001" # {{- end }} logging-format: json diff --git a/charts/kubeadm/templates/admin-aws-iam.yaml b/charts/kubeadm/templates/admin-aws-iam.yaml index 4d91b3a..fe31686 100644 --- a/charts/kubeadm/templates/admin-aws-iam.yaml +++ b/charts/kubeadm/templates/admin-aws-iam.yaml @@ -4,13 +4,13 @@ kind: Config clusters: - cluster: server: https://{{ .Values.api.endpoint }} - name: {{ .Values.clusterName }} + name: {{ .Values.global.clusterName }} contexts: - context: - cluster: {{ .Values.clusterName }} + cluster: {{ .Values.global.clusterName }} user: kubernetes-admin - name: kubernetes-admin@{{ .Values.clusterName }} -current-context: kubernetes-admin@{{ .Values.clusterName }} + name: kubernetes-admin@{{ .Values.global.clusterName }} +current-context: kubernetes-admin@{{ .Values.global.clusterName }} preferences: {} users: - name: kubernetes-admin @@ -21,7 +21,7 @@ users: args: - "token" - "-i" - - "{{ .Values.clusterName }}" + - "{{ .Values.global.clusterName }}" - "-r" - "{{ .Values.api.awsIamAuth.kubeAdminRole }}" {{- end }} diff --git a/charts/kubeadm/templates/patches/coredns0.yaml b/charts/kubeadm/templates/patches/coredns0.yaml index 102e90f..477219a 100644 --- a/charts/kubeadm/templates/patches/coredns0.yaml +++ b/charts/kubeadm/templates/patches/coredns0.yaml @@ -1,5 +1,5 @@ spec: - replicas: {{ ternary 3 1 .Values.highAvailable }} + replicas: {{ ternary 3 1 .Values.global.highAvailable }} template: spec: containers: diff --git a/charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml b/charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml index 607bfb6..d4c7c51 100644 --- a/charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml +++ b/charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml @@ -75,7 +75,7 @@ metadata: k8s-app: aws-iam-authenticator data: config.yaml: | - clusterID: {{ .Values.clusterName }} + clusterID: {{ .Values.global.clusterName }} --- apiVersion: apps/v1 diff --git a/charts/kubeadm/values.yaml b/charts/kubeadm/values.yaml index 33a9af2..6d2be7b 100644 --- a/charts/kubeadm/values.yaml +++ b/charts/kubeadm/values.yaml @@ -1,4 +1,7 @@ -clusterName: pleasechangeme +global: + clusterName: pleasechangeme + highAvailable: false + # -- set to $HOSTNAME nodeName: kubezero-node domain: changeme.org @@ -22,8 +25,6 @@ api: workerNodeRole: "arn:aws:iam::000000000000:role/KubernetesNode" kubeAdminRole: "arn:aws:iam::000000000000:role/KubernetesNode" -highAvailable: false - etcd: nodeName: etcd state: new diff --git a/charts/kubezero-addons/Chart.yaml b/charts/kubezero-addons/Chart.yaml index f18a9e6..e01a57b 100644 --- a/charts/kubezero-addons/Chart.yaml +++ b/charts/kubezero-addons/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-addons description: KubeZero umbrella chart for various optional cluster addons type: application -version: 0.6.0 +version: 0.6.1 appVersion: v1.23.10 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -23,4 +23,8 @@ dependencies: version: 1.11.0 repository: https://kubernetes-sigs.github.io/external-dns/ condition: external-dns.enabled + - name: cluster-autoscaler + version: 9.21.0 + repository: https://kubernetes.github.io/autoscaler + condition: cluster-autoscaler.enabled kubeVersion: ">= 1.20.0" diff --git a/charts/kubezero-addons/README.md b/charts/kubezero-addons/README.md index f17bc4d..71e277b 100644 --- a/charts/kubezero-addons/README.md +++ b/charts/kubezero-addons/README.md @@ -1,6 +1,6 @@ # kubezero-addons -![Version: 0.6.0](https://img.shields.io/badge/Version-0.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.23.10](https://img.shields.io/badge/AppVersion-v1.23.10-informational?style=flat-square) +![Version: 0.6.1](https://img.shields.io/badge/Version-0.6.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.23.10](https://img.shields.io/badge/AppVersion-v1.23.10-informational?style=flat-square) KubeZero umbrella chart for various optional cluster addons @@ -20,6 +20,7 @@ Kubernetes: `>= 1.20.0` |------------|------|---------| | | aws-node-termination-handler | 0.18.5 | | https://kubernetes-sigs.github.io/external-dns/ | external-dns | 1.11.0 | +| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.21.0 | # MetalLB @@ -59,7 +60,17 @@ Device plugin for [AWS Neuron](https://aws.amazon.com/machine-learning/neuron/) | aws-node-termination-handler.useProviderId | bool | `true` | | | awsNeuron.enabled | bool | `false` | | | awsNeuron.image.name | string | `"public.ecr.aws/neuron/neuron-device-plugin"` | | -| awsNeuron.image.tag | string | `"1.9.0.0"` | | +| awsNeuron.image.tag | string | `"1.9.3.0"` | | +| cluster-autoscaler.autoDiscovery.clusterName | string | `""` | | +| cluster-autoscaler.awsRegion | string | `"us-west-2"` | | +| cluster-autoscaler.enabled | bool | `false` | | +| cluster-autoscaler.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | | +| cluster-autoscaler.podDisruptionBudget | bool | `false` | | +| cluster-autoscaler.prometheusRule.enabled | bool | `false` | | +| cluster-autoscaler.serviceMonitor.enabled | bool | `false` | | +| cluster-autoscaler.serviceMonitor.interval | string | `"30s"` | | +| cluster-autoscaler.tolerations[0].effect | string | `"NoSchedule"` | | +| cluster-autoscaler.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | clusterBackup.enabled | bool | `false` | | | clusterBackup.extraEnv | list | `[]` | | | clusterBackup.image.name | string | `"public.ecr.aws/zero-downtime/kubezero-admin"` | | diff --git a/charts/kubezero-addons/values.yaml b/charts/kubezero-addons/values.yaml index 057e0c3..92dab88 100644 --- a/charts/kubezero-addons/values.yaml +++ b/charts/kubezero-addons/values.yaml @@ -81,7 +81,53 @@ awsNeuron: image: name: public.ecr.aws/neuron/neuron-device-plugin - tag: 1.9.0.0 + tag: 1.9.3.0 + +cluster-autoscaler: + enabled: false + + autoDiscovery: + clusterName: "" + awsRegion: "us-west-2" + + serviceMonitor: + enabled: false + interval: 30s + + prometheusRule: + enabled: false + + # Disable pdb for now + podDisruptionBudget: false + + #securityContext: + # runAsNonRoot: true + + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + + # On AWS enable Projected Service Accounts to assume IAM role + #extraEnv: + # AWS_ROLE_ARN: + # AWS_WEB_IDENTITY_TOKEN_FILE: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + # AWS_STS_REGIONAL_ENDPOINTS: "regional" + + #extraVolumes: + #- name: aws-token + # projected: + # sources: + # - serviceAccountToken: + # path: token + # expirationSeconds: 86400 + # audience: "sts.amazonaws.com" + + #extraVolumeMounts: + #- name: aws-token + # mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" + # readOnly: true external-dns: enabled: false diff --git a/charts/kubezero-auth/README.md b/charts/kubezero-auth/README.md index ea2b288..dc2fdcf 100644 --- a/charts/kubezero-auth/README.md +++ b/charts/kubezero-auth/README.md @@ -1,6 +1,6 @@ # kubezero-auth -![Version: 0.2.2](https://img.shields.io/badge/Version-0.2.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 18.0.1](https://img.shields.io/badge/AppVersion-18.0.1-informational?style=flat-square) +![Version: 0.2.4](https://img.shields.io/badge/Version-0.2.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 19.0.1](https://img.shields.io/badge/AppVersion-19.0.1-informational?style=flat-square) KubeZero umbrella chart for all things Authentication and Identity management @@ -18,8 +18,8 @@ Kubernetes: `>= 1.20.0` | Repository | Name | Version | |------------|------|---------| -| https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.4 | -| https://charts.bitnami.com/bitnami | postgresql | 11.6.7 | +| https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.5 | +| https://charts.bitnami.com/bitnami | postgresql | 11.8.1 | # Keycloak @@ -42,6 +42,7 @@ https://github.com/keycloak/keycloak/tree/main/operator | keycloak.istio.url | string | `""` | | | keycloak.metrics.enabled | bool | `false` | | | postgresql.auth.database | string | `"keycloak"` | | +| postgresql.auth.existingSecret | string | `"kubezero-auth-postgresql"` | | | postgresql.auth.username | string | `"keycloak"` | | | postgresql.enabled | bool | `false` | | | postgresql.primary.persistence.size | string | `"1Gi"` | | diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md index 7d5cbb8..a4103ac 100644 --- a/charts/kubezero-logging/README.md +++ b/charts/kubezero-logging/README.md @@ -88,7 +88,7 @@ Kubernetes: `>= 1.20.0` | fluent-bit.daemonSetVolumes[1].hostPath.path | string | `"/var/lib/containers/logs"` | | | fluent-bit.daemonSetVolumes[1].name | string | `"newlog"` | | | fluent-bit.enabled | bool | `false` | | -| fluent-bit.image.tag | string | `"1.9.3"` | | +| fluent-bit.image.tag | string | `"1.9.8"` | | | fluent-bit.luaScripts."kubezero.lua" | string | `"function nest_k8s_ns(tag, timestamp, record)\n if not record['kubernetes']['namespace_name'] then\n return 0, 0, 0\n end\n new_record = {}\n for key, val in pairs(record) do\n if key == 'kube' then\n new_record[key] = {}\n new_record[key][record['kubernetes']['namespace_name']] = record[key]\n else\n new_record[key] = record[key]\n end\n end\n return 1, timestamp, new_record\nend\n"` | | | fluent-bit.resources.limits.memory | string | `"64Mi"` | | | fluent-bit.resources.requests.cpu | string | `"20m"` | | diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 3dea21e..0e1968d 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -244,7 +244,7 @@ fluent-bit: image: #repository: public.ecr.aws/zero-downtime/fluent-bit - tag: 1.9.7 + tag: 1.9.8 serviceMonitor: enabled: false diff --git a/charts/kubezero-network/README.md b/charts/kubezero-network/README.md index ca8b01b..29637fa 100644 --- a/charts/kubezero-network/README.md +++ b/charts/kubezero-network/README.md @@ -34,18 +34,21 @@ Kubernetes: `>= 1.20.0` | cilium.cluster.name | string | `"default"` | | | cilium.cni.binPath | string | `"/usr/libexec/cni"` | | | cilium.cni.exclusive | bool | `false` | | +| cilium.containerRuntime.integration | string | `"crio"` | | | cilium.enabled | bool | `false` | | -| cilium.hostServices.enabled | bool | `true` | | | cilium.hubble.enabled | bool | `false` | | -| cilium.ipam.operator.clusterPoolIPv4PodCIDRList[0] | string | `"10.0.0.0/16"` | | +| cilium.ipam.operator.clusterPoolIPv4PodCIDRList[0] | string | `"10.1.0.0/16"` | | +| cilium.l2NeighDiscovery.enabled | bool | `false` | | | cilium.l7Proxy | bool | `false` | | -| cilium.nodePort.enabled | bool | `true` | | +| cilium.nodePort.enabled | bool | `false` | | | cilium.operator.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | | | cilium.operator.replicas | int | `1` | | | cilium.operator.tolerations[0].effect | string | `"NoSchedule"` | | | cilium.operator.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| cilium.policyEnforcementMode | string | `"audit"` | | | cilium.prometheus.enabled | bool | `false` | | | cilium.prometheus.port | int | `9091` | | +| cilium.securityContext.privileged | bool | `true` | | | cilium.tunnel | string | `"geneve"` | | | metallb.controller.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | | | metallb.controller.tolerations[0].effect | string | `"NoSchedule"` | | diff --git a/charts/kubezero-network/charts/calico/templates/calico.yaml b/charts/kubezero-network/charts/calico/templates/calico.yaml index bb5937a..d9ad1d7 100644 --- a/charts/kubezero-network/charts/calico/templates/calico.yaml +++ b/charts/kubezero-network/charts/calico/templates/calico.yaml @@ -424,6 +424,8 @@ spec: # Auto-detect the BGP IP address. - name: IP value: "autodetect" + - name: IP_AUTODETECTION_METHOD + value: "interface=eth.*" # Enable IPIP - name: CALICO_IPV4POOL_IPIP value: "Never" diff --git a/charts/kubezero-network/templates/multus/calico-network.yaml b/charts/kubezero-network/templates/multus/calico-network.yaml index a0c41b4..e7c8062 100644 --- a/charts/kubezero-network/templates/multus/calico-network.yaml +++ b/charts/kubezero-network/templates/multus/calico-network.yaml @@ -1,4 +1,4 @@ -{{- if .Values.calico.enabled }} +{{- if and .Values.multus.enabled .Values.calico.enabled }} apiVersion: k8s.cni.cncf.io/v1 kind: NetworkAttachmentDefinition metadata: @@ -11,7 +11,7 @@ spec: "log_level": "info", "log_file_path": "/var/log/calico/cni/cni.log", "datastore_type": "kubernetes", - "mtu": 8941, + "mtu": {{ .Values.calico.mtu }}, "ipam": { "type": "calico-ipam" }, diff --git a/charts/kubezero-network/templates/multus/cilium-network.yaml b/charts/kubezero-network/templates/multus/cilium-network.yaml index 4f05e67..48ad28b 100644 --- a/charts/kubezero-network/templates/multus/cilium-network.yaml +++ b/charts/kubezero-network/templates/multus/cilium-network.yaml @@ -1,4 +1,4 @@ -{{- if .Values.cilium.enabled }} +{{- if and .Values.multus.enabled .Values.cilium.enabled }} apiVersion: k8s.cni.cncf.io/v1 kind: NetworkAttachmentDefinition metadata: diff --git a/charts/kubezero-network/values.yaml b/charts/kubezero-network/values.yaml index 425e475..e4fc552 100644 --- a/charts/kubezero-network/values.yaml +++ b/charts/kubezero-network/values.yaml @@ -28,13 +28,30 @@ multus: cilium: enabled: false + containerRuntime: + integration: crio + + # Until we figured out AppArmore on Alpine and Gentoo + securityContext: + privileged: true + cni: binPath: "/usr/libexec/cni" #-- Ensure this is false if multus is enabled exclusive: false + # chainingMode: generic-veth + + # Until we switch to Cilium + #bpf: + # hostLegacyRouting: true + # tproxy: false + + # enableIPv4Masquerade: false + # enableIdentityMark: false + policyEnforcementMode: "audit" cluster: - # This should match the second octet + 1 of clusterPoolIPv4PodCIDRList, + # This should match the second octet of clusterPoolIPv4PodCIDRList # to prevent IP space overlap and easy tracking id: 1 name: default @@ -42,17 +59,16 @@ cilium: ipam: operator: clusterPoolIPv4PodCIDRList: - - 10.0.0.0/16 + - 10.1.0.0/16 - hostServices: - enabled: true - - # Does this conflict with Calico in parallel ? + # Should be handled by multus nodePort: - enabled: true + enabled: false # Keep it simple for now l7Proxy: false + l2NeighDiscovery: + enabled: false cgroup: autoMount: diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index f15917a..b55c061 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero description: KubeZero - Root App of Apps chart type: application -version: 1.23.10 +version: 1.23.10-1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero/README.md b/charts/kubezero/README.md index e87677a..7ec2550 100644 --- a/charts/kubezero/README.md +++ b/charts/kubezero/README.md @@ -1,6 +1,6 @@ # kubezero -![Version: 1.23.10](https://img.shields.io/badge/Version-1.23.10-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 1.23.10-1](https://img.shields.io/badge/Version-1.23.10--1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero - Root App of Apps chart @@ -25,8 +25,8 @@ Kubernetes: `>= 1.20.0` | Key | Type | Default | Description | |-----|------|---------|-------------| | HighAvailableControlplane | bool | `false` | | -| addons.enabled | bool | `false` | | -| addons.targetRevision | string | `"0.6.0"` | | +| addons.enabled | bool | `true` | | +| addons.targetRevision | string | `"0.6.1"` | | | argocd.enabled | bool | `false` | | | argocd.istio.enabled | bool | `false` | | | argocd.namespace | string | `"argocd"` | | @@ -59,7 +59,7 @@ Kubernetes: `>= 1.20.0` | metrics.istio.prometheus | object | `{}` | | | metrics.namespace | string | `"monitoring"` | | | metrics.targetRevision | string | `"0.8.1"` | | -| network.enabled | bool | `false` | | +| network.enabled | bool | `true` | | | network.retain | bool | `true` | | | network.targetRevision | string | `"0.3.2"` | | | storage.aws-ebs-csi-driver.enabled | bool | `false` | | diff --git a/charts/kubezero/clusters/README.md b/charts/kubezero/clusters/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/charts/kubezero/templates/addons.yaml b/charts/kubezero/templates/addons.yaml index 1b8c49c..bf7bd92 100644 --- a/charts/kubezero/templates/addons.yaml +++ b/charts/kubezero/templates/addons.yaml @@ -13,7 +13,9 @@ forseti: {{- with index .Values "addons" "aws-node-termination-handler" }} aws-node-termination-handler: {{- toYaml . | nindent 2 }} - enablePrometheusServer: {{ .Values.metrics.enabled }} + {{- with $.Values.metrics }} + enablePrometheusServer: {{ .enabled }} + {{- end }} {{- end }} {{- with .Values.addons.fuseDevicePlugin }} @@ -31,6 +33,44 @@ external-dns: {{- toYaml . | nindent 2 }} {{- end }} +{{- with index .Values "addons" "cluster-autoscaler" }} +cluster-autoscaler: + {{- toYaml . | nindent 2 }} + autoDiscovery: + clusterName: {{ $.Values.global.clusterName }} + + {{- with $.Values.global.aws }} + awsRegion: {{ .region }} + {{- end }} + + {{- with $.Values.metrics }} + serviceMonitor: + enabled: {{ .enabled }} + prometheusRule: + enabled: {{ .enabled }} + {{- end }} + + {{- with .IamArn }} + extraEnv: + AWS_ROLE_ARN: "{{ . }}" + AWS_WEB_IDENTITY_TOKEN_FILE: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + AWS_STS_REGIONAL_ENDPOINTS: "regional" + extraVolumes: + - name: aws-token + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 86400 + audience: "sts.amazonaws.com" + extraVolumeMounts: + - name: aws-token + mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" + readOnly: true + {{- end }} + +{{- end }} + {{- end }} {{- define "addons-argo" }} diff --git a/charts/kubezero/templates/istio.yaml b/charts/kubezero/templates/istio.yaml index 6f556c8..c862a3b 100644 --- a/charts/kubezero/templates/istio.yaml +++ b/charts/kubezero/templates/istio.yaml @@ -30,17 +30,8 @@ rateLimiting: - group: admissionregistration.k8s.io kind: ValidatingWebhookConfiguration jsonPointers: - - /webhooks/0/clientConfig/caBundle - /webhooks/0/failurePolicy - - /webhooks/1/clientConfig/caBundle - /webhooks/1/failurePolicy - - group: admissionregistration.k8s.io - kind: MutatingWebhookConfiguration - jsonPointers: - - /webhooks/0/clientConfig/caBundle - - /webhooks/1/clientConfig/caBundle - - /webhooks/2/clientConfig/caBundle - - /webhooks/3/clientConfig/caBundle {{- end }} diff --git a/charts/kubezero/templates/storage.yaml b/charts/kubezero/templates/storage.yaml index 3ab7be1..fc4a9bc 100644 --- a/charts/kubezero/templates/storage.yaml +++ b/charts/kubezero/templates/storage.yaml @@ -16,7 +16,7 @@ aws-ebs-csi-driver: enabled: {{ default false (index .Values "storage" "aws-ebs-csi-driver" "enabled")}} controller: replicaCount: {{ ternary 2 1 .Values.HighAvailableControlplane }} - k8sTagClusterId: {{ .Values.ClusterName }} + k8sTagClusterId: {{ .Values.global.clusterName }} env: - name: AWS_ROLE_ARN value: {{ index .Values "storage" "aws-ebs-csi-driver" "IamArn" | quote }} diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 9d25311..4e3c97c 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -7,11 +7,11 @@ kubezero: HighAvailableControlplane: false addons: - enabled: false - targetRevision: 0.6.0 + enabled: true + targetRevision: 0.6.1 network: - enabled: false + enabled: true retain: true targetRevision: 0.3.2 diff --git a/scripts/publish.sh b/scripts/publish.sh index e0761fa..f49c660 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -50,4 +50,8 @@ function publish_chart() { publish_chart + +CF_DIST=E1YFUJXMCXT2RN +aws cloudfront create-invalidation --distribution $CF_DIST --paths "/charts/*" + #reset_index