From 10a69e6cff15dae0ff7a401a81b74d6776a40eb2 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 25 Aug 2021 16:01:02 +0200 Subject: [PATCH] feat: kubezero now installs and tracks Helm charts for each module --- charts/kubezero/Chart.yaml | 5 +- charts/kubezero/README.md | 13 ++-- charts/kubezero/bootstrap.sh | 61 +++++++++------ charts/kubezero/templates/_app.tpl | 14 ++-- charts/kubezero/templates/argocd.yaml | 7 +- charts/kubezero/templates/argoless.yaml | 8 +- .../aws-node-termination-handler.yaml | 15 ---- charts/kubezero/templates/istio-ingress.yaml | 1 - charts/kubezero/templates/istio.yaml | 10 +++ charts/kubezero/templates/kubezero.yaml | 77 +++++++++++++++++++ charts/kubezero/values.yaml | 33 ++++---- 11 files changed, 166 insertions(+), 78 deletions(-) delete mode 100644 charts/kubezero/templates/aws-node-termination-handler.yaml create mode 100644 charts/kubezero/templates/kubezero.yaml diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index 381ee7cc..1d12b6e3 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,17 +2,16 @@ apiVersion: v2 name: kubezero description: KubeZero - Bootstrap and ArgoCD Root App of Apps chart type: application -version: 0.5.1 +version: 1.20.8 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: - kubezero - argocd - - gitops maintainers: - name: Quarky9 dependencies: - name: kubezero-lib - version: ">= 0.1.3" + version: ">= 0.1.4" repository: https://zero-down-time.github.io/kubezero/ kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero/README.md b/charts/kubezero/README.md index 65cac089..248d1df5 100644 --- a/charts/kubezero/README.md +++ b/charts/kubezero/README.md @@ -1,6 +1,6 @@ # kubezero -![Version: 0.5.1](https://img.shields.io/badge/Version-0.5.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 1.20.8-alpha](https://img.shields.io/badge/Version-1.20.8--alpha-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero - Bootstrap and ArgoCD Root App of Apps chart @@ -18,14 +18,13 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| -| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.4 | ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| | HighAvailableControlplane | bool | `false` | | -| argo | object | `{}` | | | argocd.crds | bool | `true` | | | argocd.enabled | bool | `false` | | | argocd.istio.enabled | bool | `false` | | @@ -40,16 +39,16 @@ Kubernetes: `>= 1.18.0` | cert-manager.crds | bool | `true` | | | cert-manager.enabled | bool | `false` | | | cert-manager.namespace | string | `"cert-manager"` | | -| global.kubezero.pathPrefix | string | `""` | | -| global.kubezero.repoURL | string | `"https://github.com/zero-down-time/kubezero"` | | -| global.kubezero.server | string | `"https://kubernetes.default.svc"` | | -| global.kubezero.targetRevision | string | `"HEAD"` | | | istio-ingress.enabled | bool | `false` | | | istio-ingress.namespace | string | `"istio-ingress"` | | | istio.crds | bool | `true` | | | istio.enabled | bool | `false` | | | istio.namespace | string | `"istio-system"` | | | kiam.enabled | bool | `false` | | +| kubezero.repoURL | string | `"https://zero-down-time.github.io/kubezero"` | | +| kubezero.server | string | `"https://kubernetes.default.svc"` | | +| kubezero.sourceRepos[0] | string | `"https://zero-down-time.github.io/kubezero"` | | +| kubezero.targetRevision | string | `"*"` | | | logging.crds | bool | `true` | | | logging.enabled | bool | `false` | | | logging.namespace | string | `"logging"` | | diff --git a/charts/kubezero/bootstrap.sh b/charts/kubezero/bootstrap.sh index 7686c098..77632997 100755 --- a/charts/kubezero/bootstrap.sh +++ b/charts/kubezero/bootstrap.sh @@ -6,32 +6,18 @@ ARTIFACTS=($(echo $2 | tr "," "\n")) CLUSTER=$3 LOCATION=${4:-""} -API_VERSIONS="-a monitoring.coreos.com/v1" - -DEPLOY_DIR=$( dirname $( realpath $0 )) which yq || { echo "yq not found!"; exit 1; } which helm || { echo "helm not found!"; exit 1; } helm_version=$(helm version --short) echo $helm_version | grep -qe "^v3.[3-9]" || { echo "Helm version >= 3.3 required!"; exit 1; } +# Simulate well-known CRDs being available +API_VERSIONS="-a monitoring.coreos.com/v1" +KUBE_VERSION="--kube-version $(kubectl version -o json | jq -r .serverVersion.gitVersion)" + TMPDIR=$(mktemp -d kubezero.XXX) [ -z "$DEBUG" ] && trap 'rm -rf $TMPDIR' ERR EXIT -# First lets generate kubezero.yaml -# Add all yaml files in $CLUSTER -VALUES="$(find $CLUSTER -name '*.yaml' | tr '\n' ',')" -helm template $DEPLOY_DIR -f ${VALUES%%,} --set argo=false > $TMPDIR/kubezero.yaml - -# Resolve all the all enabled artifacts in order of their appearance -if [ ${ARTIFACTS[0]} == "all" ]; then - ARTIFACTS=($(yq r -p p $TMPDIR/kubezero.yaml "*.enabled" | awk -F "." '{print $1}')) -fi - -# Update only if we use upstream -if [ -z "$LOCATION" ]; then - helm repo add kubezero https://zero-down-time.github.io/kubezero - helm repo update -fi # Waits for max 300s and retries function wait_for() { @@ -72,8 +58,8 @@ function delete_ns() { # Extract crds via helm calls and apply delta=crds only function _crds() { - helm template $(chart_location $chart) --namespace $namespace --name-template $release --skip-crds --set ${release}.installCRDs=false -f $TMPDIR/values.yaml > $TMPDIR/helm-no-crds.yaml - helm template $(chart_location $chart) --namespace $namespace --name-template $release --include-crds --set ${release}.installCRDs=true -f $TMPDIR/values.yaml > $TMPDIR/helm-crds.yaml + helm template $(chart_location $chart) -n $namespace --name-template $release --skip-crds --set ${release}.installCRDs=false -f $TMPDIR/values.yaml $API_VERSIONS $KUBE_VERSION > $TMPDIR/helm-no-crds.yaml + helm template $(chart_location $chart) -n $namespace --name-template $release --include-crds --set ${release}.installCRDs=true -f $TMPDIR/values.yaml $API_VERSIONS $KUBE_VERSION > $TMPDIR/helm-crds.yaml diff -e $TMPDIR/helm-no-crds.yaml $TMPDIR/helm-crds.yaml | head -n-1 | tail -n+2 > $TMPDIR/crds.yaml [ -s $TMPDIR/crds.yaml ] && kubectl apply -f $TMPDIR/crds.yaml } @@ -82,9 +68,9 @@ function _crds() { # helm template | kubectl apply -f - # confine to one namespace if possible function apply(){ - helm template $(chart_location $chart) --namespace $namespace --name-template $release --skip-crds -f $TMPDIR/values.yaml $API_VERSIONS $@ > $TMPDIR/helm.yaml + helm template $(chart_location $chart) -n $namespace --name-template $release --skip-crds -f $TMPDIR/values.yaml $API_VERSIONS $KUBE_VERSION $@ > $TMPDIR/helm.yaml - # If resources are out of the single $namespace, apply without restrictions + # If resources are in more than ONE $namespace, apply without restrictions nr_ns=$(grep -e '^ namespace:' $TMPDIR/helm.yaml | sed "s/\"//g" | sort | uniq | wc -l) if [ $nr_ns -gt 1 ]; then kubectl $action -f $TMPDIR/helm.yaml && rc=$? || rc=$? @@ -133,6 +119,7 @@ function is_enabled() { local enabled=$(yq r $TMPDIR/kubezero.yaml ${chart}.enabled) if [ "$enabled" == "true" ]; then + # slice values for this chart only from kubezero.yaml yq r $TMPDIR/kubezero.yaml ${chart}.values > $TMPDIR/values.yaml return 0 fi @@ -155,6 +142,11 @@ function get_namespace() { } +function update_kubezero_argo() { + helm template $(chart_location kubezero) -f ${VALUES%%,} --set installKubeZero=true > $TMPDIR/kubezero-argocd.yaml + kubectl apply -f $TMPDIR/kubezero-argocd.yaml +} + ################ # cert-manager # ################ @@ -198,7 +190,28 @@ function metrics-pre() { } +########## +# ArgoCD # +########## +# Install KubeZero app and project +function argocd-post() { + update_kubezero_argo +} + + ## MAIN ## +# First lets generate kubezero.yaml +# Add all yaml files in $CLUSTER +VALUES="$(find $CLUSTER -maxdepth 1 -name '*.yaml' | sort | tr '\n' ',')" +helm template $(chart_location kubezero) -f ${VALUES%%,} > $TMPDIR/kubezero.yaml + +# Resolve all the all enabled artifacts in order of their appearance +if [ ${ARTIFACTS[0]} == "all" ]; then + ARTIFACTS=($(yq r -p p $TMPDIR/kubezero.yaml "*.enabled" | awk -F "." '{print $1}')) +fi +echo "Artifacts: ${ARTIFACTS[@]}" + + if [ $1 == "deploy" ]; then for t in ${ARTIFACTS[@]}; do is_enabled $t && _helm apply $t || true @@ -216,4 +229,8 @@ elif [ $1 == "delete" ]; then for (( idx=${#ARTIFACTS[@]}-1 ; idx>=0 ; idx-- )) ; do is_enabled ${ARTIFACTS[idx]} && _helm delete ${ARTIFACTS[idx]} || true done + +# Update ArgoCD Kubezero app +elif [ $1 == "argo" -a $2 == 'kubezero' ]; then + update_kubezero_argo fi diff --git a/charts/kubezero/templates/_app.tpl b/charts/kubezero/templates/_app.tpl index 3404a9e0..269aa8fa 100644 --- a/charts/kubezero/templates/_app.tpl +++ b/charts/kubezero/templates/_app.tpl @@ -1,14 +1,14 @@ {{- define "kubezero-app.app" }} {{- $name := regexReplaceAll "kubezero/templates/([a-z-]*)..*" .Template.Name "${1}" }} -{{- if and .Values.argo ( index .Values $name "enabled" ) }} +{{- if and .Values.argocdAppName ( index .Values $name "enabled" ) }} apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: {{ $name }} namespace: argocd labels: -{{ include "kubezero-lib.labels" . | indent 4 }} + {{- include "kubezero-lib.labels" . | nindent 4 }} {{- if not ( index .Values $name "retain" ) }} finalizers: - resources-finalizer.argocd.argoproj.io @@ -17,18 +17,18 @@ spec: project: kubezero source: - repoURL: {{ .Values.global.kubezero.repoURL }} - targetRevision: {{ .Values.global.kubezero.targetRevision }} - path: {{ .Values.global.kubezero.pathPrefix}}charts/kubezero-{{ $name }} + chart: kubezero-{{ $name }} + repoURL: {{ .Values.kubezero.repoURL }} + targetRevision: {{ default .Values.kubezero.targetRevision ( index .Values $name "targetRevision" ) | quote }} helm: values: | {{- include (print $name "-values") $ | nindent 8 }} destination: - server: {{ .Values.global.kubezero.server }} + server: {{ .Values.kubezero.server }} namespace: {{ default "kube-system" ( index .Values $name "namespace" ) }} - {{- with .Values.global.kubezero.syncPolicy }} + {{- with .Values.kubezero.syncPolicy }} syncPolicy: {{- toYaml . | nindent 4 }} {{- end }} diff --git a/charts/kubezero/templates/argocd.yaml b/charts/kubezero/templates/argocd.yaml index fc82caf1..5cf29326 100644 --- a/charts/kubezero/templates/argocd.yaml +++ b/charts/kubezero/templates/argocd.yaml @@ -1,4 +1,5 @@ {{- define "argocd-values" }} + argo-cd: controller: metrics: @@ -23,9 +24,9 @@ istio: {{- end }} {{- end }} -{{- with index .Values "argocd" "kubezero" }} -kubezero: - {{- toYaml . | nindent 2 }} {{- end }} +{{- define "argocd-argo" }} {{- end }} + +{{ include "kubezero-app.app" . }} diff --git a/charts/kubezero/templates/argoless.yaml b/charts/kubezero/templates/argoless.yaml index f67faf4e..1ed9084d 100644 --- a/charts/kubezero/templates/argoless.yaml +++ b/charts/kubezero/templates/argoless.yaml @@ -1,12 +1,8 @@ -{{- if not .Values.argo }} +{{- if and ( not .Values.argocdAppName ) ( not .Values.installKubeZero ) }} +# if no ArgoCD is used, only render the global values.yaml for all kubezero modules {{- $artifacts := list "calico" "cert-manager" "kiam" "aws-ebs-csi-driver" "aws-efs-csi-driver" "istio" "istio-ingress" "metrics" "logging" "argocd" "timecapsule" "storage" }} -{{- if .Values.global }} -global: - {{- toYaml .Values.global | nindent 2 }} -{{- end }} - {{- range $artifacts }} {{- if index $.Values . }} {{ . }}: diff --git a/charts/kubezero/templates/aws-node-termination-handler.yaml b/charts/kubezero/templates/aws-node-termination-handler.yaml deleted file mode 100644 index 97c1162b..00000000 --- a/charts/kubezero/templates/aws-node-termination-handler.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- define "aws-node-termination-handler-values" }} - -aws-node-termination-handler: - queueURL: "{{ index .Values "aws-node-termination-handler" "queueURL" }}" -{{ with index .Values "aws-node-termination-handler" "IamArn" }} - podAnnotations: - iam.amazonaws.com/role: "{{ . }}" -{{- end }} - -{{- end }} - -{{- define "aws-node-termination-handler-argo" }} -{{- end }} - -{{ include "kubezero-app.app" . }} diff --git a/charts/kubezero/templates/istio-ingress.yaml b/charts/kubezero/templates/istio-ingress.yaml index a7d1b221..cbc5490a 100644 --- a/charts/kubezero/templates/istio-ingress.yaml +++ b/charts/kubezero/templates/istio-ingress.yaml @@ -26,7 +26,6 @@ istio-ingress: dnsNames: {{- toYaml $cert.dnsNames | nindent 4 }} {{- end }} - proxyProtocol: {{ default false (index .Values "istio-ingress" "public" "proxyProtocol") }} {{- end }} {{- if index .Values "istio-ingress" "private" }} diff --git a/charts/kubezero/templates/istio.yaml b/charts/kubezero/templates/istio.yaml index abd6a4c5..f23d0f84 100644 --- a/charts/kubezero/templates/istio.yaml +++ b/charts/kubezero/templates/istio.yaml @@ -9,6 +9,14 @@ global: defaultPodDisruptionBudget: enabled: true {{- end }} +{{- with index .Values "istio" "kiali-server" }} +kiali-server: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- with .Values.istio.rateLimiting }} +rateLimiting: + {{- toYaml . | nindent 2 }} +{{- end }} {{- end }} @@ -24,6 +32,8 @@ global: jsonPointers: - /webhooks/0/clientConfig/caBundle - /webhooks/0/failurePolicy + - /webhooks/1/clientConfig/caBundle + - /webhooks/1/failurePolicy - group: admissionregistration.k8s.io kind: MutatingWebhookConfiguration jsonPointers: diff --git a/charts/kubezero/templates/kubezero.yaml b/charts/kubezero/templates/kubezero.yaml new file mode 100644 index 00000000..762dc3e1 --- /dev/null +++ b/charts/kubezero/templates/kubezero.yaml @@ -0,0 +1,77 @@ +{{- if .Values.installKubeZero }} +# Add KubeZero app and project + +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: kubezero + namespace: argocd + labels: + {{- include "kubezero-lib.labels" . | nindent 4 }} +spec: + description: KubeZero - ZeroDownTime Kubernetes Platform + + # Allow manifests to deploy from + {{- with .Values.kubezero.sourceRepos }} + sourceRepos: + {{- toYaml . | nindent 2 }} + {{- end }} + + # platform namespaces in the local cluster + destinations: + - namespace: argocd + server: https://kubernetes.default.svc + - namespace: kube-system + server: https://kubernetes.default.svc + - namespace: cert-manager + server: https://kubernetes.default.svc + - namespace: istio-system + server: https://kubernetes.default.svc + - namespace: istio-ingress + server: https://kubernetes.default.svc + - namespace: monitoring + server: https://kubernetes.default.svc + - namespace: elastic-system + server: https://kubernetes.default.svc + - namespace: logging + server: https://kubernetes.default.svc + + clusterResourceWhitelist: + - group: '*' + kind: '*' + +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kubezero + namespace: argocd + labels: + {{- include "kubezero-lib.labels" . | nindent 4 }} +spec: + project: kubezero + source: + repoURL: https://zero-down-time.github.io/kubezero + chart: kubezero + # targetRevision: '>=1.20.8-0' + targetRevision: '{{ .Chart.Version }}' + + helm: + parameters: + # We use this to detect if we are called from ArgoCD + - name: argocdAppName + value: $ARGOCD_APP_NAME + # This breaks the recursion, otherwise we install another kubezero project and app + - name: installKubeZero + value: "false" + values: | + {{- toYaml .Values | nindent 8 }} + + destination: + server: https://kubernetes.default.svc + namespace: argocd + + syncPolicy: + automated: + prune: true +{{- end }} diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 3d10a9d6..e920bb11 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -1,9 +1,9 @@ -global: - kubezero: - server: https://kubernetes.default.svc - repoURL: https://github.com/zero-down-time/kubezero - targetRevision: HEAD - pathPrefix: '' +kubezero: + server: https://kubernetes.default.svc + repoURL: https://zero-down-time.github.io/kubezero + targetRevision: '*' + sourceRepos: + - 'https://zero-down-time.github.io/kubezero' HighAvailableControlplane: false @@ -11,45 +11,50 @@ calico: enabled: false crds: true retain: true + targetRevision: 0.2.2 cert-manager: enabled: false crds: true namespace: cert-manager + targetRevision: 0.6.1 kiam: enabled: false - -aws-node-termination-handler: - enabled: false - -timecapsule: - enabled: false + targetRevision: 0.3.5 storage: enabled: false crds: true +timecapsule: + enabled: false + aws-ebs-csi-driver: enabled: false crds: true + targetRevision: 0.6.4 aws-efs-csi-driver: enabled: false + targetRevision: 0.4.2 istio: enabled: false crds: true namespace: istio-system + targetRevision: 0.7.2 istio-ingress: enabled: false namespace: istio-ingress + targetRevision: 0.7.2 metrics: enabled: false crds: true namespace: monitoring + targetRevision: 0.4.6 istio: grafana: {} prometheus: {} @@ -58,12 +63,12 @@ logging: enabled: false crds: true namespace: logging + targetRevision: 0.7.6 argocd: enabled: false crds: true namespace: argocd + targetRevision: 0.8.1 istio: enabled: false - -argo: {}