From 50c9a6e729a6e29d02e5fe69454c4cbabdb6e819 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 9 Aug 2024 11:45:27 +0100 Subject: [PATCH] feat: first working version of KubeZero on GKE --- admin/dev_apply.sh | 18 ++++++++- admin/libhelm.sh | 10 +++-- charts/kubezero-argo/Chart.yaml | 2 +- charts/kubezero-argo/README.md | 2 +- charts/kubezero-cert-manager/Chart.yaml | 2 +- charts/kubezero-cert-manager/values.yaml | 16 -------- charts/kubezero-istio-gateway/Chart.yaml | 4 +- charts/kubezero-istio-gateway/README.md | 2 +- charts/kubezero-istio-gateway/values.yaml | 1 - charts/kubezero-istio/Chart.yaml | 6 +-- charts/kubezero-istio/README.md | 6 +-- charts/kubezero-istio/values.yaml | 10 +---- charts/kubezero/Chart.yaml | 4 +- charts/kubezero/templates/cert-manager.yaml | 41 +++++++++++++++++--- charts/kubezero/templates/istio-ingress.yaml | 6 ++- charts/kubezero/templates/istio.yaml | 22 +++++++++-- charts/kubezero/values.yaml | 8 ++-- 17 files changed, 100 insertions(+), 60 deletions(-) diff --git a/admin/dev_apply.sh b/admin/dev_apply.sh index 53b5555c..1e75ba9f 100755 --- a/admin/dev_apply.sh +++ b/admin/dev_apply.sh @@ -19,6 +19,22 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) . "$SCRIPT_DIR"/libhelm.sh CHARTS="$(dirname $SCRIPT_DIR)/charts" +# Guess platform from current context +_auth_cmd=$(kubectl config view | yq .users[0].user.exec.command) +if [ "$_auth_cmd" == "gke-gcloud-auth-plugin" ]; then + PLATFORM=gke +elif [ "$_auth_cmd" == "aws-iam-authenticator" ]; then + PLATFORM=aws +else + PLATFORM=nocloud +fi + +parse_version() { + echo $([[ $1 =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]] && echo "${BASH_REMATCH[0]//v/}") +} + +KUBE_VERSION=$(parse_version $KUBE_VERSION) + ### Various hooks for modules ################ @@ -71,7 +87,7 @@ if [ ${ARTIFACTS[0]} == "all" ]; then fi # Delete in reverse order, continue even if errors -if [ $ACTION == "delete" ]; then +if [ "$ACTION" == "delete" ]; then set +e for (( idx=${#ARTIFACTS[@]}-1 ; idx>=0 ; idx-- )) ; do _helm delete ${ARTIFACTS[idx]} || true diff --git a/admin/libhelm.sh b/admin/libhelm.sh index b1aca74d..1774adb6 100644 --- a/admin/libhelm.sh +++ b/admin/libhelm.sh @@ -34,9 +34,11 @@ function argo_used() { # get kubezero-values from ArgoCD if available or use in-cluster CM without Argo function get_kubezero_values() { + local _namespace="kube-system" + [ "$PLATFORM" == "gke" ] && _namespace=kubezero argo_used && \ { kubectl get application kubezero -n argocd -o yaml | yq .spec.source.helm.values > ${WORKDIR}/kubezero-values.yaml; } || \ - { kubectl get configmap -n kube-system kubezero-values -o yaml | yq '.data."values.yaml"' > ${WORKDIR}/kubezero-values.yaml ;} + { kubectl get configmap -n $_namespace kubezero-values -o yaml | yq '.data."values.yaml"' > ${WORKDIR}/kubezero-values.yaml ;} } @@ -169,14 +171,14 @@ function _helm() { yq eval '.spec.source.helm.values' $WORKDIR/kubezero/templates/${module}.yaml > $WORKDIR/values.yaml - echo "using values to $action of module $module: " - cat $WORKDIR/values.yaml - if [ $action == "crds" ]; then # Allow custom CRD handling declare -F ${module}-crds && ${module}-crds || _crds elif [ $action == "apply" ]; then + echo "using values to $action of module $module: " + cat $WORKDIR/values.yaml + # namespace must exist prior to apply create_ns $namespace diff --git a/charts/kubezero-argo/Chart.yaml b/charts/kubezero-argo/Chart.yaml index baedc483..bb243482 100644 --- a/charts/kubezero-argo/Chart.yaml +++ b/charts/kubezero-argo/Chart.yaml @@ -33,4 +33,4 @@ dependencies: version: 0.11.0 repository: https://argoproj.github.io/argo-helm condition: argocd-image-updater.enabled -kubeVersion: ">= 1.26.0" +kubeVersion: ">= 1.26.0-0" diff --git a/charts/kubezero-argo/README.md b/charts/kubezero-argo/README.md index f58b1ae9..2dda335c 100644 --- a/charts/kubezero-argo/README.md +++ b/charts/kubezero-argo/README.md @@ -65,7 +65,7 @@ Kubernetes: `>= 1.26.0` | argo-cd.repoServer.initContainers[0].command[0] | string | `"/usr/local/bin/sa2kubeconfig.sh"` | | | argo-cd.repoServer.initContainers[0].command[1] | string | `"/home/argocd/.kube/config"` | | | argo-cd.repoServer.initContainers[0].image | string | `"{{ default .Values.global.image.repository .Values.repoServer.image.repository }}:{{ default (include \"argo-cd.defaultTag\" .) .Values.repoServer.image.tag }}"` | | -| argo-cd.repoServer.initContainers[0].imagePullPolicy | string | `"IfNotPresent"` | | +| argo-cd.repoServer.initContainers[0].imagePullPolicy | string | `"{{ default .Values.global.image.imagePullPolicy .Values.repoServer.image.imagePullPolicy }}"` | | | argo-cd.repoServer.initContainers[0].name | string | `"create-kubeconfig"` | | | argo-cd.repoServer.initContainers[0].securityContext.allowPrivilegeEscalation | bool | `false` | | | argo-cd.repoServer.initContainers[0].securityContext.capabilities.drop[0] | string | `"ALL"` | | diff --git a/charts/kubezero-cert-manager/Chart.yaml b/charts/kubezero-cert-manager/Chart.yaml index 6dd2effb..5224d0df 100644 --- a/charts/kubezero-cert-manager/Chart.yaml +++ b/charts/kubezero-cert-manager/Chart.yaml @@ -18,4 +18,4 @@ dependencies: - name: cert-manager version: v1.15.1 repository: https://charts.jetstack.io -kubeVersion: ">= 1.26.0" +kubeVersion: ">= 1.26.0-0" diff --git a/charts/kubezero-cert-manager/values.yaml b/charts/kubezero-cert-manager/values.yaml index a0913d98..2706d22b 100644 --- a/charts/kubezero-cert-manager/values.yaml +++ b/charts/kubezero-cert-manager/values.yaml @@ -61,31 +61,15 @@ cert-manager: # mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" # readOnly: true - tolerations: - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - nodeSelector: - node-role.kubernetes.io/control-plane: "" - ingressShim: defaultIssuerName: letsencrypt-dns-prod defaultIssuerKind: ClusterIssuer webhook: - tolerations: - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - nodeSelector: - node-role.kubernetes.io/control-plane: "" extraArgs: - "--logging-format=json" cainjector: - tolerations: - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - nodeSelector: - node-role.kubernetes.io/control-plane: "" extraArgs: - "--logging-format=json" - "--leader-elect=false" diff --git a/charts/kubezero-istio-gateway/Chart.yaml b/charts/kubezero-istio-gateway/Chart.yaml index 06f0ed17..30e8ff90 100644 --- a/charts/kubezero-istio-gateway/Chart.yaml +++ b/charts/kubezero-istio-gateway/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-istio-gateway description: KubeZero Umbrella Chart for Istio gateways type: application -version: 0.22.3 +version: 0.22.3-1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -19,4 +19,4 @@ dependencies: - name: gateway version: 1.22.3 repository: https://istio-release.storage.googleapis.com/charts -kubeVersion: ">= 1.26.0" +kubeVersion: ">= 1.26.0-0" diff --git a/charts/kubezero-istio-gateway/README.md b/charts/kubezero-istio-gateway/README.md index 7d723183..89ffceb9 100644 --- a/charts/kubezero-istio-gateway/README.md +++ b/charts/kubezero-istio-gateway/README.md @@ -16,7 +16,7 @@ Installs Istio Ingress Gateways, requires kubezero-istio to be installed ! ## Requirements -Kubernetes: `>= 1.26.0` +Kubernetes: `>= 1.26.0-0` | Repository | Name | Version | |------------|------|---------| diff --git a/charts/kubezero-istio-gateway/values.yaml b/charts/kubezero-istio-gateway/values.yaml index ae51063b..86c859c5 100644 --- a/charts/kubezero-istio-gateway/values.yaml +++ b/charts/kubezero-istio-gateway/values.yaml @@ -8,7 +8,6 @@ gateway: replicaCount: 1 terminationGracePeriodSeconds: 120 - priorityClassName: system-cluster-critical resources: requests: diff --git a/charts/kubezero-istio/Chart.yaml b/charts/kubezero-istio/Chart.yaml index ddc24d02..39507c0a 100644 --- a/charts/kubezero-istio/Chart.yaml +++ b/charts/kubezero-istio/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-istio description: KubeZero Umbrella Chart for Istio type: application -version: 0.22.3 +version: 0.22.3-1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -22,7 +22,7 @@ dependencies: version: 1.22.3 repository: https://istio-release.storage.googleapis.com/charts - name: kiali-server - version: "1.87.0" + version: "1.88.0" repository: https://kiali.org/helm-charts condition: kiali-server.enabled -kubeVersion: ">= 1.26.0" +kubeVersion: ">= 1.26.0-0" diff --git a/charts/kubezero-istio/README.md b/charts/kubezero-istio/README.md index 3a84a75d..b8d76e9a 100644 --- a/charts/kubezero-istio/README.md +++ b/charts/kubezero-istio/README.md @@ -16,7 +16,7 @@ Installs the Istio control plane ## Requirements -Kubernetes: `>= 1.26.0` +Kubernetes: `>= 1.26.0-0` | Repository | Name | Version | |------------|------|---------| @@ -31,19 +31,15 @@ Kubernetes: `>= 1.26.0` |-----|------|---------|-------------| | global.defaultPodDisruptionBudget.enabled | bool | `false` | | | global.logAsJson | bool | `true` | | -| global.priorityClassName | string | `"system-cluster-critical"` | | | global.variant | string | `"distroless"` | | | istiod.meshConfig.accessLogEncoding | string | `"JSON"` | | | istiod.meshConfig.accessLogFile | string | `"/dev/stdout"` | | | istiod.meshConfig.tcpKeepalive.interval | string | `"60s"` | | | istiod.meshConfig.tcpKeepalive.time | string | `"120s"` | | | istiod.pilot.autoscaleEnabled | bool | `false` | | -| istiod.pilot.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | | | istiod.pilot.replicaCount | int | `1` | | | istiod.pilot.resources.requests.cpu | string | `"100m"` | | | istiod.pilot.resources.requests.memory | string | `"128Mi"` | | -| istiod.pilot.tolerations[0].effect | string | `"NoSchedule"` | | -| istiod.pilot.tolerations[0].key | string | `"node-role.kubernetes.io/control-plane"` | | | istiod.telemetry.enabled | bool | `false` | | | kiali-server.auth.strategy | string | `"anonymous"` | | | kiali-server.deployment.ingress_enabled | bool | `false` | | diff --git a/charts/kubezero-istio/values.yaml b/charts/kubezero-istio/values.yaml index bc0716fb..71b44e3b 100644 --- a/charts/kubezero-istio/values.yaml +++ b/charts/kubezero-istio/values.yaml @@ -6,19 +6,11 @@ global: defaultPodDisruptionBudget: enabled: false - priorityClassName: "system-cluster-critical" - istiod: pilot: autoscaleEnabled: false replicaCount: 1 - nodeSelector: - node-role.kubernetes.io/control-plane: "" - tolerations: - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - resources: requests: cpu: 100m @@ -57,7 +49,7 @@ kiali-server: prometheus: url: "http://metrics-kube-prometheus-st-prometheus.monitoring:9090" - + istio: enabled: false gateway: istio-ingress/private-ingressgateway diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index cfed442b..862dcabd 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero description: KubeZero - Root App of Apps chart type: application -version: 1.29.7 +version: 1.29.7-1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -15,4 +15,4 @@ dependencies: - name: kubezero-lib version: ">= 0.1.6" repository: https://cdn.zero-downtime.net/charts -kubeVersion: ">= 1.26.0" +kubeVersion: ">= 1.26.0-0" diff --git a/charts/kubezero/templates/cert-manager.yaml b/charts/kubezero/templates/cert-manager.yaml index b920d150..e0083796 100644 --- a/charts/kubezero/templates/cert-manager.yaml +++ b/charts/kubezero/templates/cert-manager.yaml @@ -9,10 +9,29 @@ cert-manager: type: Recreate {{- end }} - prometheus: - servicemonitor: - enabled: {{ $.Values.metrics.enabled }} -{{ with index .Values "cert-manager" "IamArn" }} + {{- if eq .Values.global.platform "aws" }} + # map everything to the control-plane + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + + webhook: + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + nodeSelector: + node-role.kubernetes.io/control-plane: "" + + cainjector: + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + nodeSelector: + node-role.kubernetes.io/control-plane: "" + + {{ with index .Values "cert-manager" "IamArn" }} extraEnv: - name: AWS_ROLE_ARN value: "{{ . }}" @@ -34,7 +53,19 @@ cert-manager: - name: aws-token mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" readOnly: true -{{- end }} + {{- end }} + + {{- end }} + + {{- if eq .Values.global.platform "gke" }} + serviceAccount: + annotations: + iam.gke.io/gcp-service-account: "dns01-solver@{{ .Values.global.gcp.projectId }}.iam.gserviceaccount.com" + {{- end }} + + prometheus: + servicemonitor: + enabled: {{ $.Values.metrics.enabled }} {{- with index .Values "cert-manager" "clusterIssuer" }} clusterIssuer: diff --git a/charts/kubezero/templates/istio-ingress.yaml b/charts/kubezero/templates/istio-ingress.yaml index 1c170e8f..cdfa3e45 100644 --- a/charts/kubezero/templates/istio-ingress.yaml +++ b/charts/kubezero/templates/istio-ingress.yaml @@ -3,6 +3,10 @@ gateway: name: istio-ingressgateway + {{- if ne .Values.global.platform "gke" }} + priorityClassName: "system-cluster-critical" + {{- end }} + {{- with index .Values "istio-ingress" "gateway" "replicaCount" }} replicaCount: {{ . }} {{- if gt (int .) 1 }} @@ -11,7 +15,7 @@ gateway: {{- end }} {{- end }} - {{- if not (index .Values "istio-ingress" "gateway" "affinity") }} + {{- if eq .Values.global.platform "aws" }} # Only nodes who are fronted with matching LB affinity: nodeAffinity: diff --git a/charts/kubezero/templates/istio.yaml b/charts/kubezero/templates/istio.yaml index 0b0719ac..9705bf96 100644 --- a/charts/kubezero/templates/istio.yaml +++ b/charts/kubezero/templates/istio.yaml @@ -1,21 +1,37 @@ {{- define "istio-values" }} + +{{- if .Values.global.highAvailable }} +global: + defaultPodDisruptionBudget: + enabled: true + {{- if ne .Values.global.platform "gke" }} + priorityClassName: "system-cluster-critical" + {{- end }} +{{- end }} + istiod: telemetry: enabled: {{ $.Values.metrics.enabled }} pilot: +{{- if eq .Values.global.platform "aws" }} + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule +{{- end }} {{- if .Values.global.highAvailable }} replicaCount: 2 -global: - defaultPodDisruptionBudget: - enabled: true {{- else }} extraContainerArgs: - --leader-elect=false {{- end }} + {{- with index .Values "istio" "kiali-server" }} kiali-server: {{- toYaml . | nindent 2 }} {{- end }} + {{- with .Values.istio.rateLimiting }} rateLimiting: {{- toYaml . | nindent 2 }} diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 47d66262..65d10005 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -43,7 +43,7 @@ network: cert-manager: enabled: false namespace: cert-manager - targetRevision: 0.9.8 + targetRevision: 0.9.9 storage: enabled: false @@ -64,13 +64,13 @@ storage: istio: enabled: false namespace: istio-system - targetRevision: 0.22.3 + targetRevision: 0.22.3-1 istio-ingress: enabled: false chart: kubezero-istio-gateway namespace: istio-ingress - targetRevision: 0.22.3 + targetRevision: 0.22.3-1 gateway: service: {} @@ -78,7 +78,7 @@ istio-private-ingress: enabled: false chart: kubezero-istio-gateway namespace: istio-ingress - targetRevision: 0.22.3 + targetRevision: 0.22.3-1 gateway: service: {}