Add cert-manager backup support in bootstrap, enable schedule and controller metrics

charts/kubezero-cert-manager/backup-all.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+kubectl get -A -o yaml issuer,clusterissuer,certificates,certificaterequests > cert-manager-backup.yaml
+echo '---' >> cert-manager-backup.yaml
+kubectl get -A -o yaml secrets --field-selector type=kubernetes.io/tls >> cert-manager-backup.yaml
+echo '---' >> cert-manager-backup.yaml
+kubectl get -o yaml secrets -n cert-manager letsencrypt-dns-prod >> cert-manager-backup.yaml

charts/kubezero-metrics/values.yaml

@@ -23,20 +23,32 @@ prometheus-operator:
     enabled: true
 
   # Disabled until we figure out how to scrape etcd with ssl client certs, scheduler/proxy/controller require https since 1.17
-  kubeControllerManager:
-    enabled: false
   kubeDns:
     enabled: false
   kubeEtcd:
     enabled: false
+
+  kubeControllerManager:
+    enabled: true
+    service:
+      port: 10257
+      targetPort: 10257
+    serviceMonitor:
+      https: true
+
   kubeScheduler:
-    enabled: false
+    enabled: true
+    service:
+      port: 10259
+      targetPort: 10259
+    serviceMonitor:
+      https: true
 
   kubelet:
     enabled: true
     serviceMonitor:
-      # removed with 1.18 anyways
+      # removed with 1.18, but still required for all container metrics ??
-      cAdvisor: false
+      cAdvisor: true
 
   prometheusOperator:
     enabled: true

deploy/deploy.sh

@@ -41,7 +41,7 @@ EOF
   fi
 
   # Deploy initial argo-cad
-  helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set cert-manager.not_ready=true --set istio.enabled=false --set prometheus.enabled=false > generated-values.yaml
+  helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set cert-manager.not_ready=true --set istio.enabled=false --set metrics.enabled=false > generated-values.yaml
   helm install -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml
   # Wait for argocd-server to be running
   kubectl rollout status deployment -n argocd kubezero-argocd-server
@@ -51,20 +51,30 @@ EOF
   wait_for kubectl get deployment -n cert-manager cert-manager-webhook 2>/dev/null 1>&2
   kubectl rollout status deployment -n cert-manager cert-manager-webhook
 
+  # Either inject cert-manager backup or bootstrap
+  if [ -f cert-manager-backup.yaml ]; then
+    kubectl apply -f cert-manager-backup.yaml
+  else
+    helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set istio.enabled=false --set metrics.enabled=false > generated-values.yaml
+    helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml
+    wait_for kubectl get Issuer -n kube-system kubezero-local-ca-issuer 2>/dev/null 1>&2
+    wait_for kubectl get ClusterIssuer letsencrypt-dns-prod 2>/dev/null 1>&2
+    kubectl wait --for=condition=Ready -n kube-system Issuer/kubezero-local-ca-issuer
+    kubectl wait --for=condition=Ready ClusterIssuer/letsencrypt-dns-prod
+  fi
+
   # Now that we have the cert-manager webhook, get the kiam certs in place but do NOT deploy kiam yet
-  helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set kiam.enabled=false --set istio.enabled=false --set prometheus.enabled=false > generated-values.yaml
+  helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set kiam.enabled=false > generated-values.yaml
   helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml
-  wait_for kubectl get Issuer -n kube-system kubezero-local-ca-issuer 2>/dev/null 1>&2
-  kubectl wait --for=condition=Ready -n kube-system Issuer/kubezero-local-ca-issuer
 
   # Now lets make sure kiam is working
-  helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true --set istio.enabled=false --set prometheus.enabled=false > generated-values.yaml
+  helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set kiam.not_ready=true > generated-values.yaml
   helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd --create-namespace -f generated-values.yaml
   wait_for kubectl get daemonset -n kube-system kiam-agent 2>/dev/null 1>&2
   kubectl rollout status daemonset -n kube-system kiam-agent
 
   # Install Istio if enabled, but keep ArgoCD istio support disabled for now in case
-  helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set argo-cd.istio.enabled=false > generated-values.yaml
+  helm template $DEPLOY_DIR -f values.yaml -f cloudbender.yaml --set argo-cd.istio.enabled=false --set metrics.istio.prometheus.enabled=false --set metrics.istio.grafana.enabled=false > generated-values.yaml
   helm upgrade -n argocd kubezero kubezero/kubezero-argo-cd -f generated-values.yaml
   wait_for kubectl get deployment -n istio-operator istio-operator 2>/dev/null 1>&2
   kubectl rollout status deployment -n istio-operator istio-operator

deploy/templates/values.yaml

@@ -92,10 +92,12 @@ kubezero:
     values:
       istiod:
         replicaCount: {{ ternary 2 1 .Values.HighAvailableControlplane }}
+      {{- if not ( index .Values "cert-manager" "not_ready" ) }}
       {{- if .Values.istio.ingress }}
       ingress:
         {{- toYaml .Values.istio.ingress | nindent 8 }}
       {{- end }}
+      {{- end }}
 
   metrics:
     enabled: {{ .Values.metrics.enabled }}