From 509f8d59fb7f45ad4ff2caf981d7e0d4bb74585e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 9 Oct 2020 17:58:44 -0700 Subject: [PATCH 1/8] First stab at new prometheus charts --- charts/kubezero-metrics/Chart.yaml | 12 ++++++------ charts/kubezero-metrics/values.yaml | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index d2c2a90a..4cee1e89 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-metrics description: KubeZero Umbrella Chart for prometheus-operator type: application -version: 0.1.4 +version: 0.2.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -15,10 +15,10 @@ dependencies: - name: kubezero-lib version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - - name: prometheus-operator - version: 9.3.1 - repository: https://kubernetes-charts.storage.googleapis.com/ + - name: kube-prometheus-stack + version: 10.0.1 + repository: https://prometheus-community.github.io/helm-charts - name: prometheus-adapter - version: 2.5.0 - repository: https://kubernetes-charts.storage.googleapis.com/ + version: 2.7.0 + repository: https://prometheus-community.github.io/helm-charts kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 13c93f20..0586f426 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -11,7 +11,7 @@ prometheus: url: "" gateway: istio-system/ingressgateway -prometheus-operator: +kube-prometheus-stack: defaultRules: create: true @@ -60,9 +60,9 @@ prometheus-operator: prometheusOperator: enabled: true #image: - # tag: v0.41.0 + # tag: v0.42.1 #prometheusConfigReloaderImage: - # tag: v0.41.0 + # tag: v0.42.1 # Run on controller nodes tolerations: @@ -160,11 +160,11 @@ prometheus-adapter: default: false resource: cpu: - containerQuery: sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}[5m])) by (<<.GroupBy>>) - nodeQuery: sum(1 - irate(node_cpu_seconds_total{mode="idle"}[5m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>) + containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) + nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) resources: overrides: - node: + instance: resource: node namespace: resource: namespace @@ -172,15 +172,15 @@ prometheus-adapter: resource: pod containerLabel: container memory: - containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}) by (<<.GroupBy>>) - nodeQuery: sum(node_memory_MemTotal_bytes{job="node-exporter",<<.LabelMatchers>>} - node_memory_MemAvailable_bytes{job="node-exporter",<<.LabelMatchers>>}) by (<<.GroupBy>>) + containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) + nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) resources: overrides: - node: + instance: resource: node namespace: resource: namespace pod: resource: pod containerLabel: container - window: 5m + window: 3m From 19d915cb9251ca22d337feb6da206af82aefab51 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 9 Oct 2020 18:41:43 -0700 Subject: [PATCH 2/8] Adjust prometheus URLs --- charts/kubezero-metrics/templates/istio-service.yaml | 2 +- charts/kubezero-metrics/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-metrics/templates/istio-service.yaml b/charts/kubezero-metrics/templates/istio-service.yaml index 05d8f5e7..01849889 100644 --- a/charts/kubezero-metrics/templates/istio-service.yaml +++ b/charts/kubezero-metrics/templates/istio-service.yaml @@ -31,5 +31,5 @@ spec: http: - route: - destination: - host: metrics-prometheus-operato-prometheus + host: metrics-kube-prometheus-st-prometheus {{- end }} diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 0586f426..0ae68c94 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -148,7 +148,7 @@ kube-prometheus-stack: # Metrics adapter prometheus-adapter: prometheus: - url: http://metrics-prometheus-operato-prometheus + url: http://metrics-kube-prometheus-st-prometheus tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule From 44d08c7abc5cfe10d76be2f2d9bfeb7b3d771e17 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 21 Oct 2020 04:37:33 -0700 Subject: [PATCH 3/8] More EFS fixes, cert-manager version bump --- charts/kubezero-aws-efs-csi-driver/templates/efs_pv.yaml | 2 ++ charts/kubezero-aws-efs-csi-driver/values.yaml | 8 ++++---- charts/kubezero-cert-manager/Chart.yaml | 4 ++-- deploy/templates/values.yaml | 6 +++++- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/charts/kubezero-aws-efs-csi-driver/templates/efs_pv.yaml b/charts/kubezero-aws-efs-csi-driver/templates/efs_pv.yaml index ed56adfd..3180d4a1 100644 --- a/charts/kubezero-aws-efs-csi-driver/templates/efs_pv.yaml +++ b/charts/kubezero-aws-efs-csi-driver/templates/efs_pv.yaml @@ -1,3 +1,4 @@ +{{- if .Values.PersistentVolume }} {{- if .Values.PersistentVolume.create }} apiVersion: v1 kind: PersistentVolume @@ -17,3 +18,4 @@ spec: driver: efs.csi.aws.com volumeHandle: {{ .Values.PersistentVolume.EfsId }} {{- end }} +{{- end }} diff --git a/charts/kubezero-aws-efs-csi-driver/values.yaml b/charts/kubezero-aws-efs-csi-driver/values.yaml index 8650a119..e02877c7 100644 --- a/charts/kubezero-aws-efs-csi-driver/values.yaml +++ b/charts/kubezero-aws-efs-csi-driver/values.yaml @@ -19,7 +19,7 @@ StorageClass: # volumeHandle: ":/path2" # Deprecated and removed with next release -PersistentVolume: - create: false - EfsId: "" - Name: "" +# PersistentVolume: +# create: false +# EfsId: "" +# Name: "" diff --git a/charts/kubezero-cert-manager/Chart.yaml b/charts/kubezero-cert-manager/Chart.yaml index 8a15bf22..9096c450 100644 --- a/charts/kubezero-cert-manager/Chart.yaml +++ b/charts/kubezero-cert-manager/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-cert-manager description: KubeZero Umbrella Chart for cert-manager type: application -version: 0.3.6 +version: 0.4.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -15,6 +15,6 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: cert-manager - version: 0.15.1 + version: 1.0.3 repository: https://charts.jetstack.io kubeVersion: ">= 1.16.0" diff --git a/deploy/templates/values.yaml b/deploy/templates/values.yaml index 00cd590c..0e5698b2 100644 --- a/deploy/templates/values.yaml +++ b/deploy/templates/values.yaml @@ -78,17 +78,21 @@ kubezero: aws-efs-csi-driver: enabled: {{ index .Values "aws-efs-csi-driver" "enabled" }} values: + {{- with index .Values "aws-efs-csi-driver" "nodeSelector" }} aws-efs-csi-driver: nodeSelector: - node.kubernetes.io/csi.efs.fs: {{ index .Values "aws-efs-csi-driver" "EfsId" }} + {{- toYaml . | nindent 10 }} + {{- end }} {{- with index .Values "aws-efs-csi-driver" "PersistentVolumes" }} PersistentVolumes: {{- toYaml . | nindent 8 }} {{- end }} + {{- if index .Values "aws-efs-csi-driver" "PersistentVolume" }} PersistentVolume: create: true EfsId: {{ index .Values "aws-efs-csi-driver" "EfsId" }} Name: {{ default "kubezero-efs-pv" ( index .Values "aws-efs-csi-driver" "PVName" ) }} + {{- end }} {{- end }} istio: From 72a917bdaed45f98b2ead518d0351ffae218a728 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 21 Oct 2020 12:51:15 +0100 Subject: [PATCH 4/8] Revert prometheus adapter changes --- charts/kubezero-metrics/values.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index 0ae68c94..beba0fe8 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -160,11 +160,11 @@ prometheus-adapter: default: false resource: cpu: - containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) - nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) + containerQuery: sum(irate(container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}[3m])) by (<<.GroupBy>>) + nodeQuery: sum(1 - irate(node_cpu_seconds_total{mode="idle"}[3m]) * on(namespace, pod) group_left(node) node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>}) by (<<.GroupBy>>) resources: overrides: - instance: + node: resource: node namespace: resource: namespace @@ -172,11 +172,11 @@ prometheus-adapter: resource: pod containerLabel: container memory: - containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) + containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}) by (<<.GroupBy>>) nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) resources: overrides: - instance: + node: resource: node namespace: resource: namespace From 8874c9869d9b99047b2e3798ad454038fbc08196 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 21 Oct 2020 13:05:08 +0100 Subject: [PATCH 5/8] Revert more prometheus-adapter config --- charts/kubezero-metrics/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-metrics/values.yaml b/charts/kubezero-metrics/values.yaml index beba0fe8..161ce05e 100644 --- a/charts/kubezero-metrics/values.yaml +++ b/charts/kubezero-metrics/values.yaml @@ -173,7 +173,7 @@ prometheus-adapter: containerLabel: container memory: containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,container!="POD",container!="",pod!=""}) by (<<.GroupBy>>) - nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) + nodeQuery: sum(node_memory_MemTotal_bytes{job="node-exporter",<<.LabelMatchers>>} - node_memory_MemAvailable_bytes{job="node-exporter",<<.LabelMatchers>>}) by (<<.GroupBy>>) resources: overrides: node: From 4fb425676dc10f6b1e930a5e79c2dea550caae59 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 21 Oct 2020 13:12:23 +0100 Subject: [PATCH 6/8] Bump argo-cd version --- charts/kubezero-argo-cd/Chart.yaml | 2 +- charts/kubezero-argo-cd/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-argo-cd/Chart.yaml b/charts/kubezero-argo-cd/Chart.yaml index 9274084f..76ffdc0e 100644 --- a/charts/kubezero-argo-cd/Chart.yaml +++ b/charts/kubezero-argo-cd/Chart.yaml @@ -15,6 +15,6 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: argo-cd - version: 2.8.0 + version: 2.9.3 repository: https://argoproj.github.io/argo-helm kubeVersion: ">= 1.17.0" diff --git a/charts/kubezero-argo-cd/values.yaml b/charts/kubezero-argo-cd/values.yaml index 329b933e..338d1817 100644 --- a/charts/kubezero-argo-cd/values.yaml +++ b/charts/kubezero-argo-cd/values.yaml @@ -31,7 +31,7 @@ argo-cd: global: image: - tag: v1.7.7 + tag: v1.7.8 controller: args: From d7006faa60d46e4c647397e299060d6f279ef146 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 21 Oct 2020 13:14:23 +0100 Subject: [PATCH 7/8] Bump argo-cd chart version --- charts/kubezero-argo-cd/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-argo-cd/Chart.yaml b/charts/kubezero-argo-cd/Chart.yaml index 76ffdc0e..1c2f85e2 100644 --- a/charts/kubezero-argo-cd/Chart.yaml +++ b/charts/kubezero-argo-cd/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application name: kubezero-argo-cd -version: 0.5.6 +version: 0.5.7 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: From 74c47d73913f2a1be08446d1e35d64c998470016 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 21 Oct 2020 13:29:49 +0100 Subject: [PATCH 8/8] Enable json logs for argo-cd finally --- charts/kubezero-argo-cd/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/kubezero-argo-cd/values.yaml b/charts/kubezero-argo-cd/values.yaml index 338d1817..e41e907a 100644 --- a/charts/kubezero-argo-cd/values.yaml +++ b/charts/kubezero-argo-cd/values.yaml @@ -38,7 +38,7 @@ argo-cd: statusProcessors: "4" operationProcessors: "2" appResyncPeriod: "300" - # logFormat: json + logFormat: json metrics: enabled: false @@ -63,7 +63,7 @@ argo-cd: memory: 256Mi repoServer: - # logFormat: json + logFormat: json metrics: enabled: false serviceMonitor: @@ -79,7 +79,7 @@ argo-cd: effect: NoSchedule server: - # logFormat: json + logFormat: json config: # argo-cd.server.config.url -- ArgoCD hostname to be exposed via Istio url: argocd.example.com