From 41f8c6fb5fa234ed985ceff16a6f403fa143e196 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 4 May 2021 15:00:24 +0200 Subject: [PATCH 01/19] feat: add file backed persistence and grafana dashboard to NATS --- charts/kubezero-nats/Chart.yaml | 5 ++++- charts/kubezero-nats/dashboards.yaml | 8 ++++++++ .../kubezero-nats/templates/grafana-dashboards.yaml | 13 +++++++++++++ charts/kubezero-nats/update.sh | 3 +++ charts/kubezero-nats/values.yaml | 6 ------ 5 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 charts/kubezero-nats/dashboards.yaml create mode 100644 charts/kubezero-nats/templates/grafana-dashboards.yaml diff --git a/charts/kubezero-nats/Chart.yaml b/charts/kubezero-nats/Chart.yaml index 67b2eed..6d40b64 100644 --- a/charts/kubezero-nats/Chart.yaml +++ b/charts/kubezero-nats/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-nats description: KubeZero umbrella chart for NATS type: application -version: 0.1.0 +version: 0.1.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -11,6 +11,9 @@ keywords: maintainers: - name: Quarky9 dependencies: + - name: kubezero-lib + version: ">= 0.1.3" + repository: https://zero-down-time.github.io/kubezero/ - name: nats version: 0.8.3 #repository: https://nats-io.github.io/k8s/helm/charts/ diff --git a/charts/kubezero-nats/dashboards.yaml b/charts/kubezero-nats/dashboards.yaml new file mode 100644 index 0000000..a02ec1f --- /dev/null +++ b/charts/kubezero-nats/dashboards.yaml @@ -0,0 +1,8 @@ +configmap: grafana-dashboards-nats +condition: '.Values.nats.exporter.serviceMonitor.enabled' +gzip: true +# folder: +dashboards: +- name: nats + url: https://grafana.com/api/dashboards/13707/revisions/1/download + tags: ['NATS'] diff --git a/charts/kubezero-nats/templates/grafana-dashboards.yaml b/charts/kubezero-nats/templates/grafana-dashboards.yaml new file mode 100644 index 0000000..9e17afd --- /dev/null +++ b/charts/kubezero-nats/templates/grafana-dashboards.yaml @@ -0,0 +1,13 @@ +{{- if .Values.nats.exporter.serviceMonitor.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" (include "kubezero-lib.fullname" $) "grafana-dashboards-nats" | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} + labels: + grafana_dashboard: "1" +{{ include "kubezero-lib.labels" . | indent 4 }} +binaryData: + nats.json.gz: + H4sIAAAAAAAC/+2cW2/bNhSA3/MrBD2tgBNYci7b3lI3XQssF9TuXobAoCVaIkKRGknFTgP/95GUbFEWlWZFFystXwyLh+Q5pM7lCx3p8cDz/NkMkbwQ3P/d+1tee96j/pQSAjIoW/13k9nV+XRyePPp+vJi+uHi88QfbPpgMIdYdSp7MJpBkcKC1z1iyCOGcoEo0ZPBBSiw8Oqu3ofp9MbzLkicU0REPVI85Fp/DATgtGARrGU5LhJEPsZKnlu0lvKragWGXbrDWn7eDsrVM/hPgRi0rH+jPwdErnE7NdJKEwbytG7c7NUfzeZ7yHi18ErzwK5DTrcABFi0NJsNPU2BoWl0FBwFT6uzbSnq3EzS3kab3uBoeDTc3WA9Kykw1lcCCQw37uJNIJPDvXeAp3MKWKxn3XUYa09vQZlXcOgtkUi9eYGwOETEdCo97GKVUybkQOlX1CNAcI8uFihCAHspxJkXpYCJcjW+AEntBFqtv10FFw+42jh2V3VHGfxCiW6dM7rkkJUCGCMB5rq7YAXUbSmK4ZgSwShWOhYA81LApQEwHjPK5TfEGjI1adsrI4oxyDk0u2qBRa9uTyFKUqHMDE+G+cqIEeXXtQZTi5YDjAAfU0yZ6vS4HpjCOWB814Qy3mvXsrvMU8aWMsYos869QFglm6DRmDDtY6bp6vakMqZTiuPA8D+rXK9Q2cqSOfglDE4HXjgcyo+zgTc8Cs/e+F1Dw69MHe5MPToeeEEQVh9y7vCNbwxtbrBeVHOliF/BpW3HMEwgsWyCvIMJOefTjp1WPe4TuyAqGINE2IUZWHUIELHdOO3odGkfI6gAuGPUPcCFTs1qXOdWYUS2nVqCJYpFKoXhruROu/5to1ndzRtVhS5prP03ooTASMC46b45lP5NZMaAVj/VhYyBGBVKx0lbZg8dueExZFB7zAJT0dQpEwyC/FrmQenw0GI8lwEtW0+bjQJEd1ZtXMA8h/GfiNgXIQBLoGjmh90cUcbrKtcmq/w6uwfsyyzKi52gUd5LZCaWN/Q9iISOirDVpfTj95RlQCesx0euE/8Mxet1e0aZWRiKnqGawUWJCudtmdoGKTputK+Nq+Ymq7T/XiY1S+hr2SRFC2EXVqWvqmXjm8/N+ysoxQLl7SjO+CfIKS6qimiPLlVG7PHFZQmUkqEttGYbIIiKrMBAoHvYnZGkRxPpY1VasDhMzTIGA2nRCqwQb6/MSAqdWh/ACj7DCRdbp6lis32nN6zaTtlKSJO3QFfVoO1nOt1Zh5UJzyqyL25neU+uRM7AerwOM0YOLKtzMNEvmBg5mHAw8fpgIoPZvmDCqrqPMHEJM8oeXhtP9IMX5g8CckcL+6GFg52w0OOnlXc3jjG2Ln89ke6uYpXbz7jcAclPxDRnL8M0thz5DKixDXNUs3+qGe2fahCZdRSel0Gbbv3P4Jtg+GKA81YZ6X0kjm0c27iTEEcN34UafnXU4KjhlVJDxpN9QoNdfb+YQf/2fykNddzwjdzQ83rruMFxwz644cRxg+OG18gNtBD7PW54woA+njdcF8KBgztwcODgwOG7gMPpDwQOTP1aNkExdGDhwELV9X2eSHTr7+uRhEMLdybxU/yfxjRltEjSXPq7+z+Nnx2AQve/p/8Bb9oO0w++Ce1807qN3wdvjnvwIEu5abIAP4dwgv/lgZanTHgG5IQvxjjjDksd4fSVDIYOb9zJSb/B4diBgwOH1wcOvJhv32Jgqdtp5/HZi52cfMXCfj3l0mmrYwvHFo4tHFt8E1v85tjCscUrZAtMl+pYgBcZZHs7l/iaFb06mphIY72x1VhHEI4gfrwfX8YYyQS686Ds9nVmKmi2XuMvytDyCV0eBhsHk2FRtanB6+2wHEV3OodWg2WYq3o82yQd06X8E+NFccHQuBiZF0H9kL1/YnwPzIvR0JQYr9gLje9BXK70drMGafCMVn82mHZ1aTEnPjUnNrWEx+ZFXbr8s9i0d2NLvX0wy1Wsk6TePoy40LVm2wsQIkv2xuTObtW+6/f9VZvp8yiFGfhr+0a+QJdG4xV95XWzOPsJgUJn6mB0Njw7WP8LyPVbsRZSAAA= +{{- end }} diff --git a/charts/kubezero-nats/update.sh b/charts/kubezero-nats/update.sh index 0e89e6f..ce0a46e 100755 --- a/charts/kubezero-nats/update.sh +++ b/charts/kubezero-nats/update.sh @@ -7,3 +7,6 @@ rm -rf charts/nats && mkdir -p charts/nats git clone --depth=1 https://github.com/nats-io/k8s.git cp -r k8s/helm/charts/nats/* charts/nats/ rm -rf k8s + +# Fetch dashboards +../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/grafana-dashboards.yaml diff --git a/charts/kubezero-nats/values.yaml b/charts/kubezero-nats/values.yaml index 8af407c..ad96899 100644 --- a/charts/kubezero-nats/values.yaml +++ b/charts/kubezero-nats/values.yaml @@ -7,15 +7,9 @@ nats: jetstream: enabled: true - memStorage: - enabled: true - size: 128Mi - natsbox: enabled: false exporter: serviceMonitor: enabled: true - labels: - release: metrics -- 2.40.1 From 07293f6ed69818271143cc8b36633b46e088df1b Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 5 May 2021 13:11:16 +0200 Subject: [PATCH 02/19] chore: remove NATS from KubeZero platform --- charts/kubezero/templates/argoless.yaml | 2 +- charts/kubezero/templates/istio.yaml | 4 ---- charts/kubezero/templates/metrics.yaml | 2 +- charts/kubezero/templates/nats.yaml | 8 -------- charts/kubezero/values.yaml | 4 ---- 5 files changed, 2 insertions(+), 18 deletions(-) delete mode 100644 charts/kubezero/templates/nats.yaml diff --git a/charts/kubezero/templates/argoless.yaml b/charts/kubezero/templates/argoless.yaml index 6fb2f25..b6796fa 100644 --- a/charts/kubezero/templates/argoless.yaml +++ b/charts/kubezero/templates/argoless.yaml @@ -1,6 +1,6 @@ {{- if not .Values.argo }} -{{- $artifacts := list "calico" "cert-manager" "kiam" "aws-node-termination-handler" "aws-ebs-csi-driver" "aws-efs-csi-driver" "local-volume-provisioner" "local-path-provisioner" "istio" "istio-ingress" "metrics" "logging" "argocd" "timecapsule" "nats" }} +{{- $artifacts := list "calico" "cert-manager" "kiam" "aws-node-termination-handler" "aws-ebs-csi-driver" "aws-efs-csi-driver" "local-volume-provisioner" "local-path-provisioner" "istio" "istio-ingress" "metrics" "logging" "argocd" "timecapsule" }} {{- if .Values.global }} global: diff --git a/charts/kubezero/templates/istio.yaml b/charts/kubezero/templates/istio.yaml index f07e0df..021a36a 100644 --- a/charts/kubezero/templates/istio.yaml +++ b/charts/kubezero/templates/istio.yaml @@ -1,22 +1,18 @@ {{- define "istio-values" }} - istio-discovery: telemetry: enabled: {{ .Values.metrics.enabled }} {{- if .Values.HighAvailableControlplane }} pilot: replicaCount: 2 - global: defaultPodDisruptionBudget: enabled: true {{- end }} - {{- end }} {{- define "istio-argo" }} - ignoreDifferences: - group: apiextensions.k8s.io kind: CustomResourceDefinition diff --git a/charts/kubezero/templates/metrics.yaml b/charts/kubezero/templates/metrics.yaml index 906bad9..02c3409 100644 --- a/charts/kubezero/templates/metrics.yaml +++ b/charts/kubezero/templates/metrics.yaml @@ -2,7 +2,7 @@ {{- with .Values.metrics.istio }} istio: - {{- toYaml . | nindent 2 }} +{{- toYaml . | nindent 2 }} {{- end }} {{- if index .Values "metrics" "kube-prometheus-stack" }} kube-prometheus-stack: diff --git a/charts/kubezero/templates/nats.yaml b/charts/kubezero/templates/nats.yaml deleted file mode 100644 index 9d1bc42..0000000 --- a/charts/kubezero/templates/nats.yaml +++ /dev/null @@ -1,8 +0,0 @@ -{{- define "nats-values" }} -{{- end }} - - -{{- define "nats-argo" }} -{{- end }} - -{{ include "kubezero-app.app" . }} diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index eb266c6..826642b 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -69,7 +69,3 @@ argocd: enabled: false argo: {} - -nats: - enabled: false - namespace: nats -- 2.40.1 From 1077359f93f523bd3df6b6c42de00ddbc17c11e9 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 5 May 2021 14:05:17 +0200 Subject: [PATCH 03/19] chore: rename kubezero-nats to kubezero-mq --- .../{kubezero-nats => kubezero-mq}/Chart.yaml | 5 ++- charts/kubezero-mq/README.md | 38 +++++++++++++++++++ .../README.md.gotmpl | 2 + .../charts/nats/Chart.yaml | 0 .../charts/nats/README.md | 0 .../charts/nats/templates/NOTES.txt | 0 .../charts/nats/templates/_helpers.tpl | 0 .../charts/nats/templates/configmap.yaml | 0 .../charts/nats/templates/nats-box.yaml | 0 .../charts/nats/templates/pdb.yaml | 0 .../charts/nats/templates/rbac.yaml | 0 .../charts/nats/templates/service.yaml | 0 .../charts/nats/templates/serviceMonitor.yaml | 0 .../charts/nats/templates/statefulset.yaml | 0 .../charts/nats/values.yaml | 0 .../dashboards.yaml | 0 .../templates/grafana-dashboards.yaml | 0 .../{kubezero-nats => kubezero-mq}/update.sh | 2 + .../values.yaml | 4 +- charts/kubezero-nats/README.md | 24 ------------ 20 files changed, 48 insertions(+), 27 deletions(-) rename charts/{kubezero-nats => kubezero-mq}/Chart.yaml (80%) create mode 100644 charts/kubezero-mq/README.md rename charts/{kubezero-nats => kubezero-mq}/README.md.gotmpl (98%) rename charts/{kubezero-nats => kubezero-mq}/charts/nats/Chart.yaml (100%) rename charts/{kubezero-nats => kubezero-mq}/charts/nats/README.md (100%) rename charts/{kubezero-nats => kubezero-mq}/charts/nats/templates/NOTES.txt (100%) rename charts/{kubezero-nats => kubezero-mq}/charts/nats/templates/_helpers.tpl (100%) rename charts/{kubezero-nats => kubezero-mq}/charts/nats/templates/configmap.yaml (100%) rename charts/{kubezero-nats => kubezero-mq}/charts/nats/templates/nats-box.yaml (100%) rename charts/{kubezero-nats => kubezero-mq}/charts/nats/templates/pdb.yaml (100%) rename charts/{kubezero-nats => kubezero-mq}/charts/nats/templates/rbac.yaml (100%) rename charts/{kubezero-nats => kubezero-mq}/charts/nats/templates/service.yaml (100%) rename charts/{kubezero-nats => kubezero-mq}/charts/nats/templates/serviceMonitor.yaml (100%) rename charts/{kubezero-nats => kubezero-mq}/charts/nats/templates/statefulset.yaml (100%) rename charts/{kubezero-nats => kubezero-mq}/charts/nats/values.yaml (100%) rename charts/{kubezero-nats => kubezero-mq}/dashboards.yaml (100%) rename charts/{kubezero-nats => kubezero-mq}/templates/grafana-dashboards.yaml (100%) rename charts/{kubezero-nats => kubezero-mq}/update.sh (97%) rename charts/{kubezero-nats => kubezero-mq}/values.yaml (81%) delete mode 100644 charts/kubezero-nats/README.md diff --git a/charts/kubezero-nats/Chart.yaml b/charts/kubezero-mq/Chart.yaml similarity index 80% rename from charts/kubezero-nats/Chart.yaml rename to charts/kubezero-mq/Chart.yaml index 6d40b64..e4aa269 100644 --- a/charts/kubezero-nats/Chart.yaml +++ b/charts/kubezero-mq/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -name: kubezero-nats -description: KubeZero umbrella chart for NATS +name: kubezero-mq +description: KubeZero umbrella chart for MQ systems like NATS type: application version: 0.1.1 home: https://kubezero.com @@ -17,4 +17,5 @@ dependencies: - name: nats version: 0.8.3 #repository: https://nats-io.github.io/k8s/helm/charts/ + condition: nats.enabled kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-mq/README.md b/charts/kubezero-mq/README.md new file mode 100644 index 0000000..c0f5cf0 --- /dev/null +++ b/charts/kubezero-mq/README.md @@ -0,0 +1,38 @@ +# kubezero-mq + +![Version: 0.1.1](https://img.shields.io/badge/Version-0.1.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) + +KubeZero umbrella chart for MQ systems like NATS + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| Quarky9 | | | + +## Requirements + +Kubernetes: `>= 1.18.0` + +| Repository | Name | Version | +|------------|------|---------| +| | nats | 0.8.3 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| nats.enabled | bool | `false` | | +| nats.exporter.serviceMonitor.enabled | bool | `false` | | +| nats.nats.advertise | bool | `false` | | +| nats.nats.image | string | `"nats:2.2.1-alpine3.13"` | | +| nats.nats.jetstream.enabled | bool | `true` | | +| nats.natsbox.enabled | bool | `false` | | + +## Resources + +### NATS +- https://grafana.com/grafana/dashboards/13707 diff --git a/charts/kubezero-nats/README.md.gotmpl b/charts/kubezero-mq/README.md.gotmpl similarity index 98% rename from charts/kubezero-nats/README.md.gotmpl rename to charts/kubezero-mq/README.md.gotmpl index 099d661..7b7b5f8 100644 --- a/charts/kubezero-nats/README.md.gotmpl +++ b/charts/kubezero-mq/README.md.gotmpl @@ -16,4 +16,6 @@ {{ template "chart.valuesSection" . }} ## Resources + +### NATS - https://grafana.com/grafana/dashboards/13707 diff --git a/charts/kubezero-nats/charts/nats/Chart.yaml b/charts/kubezero-mq/charts/nats/Chart.yaml similarity index 100% rename from charts/kubezero-nats/charts/nats/Chart.yaml rename to charts/kubezero-mq/charts/nats/Chart.yaml diff --git a/charts/kubezero-nats/charts/nats/README.md b/charts/kubezero-mq/charts/nats/README.md similarity index 100% rename from charts/kubezero-nats/charts/nats/README.md rename to charts/kubezero-mq/charts/nats/README.md diff --git a/charts/kubezero-nats/charts/nats/templates/NOTES.txt b/charts/kubezero-mq/charts/nats/templates/NOTES.txt similarity index 100% rename from charts/kubezero-nats/charts/nats/templates/NOTES.txt rename to charts/kubezero-mq/charts/nats/templates/NOTES.txt diff --git a/charts/kubezero-nats/charts/nats/templates/_helpers.tpl b/charts/kubezero-mq/charts/nats/templates/_helpers.tpl similarity index 100% rename from charts/kubezero-nats/charts/nats/templates/_helpers.tpl rename to charts/kubezero-mq/charts/nats/templates/_helpers.tpl diff --git a/charts/kubezero-nats/charts/nats/templates/configmap.yaml b/charts/kubezero-mq/charts/nats/templates/configmap.yaml similarity index 100% rename from charts/kubezero-nats/charts/nats/templates/configmap.yaml rename to charts/kubezero-mq/charts/nats/templates/configmap.yaml diff --git a/charts/kubezero-nats/charts/nats/templates/nats-box.yaml b/charts/kubezero-mq/charts/nats/templates/nats-box.yaml similarity index 100% rename from charts/kubezero-nats/charts/nats/templates/nats-box.yaml rename to charts/kubezero-mq/charts/nats/templates/nats-box.yaml diff --git a/charts/kubezero-nats/charts/nats/templates/pdb.yaml b/charts/kubezero-mq/charts/nats/templates/pdb.yaml similarity index 100% rename from charts/kubezero-nats/charts/nats/templates/pdb.yaml rename to charts/kubezero-mq/charts/nats/templates/pdb.yaml diff --git a/charts/kubezero-nats/charts/nats/templates/rbac.yaml b/charts/kubezero-mq/charts/nats/templates/rbac.yaml similarity index 100% rename from charts/kubezero-nats/charts/nats/templates/rbac.yaml rename to charts/kubezero-mq/charts/nats/templates/rbac.yaml diff --git a/charts/kubezero-nats/charts/nats/templates/service.yaml b/charts/kubezero-mq/charts/nats/templates/service.yaml similarity index 100% rename from charts/kubezero-nats/charts/nats/templates/service.yaml rename to charts/kubezero-mq/charts/nats/templates/service.yaml diff --git a/charts/kubezero-nats/charts/nats/templates/serviceMonitor.yaml b/charts/kubezero-mq/charts/nats/templates/serviceMonitor.yaml similarity index 100% rename from charts/kubezero-nats/charts/nats/templates/serviceMonitor.yaml rename to charts/kubezero-mq/charts/nats/templates/serviceMonitor.yaml diff --git a/charts/kubezero-nats/charts/nats/templates/statefulset.yaml b/charts/kubezero-mq/charts/nats/templates/statefulset.yaml similarity index 100% rename from charts/kubezero-nats/charts/nats/templates/statefulset.yaml rename to charts/kubezero-mq/charts/nats/templates/statefulset.yaml diff --git a/charts/kubezero-nats/charts/nats/values.yaml b/charts/kubezero-mq/charts/nats/values.yaml similarity index 100% rename from charts/kubezero-nats/charts/nats/values.yaml rename to charts/kubezero-mq/charts/nats/values.yaml diff --git a/charts/kubezero-nats/dashboards.yaml b/charts/kubezero-mq/dashboards.yaml similarity index 100% rename from charts/kubezero-nats/dashboards.yaml rename to charts/kubezero-mq/dashboards.yaml diff --git a/charts/kubezero-nats/templates/grafana-dashboards.yaml b/charts/kubezero-mq/templates/grafana-dashboards.yaml similarity index 100% rename from charts/kubezero-nats/templates/grafana-dashboards.yaml rename to charts/kubezero-mq/templates/grafana-dashboards.yaml diff --git a/charts/kubezero-nats/update.sh b/charts/kubezero-mq/update.sh similarity index 97% rename from charts/kubezero-nats/update.sh rename to charts/kubezero-mq/update.sh index ce0a46e..5455a72 100755 --- a/charts/kubezero-nats/update.sh +++ b/charts/kubezero-mq/update.sh @@ -1,6 +1,8 @@ #!/bin/bash set -ex +### NATS + # get latest chart until they have upstream repo fixed rm -rf charts/nats && mkdir -p charts/nats diff --git a/charts/kubezero-nats/values.yaml b/charts/kubezero-mq/values.yaml similarity index 81% rename from charts/kubezero-nats/values.yaml rename to charts/kubezero-mq/values.yaml index ad96899..218f301 100644 --- a/charts/kubezero-nats/values.yaml +++ b/charts/kubezero-mq/values.yaml @@ -1,4 +1,6 @@ nats: + enabled: false + nats: image: nats:2.2.1-alpine3.13 @@ -12,4 +14,4 @@ nats: exporter: serviceMonitor: - enabled: true + enabled: false diff --git a/charts/kubezero-nats/README.md b/charts/kubezero-nats/README.md deleted file mode 100644 index d294049..0000000 --- a/charts/kubezero-nats/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# kubezero-nats - -![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) - -KubeZero umbrella chart for NATS - -**Homepage:** - -## Maintainers - -| Name | Email | Url | -| ---- | ------ | --- | -| Quarky9 | | | - -## Requirements - -Kubernetes: `>= 1.18.0` - -| Repository | Name | Version | -|------------|------|---------| -| | nats | 0.8.3 | - ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0) -- 2.40.1 From 5370790b137cf08c284265a1cca03a65624489ca Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 5 May 2021 18:35:43 +0200 Subject: [PATCH 04/19] feat: add rabbitmq support to kubezero-mq --- charts/kubezero-mq/Chart.yaml | 9 +++- charts/kubezero-mq/README.md | 24 ++++++++- .../{ => nats}/grafana-dashboards.yaml | 0 .../templates/rabbitmq/istio-service.yaml | 35 +++++++++++++ .../rabbitmq/server-certificate.yaml | 16 ++++++ charts/kubezero-mq/update.sh | 2 +- charts/kubezero-mq/values.yaml | 52 +++++++++++++++++++ 7 files changed, 133 insertions(+), 5 deletions(-) rename charts/kubezero-mq/templates/{ => nats}/grafana-dashboards.yaml (100%) create mode 100644 charts/kubezero-mq/templates/rabbitmq/istio-service.yaml create mode 100644 charts/kubezero-mq/templates/rabbitmq/server-certificate.yaml diff --git a/charts/kubezero-mq/Chart.yaml b/charts/kubezero-mq/Chart.yaml index e4aa269..b8d74fd 100644 --- a/charts/kubezero-mq/Chart.yaml +++ b/charts/kubezero-mq/Chart.yaml @@ -1,13 +1,14 @@ apiVersion: v2 name: kubezero-mq -description: KubeZero umbrella chart for MQ systems like NATS +description: KubeZero umbrella chart for MQ systems like NATS, RabbitMQ type: application -version: 0.1.1 +version: 0.2.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: - kubezero - nats + - rabbitmq maintainers: - name: Quarky9 dependencies: @@ -18,4 +19,8 @@ dependencies: version: 0.8.3 #repository: https://nats-io.github.io/k8s/helm/charts/ condition: nats.enabled + - name: rabbitmq + version: 8.13.1 + repository: https://charts.bitnami.com/bitnami + condition: rabbitmq.enabled kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-mq/README.md b/charts/kubezero-mq/README.md index c0f5cf0..a28e836 100644 --- a/charts/kubezero-mq/README.md +++ b/charts/kubezero-mq/README.md @@ -1,8 +1,8 @@ # kubezero-mq -![Version: 0.1.1](https://img.shields.io/badge/Version-0.1.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.2.0](https://img.shields.io/badge/Version-0.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) -KubeZero umbrella chart for MQ systems like NATS +KubeZero umbrella chart for MQ systems like NATS, RabbitMQ **Homepage:** @@ -19,6 +19,7 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| | | nats | 0.8.3 | +| https://charts.bitnami.com/bitnami | rabbitmq | 8.13.1 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Values @@ -31,6 +32,25 @@ Kubernetes: `>= 1.18.0` | nats.nats.image | string | `"nats:2.2.1-alpine3.13"` | | | nats.nats.jetstream.enabled | bool | `true` | | | nats.natsbox.enabled | bool | `false` | | +| rabbitmq.auth.erlangCookie | string | `"randomlongerlangcookie"` | | +| rabbitmq.auth.password | string | `"supersecret"` | | +| rabbitmq.auth.tls.enabled | bool | `false` | | +| rabbitmq.auth.tls.existingSecret | string | `"rabbitmq-server-certificate"` | | +| rabbitmq.auth.tls.existingSecretFullChain | bool | `true` | | +| rabbitmq.auth.tls.failIfNoPeerCert | bool | `false` | | +| rabbitmq.clustering.forceBoot | bool | `true` | | +| rabbitmq.enabled | bool | `false` | | +| rabbitmq.hosts | list | `[]` | hostnames of rabbitmq services, used for Istio and TLS | +| rabbitmq.istio.enabled | bool | `false` | | +| rabbitmq.istio.gateway | string | `"istio-ingress/private-ingressgateway"` | | +| rabbitmq.metrics.enabled | bool | `false` | | +| rabbitmq.metrics.serviceMonitor.enabled | bool | `false` | | +| rabbitmq.pdb.create | bool | `true` | | +| rabbitmq.podAntiAffinityPreset | string | `""` | | +| rabbitmq.replicaCount | int | `1` | | +| rabbitmq.resources.requests.cpu | string | `"100m"` | | +| rabbitmq.resources.requests.memory | string | `"256Mi"` | | +| rabbitmq.topologySpreadConstraints | string | `"- maxSkew: 1\n topologyKey: topology.kubernetes.io/zone\n whenUnsatisfiable: DoNotSchedule\n labelSelector:\n matchLabels: {{- include \"common.labels.matchLabels\" . | nindent 6 }}\n- maxSkew: 1\n topologyKey: kubernetes.io/hostname\n whenUnsatisfiable: DoNotSchedule\n labelSelector:\n matchLabels: {{- include \"common.labels.matchLabels\" . | nindent 6 }}"` | | ## Resources diff --git a/charts/kubezero-mq/templates/grafana-dashboards.yaml b/charts/kubezero-mq/templates/nats/grafana-dashboards.yaml similarity index 100% rename from charts/kubezero-mq/templates/grafana-dashboards.yaml rename to charts/kubezero-mq/templates/nats/grafana-dashboards.yaml diff --git a/charts/kubezero-mq/templates/rabbitmq/istio-service.yaml b/charts/kubezero-mq/templates/rabbitmq/istio-service.yaml new file mode 100644 index 0000000..038510e --- /dev/null +++ b/charts/kubezero-mq/templates/rabbitmq/istio-service.yaml @@ -0,0 +1,35 @@ +{{- if .Values.rabbitmq.istio.enabled }} +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: rabbit-amqp + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +spec: + gateways: + - {{ .Values.rabbitmq.istio.gateway }} + hosts: +{{ toYaml .Values.rabbitmq.hosts | indent 2 }} + tcp: + - match: + - port: 5672 + route: + - destination: + host: rabbitmq-headless + port: + number: 5672 + - match: + - port: 5671 + route: + - destination: + host: rabbitmq-headless + port: + number: 5671 + http: + - route: + - destination: + host: rabbitmq-headless + port: + number: 15672 +{{- end }} diff --git a/charts/kubezero-mq/templates/rabbitmq/server-certificate.yaml b/charts/kubezero-mq/templates/rabbitmq/server-certificate.yaml new file mode 100644 index 0000000..5a70471 --- /dev/null +++ b/charts/kubezero-mq/templates/rabbitmq/server-certificate.yaml @@ -0,0 +1,16 @@ +{{- if .Values.rabbitmq.auth.tls.enabled }} +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: rabbitmq-server-certificate + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +spec: + secretName: rabbitmq-server-certificate + issuerRef: + name: letsencrypt-dns-prod + kind: ClusterIssuer + dnsNames: +{{ toYaml .Values.rabbitmq.hosts | indent 2 }} +{{- end }} diff --git a/charts/kubezero-mq/update.sh b/charts/kubezero-mq/update.sh index 5455a72..121b645 100755 --- a/charts/kubezero-mq/update.sh +++ b/charts/kubezero-mq/update.sh @@ -11,4 +11,4 @@ cp -r k8s/helm/charts/nats/* charts/nats/ rm -rf k8s # Fetch dashboards -../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/grafana-dashboards.yaml +../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/nats/grafana-dashboards.yaml diff --git a/charts/kubezero-mq/values.yaml b/charts/kubezero-mq/values.yaml index 218f301..8f0ba34 100644 --- a/charts/kubezero-mq/values.yaml +++ b/charts/kubezero-mq/values.yaml @@ -1,3 +1,4 @@ +# nats nats: enabled: false @@ -15,3 +16,54 @@ nats: exporter: serviceMonitor: enabled: false + +# rabbitmq +rabbitmq: + enabled: false + + # rabbitmq.hosts -- hostnames of rabbitmq services, used for Istio and TLS + hosts: [] + + istio: + enabled: false + gateway: istio-ingress/private-ingressgateway + + auth: + password: "supersecret" + erlangCookie: "randomlongerlangcookie" + tls: + enabled: false + failIfNoPeerCert: false + existingSecret: rabbitmq-server-certificate + existingSecretFullChain: true + + clustering: + forceBoot: true + + resources: + requests: + memory: 256Mi + cpu: 100m + + replicaCount: 1 + + podAntiAffinityPreset: "" + topologySpreadConstraints: |- + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }} + + pdb: + create: true + + metrics: + enabled: false + serviceMonitor: + enabled: false -- 2.40.1 From 1c6ac7c36abbb313001770552e8c22cdc7cdb308 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 5 May 2021 19:06:12 +0200 Subject: [PATCH 05/19] feat: add grafana dashboard to kubezero-mq for rabbitmq --- .../{dashboards.yaml => dashboards-nats.yaml} | 0 charts/kubezero-mq/dashboards-rabbitmq.yaml | 8 ++++ charts/kubezero-mq/example-nats-argocd.yaml | 35 +++++++++++++++ .../kubezero-mq/example-rabbitmq-argocd.yaml | 44 +++++++++++++++++++ .../rabbitmq/grafana-dashboards.yaml | 13 ++++++ charts/kubezero-mq/update.sh | 3 +- 6 files changed, 102 insertions(+), 1 deletion(-) rename charts/kubezero-mq/{dashboards.yaml => dashboards-nats.yaml} (100%) create mode 100644 charts/kubezero-mq/dashboards-rabbitmq.yaml create mode 100644 charts/kubezero-mq/example-nats-argocd.yaml create mode 100644 charts/kubezero-mq/example-rabbitmq-argocd.yaml create mode 100644 charts/kubezero-mq/templates/rabbitmq/grafana-dashboards.yaml diff --git a/charts/kubezero-mq/dashboards.yaml b/charts/kubezero-mq/dashboards-nats.yaml similarity index 100% rename from charts/kubezero-mq/dashboards.yaml rename to charts/kubezero-mq/dashboards-nats.yaml diff --git a/charts/kubezero-mq/dashboards-rabbitmq.yaml b/charts/kubezero-mq/dashboards-rabbitmq.yaml new file mode 100644 index 0000000..25a039d --- /dev/null +++ b/charts/kubezero-mq/dashboards-rabbitmq.yaml @@ -0,0 +1,8 @@ +configmap: grafana-dashboards-rabbitmq +condition: '.Values.rabbitmq.metrics.enabled' +gzip: true +# folder: +dashboards: +- name: rabbitmq + url: https://grafana.com/api/dashboards/10991/revisions/11/download + tags: ['RabbitMQ'] diff --git a/charts/kubezero-mq/example-nats-argocd.yaml b/charts/kubezero-mq/example-nats-argocd.yaml new file mode 100644 index 0000000..3a24206 --- /dev/null +++ b/charts/kubezero-mq/example-nats-argocd.yaml @@ -0,0 +1,35 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: nats + namespace: argocd +spec: + project: default + source: + repoURL: 'https://github.com/zero-down-time/kubezero' + path: charts/kubezero-mq + targetRevision: master + helm: + values: | + nats: + enabled: true + nats: + jetstream: + memStorage: + enabled: true + size: 128Mi + fileStorage: + enabled: true + storageClassName: ebs-sc-gp3-xfs + exporter: + serviceMonitor: + enabled: true + + destination: + server: 'https://kubernetes.default.svc' + namespace: nats + syncPolicy: + automated: + prune: true + syncOptions: + - CreateNamespace=true diff --git a/charts/kubezero-mq/example-rabbitmq-argocd.yaml b/charts/kubezero-mq/example-rabbitmq-argocd.yaml new file mode 100644 index 0000000..7fc8d55 --- /dev/null +++ b/charts/kubezero-mq/example-rabbitmq-argocd.yaml @@ -0,0 +1,44 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: rabbitmq + namespace: argocd +spec: + project: default + source: + repoURL: 'https://github.com/zero-down-time/kubezero' + path: charts/kubezero-mq + targetRevision: master + helm: + values: | + rabbitmq: + enabled: true + + replicaCount: 3 + + hosts: + - mq.example.com + auth: + password: blablabla + erlangCookie: changemeplease + tls: + enabled: true + + # Some custom plugin to be installed at boot + communityPlugins: "https://github.com/rabbitmq/rabbitmq-delayed-message-exchange/releases/download/3.8.9/rabbitmq_delayed_message_exchange-3.8.9-0199d11c.ez" + extraPlugins: "rabbitmq_delayed_message_exchange" + + # Enabled metrics + metrics: + enabled: true + serviceMonitor: + enabled: true + + destination: + server: 'https://kubernetes.default.svc' + namespace: rabbitmq + syncPolicy: + automated: + prune: true + syncOptions: + - CreateNamespace=true diff --git a/charts/kubezero-mq/templates/rabbitmq/grafana-dashboards.yaml b/charts/kubezero-mq/templates/rabbitmq/grafana-dashboards.yaml new file mode 100644 index 0000000..c66a513 --- /dev/null +++ b/charts/kubezero-mq/templates/rabbitmq/grafana-dashboards.yaml @@ -0,0 +1,13 @@ +{{- if .Values.rabbitmq.metrics.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" (include "kubezero-lib.fullname" $) "grafana-dashboards-rabbitmq" | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} + labels: + grafana_dashboard: "1" +{{ include "kubezero-lib.labels" . | indent 4 }} +binaryData: + rabbitmq.json.gz: + H4sIAAAAAAAC/+1da1MbSZb93r8iQ97ZMT0ISwIJ6IiJWQy47RjzsMEzsdN4NalSSqqlXl1ZhVB7/d/33sx6VwkJIbAEtyPatrKq8v045+TNm99+YqzW7fri99D0haz9wn6DEMa+qT/hWTDxBITWhj4fcIfXNuMHZr8y2OG2ev/X4oMb4UvTdfDZ7lZjq1FTD75vVifX5wGXbugbopii57u2CEYilOVEq55l0m3NTjfgPauUZCEwTu0yH5xJ6O40PO4Iq6IivVFlNWaDH5aGNJ2hJWTAg3JCFxXPyqnBn1/xcY07jgvvwlPsMzr5mmXKIOlBaabgSS80reADxtTcTEMzjQxJnJcbD94RjqrlX1jghyITPjL7FaGm4TqHruX6GKE/7PHXjU3Wajbhj3Z7kzU3slHHRT9Iy8L+kx1Ywg9yWUj7pBz1XO73a9Gz7+rvrz9FzVDrC2n4phdEdXbAHDFmn3mvZwYnn9gJDIehsIUTsDOo2RtTjFUqNdE3g0Iha0NHBB+w0ZqN/f2mDsKucOm6VmB6cUXWzED4PEqw2YGXd/f3dzrN3X311DKd64oxjbWkCuQaaWMHfKhe/ZoJ8YcieGtx57pQ07XADFR+ayeuYwauD52Hjc1gxNJGhKoszQFBkvtaJjCqXsxsGhr6FgaOgsCTv7x5Mx6Pt3xVk/bvW4Zrv0mH+tYosK1i/1T9v6LoBjdG4tK0hRtiV3VCy0qSNLDjvOXG9dB3Q6dfLLN6fO6LgXkLjwbckoVn/+BWKKofycywgLBX27t7u62jbCd71XzXaRzu5IIOd5qd1kHc29KGmTluij0xfTAwhdU/dJ2BOUzGbfTFgIdWIHOhmP1QBq6Nod+T4O+ZTLrQlX0Yi7rv/FR4oTZwfZsHavIZuX5mbhnycCjyWbD5bVyFzUYjk4ZtOvGDbDDEOC5WN/anESxkI9fqf+Q93QemvnHC/WuhGgdbupR5NWt2dtLfDoy2G24VO04yzNImsrnnwaC41H27WRWe7xO5So9nphssNQtcFojbINM1WPQIo65qlcrIfO4MZ0TWSiMrdTponSPod+cu1ILMN1ENa0M9OHHVrAyd3nGEEYh+LffOJaZcqDzPCofQvulS09lqbm2n33muDPSQq5UC37lOcGH+oZJsN/6UeR4P01oxbOonqnpOuHdHswx8NRJUUfIVGOiC1U7fHBQeuMkHd9St9KAjQi8qDIiBaVn51UytYukfjdx6NsBUyr0d483Gko8kF0M0oioHg1qiIJ7QLkwpepm4o9rErafSlqH9Op7Du7+HIhRdW0gJi6IEAMr7E/Yzc53XpgMoxDHEBsN52OtaYhCk3xkWTEjC32BJCEw+DqxHk67pDNxvxRf/elX7j2LYVW2T4aiAajcEvpD8uKp938g3YDqBBbBqdKXwTSHzr0RQpFDvar7AkgRTniVzSa1W+eQdNwLVaM3cY0sMhdN/l2Qr/zH0cQUbagd39bdkAsRWq8FIhrGs/mxkmhWK+053+Nx4xfCLkTkIyg8iTPBZtWXctOWFvgqGqhkoOzb3smNTPb17bLoKVfx1zmEZT3gzR6Z68TSaQY3Q96GvVSPvhyKLe6CHCBfMQA8RxlgIPRimzdXSmSsGoYploIohFOzczVdJbQQ/tzORYRY6md+3hd8TzPUUwNIiwEKAhQDLsgBLIDLoY8Shn1gpbvHCnmXKkeh3Azfg1m+dhvy6saY4Zk3gSr252V4KTvngGK6NykXcmuwNI7iyjnBl5WCJ4zrimaKSnQIqabbmhCUdgiUESwiWLFlHiRCJXA3MweqsKndd6McQDn3zGUOjJwNAqNUsA/+cR9jVJ9BDoIdAz71Az5xazPYugR4CPQR6lg16dA9RFhuktawB1DhMG4ywBmENwhr3wBqtxnxYY6dBWIOwBmGNxzBUId1ihcDEJ9UihCOWjiPIKvVZ248UccT2lI2aNuEIwhGEIx7X4DV0YNoW/Zdo8vqUhq2b7eXYtH7B5nLcsSX6Q2g1Mm4lMYNAyDKMWKeBkG0CIQRCCIQ8iRGrL/rCMmG2WkUzVvaXK2eOMjynEsBaum6lgL64jlleyZp+pmh8aXbbZ2EwdMlum5A4bSsu0257ChLfaRISJyROSPxx7LbJMnpFzZV0oxCqIFRBqOIhhtFTUMX2HqEKQhWEKp71aTACEwpMRG1CWGLpWGI6cHi2Bksv2fB52l4hHbIiLEFYYtlYAn329hUUIDTxcDSxvbm3DDBx6vZfkNWRa1mQQ9EvoYHZS/fUVadZWHVaOzPMZDtTdPF02kp8DX+taLCzo+OLcoP5MIynlRnGbyG22cV9ekCS9itYZ/40R8Xv3Lfid6cs9/t3LeseH4ooY7kRBEDOVZNvDnDidPpe8L7wS0/QJCtXBmgZCNjL+i8HeBh3zVJeZTCx7kIQ3DK5mhmOfQsWtjdnl+f5AQUvDNVEzkNYo3KPFBiOVvJsKachZT1r4fLUbmyy5m5rk+3g4rS1v5ufVuPXWtu78F5rfxNPlcF7e/tT3sNo2jv6f4huo5Z56WsuW9CHRTqh/jf8Vz85qR8dsffvf7HtX2Rh8p7igzJp9spZfQqmizpGAFO+qlCharwbu7fPz2/ZWbtQg9MLl8zDAbpCz8cYOmaK3GdiwaRbxH7jn6BTlFvxIX2ioos9QqdoLbVHJEhktfvEewDV1B+eoD/EWDefHPATwJk3aQ7L8ee7yv07RN4UuIzUvs7fWRAqKjhNPeYpZxAH6++Jpg/dW+bvEtQT5ugJczY4Kpy1e4/9kdkHWkytuWqtqZVRas5n0pz/6/Z+bGMeakWL2nR5bVpUCx+jgZcCvDJXUwHTpC6wvC6Q1ix0AlM4wTx87fFhVdIzvaor7KjVl9fqeivqaVq9LOjPu43ysC2UTZbjEo+3ozJ1QyV3r2VhKyV/4eLsnZQ590oW3hDJSOs+dySWouLKztz1nnXoI9XiuxrRh/Gg+5aKuT3uy7JJwh2GDnhX5EfhDAO15dDIhQu5wH5GwRThw4DBC0xvP5uS/SF8l7k+s4SUm+qRLWzXnzBucd9mY9OyWE9A65nDIZ4xY9yB/yEwusACj9NkvSpyw3elVG9EXYlhZ5RJTD3LRf8FW1fOlXM5ghzorBjcYYGwPdfnvmlN2NBljhjywLyBT4TBQynKuYOv04ypayQ5kzBljgLWFxafRIkIdg0zgrD+DImZYsxcXQfcdkMnwF9RpJBGn/Um6iFmWmWqbw4GUAjcIGbjEQ/Sp2YghTVQL7k9pe9sMVUiW0CHgvfUy7kSQvmTUkGPw+xCHXHTgYQ94ZtuX2UOuobK+tsJi7ZhokrEWoDJSo1nJgX8b/4h2OvPFxcbuP1vuLYXBoKN3DGzQ2MUFwyyrEoX5Q8mM5iIh5M4S2j8hc4JXkuhK3kY4gYOPLLc8YbKyc/stwOscvn19dSbNVWb6Fs1N9QXJzr1mR/qXGY+/CygUzvYtQ56MEZYFNEXKWbGUsdipjG91b2NpX4k2akbmAPT0Pe23hFf2qvrcZ9V8S5n76ykIU0HizO31Ey1P9XIBfzq8z4CndyDqVtse4UtttwZn6ottmazlA0Nki/0fnpxnlIbcKm5jd4mz+dDYZ0DeVl9bS+/GVYYQMS7wKXXbX5bEWg65cBpZkx4uLMiXI3lqSZLaJAhizuCGDg2+0FuE7lqC7Jka5OzRqm5Xv7a5KjWhB9cxghmWr5gbjGgnviwbB9XMtLZ3dreamXtcSBD2JnCHA7T4eWGhubow3Ss4MzAcjNmBdrS4izbl2fA4zf/owfkf/3G638c1P91dbV1dVX/+vPfGq+vrnr/Bz833lSAYvz0Vbtz0Nl5ew8sPjWx5uzE3rUODxuHy0isNTux7dbuztH+MhLbnp3YwXYbyraMxHbmqMZ3u3uNg2Uk1p6d2H7naH/vaBmJdeYp2fFxu7WMxHZnJ7Z38Hbv3btlJLY3O7HDg/32cXsZie3PU41vt9u7M8ztDFGFnwFlGaV9GggVnif6H7V9Xv7Z3KwtJWIxNOtqLNK1TNsMur1JcC8/oE/G4zZY/cp5nRUhDGABpVKsbv5/pGXft2+5Un7//mBTvyndLCv01Mbcd0pCRAQASww7MjsthWv7PGuKpXB7u7O329hv5iWXCb811fjFJp893HOZBv4ZAOC2HjPXjblyey/94LMYRnjrt6/3EBYissJvuGkhlgUaBT1TxIQZT+xGfCIjM7iuFZheHtrJEfdFvwKxaguzZhGVdmOxwnT65o3ZD3liTvm9JGgMfe6N0gzcRhWWSb4XQh6Dsh1VzY6aNchvZMbW6IW3M4bHVTC6gs9A490FDzOqYHPK+FczVr6fWHgaodImzB2+5bKsI2oGUXpdc4hScNm8evrYmOa/7YflszQqJuXeECvRRT6kwj+KmyTTP2UjXV+dLBJyAt80UGbyhQdNJfpKtUGJxOM+jHjUE8YwpLVsElveMUyN9U1f4EoyYUpygSlAi16zFbi+Ka9XVX8b+CLKoFqF7yPELaQhHWFSFyqpmR9jtup3xHDHp4B9+qEWe6DPGddQtYGK4xUs0Krf1BWYk3WViCq7jv4cmTPUM6AiprWg0Ffa0l2ppd/UofEGJFg9VLDam+WkZiHBao8EKxKsSLAiwYoEKxKsno9glQgXiGW6KgfdhK6urtzz8tSepxBOmo3d7d2d5l72VN0D9Z6nFKlWR/c5ypAi0n5I+yHtZ121n3+OhJO13PEFZkYrNbA+iXFOcVHCiuMGKK5wwxBeEKk0SiUZwxQumAt8RvN3ZvMJ5M+0lCRyqIxRmWX2UHYRcpN5IlFsMI7Djx8YThCFZHBqUfY+KhsoQTmpXZIfQjJoNuMOMjMRzL+CxQWFNoglmTMPvn2HD99Dghak+xGFjqVoJZhmfaSi1UHPUovYua87moW0iNYqahE+SnwXlV7ESacgnYJ0CtIpSKcgneLBhjWxaQqsTN1Bf52MaQD5OSubZVJUHkWbaDTWSUppNhqrI6K8K7CUlMCQYPJAwaQ+TTHJeQbNCRGFo+Ukl5Bc8ihyyapoIaciGLv+NdrUYIKxPc8dWoiTfPF8bS3ur2+0F9A32mRrQRoGaRikYZCGQRrGc9cwAsPrSldRhnXTMlY966RpkKaxWprG5eE5i0cMyRk/UM4g+48fKWishlP+5n61c/jW7nxu+T99Of5yfMROji8uDn69l4P+NTr7hPQWkrR7wkfhxBe8P0kvzdY/AxfVl+Tu+UiC0bdfKj3lwLQx8FoIL6sOccksd4x/ea6UJkyFW+nBqZ4Y8RvleUZGSk6SKsyHDDjjGMUZWCLdcDhiZrDFPgR/luzs7+pgVsbMDlJ2DYMjb+WWNUENSJnmJXncZL1QO/MBqCaMQJnnqINM6LcnvV1VaU3Qx50gVBFFYEidsMpXTHq6Kz06hqWXzHQMeBXv2thkE2gJJicAZOxS3H7oxGJV7OfI6b+BgqUHrrbwZBCez/ZZiBGyy8uPeHrsUyigcnWv0BY8WAWer2JnodPDa7OgmaLMIlYbB6NY+FJf33W26Hf1QuacUnLT6d0HkqL+kH5XzuddLn34raXezESAY6J+6dY/4sk0VOmObz1z5omrILBWW6BrPkSha9/XfU+rsYBCt08WSMtV73ASRJGcVDxS8UjFIxWPVLylqHj565iWIeLlrrdTUKgbg76uxoArKeL1Jq/zaT0zbeyR3bjMTXcepuW0SMtpbN5bI1mGaUqtUSMh53k7cUHTkLyYkQgKiuejqiBMiMUHkl4faJcl2bG9ySKSAvS8J5BwRwKAdhWc6gTAwaVpe/DamJsBvhi7ionfYbA8Oe7YEv2hsCFGGU0quQjxA7XCbLEvsBpYaUDynowcIWsiX4h0M/dQCQy+sLnpsNDJvNp/EPMfmL6tbXIOCkW6Ww1Q31WICOwcpn8RGKM5xIS6F737kvl8yb3JQoS+SYyeGD0xemL0xOiJ0ROjjxm9AkqAxojTP1tO70ExIs+JldCcGD0xejLNeGLTjFa72jSj2ZzPNOPD6eHZyYfTX5+5cQbQe9MBRozzV8zz8T6f2NcQdyYMliolQfihFZlOcM+zzKwH2sQgw1QGGQK1Ch6dnCmLJbFlhRI+tSSwycxAeTPBSc3ADChFRUIJ8lYbOdHUdpWzXLwzyYmVjso8jYAGLpKp5KnBneiIENZG5JNFW1xowQf1XN9FLIHijS+C0Hd0XKkdSSyTnCchdzlGST8jW4N0WHcWkSa2SZogaYKkCZImSJogaeIFShOByKgMgIEA96YKRQJ8ump5+a3TkF83SK94xnpFgnTfMDI3IHGCzA1WlZorJp4lqdG2d7q73/PdayS2OZrJUoLJbJhXGYDzuungW74wBNq6x1w/ilC7E606c2CqRPUFI2i2AKRc3TET0XUZDod4vEKx9SQLcbSasiNxjm9DNpRfDol3Bwsm+UAEEyxgOiUlRUUlwbwWVmwtBfH7prwucWgWmxDMay3wKslmPQ7fmBKlYvlHeP/Ohcrq/HT9FbZ3XRdQx64cm/wTFYPzpLDJlPxWxGmmTftWNe3f5i4WnnCpx9mQdaj5etJdyLLhIfLBHskHJB+QfEDyAckHJB+QfJCXDxKMQfLBy5APUg6SYx0kJpCYQGLCOokJkRbQZwOIIDuSkfXK0MDzAYNQuRLQe94w3tV5AK5ugtXnCHzhWabB5eJHAGg3fOHd8O1F7v7oNInOEp0lOkt0lugs0dkXTmcL1vq0F/7SzuMnyFbjUmKxxGKJxT4LFqu2pNEDYNW+tHCQ32gz7sw++giQPG5XpxrXRAQPMtte9il32kZegW3khXh3ky7dpAsriG8T3ya+TXybto+T8/EJuiC2/dzZdqaxafOYaDfR7vWl3dqxfdWpaHVKWx+F3mLKd78zid3194DOaftxW3BHJmbolis1OY5MviOzcghLTp97vgvMyIbFQdHalNPD1Bgz9C8Ovq1uZYzmHPYeYgU+MbyH6XaYxELbzJruLmI1vb1DdJe2mZdNe/XAnQ65d5qd1sE6ouB4yulGUydtQL2wy9KGU24fW52rxzKrawIDYp8nhN8fjt9LCJUgPEH4HwbhExdGPVgXK/wYXcAawU18Qff+HLjPOnL6wRCeds1WZtdsocOXHaIRRCOIRtyXRsTTN/EI4hFrwCOqHSYSrSBaQQ50f4wD3Z3mlLuN9+dzoHv25fLXsxfhQLdErKbcY8w+BOhs1wohjgILU2aKPQEkBb6Kv1aU6DLji6fvCm2eGMUyNRJ1pU3xTiF9w++/e1yaxhZAjn+z19FD9aIHvQ5e2IiJ2AJ38tJuiRo4rUVoDu2WEM0hI0EyEiQjQTISfHlGglcOY7PEjQQXrb6k8Zf7lQdqduXLdOWQEeRyjCBTdkDqBhk90o7puu2YVnL1hZi+OnLcZzwE6suVAIwOdp1+LuJ+KFTkevlg4tYQmsfgPTOQpCMM/MkMy5UhzGaKv7/nN0j21fU5SWSm9qoL6y58IvqbrBdCNu929+u4Th1ZlLqfBx6OocOOoGw3QgbmkKtt3fFoEosGpwLDoISFQ46qVJ9VgfELqNOjJE9z77Eml/bilXqyHtXfBqkVD9qVXUyuaJNcQXIFyRUkV5BcQXIFnWnMM/sMfCPzizUwvxhz3wFY+pjWF635zC82V8RYpNlYHXOR1FeTIOmEDENIPVlP9YRlVIisWKJlkVCiX2EHBl3x9nSGA0bpC8qIW4sVEIRqBN4qtMk+qyXj5BMbczOQbKB1kTh2NxthKsZE9xurG4Szt/yqi3zLFhimVGbheKGPDCAhdWuRTiVzdbIsGk880NlTEs05rHxo0zGHulH3oner4inKMrimyM0keiXTALt0w+HIC4P5lRmNgOq/uzJJvh4k8ZBE8zCDkt0FFJoUcJBCQwoNKTSk0JBCQwoNKTTraHtBhhdLNrwYm8EoQzdITSBDDJISnpOUkJhUPEBNSE5boKwwl6qQyVfm7EYUO9BlUQd+XYeYYB1TVDtJwh0MgJ1zZxJHMYkvRx6G3IeJWwhtVRLAVKyS93wXphXBRsCo8ALohGyriKPbmS3Bc5YpDOmt5UKgeiv6NnlqC9uFlEMsyywlg6SD9bDuWEg7oBuiSDsg7YC0A9IOSDsg7WB9z6GQbrBc3QBpBQkGJBiQYLBmgkFBBcBDDmrfXt0ZtYApAm3xE09f5hZ/e5FDGC26UIpoOtF0oulE04mmE00v0HRImij6S6HoGaBOZwOInBM5Xyu3CjkvCpEfwgV39TF+fBWiL0dkSrXlbaFzejEYmIYZuyaATPv6QIAMfNcZwit9UxpQMshg4pv+HZJejDKdBpJCzM14Xw2iWIgyP2xrezHOTH4WiTMzcid/F4SG7wk3k9f4FfMafx4t5y7MEVw7QFJmvLQnR0eCCfmvF/IvQvPsoFZof4LAljkugykRcOzWjGtk1TeJ4S58ozIm1T1WYy6DnIfzjJ82jPbQtT18L+sFDWYUPO3rIi0oZ1a/pKxwpRDqzZRPGBNVSiIjL2v/biEz2yd0oga5Ece2F0wqPiKeQjxlXXmKwD5NbIXYyuqzlSpgQ6SFSAuRlme6XTHLiJDowcvZq1iIH2zTXgVxAOIAszgAufAgBrA++xXkdoTAP91u+6Nvt+20qm+3bW/Pd7vtpy/HX57rlba4lEKSdg83ZgZM3bIC0xYucpIxmM4YLhpbrHg9rc2v8YaZgAkuJ/HuiAx7dQRZMFyjmICswOu9UN1aYzqMs2gFjanLJ/XaiU5wkx1BAdhHFxYOWI/vYB4jrjmLzmndNod63q1jDdStKIKNNIm7aIzKKm1xZEfM9gIUpr1LFIYoDB1RoiNKdESJjii9qCNKCcPWUGIVVYlHkSDK7OqZH0LSeFm3MtH5B9L5epPIPO3kPfJOXsRDhWHxWJoDJIzNqA0FDQt5lDYP/OiOgcPCtAeLQV/bH8qsAaIyHXTx1pD4XtUt9h74hUpMKi+cPaE24Qx9M2qSvjEKfQcvcR2lr+tLReIrXX1hIBOcbLFD7Yuj6mH0ZQ/9dqLeiOaI3cPkPYkCJGS9G7H0HMkmBpxhwDuzNvEWY8B7xICJARMDJgZMDJgY8At30qExRVcjL/LOQZejRpejrtndqCtjY6BBPIvHExkUkEEBaRDroUEAA4o0Q2b4guMRxdccybrnSm1ZjNcdpwrFBqkRL1uN6CxDjejQraCkRpAaQWoEqRGkRpAaodSICH6RGEFiBIkRDxcjYjZDWgRpEaRFrI89hDrXLILoBDEJDS9VaGgtx/B/n4QGEhpIaCChgYQGEhpIaNBmDwpfkdBAQgMJDcuwelDDiYQGEhrIi8IP8qKwN8WLQnM+LwqH7w9OT48/vgw/CpHLIgkrPuPA5yI+a00iqSKjcuijJx8GOafUgA3G8MN13MB1cLpH33BBxlGcZV4L+IvHKTFL8Gv0sOA6Sn9BiYR7nqUkGUwk1VcyH5Q0lSTbkaCCPujSUMNCe5VYZpHJ7bhRjB8hxru8xUXR5O+YVdnYyMYj542DnDXogbm/iGZDd+OQZkOaDWk2pNmQZvNCnTUkIO2luGt41k4Z4tYkfYDcMpAZwjociTDybBO5aJayMm74rpSaP2foMkv4KBomxCYJkYWC5TrDOrpsfzzThphAJ8YNNodcPpFNg8pcTLgPMQdzM25UDuox7VaZJ9o9nXZ3ZnqIWIh2t4l2E+0m2k20m2g30e4XbioRo4puBHXIVoJsJchWYnEdpLB7R8YSZCxBcsh6yCFGfoudpBCSQlZYCtlbihTSISmEpBCSQkgKISmEpBCSQrQUEplYkhRCUghJIUuQQiI+RVIISSF0buSHnBvZ3ak+N7LTmfPcyNnp6fHh5Yez0xdydERdSLLEAyKp4DL/GZH8N/NoM/qkSObBtMMiadSzzotkakBf9gklBTgA2dgoRCXvEQ2dGtHDsr2AZrNDV3ySZkOaDWk2pNmQZvNST41kMA4dHHkOB0fSBiWNgM6OkLHEWpwdKVHQ7GWeLEtEf5BpRMpzZ1tHEDdeqaMdi3FjuvyTuDFxY+LGxI2JG794e4YUWNDpjhyVLhMeMnYgY4eHGTuUuRDZO5C9A6kZ63H0o7Rnvv7yBWkO9z5DsZjmQDdvkOZAmgNpDqQ5kOZAmkOqOdAxClIWSFlYqrJAJylIWXgxysJPUbQ46eF8hqXebuieX5PGSNg8xe6tjg4OJnrU9Ll/rd8E6J/2iNpnNbuefKolsQfC9izg3E7Kd2HSkEGmF32roma5dpHCUqS+cgkIxK1qsohBV884ydNKtjzSTC47bk0Hlre+OLCqOFvSXzOySHYYQkJmxWfRMKwdXXTPP5+dHF++P/5ykf0wJWLpHAbBv4fCRwJd88q6S64Bm7nQobgtjKmavDa9L751MXGMKooazUDVpZoqARWniVzftP4RtUBh8sm09Pd5EtnMSSWmY8ZCk2qLrp61XlfDmAw+yaKPuNkb92720zi6+7S6U/XRrBZ/QOnSbtF6WLeIVrVs54JRr9pVfoozWss/LZUIw6pfjjqdLnLmQSjFpY5IZWjle9m3qSA4g7YjzJxrKOH7CroW8r5w74wnYXaoE7tPJy1mdMl9deE6eqadWf2NOOu7XixN1QrRMjnQSLfmuON6M0aEAEijsFruM88ETOinH0cV1o0JUha21ZrtDMKNF339xE7/3c78u9mw9Tz/Nc4GsrVMj6j8KPcjg2k7mX83W5kfrZ3sj376791+NsP9WkXF/eEqKlKLcEkM8OPRUEeB8MYUY/08VOpy7e9O+9auD/8VoZmbFPE0Ws3GdkNp2LUuVKMXZjh7siFz95Kejklv9o5J+ehnxTocqaqapVZFqp+fRtkqrtIK9f30/f8BeKHTQKFaAgA= +{{- end }} diff --git a/charts/kubezero-mq/update.sh b/charts/kubezero-mq/update.sh index 121b645..c875b71 100755 --- a/charts/kubezero-mq/update.sh +++ b/charts/kubezero-mq/update.sh @@ -11,4 +11,5 @@ cp -r k8s/helm/charts/nats/* charts/nats/ rm -rf k8s # Fetch dashboards -../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/nats/grafana-dashboards.yaml +../kubezero-metrics/sync_grafana_dashboards.py dashboards-nats.yaml templates/nats/grafana-dashboards.yaml +../kubezero-metrics/sync_grafana_dashboards.py dashboards-rabbitmq.yaml templates/rabbitmq/grafana-dashboards.yaml -- 2.40.1 From fcdfadce41440d631ea8fb27b07c8c3f372b400c Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 11 May 2021 10:07:38 +0200 Subject: [PATCH 06/19] feat: metrics version bump, re-add latest node-exporter dashboard --- charts/kubezero-metrics/Chart.yaml | 6 +- charts/kubezero-metrics/README.md | 38 +++- .../charts/kube-prometheus-stack/Chart.yaml | 4 +- .../charts/kube-prometheus-stack/README.md | 6 + .../charts/grafana/Chart.yaml | 4 +- .../charts/grafana/README.md | 5 +- .../charts/grafana/templates/_pod.tpl | 6 +- .../charts/grafana/templates/secret.yaml | 4 + .../charts/grafana/values.yaml | 8 +- .../charts/kube-state-metrics/LICENSE | 202 ------------------ .../charts/kube-state-metrics/values.yaml | 2 +- .../templates/_helpers.tpl | 25 ++- .../templates/alertmanager/ingress.yaml | 15 +- .../alertmanager/ingressperreplica.yaml | 13 +- .../alertmanager/serviceaccount.yaml | 2 + .../exporters/kubelet/servicemonitor.yaml | 1 + .../job-patch/serviceaccount.yaml | 2 + .../prometheus-operator/serviceaccount.yaml | 2 + .../templates/prometheus/_rules.tpl | 2 +- .../prometheus/additionalPrometheusRules.yaml | 3 + .../templates/prometheus/ingress.yaml | 15 +- .../prometheus/ingressThanosSidecar.yaml | 15 +- .../prometheus/ingressperreplica.yaml | 13 +- .../templates/prometheus/prometheus.yaml | 4 +- .../templates/prometheus/rules-1.14/etcd.yaml | 2 +- .../prometheus/rules-1.14/k8s.rules.yaml | 36 ++-- .../rules-1.14/kubernetes-apps.yaml | 6 +- .../rules-1.14/kubernetes-resources.yaml | 18 +- .../kubernetes-system-apiserver.yaml | 2 +- .../rules-1.14/kubernetes-system-kubelet.yaml | 2 +- .../rules-1.14/kubernetes-system.yaml | 2 +- .../prometheus/rules-1.14/node.rules.yaml | 2 +- .../templates/prometheus/rules/etcd.yaml | 2 +- .../serviceThanosSidecarExternal.yaml | 28 +++ .../templates/prometheus/serviceaccount.yaml | 2 + .../charts/kube-prometheus-stack/values.yaml | 22 ++ .../dashboards/k8s-dashboards.yaml | 3 + .../kubezero-metrics/dashboards/zdt/home.json | 2 +- .../templates/grafana-dashboards-k8s.yaml | 4 +- .../templates/grafana-dashboards-zdt.yaml | 2 +- charts/kubezero-metrics/update.sh | 2 +- 41 files changed, 216 insertions(+), 318 deletions(-) delete mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/LICENSE create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecarExternal.yaml diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index 28f7890..0dedf46 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-metrics description: KubeZero Umbrella Chart for prometheus-operator type: application -version: 0.4.0 +version: 0.4.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -16,11 +16,11 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: kube-prometheus-stack - version: 15.2.0 + version: 15.4.4 # Switch back to upstream once all alerts are fixed eg. etcd gpcr # repository: https://prometheus-community.github.io/helm-charts - name: prometheus-adapter - version: 2.12.1 + version: 2.12.3 repository: https://prometheus-community.github.io/helm-charts condition: prometheus-adapter.enabled kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-metrics/README.md b/charts/kubezero-metrics/README.md index 541a4b7..335ab29 100644 --- a/charts/kubezero-metrics/README.md +++ b/charts/kubezero-metrics/README.md @@ -1,6 +1,6 @@ # kubezero-metrics -![Version: 0.3.4](https://img.shields.io/badge/Version-0.3.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.4.1](https://img.shields.io/badge/Version-0.4.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero Umbrella Chart for prometheus-operator @@ -18,8 +18,8 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| -| | kube-prometheus-stack | 14.3.0 | -| https://prometheus-community.github.io/helm-charts | prometheus-adapter | 2.12.1 | +| | kube-prometheus-stack | 15.4.4 | +| https://prometheus-community.github.io/helm-charts | prometheus-adapter | 2.12.3 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Values @@ -45,15 +45,32 @@ Kubernetes: `>= 1.18.0` | kube-prometheus-stack.alertmanager.enabled | bool | `false` | | | kube-prometheus-stack.coreDns.enabled | bool | `true` | | | kube-prometheus-stack.defaultRules.create | bool | `true` | | +| kube-prometheus-stack.global.rbac.pspEnabled | bool | `false` | | +| kube-prometheus-stack.grafana."grafana.ini"."auth.anonymous".enabled | bool | `true` | | +| kube-prometheus-stack.grafana."grafana.ini".alerting.enabled | bool | `false` | | +| kube-prometheus-stack.grafana."grafana.ini".analytics.check_for_updates | bool | `false` | | +| kube-prometheus-stack.grafana."grafana.ini".dashboards.default_home_dashboard_path | string | `"/tmp/dashboards/zdt-home.json"` | | +| kube-prometheus-stack.grafana."grafana.ini".dashboards.min_refresh_interval | string | `"30s"` | | +| kube-prometheus-stack.grafana."grafana.ini".date_formats.default_timezone | string | `"UTC"` | | +| kube-prometheus-stack.grafana."grafana.ini".security.cookie_secure | bool | `true` | | +| kube-prometheus-stack.grafana."grafana.ini".security.disable_gravatar | bool | `true` | | +| kube-prometheus-stack.grafana."grafana.ini".security.strict_transport_security | bool | `true` | | +| kube-prometheus-stack.grafana."grafana.ini".server.enable_gzip | bool | `true` | | +| kube-prometheus-stack.grafana.defaultDashboardsEnabled | bool | `false` | | | kube-prometheus-stack.grafana.enabled | bool | `true` | | +| kube-prometheus-stack.grafana.extraContainerVolumes[0].configMap.defaultMode | int | `511` | | +| kube-prometheus-stack.grafana.extraContainerVolumes[0].configMap.name | string | `"script-configmap"` | | +| kube-prometheus-stack.grafana.extraContainerVolumes[0].name | string | `"script-volume"` | | | kube-prometheus-stack.grafana.initChownData.enabled | bool | `false` | | -| kube-prometheus-stack.grafana.persistence.enabled | bool | `true` | | -| kube-prometheus-stack.grafana.persistence.size | string | `"4Gi"` | | -| kube-prometheus-stack.grafana.persistence.storageClassName | string | `"ebs-sc-gp2-xfs"` | | | kube-prometheus-stack.grafana.plugins[0] | string | `"grafana-piechart-panel"` | | +| kube-prometheus-stack.grafana.rbac.pspEnabled | bool | `false` | | | kube-prometheus-stack.grafana.service.portName | string | `"http-grafana"` | | +| kube-prometheus-stack.grafana.sidecar.dashboards.provider.foldersFromFilesStructure | bool | `true` | | +| kube-prometheus-stack.grafana.sidecar.dashboards.searchNamespace | string | `"ALL"` | | +| kube-prometheus-stack.grafana.sidecar.image.tag | string | `"1.12.0"` | | | kube-prometheus-stack.grafana.testFramework.enabled | bool | `false` | | | kube-prometheus-stack.kube-state-metrics.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| kube-prometheus-stack.kube-state-metrics.podSecurityPolicy.enabled | bool | `false` | | | kube-prometheus-stack.kube-state-metrics.tolerations[0].effect | string | `"NoSchedule"` | | | kube-prometheus-stack.kube-state-metrics.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | kube-prometheus-stack.kubeApiServer.enabled | bool | `true` | | @@ -82,26 +99,27 @@ Kubernetes: `>= 1.18.0` | kube-prometheus-stack.nodeExporter.serviceMonitor.relabelings[0].separator | string | `";"` | | | kube-prometheus-stack.nodeExporter.serviceMonitor.relabelings[0].sourceLabels[0] | string | `"__meta_kubernetes_pod_node_name"` | | | kube-prometheus-stack.nodeExporter.serviceMonitor.relabelings[0].targetLabel | string | `"node"` | | +| kube-prometheus-stack.prometheus-node-exporter.rbac.pspEnabled | bool | `false` | | | kube-prometheus-stack.prometheus-node-exporter.resources.requests.cpu | string | `"20m"` | | | kube-prometheus-stack.prometheus-node-exporter.resources.requests.memory | string | `"16Mi"` | | | kube-prometheus-stack.prometheus.enabled | bool | `true` | | | kube-prometheus-stack.prometheus.prometheusSpec.logFormat | string | `"json"` | | +| kube-prometheus-stack.prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues | bool | `false` | | | kube-prometheus-stack.prometheus.prometheusSpec.portName | string | `"http-prometheus"` | | | kube-prometheus-stack.prometheus.prometheusSpec.resources.limits.memory | string | `"3Gi"` | | | kube-prometheus-stack.prometheus.prometheusSpec.resources.requests.cpu | string | `"500m"` | | | kube-prometheus-stack.prometheus.prometheusSpec.resources.requests.memory | string | `"512Mi"` | | | kube-prometheus-stack.prometheus.prometheusSpec.retention | string | `"8d"` | | +| kube-prometheus-stack.prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues | bool | `false` | | | kube-prometheus-stack.prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.accessModes[0] | string | `"ReadWriteOnce"` | | | kube-prometheus-stack.prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage | string | `"16Gi"` | | -| kube-prometheus-stack.prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.storageClassName | string | `"ebs-sc-gp2-xfs"` | | +| kube-prometheus-stack.prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.storageClassName | string | `"ebs-sc-gp3-xfs"` | | +| kube-prometheus-stack.prometheus.prometheusSpec.walCompression | bool | `true` | | | kube-prometheus-stack.prometheusOperator.admissionWebhooks.patch.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | kube-prometheus-stack.prometheusOperator.admissionWebhooks.patch.tolerations[0].effect | string | `"NoSchedule"` | | | kube-prometheus-stack.prometheusOperator.admissionWebhooks.patch.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | kube-prometheus-stack.prometheusOperator.enabled | bool | `true` | | | kube-prometheus-stack.prometheusOperator.logFormat | string | `"json"` | | -| kube-prometheus-stack.prometheusOperator.namespaces.additional[0] | string | `"kube-system"` | | -| kube-prometheus-stack.prometheusOperator.namespaces.additional[1] | string | `"logging"` | | -| kube-prometheus-stack.prometheusOperator.namespaces.releaseNamespace | bool | `true` | | | kube-prometheus-stack.prometheusOperator.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | kube-prometheus-stack.prometheusOperator.resources.limits.memory | string | `"64Mi"` | | | kube-prometheus-stack.prometheusOperator.resources.requests.cpu | string | `"20m"` | | diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml index 2973939..54a73d5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml @@ -10,7 +10,7 @@ appVersion: 0.47.0 dependencies: - condition: kubeStateMetrics.enabled name: kube-state-metrics - repository: https://kubernetes.github.io/kube-state-metrics + repository: https://prometheus-community.github.io/helm-charts version: 2.13.* - condition: nodeExporter.enabled name: prometheus-node-exporter @@ -44,4 +44,4 @@ sources: - https://github.com/prometheus-community/helm-charts - https://github.com/prometheus-operator/kube-prometheus type: application -version: 15.2.0 +version: 15.4.4 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md index 93b23c0..eeaad6b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md @@ -122,6 +122,12 @@ kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheu ### From 11.x to 12.x +Version 12 upgrades prometheus-operator from 0.43.x to 0.44.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.44/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +``` + The chart was migrated to support only helm v3 and later. ### From 10.x to 11.x diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml index 7b52c49..167090c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 7.5.3 +appVersion: 7.5.5 description: The leading tool for querying and visualizing time series and metrics. home: https://grafana.net icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png @@ -19,4 +19,4 @@ name: grafana sources: - https://github.com/grafana/grafana type: application -version: 6.8.0 +version: 6.8.3 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md index c2513a8..8719100 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md @@ -158,13 +158,16 @@ This version requires Helm >= 3.1.0. | `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | | `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | | `sidecar.dashboards.searchNamespace` | If specified, the sidecar will search for dashboard config-maps inside this namespace. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces | `nil` | +| `sidecar.dashboards.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | | `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | | `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | -| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `nil` | +| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `nil` | | `sidecar.datasources.searchNamespace` | If specified, the sidecar will search for datasources config-maps inside this namespace. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces | `nil` | +| `sidecar.datasources.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | | `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | | `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | | `sidecar.notifiers.searchNamespace` | If specified, the sidecar will search for notifiers config-maps (or secrets) inside this namespace. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces | `nil` | +| `sidecar.notifiers.resource` | Should the sidecar looks into secrets, configmaps or both. | `both` | | `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | | `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | | `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` | diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl index 4d55ac9..98d5d4e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl @@ -96,7 +96,7 @@ initContainers: - name: FOLDER value: "/etc/grafana/provisioning/datasources" - name: RESOURCE - value: "both" + value: {{ quote .Values.sidecar.datasources.resource }} {{- if .Values.sidecar.enableUniqueFilenames }} - name: UNIQUE_FILENAMES value: "{{ .Values.sidecar.enableUniqueFilenames }}" @@ -131,7 +131,7 @@ initContainers: - name: FOLDER value: "/etc/grafana/provisioning/notifiers" - name: RESOURCE - value: "both" + value: {{ quote .Values.sidecar.notifiers.resource }} {{- if .Values.sidecar.enableUniqueFilenames }} - name: UNIQUE_FILENAMES value: "{{ .Values.sidecar.enableUniqueFilenames }}" @@ -180,7 +180,7 @@ containers: - name: FOLDER value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" - name: RESOURCE - value: "both" + value: {{ quote .Values.sidecar.dashboards.resource }} {{- if .Values.sidecar.enableUniqueFilenames }} - name: UNIQUE_FILENAMES value: "{{ .Values.sidecar.enableUniqueFilenames }}" diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml index 4fdd817..57d2e5f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml @@ -6,6 +6,10 @@ metadata: namespace: {{ template "grafana.namespace" . }} labels: {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} type: Opaque data: {{- if and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml index 40f6a26..5782ae0 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml @@ -53,7 +53,7 @@ livenessProbe: image: repository: grafana/grafana - tag: 7.5.3 + tag: 7.5.5 sha: "" pullPolicy: IfNotPresent @@ -615,6 +615,8 @@ sidecar: # Otherwise the namespace in which the sidecar is running will be used. # It's also possible to specify ALL to search in all namespaces searchNamespace: null + # search in configmap, secret or both + resource: both # If specified, the sidecar will look for annotation with this name to create folder and put graph here. # You can use this parameter together with `provider.foldersFromFilesStructure`to annotate configmaps and create folder structure. folderAnnotation: null @@ -644,6 +646,8 @@ sidecar: # Otherwise the namespace in which the sidecar is running will be used. # It's also possible to specify ALL to search in all namespaces searchNamespace: null + # search in configmap, secret or both + resource: both notifiers: enabled: false # label that the configmaps with notifiers are marked with @@ -652,6 +656,8 @@ sidecar: # Otherwise the namespace in which the sidecar is running will be used. # It's also possible to specify ALL to search in all namespaces searchNamespace: null + # search in configmap, secret or both + resource: both ## Override the deployment namespace ## diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/LICENSE b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/LICENSE deleted file mode 100644 index 393b7a3..0000000 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright The Helm Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml index 9522cfe..aaf97bd 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml @@ -172,7 +172,7 @@ resources: {} kubeTargetVersionOverride: "" # Enable self metrics configuration for service and Service Monitor -# Default values for telemetry configuration can be overriden +# Default values for telemetry configuration can be overridden selfMonitor: enabled: false # telemetryHost: 0.0.0.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl index 66299eb..8336cb8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/_helpers.tpl @@ -49,7 +49,7 @@ The longest name that gets created adds and extra 37 characters, so truncation s app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/version: "{{ .Chart.Version }}" -app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }} +app.kubernetes.io/part-of: {{ template "kube-prometheus-stack.name" . }} chart: {{ template "kube-prometheus-stack.chartref" . }} release: {{ $.Release.Name | quote }} heritage: {{ $.Release.Service | quote }} @@ -94,4 +94,25 @@ Allow the release namespace to be overridden for multi-namespace deployments in {{- else -}} {{- .Release.Namespace -}} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} + +{{/* Allow KubeVersion to be overridden. */}} +{{- define "kube-prometheus-stack.ingress.kubeVersion" -}} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} +{{- end -}} + +{{/* Get Ingress API Version */}} +{{- define "kube-prometheus-stack.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19.x" (include "kube-prometheus-stack.ingress.kubeVersion" .)) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* Check Ingress stability */}} +{{- define "kube-prometheus-stack.ingress.isStable" -}} + {{- eq (include "kube-prometheus-stack.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml index 0085e73..8ade270 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml @@ -4,13 +4,8 @@ {{- $servicePort := .Values.alertmanager.service.port -}} {{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} {{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} -{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" -}} -apiVersion: networking.k8s.io/v1 - {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} -apiVersion: networking.k8s.io/v1beta1 - {{- else -}} -apiVersion: extensions/v1beta1 - {{- end }} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} kind: Ingress metadata: name: {{ $serviceName }} @@ -26,7 +21,7 @@ metadata: {{- end }} {{ include "kube-prometheus-stack.labels" . | indent 4 }} spec: - {{- if or (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1") }} + {{- if $apiIsStable }} {{- if .Values.alertmanager.ingress.ingressClassName }} ingressClassName: {{ .Values.alertmanager.ingress.ingressClassName }} {{- end }} @@ -43,7 +38,7 @@ spec: pathType: {{ $pathType }} {{- end }} backend: - {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + {{- if $apiIsStable }} service: name: {{ $serviceName }} port: @@ -63,7 +58,7 @@ spec: pathType: {{ $pathType }} {{- end }} backend: - {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + {{- if $apiIsStable }} service: name: {{ $serviceName }} port: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml index 51aabaa..c55ec2a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml @@ -3,6 +3,7 @@ {{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} {{- $servicePort := .Values.alertmanager.service.port -}} {{- $ingressValues := .Values.alertmanager.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} apiVersion: v1 kind: List metadata: @@ -11,13 +12,7 @@ metadata: items: {{ range $i, $e := until $count }} - kind: Ingress - {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} - apiVersion: networking.k8s.io/v1 - {{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} - apiVersion: networking.k8s.io/v1beta1 - {{- else }} - apiVersion: extensions/v1beta1 - {{- end }} + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} metadata: name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} namespace: {{ template "kube-prometheus-stack.namespace" $ }} @@ -32,7 +27,7 @@ items: {{ toYaml $ingressValues.annotations | indent 8 }} {{- end }} spec: - {{- if or ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") ($.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1") }} + {{- if $apiIsStable }} {{- if $ingressValues.ingressClassName }} ingressClassName: {{ $ingressValues.ingressClassName }} {{- end }} @@ -47,7 +42,7 @@ items: pathType: {{ $pathType }} {{- end }} backend: - {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + {{- if $apiIsStable }} service: name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} port: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceaccount.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceaccount.yaml index c5f1230..066c7fc 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceaccount.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/serviceaccount.yaml @@ -13,6 +13,8 @@ metadata: annotations: {{ toYaml .Values.alertmanager.serviceAccount.annotations | indent 4 }} {{- end }} +{{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | indent 2 }} {{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml index b24a395..e802922 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml @@ -150,5 +150,6 @@ spec: - {{ .Values.kubelet.namespace }} selector: matchLabels: + app.kubernetes.io/managed-by: prometheus-operator k8s-app: kubelet {{- end}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml index 2048f04..a91889b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml @@ -10,6 +10,8 @@ metadata: labels: app: {{ template "kube-prometheus-stack.name" $ }}-admission {{- include "kube-prometheus-stack.labels" $ | indent 4 }} +{{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | indent 2 }} {{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/serviceaccount.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/serviceaccount.yaml index f0292e9..650f53c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/serviceaccount.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/serviceaccount.yaml @@ -9,6 +9,8 @@ metadata: app.kubernetes.io/name: {{ template "kube-prometheus-stack.name" . }}-prometheus-operator app.kubernetes.io/component: prometheus-operator {{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | indent 2 }} {{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/_rules.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/_rules.tpl index 83245c0..0e33d65 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/_rules.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/_rules.tpl @@ -35,4 +35,4 @@ rules: - "prometheus" - "kubernetes-apps" - "etcd" -{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/additionalPrometheusRules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/additionalPrometheusRules.yaml index 794e9ad..cb4aaba 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/additionalPrometheusRules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/additionalPrometheusRules.yaml @@ -1,6 +1,9 @@ {{- if or .Values.additionalPrometheusRules .Values.additionalPrometheusRulesMap}} apiVersion: v1 kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-additional-prometheus-rules + namespace: {{ template "kube-prometheus-stack.namespace" . }} items: {{- if .Values.additionalPrometheusRulesMap }} {{- range $prometheusRuleName, $prometheusRule := .Values.additionalPrometheusRulesMap }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml index 9e881f8..67f6ece 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml @@ -4,13 +4,8 @@ {{- $servicePort := .Values.prometheus.service.port -}} {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix -}} {{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} - {{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" -}} -apiVersion: networking.k8s.io/v1 - {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} -apiVersion: networking.k8s.io/v1beta1 - {{- else -}} -apiVersion: extensions/v1beta1 - {{- end }} + {{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} kind: Ingress metadata: {{- if .Values.prometheus.ingress.annotations }} @@ -26,7 +21,7 @@ metadata: {{ toYaml .Values.prometheus.ingress.labels | indent 4 }} {{- end }} spec: - {{- if or (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1") }} + {{- if $apiIsStable }} {{- if .Values.prometheus.ingress.ingressClassName }} ingressClassName: {{ .Values.prometheus.ingress.ingressClassName }} {{- end }} @@ -43,7 +38,7 @@ spec: pathType: {{ $pathType }} {{- end }} backend: - {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + {{- if $apiIsStable }} service: name: {{ $serviceName }} port: @@ -63,7 +58,7 @@ spec: pathType: {{ $pathType }} {{- end }} backend: - {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + {{- if $apiIsStable }} service: name: {{ $serviceName }} port: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml index b09819e..5a4d6e1 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml @@ -4,13 +4,8 @@ {{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}} {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} {{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} -{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1" -}} -apiVersion: networking.k8s.io/v1 - {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} -apiVersion: networking.k8s.io/v1beta1 - {{- else -}} -apiVersion: extensions/v1beta1 - {{- end }} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} +apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} kind: Ingress metadata: {{- if .Values.prometheus.thanosIngress.annotations }} @@ -25,7 +20,7 @@ metadata: {{ toYaml .Values.prometheus.thanosIngress.labels | indent 4 }} {{- end }} spec: - {{- if or (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1") }} + {{- if $apiIsStable }} {{- if .Values.prometheus.thanosIngress.ingressClassName }} ingressClassName: {{ .Values.prometheus.thanosIngress.ingressClassName }} {{- end }} @@ -42,7 +37,7 @@ spec: pathType: {{ $pathType }} {{- end }} backend: - {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + {{- if $apiIsStable }} service: name: {{ $serviceName }} port: @@ -62,7 +57,7 @@ spec: pathType: {{ $pathType }} {{- end }} backend: - {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + {{- if $apiIsStable }} service: name: {{ $serviceName }} port: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml index fe74f29..a89c1a9 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml @@ -3,6 +3,7 @@ {{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} {{- $servicePort := .Values.prometheus.servicePerReplica.port -}} {{- $ingressValues := .Values.prometheus.ingressPerReplica -}} +{{- $apiIsStable := eq (include "kube-prometheus-stack.ingress.isStable" .) "true" -}} apiVersion: v1 kind: List metadata: @@ -11,13 +12,7 @@ metadata: items: {{ range $i, $e := until $count }} - kind: Ingress - {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} - apiVersion: networking.k8s.io/v1 - {{- else if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} - apiVersion: networking.k8s.io/v1beta1 - {{- else }} - apiVersion: extensions/v1beta1 - {{- end }} + apiVersion: {{ include "kube-prometheus-stack.ingress.apiVersion" . }} metadata: name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} namespace: {{ template "kube-prometheus-stack.namespace" $ }} @@ -32,7 +27,7 @@ items: {{ toYaml $ingressValues.annotations | indent 8 }} {{- end }} spec: - {{- if or ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") ($.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1") }} + {{- if $apiIsStable }} {{- if $ingressValues.ingressClassName }} ingressClassName: {{ $ingressValues.ingressClassName }} {{- end }} @@ -47,7 +42,7 @@ items: pathType: {{ $pathType }} {{- end }} backend: - {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1" }} + {{- if $apiIsStable }} service: name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} port: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml index 124399e..7b47e38 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml @@ -159,7 +159,7 @@ spec: {{- if (or .Values.prometheus.prometheusSpec.remoteRead .Values.prometheus.prometheusSpec.additionalRemoteRead) }} remoteRead: {{- if .Values.prometheus.prometheusSpec.remoteRead }} -{{ toYaml .Values.prometheus.prometheusSpec.remoteRead | indent 4 }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteRead | indent 4) . }} {{- end }} {{- if .Values.prometheus.prometheusSpec.additionalRemoteRead }} {{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteRead | indent 4 }} @@ -168,7 +168,7 @@ spec: {{- if (or .Values.prometheus.prometheusSpec.remoteWrite .Values.prometheus.prometheusSpec.additionalRemoteWrite) }} remoteWrite: {{- if .Values.prometheus.prometheusSpec.remoteWrite }} -{{ toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4 }} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4) . }} {{- end }} {{- if .Values.prometheus.prometheusSpec.additionalRemoteWrite }} {{ toYaml .Values.prometheus.prometheusSpec.additionalRemoteWrite | indent 4 }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml index 2a46523..53995c5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/website/master/content/docs/v3.4.0/op-guide/etcd3_alert.rules.yml +Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/website/master/content/en/docs/v3.4/op-guide/etcd3_alert.rules.yml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.yaml index 19511e8..011a4a7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -26,7 +26,7 @@ spec: rules: - expr: |- sum by (cluster, namespace, pod, container) ( - rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}[5m]) + rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m]) ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) ) @@ -56,27 +56,27 @@ spec: ) record: node_namespace_pod_container:container_memory_swap - expr: |- - sum by (namespace) ( - sum by (namespace, pod) ( - max by (namespace, pod, container) ( - kube_pod_container_resource_requests_memory_bytes{job="kube-state-metrics"} - ) * on(namespace, pod) group_left() max by (namespace, pod) ( - kube_pod_status_phase{phase=~"Pending|Running"} == 1 - ) - ) - ) - record: namespace:kube_pod_container_resource_requests_memory_bytes:sum - - expr: |- - sum by (namespace) ( - sum by (namespace, pod) ( - max by (namespace, pod, container) ( - kube_pod_container_resource_requests_cpu_cores{job="kube-state-metrics"} - ) * on(namespace, pod) group_left() max by (namespace, pod) ( + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod) ( kube_pod_status_phase{phase=~"Pending|Running"} == 1 ) ) ) - record: namespace:kube_pod_container_resource_requests_cpu_cores:sum + record: namespace_memory:kube_pod_container_resource_requests:sum + - expr: |- + sum by (namespace, cluster) ( + sum by (namespace, pod, cluster) ( + max by (namespace, pod, container, cluster) ( + kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"} + ) * on(namespace, pod, cluster) group_left() max by (namespace, pod) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace_cpu:kube_pod_container_resource_requests:sum - expr: |- max by (cluster, namespace, workload, pod) ( label_replace( diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml index 198bbb8..77bb40a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -82,7 +82,7 @@ spec: != kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} ) and ( - changes(kube_deployment_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) + changes(kube_deployment_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[10m]) == 0 ) @@ -103,7 +103,7 @@ spec: != kube_statefulset_status_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} ) and ( - changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) + changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[10m]) == 0 ) @@ -273,7 +273,7 @@ spec: < kube_hpa_spec_max_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) and - changes(kube_hpa_status_current_replicas[15m]) == 0 + changes(kube_hpa_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[15m]) == 0 for: 15m labels: severity: warning diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-resources.yaml index 898f8ee..27babbd 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-resources.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -30,11 +30,11 @@ spec: runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit summary: Cluster has overcommitted CPU resource requests. expr: |- - sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum{}) + sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) / - sum(kube_node_status_allocatable_cpu_cores) + sum(kube_node_status_allocatable{resource="cpu"}) > - (count(kube_node_status_allocatable_cpu_cores)-1) / count(kube_node_status_allocatable_cpu_cores) + ((count(kube_node_status_allocatable{resource="cpu"}) > 1) - 1) / count(kube_node_status_allocatable{resource="cpu"}) for: 5m labels: severity: warning @@ -47,13 +47,13 @@ spec: runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememoryovercommit summary: Cluster has overcommitted memory resource requests. expr: |- - sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum{}) + sum(namespace_memory:kube_pod_container_resource_requests:sum{}) / - sum(kube_node_status_allocatable_memory_bytes) + sum(kube_node_status_allocatable{resource="memory"}) > - (count(kube_node_status_allocatable_memory_bytes)-1) + ((count(kube_node_status_allocatable{resource="memory"}) > 1) - 1) / - count(kube_node_status_allocatable_memory_bytes) + count(kube_node_status_allocatable{resource="memory"}) for: 5m labels: severity: warning @@ -68,7 +68,7 @@ spec: expr: |- sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"}) / - sum(kube_node_status_allocatable_cpu_cores) + sum(kube_node_status_allocatable{resource="cpu"}) > 1.5 for: 5m labels: @@ -84,7 +84,7 @@ spec: expr: |- sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"}) / - sum(kube_node_status_allocatable_memory_bytes{job="kube-state-metrics"}) + sum(kube_node_status_allocatable{resource="memory",job="kube-state-metrics"}) > 1.5 for: 5m labels: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml index 2ed298b..c3110cf 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -51,7 +51,7 @@ spec: description: An aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-aggregatedapierrors summary: An aggregated API has reported errors. - expr: sum by(name, namespace)(increase(aggregator_unavailable_apiservice_count[10m])) > 4 + expr: sum by(name, namespace)(increase(aggregator_unavailable_apiservice_total[10m])) > 4 labels: severity: warning {{- if .Values.defaultRules.additionalRuleLabels }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml index 4d536ec..5671b1c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -59,7 +59,7 @@ spec: ) / max by(node) ( - kube_node_status_capacity_pods{job="kube-state-metrics"} != 1 + kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1 ) > 0.95 for: 15m labels: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system.yaml index 52230c6..ea2f258 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -29,7 +29,7 @@ spec: description: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeversionmismatch summary: Different semantic versions of Kubernetes components running. - expr: count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"gitVersion","$1","gitVersion","(v[0-9]*.[0-9]*).*"))) > 1 + expr: count(count by (git_version) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1 for: 15m labels: severity: warning diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node.rules.yaml index c841e6f..f24c555 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node.rules.yaml @@ -34,7 +34,7 @@ spec: count by (cluster, node) (sum by (node, cpu) ( node_cpu_seconds_total{job="node-exporter"} * on (namespace, pod) group_left(node) - node_namespace_pod:kube_pod_info: + topk by(namespace, pod) (1, node_namespace_pod:kube_pod_info:) )) record: node:node_num_cpu:sum - expr: |- diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/etcd.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/etcd.yaml index 28cc925..ce4e87b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/etcd.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/etcd.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/website/master/content/docs/v3.4.0/op-guide/etcd3_alert.rules.yml +Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/website/master/content/en/docs/v3.4/op-guide/etcd3_alert.rules.yml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecarExternal.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecarExternal.yaml new file mode 100644 index 0000000..f9a0331 --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSidecarExternal.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosServiceExternal.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-external + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosServiceExternal.labels }} +{{ toYaml .Values.prometheus.thanosServiceExternal.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosServiceExternal.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosServiceExternal.annotations | indent 4 }} +{{- end }} +spec: + type: {{ .Values.prometheus.thanosServiceExternal.type }} + ports: + - name: {{ .Values.prometheus.thanosServiceExternal.portName }} + port: {{ .Values.prometheus.thanosServiceExternal.port }} + targetPort: {{ .Values.prometheus.thanosServiceExternal.targetPort }} + {{- if eq .Values.prometheus.thanosServiceExternal.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosServiceExternal.nodePort }} + {{- end }} + selector: + app: prometheus + prometheus: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceaccount.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceaccount.yaml index 7657831..0b9929b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceaccount.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceaccount.yaml @@ -13,6 +13,8 @@ metadata: annotations: {{ toYaml .Values.prometheus.serviceAccount.annotations | indent 4 }} {{- end }} +{{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | indent 2 }} {{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml index 6b210fa..5178836 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml @@ -14,6 +14,10 @@ namespaceOverride: "" ## kubeTargetVersionOverride: "" +## Allow kubeVersion to be overridden while creating the ingress +## +kubeVersionOverride: "" + ## Provide a name to substitute for the full names of resources ## fullnameOverride: "" @@ -1607,6 +1611,24 @@ prometheus: ## nodePort: 30901 + # Service for external access to sidecar + # Enabling this creates a service to expose thanos-sidecar outside the cluster. + thanosServiceExternal: + enabled: false + annotations: {} + labels: {} + portName: grpc + port: 10901 + targetPort: "grpc" + + ## Service type + ## + type: LoadBalancer + + ## Port to expose on each node + ## + nodePort: 30901 + ## Configuration for Prometheus service ## service: diff --git a/charts/kubezero-metrics/dashboards/k8s-dashboards.yaml b/charts/kubezero-metrics/dashboards/k8s-dashboards.yaml index 74fd58d..f3d42c0 100644 --- a/charts/kubezero-metrics/dashboards/k8s-dashboards.yaml +++ b/charts/kubezero-metrics/dashboards/k8s-dashboards.yaml @@ -7,6 +7,9 @@ dashboards: tags: ['kubernetes', 'DNS'] - name: etcd url: https://grafana.com/api/dashboards/3070/revisions/3/download + tags: ['kubernetes', 'etcd'] +- name: node + url: https://grafana.com/api/dashboards/1860/revisions/23/download tags: ['kubernetes'] # cd kube-mixin; for f in *.json; do echo "- name: ${f%%.json}" >> ../dashboards.yaml; echo " url: file://kube-mixin/$f" >> ../dashboards.yaml; done; cd - - name: apiserver diff --git a/charts/kubezero-metrics/dashboards/zdt/home.json b/charts/kubezero-metrics/dashboards/zdt/home.json index c20dd4f..4ef7969 100644 --- a/charts/kubezero-metrics/dashboards/zdt/home.json +++ b/charts/kubezero-metrics/dashboards/zdt/home.json @@ -130,7 +130,7 @@ "type": "timepicker" }, "timezone": "browser", - "title": "Home Copy", + "title": "ZeroDownTime Home", "uid": "6QOeg59Mz", "version": 4 } diff --git a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml index 71be71c..dbcaac0 100644 --- a/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml +++ b/charts/kubezero-metrics/templates/grafana-dashboards-k8s.yaml @@ -12,7 +12,9 @@ binaryData: coreDNS.json.gz: H4sIAAAAAAAC/+1dW2/bOBZ+z68QhHaR7Hgzkm+1C/ShTaczxbZpt+kMsGgDg5Zom40sqiKVxA08v314kSzq4ji25UZO+JCLSOmI/Hgu3zmmrJsDwzAHA+QHESXmc+MLOzaMG/Gb9fhgClmr+fps8PHTh/e/ff7jtz/PzEbS7YEh9Hj/xxBPIZ3AiKSdLiROiAKKsM9PSTvoLBBCXUABwVHowLQv8KIx8t+6vD8oESr7T+NhKbcVJ8zZ7/OGnFIIv0cohCWTSu4/DsEI+CAVjtzS5gSE3/MdlzAk8eyeHVvHrXgQjfLbBcBnYBVvFkxKb6U2Kze6/R5lkKKlYPpFGMtuabO5WRvMjVBAizc7y7QWZ7ZYQuD7mJ3LevkayluaHiJ0saLpQFjPMEIefcsl2Y20VQGkfKLsHOiDocf7aRhBpX2C3JJW5GD/BHs45ALD8RAcWg2jadvsV6fTMOwjVXQy6ZfpXIx/GS89GNLMENL1I5MhBqFrxn1z8ff8IIY+b1QvjcUVxgiHBpubcYJD+Pr0zOA/BIYMYeMK0YkRBQwM6BoMgRA5RJwfw2/Yx0yBfzk2jFeAsFNYiyopvcdwZgyjCZgdi8Gb0EU0h5059iEV9ms3O62+bOLa/Bljj6KAdViiUeiIH3mePKIwBPGs7E6/1e50re6zXqsrej3kX5QY8pMnEzay/8IZhwIPv0GHPu+3FTV0JErwmkn3gWdwOYofAmMh9FxpCceQvvIAOy278CZFVMzTjEE5Rrjo0bLyo1D4xgmlAXn+668Ou9D1Cb8wp+nCekrm52DPAwFbENY1Ah5JR7NSrRnmyP2IU8uRGp2zjit23GwrDdfJ8sTHHFkrUcWseTd7qVNejP+8iFcRpRBflXuT1bMaIei5J9gfoXF2Zi4cgcij2flyCCNC8TTXyp2Lh8Z+rIBKz7yhXuwiEnhglkSbm5vYXOZzM3PeFAQB8sdEcUx595SMPxSDMa2MgBRUq9CMA24Y0tsUL6JMtZf0YOG6SzrkKjQz7XPl6DwzNzphMXSCPZcUQZxi4SBNMCTYiyjM3YwFABgUQSnCIlVdznEcQugXRs0jBfAiWFyxwqqtugEzwtvE96y88Mzx+VJliXwklsLHPjQPSs4xMVMfZpWSkpwXrGqpxbZyFrvKYO1yg20tzmNKlQ2rKULv4yWVcCiChQ9PesUclc5vzMrQaJZ0O9BnDlc9AYeItSUO3gQRxWo3W5PIgR9KRsXHBTynqEWMexJ6iukpUwdzqfYKf0EKFiKnRxK3epBf63mOb/6VJ3q5iJEdXmb08DoQahd7/wGnKS7j2yN8g3zGkHwHvvj7q/kkOfhq5pwLC9NTIDSLoikcsIiOIMmeIq+lBa4iuthKsNkWIfDgGPrum4X0pd4thCPJx1+aBaBUfz+Fb6R3S4L6ov1sgka02JENqEYCciFeCB5ZGjDqESDbSwJkt/IAyaIWIIJ8inGnNxyCkJRgQCbvoD+mYlZWph2SDSAroXuyPQyFb83K2yJWz5eGVx2c7hqcNglBI+R5qgWIBpaLutx7q5ovrCergkvt6VnOnnorzKlTGBZLxFzon0nPl1czaWtpSUK4tewowOU4f5lQtzCU08r3TMF1WSvyS1qZ+l0Vk0TKsj2v5OzSuJNOlOUPojdjXbzxCrnCiptq60XOo3Dl+4iZx1/EYez7LCdSdKo88nPLf7eQV4yBMOQRHYxhAfxV4THgw+H6E3HZnWx7cS3ZergwhMIURh5W6gQy7H1Q1bc83goXKaKlWIKMu5iBa0Qy3LcYx0gAHFjmNVkUcgpZIXciAXQZeEVs7kwNSDQ9ZBwfHiYcgf/w8hUkdODgyKcDMZmlhOFLZ3p+dMST88MgxBQfGTj86t8mlqwrcQNOsox4JD1vgCPzmuYKXiJGcAsrybTzFWHNXes2rqKGjS934zCf4Dg2nNwFt5ObTzHexqEA/EgJ9ot6yI3qTUAo2EzOnxAc0ixS0pUMEsqAfBddIjcCCzI8b5TUO9Wi4nVsDZkamnMhVVadjRJM+YRL6lu5s8t94sL3lbgYZpm3GXVZocfu2suUMghyyujhMS9sZcndwtFnR7/w9FZWr9I5lUXWOwy3ee/DLRjCrKgCSWEkH7lE+zt4Ker9Cs2ZP26iujcsq1cBy2prlnU/LKvuLKqxQgpm9hnuPxfjgXRdQsav4XzMqIyQCYk7I1tc+t5zLQbT9yxMmm5tS7es9l7RLauj6ZamW/dGt+xuBXyrq/mW5luPoWr1A/uw0qKVELgzjsSlPwSOlEVJU6RtKVJrvypSLV2R0hTp51Eku7lq30pzA47U0xxJc6RqOFJ2xLsnSS5ehyctZUdMzFoEaUe06PWHtQhR21oZtKpmmtXwyx3BV6LUCwRfrUSwHpTy9QdjiOjOGKUTTSMPUHQJHw2j7PT2ilF2+rsdbqFZk8pHTiq7+bpbswpSqSyOZpWPhVU6wfO+tfVHnUJM36hETsfaD5Y7QcxRsMg7HXyPgE+RBw+t436/YdzK3Aj6AQfDGYV8UzyPycuIV0NsAHvx1YzcIFvm86DsOzra9eaz4pquUfBrbAKfdf/wrQ+StRaD3Rajzj5i1FmG0cmeFI45qA2DAbUl0bc00V8w56a9lOgL9a0d1W/aS6k+Ex1SXT7WTH/HTL9XCdPXT45opq84Drsipm9Xw/TtR8r0GSZ3o2H38iTKQ0gG6o7w3mcSNQf4YaQhDGSdhWyUhdQpwdC5g84d7nnrSX+T3EE/D/Xwcoc67qklAQMPDkKHIbzu1lpx0a17H6R0sq7InTEUIX7N7bX12QsRoyn21+aA0htst6ySdtt7tR2i2955jVRviNBUZwXVKe6I2ITrtHSd9FFzndrUEt1Ifk/sgEAGvruqHJMrwBjf8HCL+ssdOUy//3RXe2E3Kw9uBdpPwMt6uvHO151U+2qOV2cpXif7wo4ThHX1rsIn9HvWUr5Zwwf0e7Zmx5od16wQ2OxuUgjUD6E9uk0Ekcs/tX9a0af/T6vZd1yVnP7Th7nxOC6kVrot1PgJuw1qk07sKYD1yS9+AoDxa2tyvn+7vQI5/OQrZIortH95iN7MvINExG7292s3s223LL2dWWcitarTb5SKNPWTizoV2UUqYleUitgVpSL2Y01FbtlXauy68PywcpKaIPkAkpMyJI2fD+WDz1P0dme93VnnFjq32PZTjlZrk9RCbwHS250r3+6cMBAHOBNJPZbussi+UEHuZ85ezgDkM7yLhDq9QKEufOOEgyi5hv4St+q+26G99FUELnRq+vUO7a6uh2rOUqt66GakRT+j9ejqoVNESN6n7stLFCSPmaAtXwhV1WfAfCDPt+Y0jY1AkMtYCxjKNGpPvp035nRsITnE+mG06h5G6z/bq4fR+r2H/LaHg1isSZi6T0H6RupmRzbTmbQHF4QXYrbMSY/TJTcvoiEMfZiScfP16Zm5EEvhNOBpi59SLhZoCVV0JjMJ76/4le3m8b/NUmqhvuR+NUWU7A75iMYv2vbAEHoDqdOHUXDzDQ9ffDVjD8ocY8NIvKTqExOPqBaEkO94kQtfCpKXsxlxF367t7EwVRZL5CgqWarYKE1Uck3KMlIvxpq/RzCcbTEt5oy5R81qLGsdw+tcSm+SCxT8GXpnM98pI23S0bVUxgbGYjHJ/5JBmtnewmx4W/nJsSeU01U6IgI/S0EZ+if+cm81l0qIBLCx+o1koDB9fPUfO/GrzK3HbWbmsgAxzxqmF8eADZJAqDo/07aU3KSlHtjT9P+O8r+tHrQstUfx+E3lf9s1S2bH30i0AE0JYmz9mTka9vGzY+sX2RkJfm9e/b899D69//2HbL1cGL4g/opiiIkczP8BlFHZZPGQAAA= etcd.json.gz: - H4sIAAAAAAAC/+1dbW/bOBL+vr+CJ+CAFPDmbCdpmzssDm3T3hXXNyS5XRzawqAl2iIik1qSSuIt8t9vSEq2LFOO7Dqx3OWHFrGGIueNMw+HlPTtJ4SCwYCyNFMy+Dv6DL8R+mb+BwrDEwJXg7OLwafzj+9fX/779X8vgk5BTvCQJJqeCj4hKiaZnBMjIkNBU0U5003mBDVNTacRVljyTIRkTkuTbEzZ26iuU0v/kLP1ad7CNLiD/792rEiC/J5RQRxCFeOPBR5hhued08h5uVDCv6qEayJkLt3x4fFhL2ei4x4uxQyUtTxYGjuHKl8uDbR6DJdKaa0y2bIaXUP2DruH3Q1kk5SNEyIVVstDXjhoy1LOzIkZ49AWqNqedvggoVJp6379KWcsIBFVeJjoEZTIiLk2ZkQZfzrqPuvaK1q5l5wniqZw3V6MaURecaYET/QQI5xIe7+RhWVJYn4llF0Zl7J8CX7jcLCQJwlOJSn3YwgxoeNY89w/6c5dWutu3ku5J9sbDmNySSeEZ6rEypzOEy5e4vBqLHjGouqg8za/4iQrq2aRusiBuS7GQ3zQPz7poJNj+697ePok6LhaHT3roF7/tIOOu7rZ85p2IDfqPetDsxPT27MnQanV1wW2Ss7sdlLTasTFBGvFBIwzskgb42xMZg4zuzzBt4Uuet1uhc0JZQWxSpIxv3GpV0+EGEJNzJPonQ6J8r5W77G4Ikbl2hilRncL/BvfOz5evMYUEdc4cbnConeW5E1TmG+XdrL26mjLHrCot9L0vdYaQoojRW5VxdAoJ+uhFgh3nWadC8zGDTrvL3Ze60Zg7TPwpE8cNCeXTR5oLRriex6Z8UPOGAkViYKldpeaI4fiUy7ViN4u5Lky4Q1Elgv6h+n+pPvXShtB3Pea6ytvNap6D7HmftONYP6YWQK8LytVWcGCD3974SDy2Y0NdS4hqGnXrV4UV+Cijhk5oknySkchY38dKI56ECh6z+G/56c6UvSeLwWUkWbIPdH0KOX+bHd9HXxOj5Y6Ks/r2rloEgv0mU3YsqkUFmOiGliB3KaGJ3DKA6LCaCBhMhMxiLEcJARHRDxZ1v88xClIA/oWSuRysyIyvMGhMpL3l5qAR1ls5bCyVESnw6fdbkMjz6KZljvodnoVnVBlMnHwGuREICDCyIr4z0rDHEQ48MJ8wpenwfPqNDAtmk0DrmUMfqmfAf97fVEfc4JesFZIu3e0Dx9XDNZtOt3MHQUoxtfj+X0lBh8WVNSQPapoK6roe1ThUYVHFT80qrDpdhDG2qASrhI2ULCQTvYVZNQgjMuYwOyZDIlAfJSDDJRLjbTUbQYczskxwwDrzA8PAzwMWBcGHHkY4GGAhwEeBvwgMMDIVQIDI0wTEqFUcJgtYAMPB0r3zNqDsVKCq9Env/oWTI3zbTwH+ZzfvI2qJO31l7ldFvdAZtY6wzIeciwiBD0Ei/RC0/FT975Tkx2WfuMdFpxQLF8VOOXb4tQcYuHMuIAgZPyOsLGKTdxfopG625oAD8dO1pwmhJlyy33rSFeFFjbP9xfzfELGxMC5SqjUXuIOemEmBGGqhgoRp45CWQ1lFeDRk7iGZrxa3htRdZCWLu1pwg2NjNX6TQBO0xSeErApU3jsxsGp7kLgiGZ6jJNlmttdQOkQu4mJ66OEV0KTDc8fIcgLGhEH85AWQ1LnpXkifbp4UQHcd7Kig3ZKonc2yS7T185TMpscQGQhB2ORhkWygvEF6NYmqG+GogP0L1+CjGEx/RLcfT6ZfH3yoKnLTo83s/7OP71C58DocmcweQUN7U5+jQzBZplxLXS7UqOQ9qNklUY75loI7v0XuPjxPzvT8RubrNdR9YJwK1T9srGqm2GQyjzT2nhjQfASfte0i5iOlJuYp0Onj0EkLI4pVJeZ8pxIAIx5XnYHWCxI5IiBmsaFcixCTXAdFICIsohe0yjDC5DjzgmeFg+yGNItvqVymfNhFl7ZMFFVhhYrj7FaZ1UknS/hHHflqcQh5yxbmFMibhGm+LbJ8nTu+jx1uHxxFmqZPU3k45dYLi2QS7nTeZtNnk5SSeS1IsVcCuhBqPbKUZ6G99ax/gQI7rjnEZxHcOsguCXFPQiAuwe7aQoNNdjQFQnbLh0e/oZVGBfII0cjQxpR6EIQPAEE8gT9jKr9O5DM5v0/Jrgx7KALM7bcNrR5EBS5mVnfEcgOD2jWBv0/plkNOw9l1tYj1hehotdu8T1ubSNuvQfxBYHHrQ+FW9tQcH2IMmvQP+mmt8E+l1ojEtIJTlwTe5sgfiwMjP/m2IT1yN4j+xpkf7yr2qzZP4zIMBuPKRsPJtdhOIiGFrQMJMSNAWWD4VS5kFRDwKUfuXE70uJJhGArWO3bN8pAfywkd3fo7CUysW8FZNsIcPf67YBmLvl2i8nCbJIlWANGj8nKM6RmCu0T8HooVqtJxBcFG+IJv9fbKjzR23s8saTYteFETKXiENcng98zzBRNyEH38PS0g2Y7lhZwUHk1uAGEMZJTBoAjs+sggBBgiEgObLy3m5NoOEUHRU7voIR8x35lIyiyKdj47cU7ZORZBTfWEH87yKSzfcMNwZtADYOQTyZUPZ71toAG17BPMyk3Kuy1Bj6CnOgCNILOcuk8kmx/dc+jSI8iNzsceOoBoy9AtawAlQoeEikHkOeAe6YGEzLhYvqdNafvhQrnOTfoveFmFWJYU4A9qi45ZPcbfvtUXPJH1f4kW34fyA0Sfttv6yDqxAGi+h5EeRDlBlFHOzqfN6/PMKJuuLgahAnVUMScRxIkJLCKjiwisVt7pjizM3T1ynCHLgUejWiI3rJ7KzJN5dprvLVaLR567eFZKw+9/FMCD4JC/Fkij0L2CoVI/VdbEcjHTG0EQapC/Ujwo6oTjz/aiD9e+mcUPfp49I2krkcfHn2sgz4e8y0TC9k6JUTUV0AWj6fsDo58Ai43KYfUSrfXQGSVNjwK8SjEo5C9RCEteADrqcctHrfsD25x1ky2hVl2+FjWQoJfp/bi1MqPg3XaVXLxB3DXATv+CK4/MeNPzDjfaOWrRR51tfadpOUXaM/eHj2wr5Muo67HrA19yvkwL8rMBLn3TZn3C/HY75Vyc5SCmJSNd6PMT3ZwdOnWx33azHnfypucOltxUvtkmmqBn74ynGzophUxVij41SMrGKdpQlug3hfAx7RGuzPlnLX+zbd4pFAhk3+PmH+PmK/q/lhV3e6DLiN6DZ9eTOiYvZCX+dCugPGYKw2h14UXtaU2vxL5069E8s/1NPuSz+detMNTcwY5o3f2u3uvEzA/ZGUJwFqgMzxtivxqhdvKGmVH4GYN1Xiws4dgx29i+4rwZhXh/kl3v1+bKkNB01zfQe2yHQkSchFJpBxfZwOwIAmTmSx9oG12/yG6jKlE5jubCAyVJRGiLBTmpdscMgbSgQPRkek6TDKI/ALBHTHBiYqnh+iCQCsYML+AJkQPLPXImCGde2a3TfAUxfiaoIiORkSDvZzZGTslFrFCHDJfzmBEQRMETBhOTTdD6MR+wFKLDhzo4pL+QiLS25Rw90gPaF6qDpQOSrB5xybcF0MQNsLYVNhBXMBVTZtfBM5NG0iBXBDNc0GfcKlyduUhequ0KugkhWmPjTTQgFHI79AYKDYHIxwKLoGnJLF95BqirKzUfyBsTEXhB1PJFFgGMKMZB/bUDSRqaGA/n5f3YLgEPgqmdXYJsSLaDUB7eSNgQyb8RouZscJqX9gXVlNqWduX8ru1oYi1t8UaOUEi7SBTh40R1i+EigVnPJPJ1HZQuEZIZmLXu71WQJ0YuTNnErJtYhxGTrQFDm4o2EYrcwRBR0ErCd0YLhloSCsyhuiBEo6jJ2Bi6/kOtuDyRCsEHCzTOgdXpDDLwSJVS6iKYsAk2mX1CLMZONERCzwN8C1DN7G2ti48ab8DsAo616+B/z0z4BIl9IroCXc9RfZbqwUBjGzdRt0yxNM8VsvDwLXO6zkOF2+yzHOgBLvKcxDmizwH0aY7F+F7l3iOPhut8FZ+3LfRAq+30QJv6Ruvfm237V2mnRfw18rTW3+L8g9ZtF8jXW2o0NZ/MORyXQTYcHns175+7bu/Gxb7X+hfWAjO8SUIDOshNi0HOGHwYAZIGzCpneeH6JzqtQPKN9LL3+rOxmMilbTrHY1NsQXA9llbg4ORWdDM1h0hZowXy6B5V41R5klbzyRtChO3sBHgceIucWL7TsxIdP+Zl9YX5lcL48GGBxu+0P64hfYzQCJDjkWEzn25vYqy6hHLM49YPGKpIpZevx2lrfu+aAGtP/ceuxTzRelKRAJM6i0bYFEJOszsxjwf2W9BwEIiSWCZMpyimw1PIBxto4zV6JMTO9HiSh3m66+5EnMRNipfHbXnqxiytHPhAaMHjB4wesC4b4DxuQeMHjC2FTA2flH1TvBOdffKMIQK/jTO0Qzn5Wm5O9TY6EWbLdKgtOetmmqv9VDxg9W/R4geIXqE6BHiXmzcvhgy7XD6QKLZY5UMpI+5QkW5AR0UlwYSX5P8W/DVYsST0glDXbxAVEp9BlGfh7QHCUOcSbJwZhci/5CgTD9QMkxI473aUw9kPZBtbeWTDDNzrnugZ82qGbO78p0FY7VFPM1zqYRXzP6twNodAbOLalDzEM1DNA/R9gWi/ZQzG8gwJhP8KxHSMtQ7tpfV1PYa4Xz1BWF8PHfI4CqDdSesjsFJZ30pMkn1++zYeDbBIHFKVXj6nW1FzYzJ6SMbhQLGb37uFXMVwkd+LVi4LaUwS8X8ZgiXOsYNiqhdnjDByTy8B71u6cdR+UdvMv/7pPR3r/zjqFumzCNK0C/93Yusbr8WMug8w1O7w1Hmq26UcsdPyx2XR+kfl3/Mt4GCZ1GZ34KXBfX9wU2GDoaC30ACzM1a+M9r/fAKZKcKyg2uZ75hcmEV7Zrb5r4HE7nUQ/5wlERwC06LIXOz6bu1LX66+z+oR+dIPb8AAA== + H4sIAAAAAAAC/+1dbW/bOBL+vr+CJ+CAFPDmbCdpmzssDm3T3hXXNyS5XRzawqAl2iIik1qSSuIt8t9vSEq2LFOO7Dqx3OWHFrGGIueNMw+HlPTtJ4SCwYCyNFMy+Dv6DL8R+mb+BwrDEwJXg7OLwafzj+9fX/779X8vgk5BTvCQJJqeCj4hKiaZnBMjIkNBU0U5003mBDVNTacRVljyTIRkTkuTbEzZ26iuU0v/kLP1ad7CNLiD/792rEiC/J5RQRxCFeOPBR5hhued08h5uVDCv6qEayJkLt3x4fFhL2ei4x4uxQyUtTxYGjuHKl8uDbR6DJdKaa0y2bIaXUP2DruH3Q1kk5SNEyIVVstDXjhoy1LOzIkZ49AWqNqedvggoVJp6379KWcsIBFVeJjoEZTIiLk2ZkQZfzrqPuvaK1q5l5wniqZw3V6MaURecaYET/QQI5xIe7+RhWVJYn4llF0Zl7J8CX7jcLCQJwlOJSn3YwgxoeNY89w/6c5dWutu3ku5J9sbDmNySSeEZ6rEypzOEy5e4vBqLHjGouqg8za/4iQrq2aRusiBuS7GQ3zQPz7poJNj+697ePok6LhaHT3roF7/tIOOu7rZ85p2IDfqPetDsxPT27MnQanV1wW2Ss7sdlLTasTFBGvFBIwzskgb42xMZg4zuzzBt4Uuet1uhc0JZQWxSpIxv3GpV0+EGEJNzJPonQ6J8r5W77G4Ikbl2hilRncL/BvfOz5evMYUEdc4cbnConeW5E1TmG+XdrL26mjLHrCot9L0vdYaQoojRW5VxdAoJ+uhFgh3nWadC8zGDTrvL3Ze60Zg7TPwpE8cNCeXTR5oLRriex6Z8UPOGAkViYKldpeaI4fiUy7ViN4u5Lky4Q1Elgv6h+n+pPvXShtB3Pea6ytvNap6D7HmftONYP6YWQK8LytVWcGCD3974SDy2Y0NdS4hqGnXrV4UV+Cijhk5oknySkchY38dKI56ECh6z+G/56c6UvSeLwWUkWbIPdH0KOX+bHd9HXxOj5Y6Ks/r2rloEgv0mU3YsqkUFmOiGliB3KaGJ3DKA6LCaCBhMhMxiLEcJARHRDxZ1v88xClIA/oWSuRysyIyvMGhMpL3l5qAR1ls5bCyVESnw6fdbkMjz6KZljvodnoVnVBlMnHwGuREICDCyIr4z0rDHEQ48MJ8wpenwfPqNDAtmk0DrmUMfqmfAf97fVEfc4JesFZIu3e0Dx9XDNZtOt3MHQUoxtfj+X0lBh8WVNSQPapoK6roe1ThUYVHFT80qrDpdhDG2qASrhI2ULCQTvYVZNQgjMuYwOyZDIlAfJSDDJRLjbTUbQYczskxwwDrzA8PAzwMWBcGHHkY4GGAhwEeBvwgMMDIVQIDI0wTEqFUcJgtYAMPB0r3zNqDsVKCq9Env/oWTI3zbTwH+ZzfvI2qJO31l7ldFvdAZtY6wzIeciwiBD0Ei/RC0/FT975Tkx2WfuMdFpxQLF8VOOXb4tQcYuHMuIAgZPyOsLGKTdxfopG625oAD8dO1pwmhJlyy33rSFeFFjbP9xfzfELGxMC5SqjUXuIOemEmBGGqhgoRp45CWQ1lFeDRk7iGZrxa3htRdZCWLu1pwg2NjNX6TQBO0xSeErApU3jsxsGp7kLgiGZ6jJNlmttdQOkQu4mJ66OEV0KTDc8fIcgLGhEH85AWQ1LnpXkifbp4UQHcd7Kig3ZKonc2yS7T185TMpscQGQhB2ORhkWygvEF6NYmqG+GogP0L1+CjGEx/RLcfT6ZfH3yoKnLTo83s/7OP71C58DocmcweQUN7U5+jQzBZplxLXS7UqOQ9qNklUY75loI7v0XuPjxPzvT8RubrNdR9YJwK1T9srGqm2GQyjzT2nhjQfASfte0i5iOlJuYp0Onj0EkLI4pVJeZ8pxIAIx5XnYHWCxI5IiBmsaFcixCTXAdFICIsohe0yjDC5DjzgmeFg+yGNItvqVymfNhFl7ZMFFVhhYrj7FaZ1UknS/hHHflqcQh5yxbmFMibhGm+LbJ8nTu+jx1uHxxFmqZPU3k45dYLi2QS7nTeZtNnk5SSeS1IsVcCuhBqPbKUZ6G99ax/gQI7rjnEZxHcOsguCXFPQiAuwe7aQoNNdjQFQnbLh0e/oZVGBfII0cjQxpR6EIQPAEE8gT9jKr9O5DM5v0/Jrgx7KALM7bcNrR5EBS5mVnfEcgOD2jWBv0/plkNOw9l1tYj1hehotdu8T1ubSNuvQfxBYHHrQ+FW9tQcH2IMmvQP+mmt8E+l1ojEtIJTlwTe5sgfiwMjP/m2IT1yN4j+xpkf7yr2qzZP4zIMBuPKRsPJtdhOIiGFrQMJMSNAWWD4VS5kFRDwKUfuXE70uJJhGArWO3bN8pAfywkd3fo7CUysW8FZNsIcPf67YBmLvl2i8nCbJIlWANGj8nKM6RmCu0T8HooVqtJxBcFG+IJv9fbKjzR23s8saTYteFETKXiENcng98zzBRNyEH38PS0g2Y7lhZwUHk1uAGEMZJTBoAjs+sggBBgiEgObLy3m5NoOEUHRU7voIR8x35lIyiyKdj47cU7ZORZBTfWEH87yKSzfcMNwZtADYOQTyZUPZ71toAG17BPMyk3Kuy1Bj6CnOgCNILOcuk8kmx/dc+jSI8iNzsceOoBoy9AtawAlQoeEikHkOeAe6YGEzLhYvqdNafvhQrnOTfoveFmFWJYU4A9qi45ZPcbfvtUXPJH1f4kW34fyA0Sfttv6yDqxAGi+h5EeRDlBlFHOzqfN6/PMKJuuLgahAnVUMScRxIkJLCKjiwisVt7pjizM3T1ynCHLgUejWiI3rJ7KzJN5dprvLVaLR567eFZKw+9/FMCD4JC/Fkij0L2CoVI/VdbEcjHTG0EQapC/Ujwo6oTjz/aiD9e+mcUPfp49I2krkcfHn2sgz4e8y0TC9k6JUTUV0AWj6fsDo58Ai43KYfUSrfXQGSVNjwK8SjEo5C9RCEteADrqcctHrfsD25x1ky2hVl2+FjWQoJfp/bi1MqPg3XaVXLxB3DXATv+CK4/MeNPzDjfaOWrRR51tfadpOUXaM/eHj2wr5Muo67HrA19yvkwL8rMBLn3TZn3C/HY75Vyc5SCmJSNd6PMT3ZwdOnWx33azHnfypucOltxUvtkmmqBn74ynGzophUxVij41SMrGKdpQlug3hfAx7RGuzPlnLX+zbd4pFAhk3+PmH+PmK/q/lhV3e6DLiN6DZ9eTOiYvZCX+dCugPGYKw2h14UXtaU2vxL5069E8s/1NPuSz+detMNTcwY5o3f2u3uvEzA/ZGUJwFqgMzxtivxqhdvKGmVH4GYN1Xiws4dgx29i+4rwZhXh/kl3v1+bKkNB01zfQe2yHQkSchFJpBxfZwOwIAmTmSx9oG12/yG6jKlE5jubCAyVJRGiLBTmpdscMgbSgQPRkek6TDKI/ALBHTHBiYqnh+iCQCsYML+AJkQPLPXImCGde2a3TfAUxfiaoIiORkSDvZzZGTslFrFCHDJfzmBEQRMETBhOTTdD6MR+wFKLDhzo4pL+QiLS25Rw90gPaF6qDpQOSrB5xybcF0MQNsLYVNhBXMBVTZtfBM5NG0iBXBDNc0GfcKlyduUhequ0KugkhWmPjTTQgFHI79AYKDYHIxwKLoGnJLF95BqirKzUfyBsTEXhB1PJFFgGMKMZB/bUDSRqaGA/n5f3YLgEPgqmdXYJsSLaDUB7eSNgQyb8RouZscJqX9gXVlNqWduX8ru1oYi1t8UaOUEi7SBTh40R1i+EigVnPJPJ1HZQuEZIZmLXu71WQJ0YuTNnErJtYhxGTrQFDm4o2EYrcwRBR0ErCd0YLhloSCsyhuiBEo6jJ2Bi6/kOtuDyRCsEHCzTOgdXpDDLwSJVS6iKYsAk2mX1CLMZONERCzwN8C1DN7G2ti48ab8DsAo616+B/z0z4BIl9IroCXc9RfZbqwUBjGzdRt0yxNM8VsvDwLXO6zkOF2+yzHOgBLvKcxDmizwH0aY7F+F7l3iOPhut8FZ+3LfRAq+30QJv6Ruvfm237V2mnRfw18rTW3+L8g9ZtF8jXW2o0NZ/MORyXQTYcHns175+7bu/Gxb7X+hfWAjO8SUIDOshNi0HOGHwYAZIGzCpneeH6JzqtQPKN9LL3+rOxmMilbTrHY1NsQXA9llbg4ORWdDM1h0hZowXy6B5V41R5klbzyRtChO3sBHgceIucWL7TsxIdP+Zl9YX5lcL48GGBxu+0P64hfYzQCJDjkWEzn25vYqy6hHLM49YPGKpIpZevx2lrfu+aAGtP/ceuxTzRelKRAJM6i0bYFEJOszsxjwf2W9BwEIiSWCZMpyimw1PIBxto4zV6JMTO9HiSh3m66+5EnMRNipfHbXnqxiytHPhAaMHjB4wesC4b4DxuQeMHjC2FTA2flH1TvBOdffKMIQK/jTO0Qzn5Wm5O9TY6EWbLdKgtOetmmqv9VDxg9W/R4geIXqE6BHiXmzcvhgy7XD6QKLZY5UMpI+5QkW5AR0UlwYSX5P8W/DVYsST0glDXbxAVEp9BlGfh7QHCUOcSbJwZhci/5CgTD9QMkxI473aUw9kPZBtbeWTDDNzrnugZ82qGbO78p0FY7VFPM1zqYRXzP6twNodAbOLalDzEM1DNA/R9gWi/ZQzG8gwJhP8KxHSMtQ7tpfV1PYa4Xz1BWF8PHfI4CqDdSesjmdB1DyBGsz6VWSS6nfbsfFsskESlarw+jvbiprZk9NHNiIFjN/83CvmLYSS/FqwcFtKYcaK+c0QOnW8GxQRvDx5gpN5qA963dKPo/KP3mT+90np7175x1G3TJlHl6Bf+rsXWT1/LWTQOYendrejzFfdKOWOn5Y7Lo/SPy7/mG8JBc+iMr8FLwvq+4ObbB0MBb+BZJibuPCl1/pBFshUFcQbXM/8xOTFKvI1t839ECZ1qYf8QSmJ4BacFkPmZtN3a1v8dPd/fx3uAEm/AAA= + node.json.gz: + H4sIAAAAAAAC/+192XLjyBHte39FmbbvtGa0cF8c4QeJlDy67kVuqWfCywQDAosSLBDgYJFa3bf97TerAJDYQRILATAfZlokQKAqazvnZFbWtzeENKZTSVmaht74C/k3fCbkG/8/XFGEBYVvG5Pb6c2nj+8v736+/HzbOHYuy8I9ldn1G01dUOORmvr64ozqoiYtDUlV2C3rC8brkj90JhiCrpqaSNfXlrL5ICnXM3Z9GfJQ6/oHu1iu1/IbvsP/fzu2qqTR301JoyGVct6/FBQo/erREn/pg2A+uMrjGOBv3q+fqaY79bLefBz+jgdNmAuKEHiL/+vVe/wXXG8anHZOB/GvC6+SJiwfQ1/l/nrjKoU1mxTZYEqwqcJe2TptnjZ3qJsuKQ8y1Q3BCL7yNuRasJarLiMoigr3wlXWZ6zXN2RJN1Y9aF0ouPKnPz0K+uPf6St7lnr/Xyoaf2k1e8PVu+Cee1OSjWv2tpbrW5cBww0D91BFuJfZdUMzqev7R2kW8q0kqspYlVWNPVB7uBfeNo9Ju9WC//V6x6R15H60Y57zdX3J/yHnMtUMTxHW7a0/3quCNmvY177zf397YzdVg84kw1faxoNCDT6KW8N+0/qGdbc7VZUNaQnfW1/yRlRMWbY+GVQT7Pmi1W91+81es9XvD/v8qiwpTyGjmdWdFZN+gV8rgkzYfa7JRnjgP/pt/Y1kyNYAkIxH8z44L3kfYGp8hns0jKX+l7OzB/6jU1FdnGlzTZDkzpk9mE9WhtLDO3KKkvpnhc2Kav2Kl9X++2xdxjPWMv4xwMdYiJFFVZaFpU5Zc80FWafrWT6pM0PDS7MbdT2mrH7sGxMv8LnddX3xxekj9mc2zJpOB/ROAu1+a70+rCqwNqNGl1QwXB3Na9x/mJL4RMY3n8kZeU8X8P+JpD8Fba2pL+HtKgriI72TFlQ1A29JNo9vmbww9VfC5ixK1DkRZJmXTFRhJSOG+sB+ra1/PJeoPBurylx68Np3RueCKRteq1sNyWcJ95fw9ULl00rDeIT3PKqy04V99rYeYeqGumDP8H6/EJZLmHF113TpnzTdrdY89n+tsnmh8ddG4IIBI4Zd+nB2HnLRap5W4MKzIJu8UqxBGp6r312ffvNVgnW8VrPp/VZS/CXmT/2FveK9bTv+Gs8tLmtGWly411XZNKivYg3doMugKYPGdLWpNfP3YOpvDdrHpAszf/N0NDgK2MxlHFZo39Xvx9u+st0ZwDvbI3hnk71zOIp957CX/o2scr2u9R9UMvZ9o8D7PJ9/i+zppiLxfrekMH4VY92HXLc1VIAUMMNZQPO3wAwVOft1fbNfJ2Hya0VMfs01GncWyPXUpy69eMb6UpOgNs5C23iEz19V+MLde2HSnJki/Rjyez7jyWKwcwId0I0PqvHBN968I4xPWbqHEaxbS3fWl1BTw1h6uXPG1DvGPXT/euS/672gPQHks6FJwIQWnfjFj7Fda7IGU663qh5L0C9L3ivfvn0rqqZi2P9XYHRPxaU51Sks+jN9agDUkr9JCszsikj/+p/Gn9gt/2kc/1e9Z5/gn/80vh+R+1fyFn53dHRETojw/PBWNxf8SzZfHL0FeESjHs7u+OsP0kymPxwnvejffwKCBM8C4gdYBEz/2xF75Y9s5juCFTDbungb2oawvmZj/dkuy5UgGnyse+b0hkwfqDK7UrUFX8x93Uejc4s1epcIPo9ybNF84x/+IUCLrbVsBQ6u/hYvLMf6T972CCxJ0D8ALz/QI8QDiAcQD9QcD3Q2wwMtIPwICPYPCNjazVdOWRVmvQ1WyzOS8aprreZem85Xq6cB6xU8G1pez2Z5zmYBvn3VyTuwGCxxC4Z/jsq+ErdwKcalGJfig1qK+xsuxSNcicu2ErdKsxSXZZ1t7X2h/aCy9VOQZOZEIp/O35MFXajaaznX0hkVpYXAu3czcpHFlavwlatZ8MrVrOLKNUpcudi0xAb2R7sEAYeftbT1cWkrherM1yZrupy+p4s7tjJN718NqieuT+SE+H59pVG64Y/ZIpnq5Wn5qTcCIttl8zje6lBqsF3A9ufOErapCVZqewo7HhXP7y/S4A62un9mcQR7Qxvs7eT2RVgiVUeqXmHA0yoY8LQrSdVb7Q3d6C0ENKUDNGyWToFo2M/TQJqtXx+OaYrh87+e35RhYf2kqga5usW1FddWFBNqLia0epu6pLu4uJZgcfXw1rkkU/0VxvBiyrXXzda44wVTxJcqrGnw3Rl8MdfZ9PYH+KTB1D/Xg8TW9SZd+kozfNH25LeQtdheA7NdjvnsciGITw8aWCYYh8+v/2LPJyGXvP0jeY7aZN6MnM4DttkaTHDcBRZY3FONud9Xrvc0yGINC3aZJ9fdC0at5tpTZLWopwiwSP+y4mnuSF5Yp50LTd98EDJNGIkTibHhJBI5ybf9mzD8k/xww0l+Pcc7g8zfhUPmfhuR3fkQkvv7mInN2cnEp2ViqITDsOCM7Sp1tLrnPEwTlIeEh7VjZgFo+Ql09hs2c+ne5mfXbqj2iTf1WkBnNuK3h8I0/unOgpceay5V3ZhLX7xbK+0vr2CpvIW5ll3sNf/sug4zXOA3/LvIn3B7vBeWMe0w1/iwCsOXEbi4YahB3Bs0pr6EXg3dxje6YEWRvdvdOi2Yg6CrQncdsUmoNfTMU3P2nuDgYU92P8d6TJvNZ6OO5wHu8Rno/3zzGzzHXPg3um66MucZbu0ajY1GWeKt3bTDbTB3HPbYO90762fYjk8+MCN7ML8a34PDiF10543gbcF68hudzcqiqWkrjI5Lv3vpX3nUW1F44JYDSGIuGbQrBQao6frv34PZbu7iMnZ75xwSWDw+CN+iPep6R3OmKCLilb3wV+aINdKjCh1hRcVhBYcSNhfmWGJjBf0eKOx2v0yAFu0CwcNn3yKRK3KIGPADb8UODl/EarybA4wgTEmHL5rxegNTbtI5L7KCF1wkO1SJoRPho+2gxIASA2KB3bFAflJ8RbaZeaDDoHk8ajbC9furW8KXBNQfDkp/SMIH5+8RHBQLDgL6QwQ4GPQQHJQSHAwQHFQDHOwWgN7Y0+LdCI8tx0UbF23fos1CI3HVLnjVbm+2aq+pP67auGrjqr3zqr1trHqJlm0eun5Y63Ypsun2orLpdtJk070QdEn0ZNP9QI3tc+oKsiToYweIuKrKs/PBL/94eX4x7Izcw40n7LlWXwQrJvqPw1HzqtkO3KHyFLrshtbVYDhuuW+4nlm1+GOz177oBS6RE/IrPB06JYF1nei8edgnmPgfYfGhSsSPH0yqW4UanY+7Q3cXa0jR75TWlQlWV9J+51curloXnj0nDUWy+s8fx63euDXwTEnq3HB+eNnuTroe+8CAtgKF/ng1vmxPLj3XuG7IL/aaw773h6Zum7TXGvXGl41A17oXND2kt+uP76jyYPD+2/R8T/UdBocLFLYj0zqtOqikzNVsgKE3bmoFWHbCixJfdLqeL/6mCTMWD+8Bu5EzwcA3E7TaCTNBPyykY0aVW0u3Dc0AMBisERoPFPOWA8bvg3Ku39l54f1rqPD8EPKtM2kGr1ibXgLf8k0v/m816eER1oYwedlZkX3p83W4+Vdpxrthu+dB2CzQ1zvV2d9OoFcFr/BYvZD3hm4c+O7GuFT3h82wL1/sUrXi4fAOADF014XADgFY7UuI3JBgbTYRHgKRPklbFbgTgfVlk7245/0+2NGgM8yoZs0sc1l1YQDLo/DRPZTC13q+jqyyudmLw3HYfrPVepGI+VfP5KtFxMMGlxft/mSLh/ECWmFuUc+014Atn/lZd2WM9z2xed7tDMbbPvGjJwm975H9Seuq307A5yINm/cB54lP/j7FQOaSzt5ZeN7bQTaGyk4WZQcWb5BJ+T/2igfIeZdsyrlnX2zHButG9qYM82DsYNQfGFD4obIGDQ6lbbJi5GBOCyAWatBd7BY2764sN96H5f73n8bpjwCEix3dO9nu0z/0CMtNMrFcorH+4GSVtz9YY9j+YE2Sq49Oh3Q+ar+v77SoR+k7a8jqtrL4ZWZ9Nf98/sWYLIiBVsa6SqsWeY5CWtArSzD06Q4L+ok+2FDW94PbR2kerVTwXP+MDLpkidXxVN/cnEHQ6CyENVjcoOmH+VNH3JCUmfQszUxBDvJh1/lw7kPYvghfJB8mvzfFJwvj+LjGKiOCO9h1LUn7OUso81nxkhAm+ip82Tq4vt2J6m/ePZP2Vmz79EDf1+oDNEsgY4LNARuBPm2xwIbvW1d9dwvab3d3qIrX6ImVCdxu1SXwdYiQHDZ4XoO9h/PwMALOvn9Hn1el9pzttrEad77kQiug/vbo0qtdXZjzueVagqut7mTk7hqNMXNduvmC/9rMlpNag65H9Rqri4VkGM71gBLFEr64qY3r0s+CNnuBscx2lmnmEp5BTsg5j25jW4yZ+9h4FAz4HyVPVFOoTAA8KoY0l+BWQWcbkO3fsa3tBnlRtSdJebCktqur86srj2CowAwqPVul6Q273YFHibsB/szlEd0ubq953vLfMI29A0o8XdX3snk1mnjesM4JEy4T3kKvtYhvs395MfBfmq7b6LI56LUnnhtYArWIx8Kl2NfCddezgxolv2FVr/ZVb9DzKJGfFdN5upcRl1dutLw0xNSFB3qQkmMgD9dOmuOwnppjx6M5Hp6G6G3qEouIwQgjv0Lnn4TXw6vpA8OW1BWZSCa6AHzqJD8Ra4GNVPYsZ9J2j+aTblaC5vkz+/M+WiKdXE7Or1IYqwzCYmaRcwVKWRG9OENlcF9ZiVl6I/evLSS56Y9/8rzZxq8lyMOb3JqeTENZ65L7MWjeukXSTJqhPLl7fy7CCME5P0N9cZ/pPPM2XjDl5tZa4T7lr/cWJUEFbGcFLCq0d3dxK0IPSqNtoWwVIlt9ouKzNY1MgdO33fj2OPQuWY2SYNg9M01drh/Uv5w0L5vhN9nPCWo1/BYgIq4CBZWT9U32c8bj8473ljtNUPTkqrlvi6ybdVNS5Vx3RdbOuiepeq67IuunbdAmmrtNmlbU3ajbbZ6H32Q/Z9QfdLzRgpq7TZoRhdb8bRKMOjS8bdKMaBNjkzYxPG0SVTkj0CbB2hmeNomqnhFoE2+UTPGSW4i6plCDSbE8oI8sqUb4Sj4HMojhfeyGzi5aW7dIrY2RocvF0niNuPYvqqnFS3QWvFkVOzwC0Pejvah3qNN5paez0x/5xHX645kvyp99yyARNxJ9EJiD5uSfqaLVvNXIRFVahwXYU9sUlhkKRbWXh81SQoYEBfw4zJsVsfWQfPs2o8+SSL9/z01iCpqIt+1CMspvI15Sl5FII8OTlopjkB/sZRdw23wOyzBSySyo5FLfB0fbkksulznS3bLzyD26mTtRQJhtryJ8qSLMOe6cWL5O+KMTHujgzkh/yO7nMExMGUL0blJMxsmtXrtGTumNEG9gNzk6rPcGhPeLU7M4UcQCQX/43w9WLrEfsjk+JPDU3N0B376t86Slwb37RHR8Fbnlqwg/zQsRXQaIzjmbKQtUl0f0awk8BN5yFgfs3BvxPctQkfvw282ojfi9kI34McpHiKcj9c71pN3rCTvYE3exx+9kT9jNnryjPWlXe+LO9rjd7b5BFcEH4jhBDC/YrBtGhqGGcYRG8IyzEPQfxwDiWEAUEwhYKYoRBM3pZwax7CByVK5HZttXjJewL78ET370ZDAKFjOOJKyHcycEPQTL6aML/lPOV4wh5MKaNIRctKblsAt8Yg65EMMeolfM+O300Vvq47fVx7EM7zrtWUj87RRGOKJJRwTxiCYfGxCQSBKSSESCtYnchb8JJ4njJTHcJJ6f2FdD0sMkspdYBhMZwplEZCLJjH/9TLNDL/t94hsSFA9JsbBhzA3hLCY8sMk6a+eEwIIjUl2nOqFfqGhy9CApzt4YDrADD4ugOlF0J2R1yKphst1rvkujbGFztrMcLP6BfSGTZbjhWYWSzH6xd7MzvFYVs3+AsjKzwz+zNFYf793qme4TztnqGxKSaHNP9m/ubBMv5G1wXlqfya/PPjJji+piKVMjrndf7t/cGeYNyNvW2u9gaGADz5JozSDwCLZtVI+x8NXeLZx1doacrXxrFddjabsKcXb+2/7tzMSIyliZFRZszHJMEx3mZIMtiDyBIlGBgAh8MnEcjC+PVCGaqSj2wimQZ0kzTKC0X2F1pQp8UpWFXwr1NtDPe28gLnZVpYH+xgrrbSCnAVbW57kO2YQvEF63QMvFNMf15s3h+uSldRH+izgfRqwfI9aX4fVnQM39CYSDHow4L0aEJ2Mjb0aQsod7NSI8G7HejTgPR7SXI0G3CXo7glUIej1CB1+cT8EbL+KSMYI3hfsY4vwh0T6RSH/DJpPJxvUJGnyjeoT+LNxnEh0REz8MX8N7WIQjJdaZ4n1R7EbgKJ9EdOIQfjk2eQi/IyaByPp6RBIR64b4RCL8nuhkIvxysQlFLF9ObFIRfktSYpHVTdPEu+ITjPBbopOIrC7HJRKxbopMJrK6zIAe+8eTSCP+J3E5RtY3RecZ4fcEc42E3gDFYw9aFU8RdB3Gjv1L/158dBvV020U+DILv1G7e5COow46jopwHAU73d48R3HbPEJBmXu7R+hS/OPpWRh/CUsmEo+dSuKZ2mNajd0TOfh+vF0GCV8GAAYmdvrlGhPtnntgq5LvLAI8Rk6cuykEDGhDZd47uAkGxj3/VzuxwsWF5VKWRMGaWop056Vqn9LYd11yn5UNlSwAsN5T44VSZSXGCMqMLB9fdTC57MBFYTbTuOOpUL/efvr3LiqkU7i1hTmteqIU+jasQ+ITo1tLaAnduvIoPFOwPJh9Tg1WK8IOmyI6YxD3psG51is1rFsW6owTskLdeztOa6Xp9qzMwWnFRXSZZ4m3GSMjRIfFUTRM6OVciZQMnagv3LVK3vLbdCJLT5RIrNLHhJFk7fWYQOMdFeoG3HmlKk3DOAPlhp3sN+MbgKwmeMv/FFV4pGIcWY1TqM8vo1xQ+zSuXWww74Wswqxj7TIgb+npwylMOtpsJulPyba9ytW2KVJM7dO2SUJKkU690qSt2usU75Ld1jsLC/Xdue3oMLwVwataB89NLa6t+85GF7ecYKMfb69+vGCKM68fL+p6GtcXevBK7MFzMpLAysC6dkxmq9DbnVxhoTt2Vqk8Nnx44H7v00vmeEEXynY7b1q9lC6U4eG4UMrk7ECXxtYujbCTaHrDQSu4XLmdH3dhua7WE2NcvqsM/B5Bo2Tu+CgqH1bOQqYr49MJ+WTVoFDJvbCkWUUa8s6uQjbieRl5kC/RFlKhfVCh8Kl52I3r9/5kXj7KxGVw0yBvT1i2E0khb386ahTHPbbmUBEm6MWZANnWjmyrZCFiHQwRyy1ELI7gBJOSbUF8Wr3+4QSPReYu88eIhT0Xg8Qqy6jKSVmyzF3meJSyyrBWLFSPTIu2M9spI0b3pU5DjI7uCnRXIICuB05ul8IP0B6k3UrRHu0VDrtOskA3AcLdvPdEfKLCLIVX4HjzN+kzYRr6pujjY7d/xX3SK7yHge/yCjHpFfbxVyleMUusxdWw2xmneQVNfEXALb51c7cSXmFvTU3zinZSNS7Ou53zVO0tdJLecdWd9EbDVN02yVT21ts0r2gnWcraJFvyV3SSuq21eTjN+E5qi1G/PRmmGhlikqHs7expXpFkKDtLb5pZKslQ/d64N7lI9YokQ12NRp3uZapXJPao8363n6rT0lZip+W75nM11PjyfJyquWknsS0u25N2dcMGWFz+VAM8pE+dfGezzZzdtnr21//BF+wp1kc93Au+gWLW3TZKAMq8ytE221O0ADffiyaxEIE922/L4IBfeaE3MuBFDQTI64/+g5mSpcdQ5oja4+7ao6TGuv2vPxI2E9lefz6qyu74RzUy4+BpSSW8CwcP1A32P3T919j1n1rT7LYPzcOPCWBQ7AySGLakhuZ7yUPszF/rzF/qzF/pzF/o3EXnrCJr3SY6OyXh2tPmWA9puzVFloh/DhPgq4WVI9z1xZJfgyqH3BC2CRLbouI8mqXF/8zSS3LtBRgalxAwniccpsViqnyC7kf92IDzi2Vi3E/l+He4GfpxZljEWWGvdUWKjhS9Suf+jVIy9FZ7gBQdKTqG3++RREnq1EbR7gM/dgbwpHAEH75qp8L39Y3F5xDekGTpK09riti9NNi91ey24gaKPe+ZihR3ZsifTWjcEgfvR9S8HbtPthxwvRwutcAJ8KGnynrO+QEqZ6cLOyMfqAH/Zw7x9XntzsDToJolPgu+FXUWfD+bs+Dx3BU8d2V9U/K5KzkdolKVo1Cy3v/STMNEB820TLTTx7NESs0325nxTdbx34WpMtsRzfKSyGKP23Dm/xKmx3eKtk7H/vIoiY8EMKiV4B7WGiZ5s3aWrVztp4RcG0TSCdxOCYXhLsFQZznb7ym7UxagjDOeqt067HNpaktVp/s7IOK8rNY/99nePn/ANj1Pac2NvLI/OwDC1IEkwt8Me6zNbSq8pYR7XZVNg7LrlLnfBO21vu4222y2Gc/ICmgVRNpFc2HKPCYEg1hxAz2GrCJzROaIzLHCzDE0zDgD6thD6nggrspqUcfNA0ZX68f0/NY+HXLn4FPrWe8k+F/UQZOBvdFBVmLNEc2KBah6Dsly2bSE7MzX5OuV3j5faAm8x6ZlsqyKAlvYVYUv/1aitb0RXlcHK61dne6/NqtjV3suZ3ZlieqEtbqwvZ3rQXAtixWXGA55LfJa5LXIa5HXIq+trUd0mJrWjlpIa5HWokc0hUeUZaWu2lmcnsKzM2fhn5N7dnjczKEwQE8kx7P67tNnIku6sTc2uCqvoKhKZY3NCs9wCfzzCthE575Qfii7dVR4mMmP4StRNmcARoixWM518lZ/BKscZcMYd3dFV7Hbn2/Q6Tfv8uMizFzFDn++cXe3jS1TQTdOnBCBEx418BZa4Ch8CMS0yaSe0QBkQg3BF/ON4gmKJzuLJwk1KVY8qewe2fO1AjAadbvN8zj1pDdsdZvjvagnE0kzXt1pTaLUlYvB5OL8IlbqsHOG+m55LzCkbQkYvfZFr5WjFuJXMiwBIQcl487mhFGm59eZuBRT79U95CeyKlPcraubwtW2Xxbcg/B5LZ/4U7iifLJxWEB6/aTTPEj9pEJbmDGkvKy+c55YhvGvEjp4V2XzR5VLuk1a5FdyTxklWeUFYncbKpm5990VLZSsyn23WJbZrFC8tWU537t/JVefby950L1BF0tVEzQr6ZLVDvc2lNqX7MExVAlNyssV0ktfBMngjFklMEQDvZR5whJ66rgWXHo9kpn8YCFh3BCPNBpjEDAGAWMQMAYBYxDSxSCM0nPoIcYgYAxC5ULrw7LcdFudkPw+q8D5W+a5/dl8oGzC1YMEyEKcjjPSdvRy3Gq5ftfBw+RFMh7JIzyKLNmzNg2uP96pTsNS16lSGwYsobyEPNIqGLTfZ33dXJJCFtb3vEUIP4F9tb/+mVq7vBdwD50dE92ErwHIyNI9kHdpj9vleacsoZF5uRwbB0YG/3hPjRdmVB2QtibIZKmpbBM81Y8BOZq6yxfOECWj8Pr+YkG8g78og2eWDLDIuaugsBFepZvFrNh5JtsWWRWfE6gVg7Jbw2qGM94CR07b2GE80GYbtUE9wkRuLYOw/mj7XzE+ZA/CVjhoasamJ6+6BBZR50FsikcUyzDoBINOMOjksINOMhDMWhh0gkEnuGtnB/nl9rNiJ5wrozqwKhwQnxtB46yHLTzHludIFBTmF/LkKFQVdy4H3dTo/kSXT1ap2HxTRuu6ihdu3wWbpjzmXWtafMuCXvs0DdyBh+ERGB6B4RHI+JDx1ZHxIZnLLvih1UxL5ga4gQCjH3ADwU5kzp7Jxo+m8lS1PdLusgMbecfsoAPFUxVDAiBv6uQeLj8xhvJs3UoAbgvr+O45W0z3RfXs0vNVp6KWt1bME2L9q0tfqdvWNqdmJt+bc921UFfUxjzMwR0EG96VeRxm3fch2CZBco3kGsk17j3AvQe49wD3HqT1pabn363eCHcfIP9G/r0D/76A9UUso6vPKpgvnpdtLLc4tXUCOLm37kreWH5eCwJiGQVDRJF/IP9A5x4699C5h+wi3rnXSk0u2iP07iG7qPaxYasctKGnfEFZFZ2BAm4t+sAh38k/GxXeossSFhe+p3Gb9Mru8q1JjqQQYZVqeaNdcDkfWQ2FKbMJg+YDmqjZG515nO1qTyHf9lz/M6id3oM+KuSIyBHRR4U+KvRRoY8qtY8qPY3sN9FHVWoWifv9ykrm/s5XklujnGmmXaWDxdD65MQecoOdkrtHSWdBcmzF09Y72PZG66DPi0uzcolloNjjm89s3x9wPPaXE+G5SuEzewUCIInwxSvjfLIqzPhWQaAM5iHQP7v3nTHroJcQGSAyQPQSopewjF7CcPIY6iUMvxW9hBl6Cdup6V03W3oXNjWhmxDdhPUkeGtPHVtbSsXuvEUD7vHzyivH3E1M6FwCtLZkT6aMMo73So01Kdkby1sX/ZP+PCupVVnRvFZlMZ7WtjKBiByuLGAQsfNuHJOSOaw2a9M/Crqdw1iY0WNybxrQCM7NMID5DaxJVjftbTvgut63prYsaZOwokV3dOFefab8I5/L2bUzlt/4TH/Vz54XZ4o2ZY7rDDPolphvr134Y+YDgUkZ3a57Jd0JVLUR0S2rR7obDaTcSLmRciPlrqpLNTXnHrSQcqNPFSl3SsrNJ8ByUpFAapcVC4G/SxAe+7NDdKB4JfRMu4vnEDpmV27O2vtC173o1l9dJGboDUVihsQMiRnumERfqI+XddLvmCzwMNiweYtlwrtcLPnAjbr+L6qpEZeR16ErtbK8DhYsKhqwqLT+VkJG4iqdZ+OI5WayjykVdOB5kp7EUvKldquStt9XLk7WVfbUVr4oxsrdp+paufv31Fauhzd0ZREk20i2MfQYN59uSbZx8yluPkVPaR6MvDPAzadIqJFQ70CoPyss06hR0gMRXaXzLIam63t7l6Rz/OQP/HhE/cUiKKpp8IhbgTwLUCrjlf1aozB7Kvvzq75nKV7zPzFjB3u/f8dLBra+tV3SFt2zCsyClBdOulpAFhyKLNi1t0dEfwULLQjbnFp736u7WwrKjNhmwy2pyAuRF6IbFt2wtXTDWreubgpXEtBXm6GvtpueGbYxuy1SQ6SGu1DDD1e3QA/1snJDKB5xiufJkcouWLRFt3dWMp4CbwbzO/snrU2s1v5Li9jwe+DdrF/U/ZwNsBD6sJCrIFfJiau88b0O5o8lFfxDMzAs4R9JmauN9Q32wNCgbm9creKUuCGqsiws9cAITYaBobit4VvBOVxrd11f+KAah2nt9htfn7HAV7u/kuUbS0GhsneAxNG4MmDkUkHhdictEh70DwYJI9QtF9Td/CAH1TTqe4CDJhj0LQfYzwt4rTFdPiwfJCURVP/7T9Mp++3UAcm/HRWJs/kmI4DVJ9bmKvhLhX4lWD2zSBdKqP2gx1TCgMwRZVuQ/bmRCevhLLE2qV0r5Ix8NA3kHRXIIMK76NuTI2gyGO5vfzoqcNMXMpHivSYHDbdDg5JS4+12G+E2wm2E22WC2/rLsgJwm5fSxor6KpAJhu1+sTYUqwpY2ypmwHyqaRwI0Gb+aUTbiLYRbVckRgn3ruDelSyjonDvSmFemU56r0wPd64giawcifzTnx5hhP+d8qBV9f6/VDT+0m+1hkE4sKKbNw9zwZRXIvhC+C/blqHMCHQ1+Mu6GCeL2yO/GQLVXZwyS5Ia6ElFuIS4IUpPs9I2ZrFuIihfRczqlJQFtHGjMoawjT0virDnzsYkUK9qt4+krNqHd/Vt22dcG6mBXLFa67jpaR8SQ/gC3OnEjYYkMWIebM9yxRxGVLqbotKoUmyoUuBOKvsW3EmFO6lKLVSE+rNTKxWd5gB3UqFUgTupdqb2qrqYPsH43wvJySwtINSCsFpQjUjKs30EnU7qu2Xq48f35O+8wuhMLQvT6XUGqZiOGHKaXfmpDtQaqc52VGfHjVi/8Bm7cvuwOhH7sEadKu/DglVe1CQOVQKrWMk2aXXTBo32mxUD2dWICkW3XTD28xdBkzh4+/E0JAZUZFOAFc0wal4121UL/uTAm6GZL1Oqw7/spOop2EnVACqLqjLLMZfBYyTnzAOQXzrVI7x6LITMrmHx+e0sg6vzuU6Nutn5DopCrKoxG99T44Wdt81O4Zad5HfMAwo2hhEL9SUiy4xXfO55qxFgKq9ld38PWHFhLrbp7BV3ufGOd/uqiJqqSF9hnE809hMko/uN7I2hl5FdsoJhvDlxZdw5V3eCFL6tLjVDGrWQISFDqp23wIXcZFVd8j+nsIowzGZU6NSgm0dYEk7sdNGsJoS9nwiz/5q6UV8nAcdoN+/ekfNgRRGclW7bVfGYBvdZITYrt3jdSQvNWv0hQjOEZiheZwkG9VdFnDJnpFmlsyOvdUv8JKz4j45uZahEIBqFZmQZe618vORti/yVvFL9mDThD0U92pdqPdfo7yZVxNepBVYXMCCmPJ6/QoZ/x9Voy/arCpF1heqbAyGgkt7yQYNIHJE4InFE4pVSSTvpVdIuQnGE4nVWSQ1JfMrft505QLu1SrwKH7CgGqvM3sI1DEEqLGQje6bBfqJwKgjA99xQF5JIOBZ8e3d+fWQHbNQc9r6XdBGBLsYDIMgtKIp8NfxurbgvNgqZ1FG5iPFueMR4Zw1L8eSOlLr6ICWW77cRyucA5XHjZKlB/lJTRX16bwUSlOrYOZh0RKrrFDC8HebwIkgGOw2bnTN+ffaR6dyiuljK1KDFY3rLcJqpKFCkkhpOUggroOUGMIRYM1Ucn68rjXp0ybLl9JrNw9tDCrVu4R7Smkrc5VKx0wLfTquHyBeRb1mQb/7JQFf4jQMGHvVBKyTFfvtmgTny/Xt9A329eI4inEM4h3AO4Vz9ZcxWWjjX6mJ4MMK5QxIy1xngYEJ+0qe81xSa/W2jrfk7SXi8Rvbe+0NAe+SKVRjhXmngXr+bCu5Z/feM9eCqQb5+FyEfQr4iFLwMMN8IMR/GoaZAdptvCXsvfDn98Sz62KAqnl5ri4HTZ0kzAAxMFzw95fT+1aB6NkCy2ORNazxl14hYNSK69JXyrFqsZoU6zR0TA8bkJxI6NoapY0M7lyUjlrA6btFnXOFZkGTubN/AwjmdH4Q92WfncaF23rw3l9DW2XbwSV0oqZWrGCnpfuO+Z1SM6HErrhl1vSwsE7kjcsfMU2Gnj/7oIHVEd8EeSGWY6DZojuKOtr2e6ESWFpIRl4vkqt0d9ca1Iai+aJWlVKl9gx/MxT3VGJZkjbefMG3LcAyarz5VKdNfTKevTfQ2q6PdVVhu73eB6iLixpgfjPlBEF9PB1B6FN/EGG5E8WVB8d1RvxMD489Of7S38oU6kqBOis6mWW5V+sDPIj/5Z6OKjiWOQXV23jI/mtLeLujk/9hDpFLm8HR885l8+0bEpUm+fycnzjEhRF9C1yV2hYlAbOhdKAMIawK75x1OE9y/OrZnDeHeQ2s8SjqBH9d+VyjhzW/au2B18snulWfkV8scSDXKQjW6w3481ah2+peIKiPPQJ5RBM/opPcWYMJD5BnV8BbcAe4QZgfvMDAsM1RI8T6X2WGP7KRNu+z7dhtUz4QJfb82WRntaqLrAF0H6Dqod+7HVUBe5ZI/9sKTP7b7I0z+mFEY1DAlr8Et08hqMmA1Gl1SIRSyVHAztajCyvvFmOovkiE+0n1K5duI4VapiVPqPTkcWOeqwUZ0nvZdM5eGXl8S4fSYW7vHkDMSUW1kEgdyrhFG/aOQX2ohPy3eHSDeRbyLeNeRe2VVmLUKTOLd3eCASWFGWovilW9mil75TNFbFLqXet0rSmiLVqwxxtVG47baxyqK2Ls0Kn47VYhKVVX8dr+NKj7i+fzl61ZaPN/uYQKoPBA9Rt/4gmrOTn8ca5IhiYL84+lZXFTNZbs76bY3jao53ioBVdKrr4bdzrheKaocVdnWJesQt/7tG2Gwygpahw+SMldrndR+LSuTCTUESUaEi+oyqsuIRsuWj3SYejtqH+EoCsy5ANXq7PZkS58uSyKt3y7D+mK0W2er4Lr1CP1CRZPFo9+/EiqIj8wICN0qs2VwLyAPtw0iHqxLjsG0cLDVayEcRDh4QHCQI0GwjqYuX6ertMnTe8nQS3VW+KVVRFdmZ0MlmqDM1AXMo3xTFfyAAihlk2NtQZ9tBsR05dk21h8e4rax/gghHUK6AiS+dnqJD/dMIaY7OInPOfdDXJp1SiF2B2+z8oXVF+QxAdNY1ZMdYWLqdtIA3Qo6tHe84aYfTAeGuh6CwLrrep20GLCPycAw6DC/oMNQitxpJ6QWTg4PHI6aV812bcIDHUjKMlXNd8pSldtJciogLQJmpQQaXNSkZXYy4pbHTrKSlMU6H7e0SsX311+xmk4iaoroep8qa6dziCprp4Mq63YAO5CcK3Tjb2AT33tJFyuXrasfnq2r0+xitq6sAgr6qQMKUHxG5lEw8xi0+/HMYx8boyJKOkjLkWq2hYrHYzy+LFRlCgvTcirC/A1drEIZfVno7aO0ZJul4E+dKoCB2QdWneKzRLhNCb1+KsiCtsjfqnZ6s+AsWqTFnWFOzlmli09L4bd9ffqyY9lssluksekjANhD684/Szo8K8H+k9ztz2Ssw7A8rMIxlr6sturzs6DNXgSN8gWKBRCa8Dc0ssSshgpQWRSgQS/Wv2oPxBgNyNW8VdOBBj10tKKjtZBou37qaLsmCh4oeBQqeLT6vR7KCEGsKgJeYSeIzuizJMJHU5uy7b40d7Qan9U7u42+1kTAsCpDLAypSgpZJ2spXm3wWZwxhJwtnodHfBeDVv98JtWAv4ndgsRqQcT/ZcH/rf5geIB7p6Ha6AJG6F9EjGV65N9G5I/IPwXyL/MppeoLBfRsLpfy61RVWCPWBUUDvHPXjoE+q4L13WBzw+pLrPoixEOIhxCv4lF+QWdOJSP6BhHnb476GNGXEcxtNdMmMB/hkUQIc+sHc115IXlM9GyqqwxbTAVRpEuDztgxmwosWKwJapLU2xE5x+ua1ThfpNWs5Fa1ICPiXszonQrMIkRFFTL7jN7p8RnuuEB8ll8Awiqs4EqQZDqLiym4andHvXFjt/NjrhWAMtIzjX3B1WjYae74gvPExw86F1f90Y6Pn1BefsGQlIf4OowvBpM0dUh8xXh41R63G1mGcATGUz6yswPFTWWDfJzHPNQAPq4NX6rIgrjGyjswI4UhafmMSIvfy7GbAWe0pH0xfnbKexPHbsac8wWnVGaMWgPz3oWxmwElpYQDOnqdr8sGC0d4+Mwai9yyxkDx4RDFh0xdaag+7OQgswdj5VxjwwjX2KBZZdeYk1YJplcrCpU6yezfCnMYgGRBgTboR0Sdk+uzj0Sjv8NkALOoqC6WMmUn/SxZJANPIElgSBB4SWjs6lyi8mysKnPpIYSf07lgykawL3KxQzf40vDte3AUhXH6wNBtqB7vR+QsVi4/YSulDrXfY45ZCNG/qKZmpVFp0sOjcRsZl3TQKasFmWrGnYOvkmqTJHLJ5oOk/EI13Z4TBqedU2+SjHyEsNBESbtJZGFRLu1mJyEd4icqzE5/DNmkA1VVdLbc83ahDwKDyyf/bKSOvIEydePLpM+EaWiZXCLZ5UW7P8mkML2kwtwnFeby/GLYGWVSmH5SYcSkwvQvJ83LZiaFGSQVZpZoGWt/VxaFGSYVhiYWxkqsk0VhRokduJVQmN6w2x0MsihMt5lYmHaSaS7Ou53zTHpwt5VYmk5Saa66k95omElp2omDO6mhmue95nkrk8J0EgvTTmqnq9ZFM5PB3U2cg+87SeOpOei1M5mDu4lzsJjUTKN+ezLMZHB3k+fgpGbqt7qTUSeTwiTOwWJSM43Ox93heSaFSZyDZ0nN1O+Ne5OLTAqTOAfPkprpajTqdC+zKEwvcQ6eJY6m8363n8nQ7iVOwbSVOLSvRpNMVqdeO30zjS/Px5l04F7iDEw7iX3msj3J1qtZaJjhTNKfphqAfn26UjOyjCsMaPHdDUICLenk+3dyQhgfcQkthfosfVZ60SSDFmWm9nZm+pWXbSM7VXyP+ASaglx/XOpkHFrZDfwYodIJOjK2dWSEzqmtYT/Oqyepy7jsUNdMXBVm5O3JETkjfMCRtz8dVW0zEVhhgJuJahqpGe0tUOfk/pVNw7wPz+H3RNV4Lzaowk7KXbsFXA4D9BCkC1VN6yLodNBHgD6CCvkIdg+WzV3fL0C1L0CLL0BhL0A3L0ANL0DjLkK5LkKPLkBlLkA7LkARLkDnLUC9LUCTLUBpLUA/LUAVLUDrLEDBPFRdcsop1R63OqdRLi0+uG/REthneay4g7DJ2HOSIeugan46+5VMBENAPbM0emanH5sV6CJWzXSJQRUXNDuDJuqZh6NnCs9UEx4oYYOUxzOvIp4lXTfpzKdlwqd7SmAFeKazU3L3KMF9iiib8GR+G3+MvmRJou9f+Tfr5ykE/jLhlcrMfy97oCSyNMBwYXFaVZ20U44zFFPLpAOUSd0r66q4IZfZ6cWosaLGihoraqyosaLGihoraqyosaLGWlqN1ZbwWDhKCpGQnJHiAkuzTM++pbKrTYUXQTL2HouaW7PlGeladFp9V7u9JLZbHVTkc1u8+RWqSu78MivKyfuUk7vd+CNE4w5OhRqe1kJM7obtwkMxue5isqXzyvzNLFTWowMbj4JBXqhGQxVmDJFNd5xsau23h9ovar+o/aIii4osKrKoyKIii4osKrKoyOajyEqqJey9UIZT6Sy9wtcoUiCt79k9jrL2D85kb6WvKK2VR1rrtTpxqpKihh2lulLXhN/NE/1riY/5Ca9yF4W0wxHS1rvMuQrMIiYtDXilovE8tZ5ktGtdjctvqKtlGFLZSSurtXDrOepqqKthTCUqeKjgoYKHCh4qeKjgoYJXl3yaFhsrU5ZIviXdKlYJEmmWzz48i2aygeqy1/x9sKKoYe5Vwxz20iTPPPuoVy0WsBefLhQlzPpImDcrWYJJmNQ6zcza7z0zNbbN++VREh+9J2xFhgSSt/eCMuPaDDENSZa+CuxFvtO3jk7JxLpdFwxTs25RRdHU4NGPLCkn26zORzW8hYiyqvPt7K1m88/8UdZjdGsvOhRxVTAmZgiy/HpKyIVpeO4N3iwpZClocDuVj4luQiUFnXw6v54QQdOEV51ruWyu0hRyezvRj61y2YLvTIWHKqoBz5vLMGhY/SSNqbx8oMCiSWRpIRk6bpFPFyeZWtDt9FHPRT0X9VxUWVFlRZUVVVZUWVFlRZUVVdZ84ySrFR5JTsj1x32qr/A/UdBmFbWdXfr6qrNsxza55QnwJipTEQKyHmq0+9Ro+73Yo31ssmYqUoxu2fgzk6wqFm3a7w1Rqj3EaFPVNNiyMPNImgKXIYm1Ylh/6wJLF0JeBJ0YwhNVTsm1Imp0AcOBzpjiuf6xRsmD9GwdiiQsl5q61CRYV4gOPVU0nBun69SgM+p5ErzulcwlRdIfTzGINU0Qaze15tlFzRM1T9Q8UfNEzRM1T9Q8UfNEzRM1T9Q8c9M8FfWlHEezX38kUJb6bve+tritoFDV1HHTdxnFuEF7VK6AybxFuEEHD+I5EBEOda1UwXzpha1mC5WtqotTTCz+yoyIItWWK0272QnJLlKAnBVRmF5SYXIRviIK008qTC4SWURhBkmFyUVMiyjMMKkwuchuEYUZJXbgPAS68MJ0m4mFyUXKiyhNK7E0uYh+EaVpJw7uPOTBiMJ0EguTh5AYUZjEOTgXyTGiMIlzcC7iZERhkufgPGTMiMIkzsG5CJ4RhUmcg3ORRiMKkzgH5yKihhemlzgH5yK3RhQmcQrORZiNKEw7fTPtIOFGFCZxBj4MsdeOdczpIJ584zQndtnJquwliHnNM+dAPkcWrcx4EEkJrj8u9ZCeQ87Cqo/K+z6V93Yrfud+kvRevVQFUOMBau9bae9vfK+D2WpJBf9EsJ4EbmGOZPkB2GTQWF+2B5oGNXvjaj2nvGzZt7IJ+EZCsu4eqkA3fGIkF57brvXPH0xpHbIzeuPrW5aM3B6sHtVYCgqVvQOu/E4JUVoIvNAd9FbkFYXbaqd0VnT3m0kWvREYKruBF6KcrIvTBBj5VH+FZy+mwrMgydP7V4PqSRTh2ILqf/jfD5qqGnP9h+9xCMHeccdH8m7nom5FIxaqqRi8+TiVOGfV4gM/iCeooUliOIPJkaT5DT/XKN2X3YMTTUZmv4JKZUPbdrSqLn2tn1UDcVheq46rTYavVq1H+CRJhNCxuwEDjt6nggQ4HQHuDNuxBJiPuBgGHHW9zNFnUGekwBh+hoQuOfwsPaPD6LPy8T1kdZVldfxPTjHqROoYuyCsaoRVRK/vZgwGickHVscgo0IcvFccPEpzCl0jrvOWHAyPegiGEQwjGE7ybrTTYuH2sGLeDfReIM4tGufOZ1NoUGlhLhIDm3bGsztEhb0XvhB1SRWOUvX9+BfANIIsq6Jg0FmBxknG8x83McxFDZD7xF5PwS4Yw7UP6L4dIC8zFkeEjQgb5ebMIXZrhHoz6s2IwzPVm2slNa+EOmINkUPQmnfJ94Nac/GA1WouRK11Rq3eR5/x02w/KjLX5P84HDWvmu3ogbP36PnAdIMAOEOJuZM63GL/aawvF0vjFdEx6tflwc2BNsodNrOj01WY1AtEzlmA49VatM/4ecskU2hyVdvafsdznYFL+MJYLDOxZ4Y7on3Wts/05RWtt3BuR4FLyqqHkTNyGag3spJ9RsD0B2mScjZE1rmptgt5aYTkECko+qU/aGH0y1YsZ9fd0Ot5oGp7ojvNqD3R7V33RPtwLBWp9EyngGXYdDKFqjTdiQeP42+XVXfGOT/S1QRFX0jGpg8P3O99esmYabnYWzMle+s3D2b7sw4/+tVmSZ1ms6bOC9aJ34XJFduxr9IdAXTHJonQ3FV8+mDrJ7cUfRAMmKlO/lndFFYKNV5U7Wnqn3QzzL20P0LiStH0yarfntJbOVYOrD51M/OdXcH68r0PVlOyms7nkkjuX8mN1ZjI9fbrgVrG5rKyRxw7J5O8PTkCgg50/e1PR2VOcIVErRZBVOUKQkoN41vddmm8MGH9x+WGCbtcHhpQm7NIkUMgh3BxCLDXPpFtSopgOSz0sjCFqhrTLn+yNWvGBy6D9UUugFwAuQBygdIp+t30VKCDVACpAFIBpALhVGCmqcsKUwFW/LIQgYqa0vEIJNmyZjRg4q8tkgAkAUgCkASUzyGQAQtoIQtAFoAsAFlAOAtgZ4QBMtOzPdOtYC6wrkRZGEGlzbriBRvZtWbsYBxeZ+QIyBGQIyBHKJ2joJeeIvSRIiBFQIqAFCGcIixMAD6ioBsVZgirOtQ36ZIfyL4PrTLiWMSxiGMRx5ZP684AyHYRyCKQRSCLQDYcyM6luVphDMuKXxZ9u6KmXCnbCbasmaZ95a8tsgBkAcgCkAWUTs3upycBPSQBSAKQBGyUhK43CDmCHdmCxRY0gA57wLjZn0MQxiVY5Q5HC78KVBcR8B4TfvaGo7jhUT+oHGqEURPzfyKmLkBZTw2q2x2MIkdQfZiguiqwdR16LEDhqVZBdfbWABKhG4DX/JU5HKQ6DqkwYtXSHZmVIuU8yrMIJasqzw7SI0lMUIhIEuVZjNGI3jkny3q1N81BBQ4IsAaqi3AV4SrCVYSrJVA+U+PVfutgzsVBUFgNn32/34nx2X+4AgCisKKLT0SWQs/7ENmM4T70OYiZrFHcrBistBDlfLqywBRamJk8EUkWCRs9TWQXsPhjT8PMNOUdprzGiujPtQnbvQI0bVUVAXVZYhX6g2Gas0kjx3cqxJr34aT9wQhjExCh5y8oD9MLyk1E6IjQDy4AgGM4QVuWEuF++0YsZZRwafT80022OLeM4I1V8jKkkgjeSqeGXpYekyHUQqiVuRiaHmsNh4i1EGv5Cn0gWMtxVC8Mc3r/apQbcV2wAtYXa72/+4wYax8Ya0ZFaQEDKmg2T+++D+9+KwAWdR3hF8Kvuipdo/ToC5UuRF8Hjr70JaWz8uOvW1bM+uKvYPUQgSECQwSGCKzUAlgGEGyEEAwh2GFDsNVOjd9NCsu0zItXZjBGyDV7xhxsudpYTHjhiVX4+gK1f/BavgvWEvFaufCaoio0OecNAjYEbAckmbWaqQHbqInbjTeb83C7cc3i1raP+m132vHZINnJ6QAsD2Nnsq7ODcC806Wmivs+yZPdmAk0Ht98Jt++iUuTI+Mbp2p7SivvmHhm9av6GdgeMPXdsnJrtSC5CQGoyDH2uG2l3Ryk2bZiEQ6qEzY07TSb9kRYxVyb7eYQ97MgZSlAY07PWTot5CzIWXCvTZk5AQMB+lT/3aT0ax1x661dsxqHMNjA9aNpEHVO/mFCEyJ6rQl6rUC6I0SpiFL3J6y3UoPU5ghBKoJUBKnljNEwl99UsBa83mBw1FwCFi1XhIbkhGRwvPkRCiuwziHIhBeafL7ZQ0ok23p20vk8Y1rYg43IU55iE43ePL7qkgiGYgPRslasvlydlKLuXsDOAjBRT8a8ohiYkiN+fuN7HcwbSyr4R2tk8t/G+g57cGhQuTeuZnGKzHIOysJSD4zSZPgcCoQbPlzC8W+76/rCh3057u203vg6jR0mMuisvlgKCpW9gwS5xTbcopM25WoHmUUWzALJAZIDD7zVVfGJwcXp3fhmKsiyKuYftZ2ZPr0qM8snxf4F2DsjrEqhcbF5UwWPLSXF1GnFbMnLzI4tEJeOFYk9L8qv7ERNuJxN2MrOZl3QRX5GtU/ZDc6qeRkcagPm/gwIiMCfqvZKoKTEEJcxVh4XYGVVWz4KSsV6r1VoJhxYfyTPA5MCTGm8VMyMxgvz9dmj/wWot/JARFmNHfmXVff7WQ1GoP6oLhyiuoC7XtA5V+4QsrQMuuUWQpBCI4VGCp01hf48uXl3fXdZOernKTfjI7OlLBl0FwqYN7OGolbRvG7TlpFZsyLmyqxzMWo4ezZnGbHnUjMVqD8yFWQqyFSQqZTL1ddOT1T6SFSQqCBRyY+o2Ah0CqBzViHMd+sqtkujzoeelBv96Zhqs+zwL9L9hPAP4V99herU+K/dxq3OiP8Q/+Uc67Wgi6JSpmcGAFdlZjFK45uVkgq4z3gEWJS07+G8GBm10naFOmxt14uawOr3lop8K32liK73i64xVz1ia5RWvdC6k15a7SG0RmiN0Do/aH316fxvlYtQWBcaAOCVJjyUMfSDF9Ly81fNtHZ0QjBWYQ623mPYx6fzXyvXV1dlBnN+El526al1iQFhfYucEbAIspWyJKlq9Qbt8mWpypLZRNS6g1mqtuJAu+6yd0Z/5bbZt6O22Xdxm31WBLEdTRApo0KBVXhjn0wLiWMWxLE2mb3ApDdU+6QG+yeephG3UA4H/fjjND6axmEcpaHA7xgJuV5efoH/Kx9FA76pdsJcpxaMm1CRQgvNiMq/2dNhGl4jQ9/ao5W3sOOqoCzMCwZ6shEr7oL6YLUTub4h1wpwOpaB+GOwzkjt9knthsNRGmrH+rAIXVqFtrUOz5CUKp6b0RqOmkj20OFVQDBZboRmiHwG+Qx60crNC6YACF8EbSYpD9WmBet6AJ4FiDdffa7vXgUXor0Kry6C2X2C2VEv1WkabM4Dyy30qnkqRj08T6Ou4NV1rnpgYiqVUN/NC9e2WvuP8CoLcEWp/WCk9sMW0MXFcnqtvNcfKq+eszoARCbvqa5Dj9PJy6MkPhKY6gn0Qcl4JZotq5+SD6pBre0JxqOkEztwgkiKKJvQaYggy3BF1al9aUbuX4kEtrpWLjWNrVp7luRZs0FnrX672ZWAhgu0GzSM3XCCYdDFkjWDoRIdfp+iBeGFiU1YcYfA9fj92hWAtKncW70XTrevkrSPnAc5T6Fifm6kp9dEzoOcBznP4XEeCwVWnfVYtQiDzwHaQ+4BYcwovARmD/go6ORReGaOBQYYT/QlrAZzSSTUeuTbe2HGrxDxkYpPurnQjwn7TuaNekyoIZ4e1dcJwaseQhUQQyOGRgyNGLpqfoNebhAa3QYIoRFC7wyhjzd/0a0yOyys/nnGoPrECRioOlpfVQQA+/pvB6Dv2anAbA09uSbGdtfEY209ODPXSPtnKbhQ+q8GbVnFQSFvQd5y2MH6uZGTVnOE7ATZSf3YSbUQfM3Edoay1oiSh6GIqinPiKIa5J6SGZUB0GtWwIqgEGG5lCVRYB2vBCD/g3oDY73irWFXItAYK2+HCmYnS7iHvEjGI7QMkSUwpxKWICfHrFmhTfBOAqtmOyriDRzZl1lJLBvyv7Lv1ONGhj33k/h8b87rMJm4a2L3Yfg8p5rjfttAFJgUNF/cKrOaWN1dk1Crs+DC+p5VzuqLHs1ybabqtjvtMm6myj0VAFQc876hV3TPXtF+bmnPRig8oPCAblF0i+YgqtyJTFS5pYXv/mHvMsJHYz6SC6sjT+T1sIC+v6ZFx/ZWHxa+aO3zWRF/SbHQ/Cm5W20NWm8M0v1PYkLBOgE0hRreyxIsfzP4paIAUuGDdr+qDWtvGE17aPDsvbL+FmX+2GBrrhvF3Qw8hJV+8d4K1w1BUtgXqiKzeFc+7heSYWySQq/iDl92lhU6fNHhiw5fpGQJlEwXNYlj0CCnKZc3uJ+fN7iLuduQtKG3uMTEhiUbfsc9dawGULuXYjHvY+RoyQMQ+2rKDieFwul8/5blryS/m9QEQDwngn1gDVHt2/ceMupprommLuvfVLyWjL/884POHKHvrm/vLj+sjz19UFQttl0uimsX4Aa3r8oniw9Vmzh6qmKZ/wT+Ozsf/91F+HiL3GtUeCIz9UVZX9F1Th8BbMOPjqHH6MYZqwDD4a6fZ3PwUyqab1cxO6q/XWCA6/Ve2cXNqU+s6ABJP+bzlGIu7qnGZih+sLLzG/cvPBwdJjONLGCYkKVGnyXV1IG1b8XZJ40MLc5jIfYXhREqb61ELPKWnj6cWhtvmXVXe3GPYt3U1dAt0BddBdWi/MeKoUKBIelZO4YHuaWPx+2yqCHUz/EbGuLT68YfWPVe+DJWFYX8eBrivRXZ5MRu/ONw1LxqhkVJWRNGs2L+3AACHcOAvWSOx/zO0i2WKq8qxPQMhlpdLjx2dvE6R43j5GO2oISlfJTga41c3t6dXwC3/vlywtjC+N3H28sT8uv59V3xx0i7m8rusXVpKGcAnpB3EpAv5nZlrcLnXh+vc7ehK7uQKChEN5c80P7tRAJUK4msHf/TOGn9p3FUb8frOCJCAFnMPiNq+6N0pyjHRH2U+nyC7gDP1kLv7d69t7kxpxEevIXOWyRecYGwUNqDi4Rlnq7bV0VU1Sew3JUgyXRWYzekv6rsWDAFHiHNmG+L2NfKklQo0D6fqPh8EK3DKmo5Krdpk4t9tAmbNg6iTfh50N42SUgHNa4+SYXqj3llkaJWwtGGwcFILw/a9TbMzfXWRtcbMkAM3811X+K5yLjlxyWtesihqyIhbiweAfcoPFOyEGaUCGQmAbw3rDg2id3DYhGZo4SHKl5+uLN9XHOYgvn33Kc1sb4twf7CG0HX69Fw7pqkbrlP418m/pZzYn4TWq4GDq6JZZq7lWmQQ5SdQ1TBe4WsIYY1vPG9DiaUJRX8Y3U9Tj9Q40XVnsgHazJvrO+wx4cGlXvjahanyCzCRhaWemCgJpOGUPjf8IE6jvrbrh13fsTP0X6n88bXaewEKoNVApXGUlCo7B0n6LDLim71utF0ayNa1d3vGXrb86YDcJwhN4rhRgHz5hPiBxOAsKRTNstSBl6nM1Pj6TIBCsM6PdPrEkX27duqjt+/M32dV5w41a3vGXEfoM3I5RcW/EY1p9p3fvyIMLl4mBwDke3Bh6o6quoYtJUZCGz1BrVBgQjkqrG/pDUKC/V1B0RZieB+jN1dctXujnrj/cRM7QuJ6qYoUr3uANSuZfE7Rgz6xYCpfGV43g1rbmynzs6+9Rb5K+/eNd4CEgb/EfmXZR9IqzvsptsIkse+99zzqkO1e7gNJF/F3T3u4/T2N/ZD2YzHZil2tdO0ZvCGLj7ShfAL1XSLH7T71tfGq/WWmaA9WXcCNFuPgcaTeU81hRowLlbPN+hiKQNAUR5WBmqw9FGukePS6ddA+psXk7FZnc9KIcADpndeKjoXTNmH5vhIdl8NY2sNvjAEOoSzern9Ana23nM5BIV702naJMx1eQHvl0J+Zs9Qjcnt9ObTx/eXdz9ffr51/3ANo92LQeN3k2p8fC3DWJ6rZVuebx/oF9+K29CfpOVnTWbhqCHlc3qPq1Z+x4irCQVZ/sU2us+crsZ1G38DxspaT1KkELqaedP9X/V+mzb7r/f2pJbir5lay4sV6GCyJ00lZa4eE3jYUS4taK/b7qfAyOXNpP/DKVvDezVQCfZd+M12/7Bq6bpg6vTOepB7Nt1Tp4kz/TcP2mU5ny08fJRnT/tZ1Y2/bNPXWKGz6mwb1Rj7X7oV6d/Cydfffvp/yvOC/rt5MvrtJ8X6J2KJirx9hzWrvW139D0mqS/OJP1pRp8l0cP63F3yTRQ6dNsuTOLaxnjbmc8NrUJHzAbv3HCtFE3dAHb3xv1axiW+W6BI4la04dDc4oEwvl9OWg4bAoJmf9fw/GwpAQ/S1j+2x+gq3M1t+0bPFRfRaro+dNwfWov13z3X3y33h07TfWXN2Rpt19+tmVXj35w6MG0g2Cei3+J+cN/9YPdb2l33h/WepcZg5i6vUxaP+b6qXL5q3AMU1m2IHEWbrxw9s2FyIbmh/XM2m8k3v9rg93kFkHv9N9//P4On2tnk2wUA apiserver.json.gz: H4sIAAAAAAAC/+2d62/bthbAv+evINQNSAa39StpXKD3Is2jK5C2WetuH9rCoCXa5iJLKknZ8TLfv/0eUg9Tr8RpndhJOGBtxJco8jx+OjxRL7cQsno96gWh4NZL9OVbLSph5HtIGdHKsOf5Agvqe7LwEoqg0KVcqCZwOVfNiEMF7rsESgfY5UQVDj0i3jpQ5IWuG5UwHIy6vu8KGkB5XRWOqEMOfU8w3+WZ/jTT16XeuTaxAHtEtf+i5hTNDMptGIl4cnpWd0TQp9MPaJsTNqE2QS6ZEBf5/b+JLeiE7CDsOcgXI8LQmAhGbY4cygMXzwiUe0iMKJRgPur7mDkIM4IGPkPUgz/HalGwi4KQBT4nHDq4s2dWLZmIgwXmfshsoj1FVEO4zWgg+69/mkNGnTN/sbmqcASXzdrieiqv21rBRbJ78fVMXseX83RstYHpONbYd+RaWGPMzh1/6i0mwWE3oaaxaCuoUNJkvfcFrMmiqZgFqlyQC2Gpwjn8GckEIwMQXjl5q1XnVlTmT0ulxHVxwDPymqlwCjU5gdOHi3pie0S6dEz8UOR2PBnZZ6+xfT5kfugVxl+0+RO7Iamuzk5BlT9pdjp2e8+qZYvZsI+3m60XNVjYTg216zVUf7bf2cm3e+K027iFLa30W+a+GVG2zpgPYjgiIbeyrYhNx1gtUStXkZH33/0pGmNvhgICI3oC+QMkDQ/hgqPtPgg6XGJHSf2UUQEKQD3UqoOIzzga4QlBfUI8qOdTwkAFeGjbhPMBrPhM9RpgLhDx/HA4+m92jpFGyFnENw89KrJNhjgckow6RMKLL5KNadTruRUcUy+pzFfxkT8t20wpyiMprb7rnOI+ydq+slbvQG2I2n7BQqI1mmfnv1DobIXSxuzOULCVbILdMnnN2lttHYKAesNupIaNqrqilGbXE1p7eKzkaSJXDgkfKZWu5ZtNklXPVMxryw3OsDdcYvBmdvBKTQApOAJlOPNh5XhRFCy5iqryXWzrwB15YMbBnhTadeWMShYezLQYUGlhLaus4gQ83Cf6jxp+t/5rrg3YwNK+qvzKrmqp3oHxu37rBmAC5Bhy7sVFFdGDWe+fH5RU+mnHJdc8dg7tfCE7BxEt0dQBdd1DaSnV/ksb2GqACWzswx/7HWkDG/sFGyiNR4UCyrvo40XDNcGaNjqtwkC6vlfqqGIlGDMce8WtEpgNiVhiF8hFoOaEAyrRgbBebEZf4gmmLu5Tl4pZq+5cQl3/1VcLu+5Xq4ZsN+Sg91DwS/zjV2te3KmFrRTg1XpwC0p4sVliQ06wLdQaNQtNXDIknnOSjlccBKRTkaJ1sKxUpGaRl6xgAg8H2jqgbViJHfQf1Ok8q9fzwi9SKL3M7yfwlHP9jsZYwsH+ubAFOOdVlKHR1W8/PwPVYjn18+U8rVc31LzE1t1I+VSn97ExxZPhop/2/BkOwi7F/DAhlZwP6mNW6ucsia+nxBuKkbKqhTpS1W3lZBLaI0QYA4Duhw4oIgK4dslAINf3z2F3EQZiCRmYkU6n/ivSVQ0NQwxWVBDC89xBlXnJPZgsfcOwQ6MXlvoN/HjWGkYKVhRe2Iuhd8C7+VczvQnsaXmNHTIWTaysFjxhVQ31KmoYHY5AAZyqucS2U+JNvgY6/UUdJR15h6m0V+Aq860kmF+rwNLK85Kbq4ppfOvGMoRUYICCl0zgEw/LOT+QvaVchHL43WJduTLAbjkAxMonDFw/Z4EYCQguBY7Isn8AJwEiR0qeCJytTaq0M3bP+9lCAS86pZMEdxMExDmNXHex/sbeDxAM/Ya2V+IE0dNIreG/nbt0iMreROZm5b4xt5dy6icRvhUEQdZ9GtGBKK+MneqxnOzryDauwqfmVd1nouQ9SqlxL3Gz1HPohDohzjiycpesgk7ZmV3gC8qL8+qH9nkkeyU2JoleyEXKQ1/8tlHSq9qkpXZJhdHKH2GGL5Z5k6ryb0u+8kYiKd9BS59BVvrD15gXXvg0R1DaLfIEpVXautzove4hPaquv1v50lRryw13XPoWTA2O8aWk+qM/zQRg0/l0Y2XOhrhSFT9Kg4kwgpWtTwB2tFcMyjFovaVto4m3bWy8DTta0O307adu7c1x10TaVhdp2zWRNhNpu5eRtpaJtP1spE3a18cVavsoPUox3maibI8oylaOHZWsAWCAOAG77SAH3MaIoFS/OJJvl/0ZssG633Ysbc/E0kws7fZiaVdbGaXI8i7Pm8+ePadFg2InDvHJ7t7BXvu1dSM6Wwzfum74k+bhYf3wR4dvXzd8q/mifdT50eF3rxv+uN48abdvQDrXRjRbpRHNgkyvPKDJw7G0fNvS9O1EfwFuRPb3ZYFAekpfXzIsyO54Kfy408jm5aUy4Wg+3/zApkKYT6dv0VP0MXZXJqC5loDmQh5BygN+b6OV9+Y5ykOR9589lwl5aRgqUycZESHziIOmVMQnwdB89+Jip4JDV4WhLwyGGgzdpCPd1rqOdBMCSqjn5yhIdn31v68WIGQ1E6HnaLV33Qz2SuZ+3/hLnTEb+lo3fa3pGLV+CwC2+Y/yUBksYisu8+hkjK/TAayKd4O60a+qSJspU+sUlf2rqAyjIZ0QL7Vgtwxf+wa+DHwZ+JLnqBGs9L6HWCloCfY4YZR00otV++WIcuGDYxynvTbuIOwew9BRvNwGh9aMQw8hEMVNPpzJhzP5cLngoPo9Uy06ePYBOPTsM/x/0D38/d+j49Pj7rHJjltddlzHZMeZ7DiTHfc4s+OUtX1c6XF/KQdj8uNMflwBQ5Zkj03JlsuNZEJlJlRm0uVMutx9Tpe7mkdMvpx7FdOYhDmTMGcS5h5gwtzNuXSt6XONhuFSw6XmCHd1+XMJFt1xAt1m0di9OjTWicyk0JkUOpNC92hS6KTVVCl0FZh2x/l0jaahMUNjhsZWnlC3OWdn9xmOTEqdSakzKXUmpe7qlLoNJMSVc1rrAXNaaZUBtQ0Htb01hs22ZShqe+qzcwAzcN/YcXgUpLr825f4dR72ydMU4CSHUQ+m6NkqQvZLclEJaF92x992dlSYLGlbQ9Ib33FcK7n5fI4uL+X97wXBwb6gP+TGoAMHHAfslQG4NQOcf7endLcR19rcR7jv8ayV00rb0IqhFUMrFbTikECMDKhsFKgcyT0xlLLuMNNIrt4955RNfghDKjlS2f0RUilRh2VApaTbz3JK2RHXrR5+Xfm7hAZSfg5Scqexd0gpxYOsbfkvLNVQCb7EEJM7C+tF3ue2saaGXLJj2OZGbHMK++fZM0M35hDNHKKZQ7THC3vmXxAwUamNAr72ungvYL78BorM+YbZe6I3JmOfzXr9mSC8iuFuhnDrYrTNR7N3aq0Njq0Zx5SsPwAk2/DnMCGnHIWYD8gbCjEUEn1tS5DtBEXsIExjSVem8vxAJMmwSBWLHJ59RiGX0mxwxJx9mbOvxwQi5mPqBkQMiCg/4/eGPvNDITfQhD9uFznepCttmOOxMcdtnUpt9nM85JOprfgJLG6PyBj/SRiPZhklAYOtnkW3cjA7VwOBeR4upFTZWeYRGcNKxxJkHLhY/qJ8qnXgKbnQZHshDRpkZBAu+b6pQwY4dMs+sarXlhGhNYo4QzMuFTKVfkBaQzmt1lffCshbU+t7SNgsDscXkU8abmmHsxIKpUOS/2Jzui/a7bdyj6MtGHbd5PvjucfQllJfhyUANVkrzVuB2QV/6JCDsm8lpwuZ/OK1PtYYdoSW9ElWuaTLdUusbteLrOh2xRduUle+U74Nzeu2IXZG+naBsKu15n8kM7GytYUpy7LyxvEmR8+kVYScdKOBdD69652vV+181ptVadB1e54gzqo3vTJjqRTrFolMD0NGthKnNI8MLx0vPpG++G67P33aSJgo+SR75AEW3QIKNMQWneMl6SU8q4OJtbsQIKtR1y5a+kVjvPh5V/u5oV+06nrNgtyspvZzw4ns4bfkGSSBa9Jz7V30gff0gfW7NNv6xeIb/tYLR59vMpfM8v3jq5cZ63P3MPaSiY8+OHuLIqmMKkL19m7VO8Tex7hBOp09Z28wsJ29/caLfh8PBm2n3+hHjSepP65vzf8P1OdmvlGiAAA= cluster-total.json.gz: diff --git a/charts/kubezero-metrics/templates/grafana-dashboards-zdt.yaml b/charts/kubezero-metrics/templates/grafana-dashboards-zdt.yaml index e08bec0..a77dfd4 100644 --- a/charts/kubezero-metrics/templates/grafana-dashboards-zdt.yaml +++ b/charts/kubezero-metrics/templates/grafana-dashboards-zdt.yaml @@ -8,4 +8,4 @@ metadata: {{ include "kubezero-lib.labels" . | indent 4 }} binaryData: home.json.gz: - H4sIAAAAAAAC/9VVS2/UMBC+8yuMD6hI7e5mH63ghlohKkAUWopEWyFvPEmsdexgO93dVv3v2M7L2RQqbnDZ9XwzHs/jy8z9M4QwEUIaYpgUGr9G9xayIGfaWOnKS6hGvWZZMm5OhVVG+x1KiSFalioGq8BnSuZgMig1DmxAkCV3eqNKCPCM0UdQFktxLLlUzqFKl2Rvso+mUWR/Fot9FL0MXQuS+4ffdLmgF+gNB2V6IZht4e0o0dlSEkVxrXvw/zf298GZY6DM7ESLUwHmlFpElJxXiCJFdiElN6yw+MSDzJlMX/kzZ2Llqnp148WCCOC6rWtT1X71Gu9ekzDg9FiKhKVtc6orkJCSG9+yhyBDeQtK2YJWrzbZtf5SqzuTuu8rs+Is8LF2CcwDYNMkV8tbJw98+8QPW1EWfU550DbVgHDUws+vvp9coA8ylTd7mTGFfj0ex1SM7kDJAyrXwrAcRrbmY6I1GD1elUtwyjG3dw50Tjg/OJyPCpGia3xHzTV+eS2+AY8t+ZCR6L21/27t0UcwisUaPUdtHEEyufT0wzlRK/csHiRW8DJl4hKUtgk506PRYjRrfWAX6FtL+d3mOfw8Y4kZKoynFn5nQw381Ow0sDFVFHUM/xJRosUOU6LpE0yZPc6UlnE7X8nvyZNITkH5TzB8IyebUwO5M56F+M8SlHsfh93WmVy/A0KZSPVw6DjtF4gtQ/n2ksEa6OM250BU7KqREK4HSkOUeuymIelvqv0kw5qbQwqdNKNMD4nkxpyf5P8vmXrAH9g0/3s2AdCvirs6NeNnOHqYoLAZbXI+nAptBz4QA9qgxE4AZMuJlnY+DXshYK3rPtjfah8oSBRoVwg8m1T9wzrOICcdFaZHFWy2vO6pWlWWPVLYoZEX3K4+kQ63eLfXXGKdPqmGFhZyfRBldczYyBrDvWsFi1egust2bVMQPZY3Cf1gdsqrWxLsOqtcBAyNJoEwC4Uo786L4ByFwmwSarLuPA3OUb3db5q8bBI/OipcPflK6PgwdBy+Mp2HAu3ORzSMdzeWZtR3he0V+04Kr18qudZWWSuCtYGOZbGt8NJ/Afjw8ydIF68+3lXobUuh+bOHXxBMSbrqCQAA + H4sIAAAAAAAC/9VVS28UORC+8yuMDwikZGZ6HonghogQEaBllyxIJBHytKu7rXHbvbY7MwnKf9+y++WZDkR7Wy4zrq/K5Xp8XfXjCSGUKaUdc0IrS1+RHwghKIV1KF0GibRo0KxrId25QmVyNKCcOWZ1bVJABf1kdAmugNrSyAYUW0uvd6aGCC8EfwAVqVZvtNTGOzT5mj2fHZF5kuDPanVEkhexa8XK8PDrIRfyjLyWYNxeCO62Cnac2WKtmeG01d2H/2v8vffmFLhwB9HSXIE754ioWsoGMawqLrSWTlSIzwIovMn8ZThLoTa+qpfXQayYAmn7unZV3a9e5z1oMgGSv9EqE3nfnOYKZKyWLrTsPspQ34AxWNDm1S673l+Ouk/a7vsqUFxEPrY+gWUE7LrkWvnWyyPfIfGTXtTVPqcCiE11oDy16NPLb2cX5IPO9fXzwrnKvppOU64md2D0Mddb5UQJE6z5lFkLzk439Rq8cirxzrEtmZTHJ8tJpXJyRe+4u6IvrtRXkCmSjzhN3qP9N7QnH8EZkVrylPRxRMmUOtCPlsxs/LN0lFgl61yoL2AsJuRNTyeryaL3QX2gb5Hyh83z+OdCZG6scIFa9B2GGvlp2elg55oo2hj+T0RJVgdMSeaPMGXxMFN6xh18JT8nT6YlBxM+wfiNku3OHZTeeBHj/9Rg/Ps07rYt9PYdMC5UbsdDx2v/ghQZKm+/CNgCf9jmMzCT+mpkTNqR0jFjHrrpWP6Taj/KsO7mmEJn3SizYyL5MRcm+e9Lpj3gF2xa/nc2AfC/jfR16sbPePQIxWE32ZVyPBX6DnxgDqwjGU4AguUka5xP414o2Nq2D/jb7AMDmQHrC0EXs6Z/1KYFlGygwvy0gd2tbHtqNo3lHilwaJSVxNWn8vEWH/aaT2zQZ83Qokpvj5OijZk63WJ071ol0g2Y4TKubQ5qj+VdQt8FTnlzw6Jdh8pVxNBkFgmLWEjK4byKzkksLGaxphjO8+ictNv9ussLk/g+UOHy0Vdixyex4/iV+TIW+HA+5XG8h7F0o34o7F6x77QK+rXRW4vKVtEyzm+1M6TpBVqSfofQOnwJ9OTPPyBfvfx416A3PZWWT+7/BfIgcBPyCQAA diff --git a/charts/kubezero-metrics/update.sh b/charts/kubezero-metrics/update.sh index 204398f..e3ea8f3 100755 --- a/charts/kubezero-metrics/update.sh +++ b/charts/kubezero-metrics/update.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION=15.2.0 +VERSION=15.4.4 rm -rf charts/kube-prometheus-stack curl -L -s -o - https://github.com/prometheus-community/helm-charts/releases/download/kube-prometheus-stack-${VERSION}/kube-prometheus-stack-${VERSION}.tgz | tar xfz - -C charts -- 2.40.1 From 9a0a2602c8fa267433fccf2acf07fe84607b99a7 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 12 May 2021 16:32:24 +0200 Subject: [PATCH 07/19] chore: checkin diffs to sync desktop --- .versionrc | 2 ++ charts/kubeadm/README.md.gotmpl | 1 + charts/kubezero-lib/templates/_helpers.tpl | 1 + .../templates/eck/elasticsearch.yaml | 2 +- charts/kubezero-redis/README.md.gotmpl | 1 + charts/kubezero-timecapsule/values.yaml | 8 ++++---- charts/kubezero/Makefile | 5 +++++ docs/Upgrade-2.20.md | 5 +++-- docs/_release_hugo.md | 12 ++++++++++++ 9 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 .versionrc create mode 100644 charts/kubezero/Makefile create mode 100644 docs/_release_hugo.md diff --git a/.versionrc b/.versionrc new file mode 100644 index 0000000..5795d32 --- /dev/null +++ b/.versionrc @@ -0,0 +1,2 @@ +# template: "/tmp/doesntexist" +linkReferences: true diff --git a/charts/kubeadm/README.md.gotmpl b/charts/kubeadm/README.md.gotmpl index fce4c94..ab835c6 100644 --- a/charts/kubeadm/README.md.gotmpl +++ b/charts/kubeadm/README.md.gotmpl @@ -29,6 +29,7 @@ Installs the Istio control plane - https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm/ - https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2 +- https://pkg.go.dev/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3 - https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/config/v1beta1/types.go - https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/control-plane-flags/ - https://godoc.org/k8s.io/kube-proxy/config/v1alpha1#KubeProxyConfiguration diff --git a/charts/kubezero-lib/templates/_helpers.tpl b/charts/kubezero-lib/templates/_helpers.tpl index 808f73b..a0b2c7e 100644 --- a/charts/kubezero-lib/templates/_helpers.tpl +++ b/charts/kubezero-lib/templates/_helpers.tpl @@ -5,6 +5,7 @@ Common set of labels helm.sh/chart: {{ include "kubezero-lib.chart" . }} app.kubernetes.io/name: {{ include "kubezero-lib.name" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/part-of: kubezero {{- end -}} diff --git a/charts/kubezero-logging/templates/eck/elasticsearch.yaml b/charts/kubezero-logging/templates/eck/elasticsearch.yaml index 5fdc70c..145f3d2 100644 --- a/charts/kubezero-logging/templates/eck/elasticsearch.yaml +++ b/charts/kubezero-logging/templates/eck/elasticsearch.yaml @@ -72,7 +72,7 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: failure-domain.beta.kubernetes.io/zone + - key: topology.kubernetes.io/zone operator: In values: - {{ .zone }} diff --git a/charts/kubezero-redis/README.md.gotmpl b/charts/kubezero-redis/README.md.gotmpl index bda5861..3fc4fc7 100644 --- a/charts/kubezero-redis/README.md.gotmpl +++ b/charts/kubezero-redis/README.md.gotmpl @@ -21,6 +21,7 @@ https://grafana.com/grafana/dashboards/11835 ## Redis # Resources +- https://ot-container-kit.github.io/redis-operator/ - https://github.com/helm/charts/tree/master/stable/redis - https://github.com/rustudorcalin/deploying-redis-cluster - diff --git a/charts/kubezero-timecapsule/values.yaml b/charts/kubezero-timecapsule/values.yaml index b197d8a..3fe4cfb 100644 --- a/charts/kubezero-timecapsule/values.yaml +++ b/charts/kubezero-timecapsule/values.yaml @@ -6,8 +6,8 @@ gemini: resources: requests: - memory: 64Mi - cpu: 50m + memory: 32Mi + cpu: 20m limits: - memory: 256Mi - cpu: 200m + memory: 128Mi + cpu: 400m diff --git a/charts/kubezero/Makefile b/charts/kubezero/Makefile new file mode 100644 index 0000000..e8c15cb --- /dev/null +++ b/charts/kubezero/Makefile @@ -0,0 +1,5 @@ +.PHONY: sync + +sync: + rm -rf scripts templates + cp -r ../../kubezero/charts/kubezero/* . diff --git a/docs/Upgrade-2.20.md b/docs/Upgrade-2.20.md index 6414a88..b610787 100644 --- a/docs/Upgrade-2.20.md +++ b/docs/Upgrade-2.20.md @@ -42,11 +42,12 @@ Providing backup solutions for KubeZero clusters: ## Metrics - Added various dashboards for KubeZero modules - Updated / improved dashboard organization incl. folders and tags -- Grafana Dashboards are now all provided via configmaps, no more storing of state required +- Grafana Dashboards are now all provided via configmaps, no more state required, no manual changes persisted +- Grafana allows anonymous read-only access +- all dashboards ndefault to now-1h and prohibit less than 30s refresh - Custom dashboards can easily be provided by simple installing a ConfigMap along with workloads in any namespace - ## Upgrade - Without ArgoCD 1. Update CRDs of all enabled components: `./bootstrap.sh crds all clusters/$CLUSTER` diff --git a/docs/_release_hugo.md b/docs/_release_hugo.md new file mode 100644 index 0000000..61dce00 --- /dev/null +++ b/docs/_release_hugo.md @@ -0,0 +1,12 @@ +--- +title: "KubeZero release" +date: __now__ +author: Stefan Reimer +description : "KubeZero release" +summary: "Released KubeZero __tag__" +categories: +- News +- KubeZero +- Releases +--- +# Changelog -- 2.40.1 From 2ad57adaa7d1b0eec22ecb356adb3f639db5befb Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 12 May 2021 18:11:40 +0200 Subject: [PATCH 08/19] feat: initial kuberzero-sql module incl. MariaDB-Galera --- charts/kubezero-sql/Chart.yaml | 22 +++++++ charts/kubezero-sql/README.md | 58 +++++++++++++++++++ charts/kubezero-sql/README.md.gotmpl | 20 +++++++ .../kubezero-sql/example-mariadb-galera.yaml | 27 +++++++++ charts/kubezero-sql/update.sh | 7 +++ charts/kubezero-sql/values.yaml | 20 +++++++ 6 files changed, 154 insertions(+) create mode 100644 charts/kubezero-sql/Chart.yaml create mode 100644 charts/kubezero-sql/README.md create mode 100644 charts/kubezero-sql/README.md.gotmpl create mode 100644 charts/kubezero-sql/example-mariadb-galera.yaml create mode 100755 charts/kubezero-sql/update.sh create mode 100644 charts/kubezero-sql/values.yaml diff --git a/charts/kubezero-sql/Chart.yaml b/charts/kubezero-sql/Chart.yaml new file mode 100644 index 0000000..dc73439 --- /dev/null +++ b/charts/kubezero-sql/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +name: kubezero-sql +description: KubeZero umbrella chart for SQL databases like MariaDB, PostgreSQL +type: application +version: 0.1.0 +home: https://kubezero.com +icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png +keywords: + - kubezero + - mariadb + - postgresql +maintainers: + - name: Quarky9 +dependencies: + - name: kubezero-lib + version: ">= 0.1.3" + repository: https://zero-down-time.github.io/kubezero/ + - name: mariadb-galera + version: 5.8.0 + repository: https://charts.bitnami.com/bitnami + condition: mariadb.enabled +kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-sql/README.md b/charts/kubezero-sql/README.md new file mode 100644 index 0000000..a28e836 --- /dev/null +++ b/charts/kubezero-sql/README.md @@ -0,0 +1,58 @@ +# kubezero-mq + +![Version: 0.2.0](https://img.shields.io/badge/Version-0.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) + +KubeZero umbrella chart for MQ systems like NATS, RabbitMQ + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| Quarky9 | | | + +## Requirements + +Kubernetes: `>= 1.18.0` + +| Repository | Name | Version | +|------------|------|---------| +| | nats | 0.8.3 | +| https://charts.bitnami.com/bitnami | rabbitmq | 8.13.1 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| nats.enabled | bool | `false` | | +| nats.exporter.serviceMonitor.enabled | bool | `false` | | +| nats.nats.advertise | bool | `false` | | +| nats.nats.image | string | `"nats:2.2.1-alpine3.13"` | | +| nats.nats.jetstream.enabled | bool | `true` | | +| nats.natsbox.enabled | bool | `false` | | +| rabbitmq.auth.erlangCookie | string | `"randomlongerlangcookie"` | | +| rabbitmq.auth.password | string | `"supersecret"` | | +| rabbitmq.auth.tls.enabled | bool | `false` | | +| rabbitmq.auth.tls.existingSecret | string | `"rabbitmq-server-certificate"` | | +| rabbitmq.auth.tls.existingSecretFullChain | bool | `true` | | +| rabbitmq.auth.tls.failIfNoPeerCert | bool | `false` | | +| rabbitmq.clustering.forceBoot | bool | `true` | | +| rabbitmq.enabled | bool | `false` | | +| rabbitmq.hosts | list | `[]` | hostnames of rabbitmq services, used for Istio and TLS | +| rabbitmq.istio.enabled | bool | `false` | | +| rabbitmq.istio.gateway | string | `"istio-ingress/private-ingressgateway"` | | +| rabbitmq.metrics.enabled | bool | `false` | | +| rabbitmq.metrics.serviceMonitor.enabled | bool | `false` | | +| rabbitmq.pdb.create | bool | `true` | | +| rabbitmq.podAntiAffinityPreset | string | `""` | | +| rabbitmq.replicaCount | int | `1` | | +| rabbitmq.resources.requests.cpu | string | `"100m"` | | +| rabbitmq.resources.requests.memory | string | `"256Mi"` | | +| rabbitmq.topologySpreadConstraints | string | `"- maxSkew: 1\n topologyKey: topology.kubernetes.io/zone\n whenUnsatisfiable: DoNotSchedule\n labelSelector:\n matchLabels: {{- include \"common.labels.matchLabels\" . | nindent 6 }}\n- maxSkew: 1\n topologyKey: kubernetes.io/hostname\n whenUnsatisfiable: DoNotSchedule\n labelSelector:\n matchLabels: {{- include \"common.labels.matchLabels\" . | nindent 6 }}"` | | + +## Resources + +### NATS +- https://grafana.com/grafana/dashboards/13707 diff --git a/charts/kubezero-sql/README.md.gotmpl b/charts/kubezero-sql/README.md.gotmpl new file mode 100644 index 0000000..07df9ab --- /dev/null +++ b/charts/kubezero-sql/README.md.gotmpl @@ -0,0 +1,20 @@ +{{ template "chart.header" . }} +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} + +## Resources + +### MariaDB diff --git a/charts/kubezero-sql/example-mariadb-galera.yaml b/charts/kubezero-sql/example-mariadb-galera.yaml new file mode 100644 index 0000000..e6b9f05 --- /dev/null +++ b/charts/kubezero-sql/example-mariadb-galera.yaml @@ -0,0 +1,27 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: test-mariadb + namespace: argocd +spec: + project: default + source: + repoURL: 'https://github.com/zero-down-time/kubezero' + path: charts/kubezero-sql + targetRevision: master + helm: + values: | + mariadb-galera: + enabled: true + metrics: + serviceMonitor: + enabled: true + + destination: + server: 'https://kubernetes.default.svc' + namespace: db-sandbox + syncPolicy: + automated: + prune: true + syncOptions: + - CreateNamespace=true diff --git a/charts/kubezero-sql/update.sh b/charts/kubezero-sql/update.sh new file mode 100755 index 0000000..b0babf4 --- /dev/null +++ b/charts/kubezero-sql/update.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -ex + +### MariaDB + + +# Fetch dashboards diff --git a/charts/kubezero-sql/values.yaml b/charts/kubezero-sql/values.yaml new file mode 100644 index 0000000..c9ecfce --- /dev/null +++ b/charts/kubezero-sql/values.yaml @@ -0,0 +1,20 @@ +mariadb-galera: + enabled: true + + replicaCount: 2 + + # Passwords should be fixed otherwise helm will create random new ones each time we template|apply + rootUser: + password: 12345qwert + db: + password: 12345qwert + galera: + mariabackup: + password: 12345qwert + + metrics: + serviceMonitor: + enabled: false + + prometheusRules: + enabled: false -- 2.40.1 From bd3ababa01df7ad1617a6d01cfab0649c70f4e7b Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 12 May 2021 18:41:31 +0200 Subject: [PATCH 09/19] feat: add mysql dashboard to kubezero-sql --- charts/kubezero-sql/dashboards-mariadb.yaml | 8 ++++++++ charts/kubezero-sql/example-mariadb-galera.yaml | 1 + .../templates/mariadb/grafana-dashboards.yaml | 13 +++++++++++++ charts/kubezero-sql/update.sh | 1 + charts/kubezero-sql/values.yaml | 1 + 5 files changed, 24 insertions(+) create mode 100644 charts/kubezero-sql/dashboards-mariadb.yaml create mode 100644 charts/kubezero-sql/templates/mariadb/grafana-dashboards.yaml diff --git a/charts/kubezero-sql/dashboards-mariadb.yaml b/charts/kubezero-sql/dashboards-mariadb.yaml new file mode 100644 index 0000000..0df496b --- /dev/null +++ b/charts/kubezero-sql/dashboards-mariadb.yaml @@ -0,0 +1,8 @@ +configmap: grafana-dashboards-mariadb +condition: '.Values.mariadb-galera.metrics.enabled' +gzip: true +# folder: +dashboards: +- name: mariadb-galera + url: https://grafana.com/api/dashboards/13106/revisions/3/download + tags: ['MariaDB', 'MySQL'] diff --git a/charts/kubezero-sql/example-mariadb-galera.yaml b/charts/kubezero-sql/example-mariadb-galera.yaml index e6b9f05..4c1f882 100644 --- a/charts/kubezero-sql/example-mariadb-galera.yaml +++ b/charts/kubezero-sql/example-mariadb-galera.yaml @@ -14,6 +14,7 @@ spec: mariadb-galera: enabled: true metrics: + enabled: true serviceMonitor: enabled: true diff --git a/charts/kubezero-sql/templates/mariadb/grafana-dashboards.yaml b/charts/kubezero-sql/templates/mariadb/grafana-dashboards.yaml new file mode 100644 index 0000000..e37e1d5 --- /dev/null +++ b/charts/kubezero-sql/templates/mariadb/grafana-dashboards.yaml @@ -0,0 +1,13 @@ +{{- if index .Values "mariadb-galera" "metrics" "enabled" }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" (include "kubezero-lib.fullname" $) "grafana-dashboards-mariadb" | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} + labels: + grafana_dashboard: "1" +{{ include "kubezero-lib.labels" . | indent 4 }} +binaryData: + mariadb-galera.json.gz: + H4sIAAAAAAAC/+2dWW/buBaA3/MrBGFw0QJuKrtZOn1r0kmnmGSSTtKZe1EEBi3RMicyqZJUErfIf79ctJNeszkpX1qYlEmehed8XGL92PA8v99HOM048995X8Vnz/uh/hU1GIyhKPU/nPZP/jo++u3s99++nPqdojoBA5jI+pSSMeQjmLGqMoIspCjliGD5SFXBJ6lqNAIcMJLREFZ1aZLFCH+KpjWq6//Mh3VSPaEeuBH/nne0SBR+yxCFFqGK/lOAxejLppHqNAZZXBtPoYCPzeJLSFkhl+65Y+8jpmAIMDB6aReX/bQraj3tbvY2g9nd2UWiIB1Zu6oXLyySzWxoqsGwaSpbl93NYCXZGAfc7Oy0UbqwZNYeOBgkFoc4axabfZSuCDAmYjyiVvqi7tRPEOOlZ1ZDETWDDCX8k2yp26lKa0q3K1M8A7Ea0zuP0wzWykcospSikOB9khAqG6TxALwIOl6v2xX/bG93vO7LetOF1O8rWbz/eO8TSHljCJWPsNGAABr5ed2N+v98I1d+OziMAUUgGngxEE0CL0wyxiH1iFDrJYJXm54Ql6KQeUMhuTeesG9J5MHrlFDx2KYagQ8jxFsK8GMMuQom3TfdYEcXSbc/IyThKBUVgSpUpsZZkuhPolGQD627E3R/3Q223m4HO1uqNkH4QkUVbV7lNJYoM9di/hDBJNoneIji0jHyyDkEWcJZo1SUh0IrZNwqlR6WoBjXBKhVDYUzCWG0WoYgYbBWf1N/WNggTRGOWc0t285ZNErVKHy/065RegyMYg6vpbf7X07Mr3Bibyp3pa5RcQmSTOm06zeqbjq3GnV36qg/HP/z512OO2iNu/bpvGERPhL5a0SSiJk2HxM1qX0wYCTJOGwNRARGmJqmNNUi3aqIAjGFEBsS1YYuHaxVe9NZogMKo1nNB+22G5/PN2w1tf59GS6oCHZtuduqAzwcQWrotAz5g8mZNKJhQZIWUdzH2XggmpjlgDIdpiJAIrigHXTnepJvRoilCZgcSTPPUJnW7asBCC9iSjIc+SvYJ090lbfN6ND8+nxnnOOS08a1mPfMd9Epili022mzotVx196rpfR8Y95Tixst95YrFPHRLLP1glUmV1FajkcoA0UnpBmP/JH4uF2bh1dSHb1awXUzK/iT2mS/afJW+bXabKt1JRz06ncIIjV9W0TDBA3sTVRmNtrW64a/2zBdrUkAjSFvemcj9wrYUN6g2KOfpQ1l+0NCRVQxcVHLhQWlYm4MWFWJ7CxsZKSSsuYAhFz5YSOh+AmMIY4Oym6bX6ZwqJdQ76twUFj1vJIZjeGBTop1clDlpyM05GYFTzRfSywLoSc5u040nALMtC5y003RpnZeDScHEoPY3kQt6hpi2DwgV44gxMgWjhSp2oNMbofQGk//VvNkY9ZEvbHnH1OtOQNrT7AuOJ40G/7cjPLWQYpoOcOIz8QS5j9GSus4yHrekGXMvltR1gMTUKNgBgJtLYlA9w07cUIGIOkzlewF+kg+uG/8WQ+++dKU1fHNOvDNomQyBnLSvWmWIdzeoHpwgpkDGLfllwlMEnI1q4fuZhAE3dv0MRfCetYubkNK97kU3bbH4Z3ZcVjYMQvhsXUO+yFIQtMx/AQw7k/d71OTgpnhTmmVGZBe15VMCmeFIx/KY7ny+WlPHQF6IVLEoySRK0Zh2s+3+fsMfZ+eUEQA7jNIJaBOSSstKdc4r+zn5xqndYGXzSrq0JWdEZ1XFswoyoUOi+PaMlMsGfL14ahb0j6phDA3WK9tnDaAeUqgfjs7UCtNHOWG00LX2lXngUUtoBDUK/8VLoiGk7I646ReTURYwrw4KjSqV8kQYwjwvWQIeZrVkONRI35z4/DpB/TbbYQ+RkiXI0yFvxsrszLaq7sV1mAvIjBg6vIAaxC5PwDU4A51GeAQ4pjLSd8NGuXQ9vgaLB7ctubdJgGRmJP6IYYq+EhBhLQDBvPTxe6yWN8191dGKIogPtU42XY8fQ2gon4dWpoDAZexhavDjOYTqV2jF6FGqVqGtkspikf81HpZR4K7pVQ8+4/aJlW7ofVgT7gKlu0urHmiUk+CMGTtcCALr/JeKvtJfzwhIi4XOUUW+LMTsbzcw8v1x9SFBxRTX2TVGBoWmpehUjkg6VQZaxwlqnLT4MJmEaRqH80fJqR2ZUwvOI4bPl1VpiCEtoAmAmZ4YfQiw0EKo0OhRqNu4ZRKAYcvbHn1W6aG+rU7Pn/pr5gzf/zwivTh3dyslEHr4fLrYpn1LxgXufF8mZSr55r3+aR2EZWX97kaG6VARj3rOXErXKh50S8SH8IRukRRJnRneGftYmf99uQ1uEYtbx9k4YW2bXOhUuaP1j5qccmu9bR96pcT2RJrJ+AaznCpam0tmqa85Ro5T7TXVn5C4j3AjJtNeYAzHtcRziiuCWPLIus9TsPrJ6bRi7VpO/Cq8kN4WQ66cS/S8ZXjq/XnK2M5vhpgBbMBS06U9+zMfpXZ4VdVetFKnU+AyLafEZHZYEy4Wz9jMOqHBGMYaoUvv7F9R3fCClJ65S0NeJ0lZL+Ud/fFbNXir4XkR+B6Uan31gRrj6RKvX2L9hzcOrh1cLsWcPtkdr52VwGzntv5cui1GHrZg5qa2fL7VxS1lm3ViYhSD4wBR5fw1f9mZd/HoDo0daMNYUyiQV8GmD6FIGJfu4Gx5/aQkKMGcQ9wt5gKlIkfXQdqFE8N9ZRqPS9C7MLTRrxkntangz4HfQ76HPStth23EvVtOepz1Hcn1PeaiTG/fmbYN5iItCyAL4Ri4NHX7UdGPj2MB6U+rQFp20eWni2xmblerIchvyJU0l5uPwF8UhqHew73HO453KvfYc5xr7c1Z4+vt7UK7b19OrTn7rA9sztsnKQXL7Y73lTSCMl4DHDE+sr0ijZWvtO2T8Z9QQp5k+t/re2MpN5+Pth9kkk53W6QwwOHB/eABxEM0Rgkzfj6DJhh7rngm94KzNC7/YWtVuE8oGg9rn34bpjBmq0dMjy5a+8khRhGfTFnfqa770eT08+H3oEQ2jsWClBXax0jOEZwjOAYYVVGMI6RVoOEnoMEBwnrcxNb8oGmg1XBQCZYlWrZw12grkbdT9C49RN7q439ULazyqlJZ3ml5/eDcDa+A/1/Eo192POWN8P+WsFaNX5Hao7UHKndK6l1f6rdnK1V/vyut3V/oNZGsumcNutwyPajZQ7injvETb9/whJydSc/c3AqGvI+64Y8gr0nuPVTF8HxxCo88csvI5Fo/oAygPpk8C8M+btft6Zdalpv5rDKsr3msrhLKM9ps2g1BtlZi80iBxoONOqgAQbydXLlH+/fDjbe68aKv+hm3gvAORynnL2U7PGQF3hLuRIZCe5ILN2W90KmcZLxZYTaWyukahnK/en9XZJVL9h5PmjVC3Z/XrYqX2cqprGcoVLqN4GeKj4LR3AMqmyiE7yI15Mkfx0ovdBPimRUuZB/JHe+P+zlitMT0i97kuEyARzheJH3pi7yhlT989Dtn6MqXpBan7jlm41tb0E9yJupz6YLlH6hyekEhxbdFpMbRCMSTnslqpq2paDF2yoxuXrVLQJC/tJJUeY3vpYiERJoCbmq7DtRWdLP9V4EvY/qDauvc82LkH2cv2NVP5flb/L9b/w9+HbwMbda9Xbb3sbN/wH5nSdeM3oAAA== +{{- end }} diff --git a/charts/kubezero-sql/update.sh b/charts/kubezero-sql/update.sh index b0babf4..8cdcf32 100755 --- a/charts/kubezero-sql/update.sh +++ b/charts/kubezero-sql/update.sh @@ -5,3 +5,4 @@ set -ex # Fetch dashboards +../kubezero-metrics/sync_grafana_dashboards.py dashboards-mariadb.yaml templates/mariadb/grafana-dashboards.yaml diff --git a/charts/kubezero-sql/values.yaml b/charts/kubezero-sql/values.yaml index c9ecfce..5a42a11 100644 --- a/charts/kubezero-sql/values.yaml +++ b/charts/kubezero-sql/values.yaml @@ -13,6 +13,7 @@ mariadb-galera: password: 12345qwert metrics: + enabled: false serviceMonitor: enabled: false -- 2.40.1 From d91f196ae6ee63b1af87f65ccb05d2d17c1014b4 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 12 May 2021 22:27:53 +0200 Subject: [PATCH 10/19] feat: add istio support to mariadb-galera --- .../kubezero-sql/example-mariadb-galera.yaml | 3 +++ .../templates/mariadb/istio-service.yaml | 22 +++++++++++++++++++ charts/kubezero-sql/values.yaml | 6 +++++ 3 files changed, 31 insertions(+) create mode 100644 charts/kubezero-sql/templates/mariadb/istio-service.yaml diff --git a/charts/kubezero-sql/example-mariadb-galera.yaml b/charts/kubezero-sql/example-mariadb-galera.yaml index 4c1f882..bb199be 100644 --- a/charts/kubezero-sql/example-mariadb-galera.yaml +++ b/charts/kubezero-sql/example-mariadb-galera.yaml @@ -17,6 +17,9 @@ spec: enabled: true serviceMonitor: enabled: true + istio: + enabled: true + url: mariadb.dev.mayneinc.com destination: server: 'https://kubernetes.default.svc' diff --git a/charts/kubezero-sql/templates/mariadb/istio-service.yaml b/charts/kubezero-sql/templates/mariadb/istio-service.yaml new file mode 100644 index 0000000..d346491 --- /dev/null +++ b/charts/kubezero-sql/templates/mariadb/istio-service.yaml @@ -0,0 +1,22 @@ +{{- if index .Values "mariadb-galera" "istio" "enabled" }} +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: mariadb + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +spec: + hosts: + - {{ index .Values "mariadb-galera" "istio" "url" }} + gateways: + - {{ index .Values "mariadb-galera" "istio" "gateway" }} + tcp: + - match: + - port: {{ default 3306 ( index .Values "mariadb-galera" "service" "port" ) }} + route: + - destination: + host: {{ .Release.Name }}-mariadb-galera-headless + port: + number: {{ default 3306 ( index .Values "mariadb-galera" "service" "port" ) }} +{{- end }} diff --git a/charts/kubezero-sql/values.yaml b/charts/kubezero-sql/values.yaml index 5a42a11..8381fb1 100644 --- a/charts/kubezero-sql/values.yaml +++ b/charts/kubezero-sql/values.yaml @@ -7,6 +7,7 @@ mariadb-galera: rootUser: password: 12345qwert db: + user: mariadb password: 12345qwert galera: mariabackup: @@ -19,3 +20,8 @@ mariadb-galera: prometheusRules: enabled: false + + istio: + enabled: false + gateway: istio-system/private-ingressgateway + url: mariadb.example.com -- 2.40.1 From 074b9459f9e3943aa49fd9e210e758e1f995ba67 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 12 May 2021 22:30:03 +0200 Subject: [PATCH 11/19] chore: fix typo --- charts/kubezero-sql/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-sql/values.yaml b/charts/kubezero-sql/values.yaml index 8381fb1..7d449c3 100644 --- a/charts/kubezero-sql/values.yaml +++ b/charts/kubezero-sql/values.yaml @@ -23,5 +23,5 @@ mariadb-galera: istio: enabled: false - gateway: istio-system/private-ingressgateway + gateway: istio-ingress/private-ingressgateway url: mariadb.example.com -- 2.40.1 From c614f216a79985a34bb1eecd4fe4ba4b8fb01b18 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 13 May 2021 16:21:16 +0200 Subject: [PATCH 12/19] fix: minor performance tuning for logging pipeline --- charts/kubezero-logging/Chart.yaml | 2 +- charts/kubezero-logging/templates/eck/elasticsearch.yaml | 2 +- charts/kubezero-logging/values.yaml | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index cf6db4a..513fb71 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application -version: 0.6.4 +version: 0.6.5 appVersion: 1.4.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-logging/templates/eck/elasticsearch.yaml b/charts/kubezero-logging/templates/eck/elasticsearch.yaml index 145f3d2..f20d9d1 100644 --- a/charts/kubezero-logging/templates/eck/elasticsearch.yaml +++ b/charts/kubezero-logging/templates/eck/elasticsearch.yaml @@ -24,7 +24,7 @@ spec: {{- if .processors }} node.processors: {{ .processors }} {{- end }} - indices.memory.index_buffer_size: "30%" + indices.memory.index_buffer_size: "20%" podTemplate: {{- if $.Values.es.s3Snapshot.iamrole }} metadata: diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index efc5491..85d3fdc 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -181,8 +181,9 @@ fluentd: logstash_format true reconnect_on_error true reload_on_failure true - request_timeout 60s + request_timeout 120s suppress_type_name true + bulk_message_request_threshold 2097152 @type file_single @@ -190,7 +191,7 @@ fluentd: chunk_limit_size 8MB total_limit_size 4GB flush_mode interval - flush_thread_count 2 + flush_thread_count 8 flush_interval 10s flush_at_shutdown true retry_type exponential_backoff -- 2.40.1 From 8e75ebe30080e22c2c99f1fc0c3237dceffd0de8 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 13 May 2021 16:44:45 +0200 Subject: [PATCH 13/19] fix: bump eck operator to 1.5.0 as 1.4.1 could not upgrade ES due to license issues --- charts/kubezero-logging/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index 513fb71..4f5c239 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -19,7 +19,7 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: eck-operator - version: 1.4.1 + version: 1.5.0 repository: https://helm.elastic.co condition: eck-operator.enabled - name: fluentd -- 2.40.1 From fa3a41c3761354a79066bc70ab1a6238a25384c2 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 13 May 2021 17:39:02 +0200 Subject: [PATCH 14/19] feat: add custom my.cnf for MariaDB to kubezero-sql --- charts/kubezero-sql/Chart.yaml | 4 +- charts/kubezero-sql/README.md | 56 ++++---- charts/kubezero-sql/README.md.gotmpl | 7 + .../kubezero-sql/example-mariadb-galera.yaml | 3 - charts/kubezero-sql/files/mariadb/my.cnf | 124 ++++++++++++++++++ .../templates/mariadb/mycnf-cm.yaml | 9 ++ charts/kubezero-sql/values.yaml | 2 + 7 files changed, 168 insertions(+), 37 deletions(-) create mode 100644 charts/kubezero-sql/files/mariadb/my.cnf create mode 100644 charts/kubezero-sql/templates/mariadb/mycnf-cm.yaml diff --git a/charts/kubezero-sql/Chart.yaml b/charts/kubezero-sql/Chart.yaml index dc73439..cdf130e 100644 --- a/charts/kubezero-sql/Chart.yaml +++ b/charts/kubezero-sql/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-sql description: KubeZero umbrella chart for SQL databases like MariaDB, PostgreSQL type: application -version: 0.1.0 +version: 0.1.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -18,5 +18,5 @@ dependencies: - name: mariadb-galera version: 5.8.0 repository: https://charts.bitnami.com/bitnami - condition: mariadb.enabled + condition: mariadb-galera.enabled kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-sql/README.md b/charts/kubezero-sql/README.md index a28e836..f47dde6 100644 --- a/charts/kubezero-sql/README.md +++ b/charts/kubezero-sql/README.md @@ -1,8 +1,8 @@ -# kubezero-mq +# kubezero-sql -![Version: 0.2.0](https://img.shields.io/badge/Version-0.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.1.1](https://img.shields.io/badge/Version-0.1.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) -KubeZero umbrella chart for MQ systems like NATS, RabbitMQ +KubeZero umbrella chart for SQL databases like MariaDB, PostgreSQL **Homepage:** @@ -18,41 +18,33 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| -| | nats | 0.8.3 | -| https://charts.bitnami.com/bitnami | rabbitmq | 8.13.1 | +| https://charts.bitnami.com/bitnami | mariadb-galera | 5.8.0 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| -| nats.enabled | bool | `false` | | -| nats.exporter.serviceMonitor.enabled | bool | `false` | | -| nats.nats.advertise | bool | `false` | | -| nats.nats.image | string | `"nats:2.2.1-alpine3.13"` | | -| nats.nats.jetstream.enabled | bool | `true` | | -| nats.natsbox.enabled | bool | `false` | | -| rabbitmq.auth.erlangCookie | string | `"randomlongerlangcookie"` | | -| rabbitmq.auth.password | string | `"supersecret"` | | -| rabbitmq.auth.tls.enabled | bool | `false` | | -| rabbitmq.auth.tls.existingSecret | string | `"rabbitmq-server-certificate"` | | -| rabbitmq.auth.tls.existingSecretFullChain | bool | `true` | | -| rabbitmq.auth.tls.failIfNoPeerCert | bool | `false` | | -| rabbitmq.clustering.forceBoot | bool | `true` | | -| rabbitmq.enabled | bool | `false` | | -| rabbitmq.hosts | list | `[]` | hostnames of rabbitmq services, used for Istio and TLS | -| rabbitmq.istio.enabled | bool | `false` | | -| rabbitmq.istio.gateway | string | `"istio-ingress/private-ingressgateway"` | | -| rabbitmq.metrics.enabled | bool | `false` | | -| rabbitmq.metrics.serviceMonitor.enabled | bool | `false` | | -| rabbitmq.pdb.create | bool | `true` | | -| rabbitmq.podAntiAffinityPreset | string | `""` | | -| rabbitmq.replicaCount | int | `1` | | -| rabbitmq.resources.requests.cpu | string | `"100m"` | | -| rabbitmq.resources.requests.memory | string | `"256Mi"` | | -| rabbitmq.topologySpreadConstraints | string | `"- maxSkew: 1\n topologyKey: topology.kubernetes.io/zone\n whenUnsatisfiable: DoNotSchedule\n labelSelector:\n matchLabels: {{- include \"common.labels.matchLabels\" . | nindent 6 }}\n- maxSkew: 1\n topologyKey: kubernetes.io/hostname\n whenUnsatisfiable: DoNotSchedule\n labelSelector:\n matchLabels: {{- include \"common.labels.matchLabels\" . | nindent 6 }}"` | | +| mariadb-galera.configurationConfigMap | string | `"{{ .Release.Name }}-mariadb-galera-configuration"` | | +| mariadb-galera.db.password | string | `"12345qwert"` | | +| mariadb-galera.db.user | string | `"mariadb"` | | +| mariadb-galera.enabled | bool | `true` | | +| mariadb-galera.galera.mariabackup.password | string | `"12345qwert"` | | +| mariadb-galera.istio.enabled | bool | `false` | | +| mariadb-galera.istio.gateway | string | `"istio-ingress/private-ingressgateway"` | | +| mariadb-galera.istio.url | string | `"mariadb.example.com"` | | +| mariadb-galera.metrics.enabled | bool | `false` | | +| mariadb-galera.metrics.prometheusRules.enabled | bool | `false` | | +| mariadb-galera.metrics.serviceMonitor.enabled | bool | `false` | | +| mariadb-galera.replicaCount | int | `2` | | +| mariadb-galera.rootUser.password | string | `"12345qwert"` | | + +# Changes + +## MariaDB +- custom my.cnf, source: https://github.com/bitnami/charts/blob/70d602fea38010145c20e1ca59be06e4cf32bf80/bitnami/mariadb-galera/values.yaml#L261 ## Resources -### NATS -- https://grafana.com/grafana/dashboards/13707 +### MariaDB + diff --git a/charts/kubezero-sql/README.md.gotmpl b/charts/kubezero-sql/README.md.gotmpl index 07df9ab..4f8a04b 100644 --- a/charts/kubezero-sql/README.md.gotmpl +++ b/charts/kubezero-sql/README.md.gotmpl @@ -15,6 +15,13 @@ {{ template "chart.valuesSection" . }} +# Changes + +## MariaDB +- custom my.cnf, source: https://github.com/bitnami/charts/blob/70d602fea38010145c20e1ca59be06e4cf32bf80/bitnami/mariadb-galera/values.yaml#L261 + + ## Resources ### MariaDB + diff --git a/charts/kubezero-sql/example-mariadb-galera.yaml b/charts/kubezero-sql/example-mariadb-galera.yaml index bb199be..4c1f882 100644 --- a/charts/kubezero-sql/example-mariadb-galera.yaml +++ b/charts/kubezero-sql/example-mariadb-galera.yaml @@ -17,9 +17,6 @@ spec: enabled: true serviceMonitor: enabled: true - istio: - enabled: true - url: mariadb.dev.mayneinc.com destination: server: 'https://kubernetes.default.svc' diff --git a/charts/kubezero-sql/files/mariadb/my.cnf b/charts/kubezero-sql/files/mariadb/my.cnf new file mode 100644 index 0000000..34c89a0 --- /dev/null +++ b/charts/kubezero-sql/files/mariadb/my.cnf @@ -0,0 +1,124 @@ +[client] +port=3306 +socket=/opt/bitnami/mariadb/tmp/mysql.sock +plugin_dir=/opt/bitnami/mariadb/plugin + +[mysqld] +default_storage_engine=InnoDB +basedir=/opt/bitnami/mariadb +datadir=/bitnami/mariadb/data +plugin_dir=/opt/bitnami/mariadb/plugin +tmpdir=/opt/bitnami/mariadb/tmp +socket=/opt/bitnami/mariadb/tmp/mysql.sock +pid_file=/opt/bitnami/mariadb/tmp/mysqld.pid +bind_address=0.0.0.0 + +## Character set +## +collation_server=utf8_unicode_ci +init_connect='SET NAMES utf8' +character_set_server=utf8 + +## MyISAM +## +key_buffer_size=32M +myisam_recover_options=FORCE,BACKUP + +## Safety +## +skip_host_cache +skip_name_resolve +max_allowed_packet=16M +max_connect_errors=1000000 +sql_mode=STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_AUTO_VALUE_ON_ZERO,NO_ENGINE_SUBSTITUTION,NO_ZERO_DATE,NO_ZERO_IN_DATE,ONLY_FULL_GROUP_BY +sysdate_is_now=1 + +## Binary Logging +## +log_bin=mysql-bin +expire_logs_days=14 +# Disabling for performance per http://severalnines.com/blog/9-tips-going-production-galera-cluster-mysql +sync_binlog=0 +# Required for Galera +binlog_format=row + +## Caches and Limits +## +tmp_table_size=32M +max_heap_table_size=32M +# Re-enabling as now works with Maria 10.1.2 +query_cache_type=1 +query_cache_limit=4M +query_cache_size=256M +max_connections=500 +thread_cache_size=50 +open_files_limit=65535 +table_definition_cache=4096 +table_open_cache=4096 + +## InnoDB +## +innodb=FORCE +innodb_strict_mode=1 +# Mandatory per https://github.com/codership/documentation/issues/25 +innodb_autoinc_lock_mode=2 +# Per https://www.percona.com/blog/2006/08/04/innodb-double-write/ +innodb_doublewrite=1 +innodb_flush_method=O_DIRECT +innodb_log_files_in_group=2 +innodb_log_file_size=128M +innodb_flush_log_at_trx_commit=1 +innodb_file_per_table=1 +# 80% Memory is default reco. +# Need to re-evaluate when DB size grows +innodb_buffer_pool_size=2G +innodb_file_format=Barracuda + +## Logging +## +log_error=/opt/bitnami/mariadb/logs/mysqld.log +slow_query_log_file=/opt/bitnami/mariadb/logs/mysqld.log +log_queries_not_using_indexes=0 +slow_query_log=1 + +## SSL +## Use extraVolumes and extraVolumeMounts to mount /certs filesystem +# ssl_ca=/certs/ca.pem +# ssl_cert=/certs/server-cert.pem +# ssl_key=/certs/server-key.pem + +[galera] +wsrep_on=ON +wsrep_provider=/opt/bitnami/mariadb/lib/libgalera_smm.so +wsrep_sst_method=mariabackup +wsrep_slave_threads=4 +wsrep_cluster_address=gcomm:// +wsrep_cluster_name=galera +wsrep_sst_auth="root:" +# Enabled for performance per https://mariadb.com/kb/en/innodb-system-variables/#innodb_flush_log_at_trx_commit +innodb_flush_log_at_trx_commit=2 +# MYISAM REPLICATION SUPPORT # +wsrep_replicate_myisam=ON + +[mariadb] +plugin_load_add=auth_pam + +## Data-at-Rest Encryption +## Use extraVolumes and extraVolumeMounts to mount /encryption filesystem +# plugin_load_add=file_key_management +# file_key_management_filename=/encryption/keyfile.enc +# file_key_management_filekey=FILE:/encryption/keyfile.key +# file_key_management_encryption_algorithm=AES_CTR +# encrypt_binlog=ON +# encrypt_tmp_files=ON + +## InnoDB/XtraDB Encryption +# innodb_encrypt_tables=ON +# innodb_encrypt_temporary_tables=ON +# innodb_encrypt_log=ON +# innodb_encryption_threads=4 +# innodb_encryption_rotate_key_age=1 + +## Aria Encryption +# aria_encrypt_tables=ON +# encrypt_tmp_disk_tables=ON diff --git a/charts/kubezero-sql/templates/mariadb/mycnf-cm.yaml b/charts/kubezero-sql/templates/mariadb/mycnf-cm.yaml new file mode 100644 index 0000000..fa326b2 --- /dev/null +++ b/charts/kubezero-sql/templates/mariadb/mycnf-cm.yaml @@ -0,0 +1,9 @@ +{{- if index .Values "mariadb-galera" "enabled" }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-mariadb-galera-configuration + labels: {{- include "common.labels.standard" . | nindent 4 }} +data: +{{ (.Files.Glob "files/mariadb/my.cnf").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/kubezero-sql/values.yaml b/charts/kubezero-sql/values.yaml index 7d449c3..7a7ba60 100644 --- a/charts/kubezero-sql/values.yaml +++ b/charts/kubezero-sql/values.yaml @@ -25,3 +25,5 @@ mariadb-galera: enabled: false gateway: istio-ingress/private-ingressgateway url: mariadb.example.com + + configurationConfigMap: "{{ .Release.Name }}-mariadb-galera-configuration" -- 2.40.1 From 92729085c9c2a3632e6762ce1e6b5f67e612354a Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 14 May 2021 00:11:13 +0200 Subject: [PATCH 15/19] fix: fix broken CRDs handling for ECK operator --- charts/kubezero-logging/Chart.yaml | 4 +- charts/kubezero-logging/README.md | 28 +- .../charts/eck-operator/.helmignore | 23 + .../charts/eck-operator/Chart.yaml | 20 + .../charts/eck-operator/README.md | 20 + .../charts/eck-operator/crds/all-crds.yaml | 3539 +++++++++++++++++ .../charts/eck-operator/profile-global.yaml | 6 + .../charts/eck-operator/profile-istio.yaml | 11 + .../eck-operator/profile-restricted.yaml | 12 + .../profile-soft-multi-tenancy.yaml | 18 + .../charts/eck-operator/templates/NOTES.txt | 2 + .../eck-operator/templates/_helpers.tpl | 333 ++ .../eck-operator/templates/cluster-roles.yaml | 63 + .../eck-operator/templates/configmap.yaml | 43 + .../templates/managed-namespaces.yaml | 13 + .../templates/managed-ns-network-policy.yaml | 228 ++ .../templates/operator-namespace.yaml | 10 + .../templates/operator-network-policy.yaml | 59 + .../eck-operator/templates/role-bindings.yaml | 80 + .../templates/service-account.yaml | 14 + .../eck-operator/templates/statefulset.yaml | 118 + .../templates/validate-chart.yaml | 23 + .../eck-operator/templates/webhook.yaml | 329 ++ .../charts/eck-operator/values.yaml | 177 + charts/kubezero-logging/update.sh | 11 + 25 files changed, 5173 insertions(+), 11 deletions(-) create mode 100644 charts/kubezero-logging/charts/eck-operator/.helmignore create mode 100644 charts/kubezero-logging/charts/eck-operator/Chart.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/README.md create mode 100644 charts/kubezero-logging/charts/eck-operator/crds/all-crds.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/profile-global.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/profile-istio.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/profile-restricted.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/profile-soft-multi-tenancy.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/NOTES.txt create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/_helpers.tpl create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/cluster-roles.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/configmap.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/managed-namespaces.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/managed-ns-network-policy.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/operator-namespace.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/operator-network-policy.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/role-bindings.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/service-account.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/statefulset.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/validate-chart.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/templates/webhook.yaml create mode 100644 charts/kubezero-logging/charts/eck-operator/values.yaml diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index 4f5c239..38c97ef 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -3,7 +3,7 @@ name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application version: 0.6.5 -appVersion: 1.4.1 +appVersion: 1.5.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -20,7 +20,7 @@ dependencies: repository: https://zero-down-time.github.io/kubezero/ - name: eck-operator version: 1.5.0 - repository: https://helm.elastic.co + # repository: https://helm.elastic.co condition: eck-operator.enabled - name: fluentd version: 0.2.2 diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md index 89edc99..171c670 100644 --- a/charts/kubezero-logging/README.md +++ b/charts/kubezero-logging/README.md @@ -1,6 +1,6 @@ # kubezero-logging -![Version: 0.6.2](https://img.shields.io/badge/Version-0.6.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.4.1](https://img.shields.io/badge/AppVersion-1.4.1-informational?style=flat-square) +![Version: 0.6.5](https://img.shields.io/badge/Version-0.6.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.5.0](https://img.shields.io/badge/AppVersion-1.5.0-informational?style=flat-square) KubeZero Umbrella Chart for complete EFK stack @@ -14,13 +14,13 @@ KubeZero Umbrella Chart for complete EFK stack ## Requirements -Kubernetes: `>= 1.16.0` +Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| -| | fluent-bit | 0.12.3 | +| | fluent-bit | 0.15.4 | | | fluentd | 0.2.2 | -| https://helm.elastic.co | eck-operator | 1.4.1 | +| https://helm.elastic.co | eck-operator | 1.5.0 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Changes from upstream @@ -70,21 +70,31 @@ Kubernetes: `>= 1.16.0` | fluent-bit.config.customParsers | string | `"[PARSER]\n Name cri-log\n Format regex\n Regex ^(?