From 77a20df965a2343a6c847bba06b6e9d8639bdcd5 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 21 Jan 2021 12:31:06 +0000 Subject: [PATCH 01/65] Version bump for aws-ebs-csi and kiam, ES bugfix bump, fluentd tuning --- .gitignore | 1 - Makefile | 6 +----- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 6 +++--- charts/kubezero-aws-ebs-csi-driver/README.md | 4 ++-- charts/kubezero-kiam/Chart.yaml | 6 +++--- charts/kubezero-kiam/README.md | 6 +++--- charts/kubezero-kiam/values.yaml | 2 +- charts/kubezero-logging/Chart.yaml | 2 +- charts/kubezero-logging/README.md | 9 +++++---- charts/kubezero-logging/values.yaml | 11 +++++++---- charts/kubezero-metrics/README.md | 7 ++++--- charts/kubezero/templates/metrics.yaml | 6 ++++++ 12 files changed, 36 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 04f73a7..3bb249e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ # Vim *.swp -output .vscode .DS_Store .idea diff --git a/Makefile b/Makefile index bc4204f..b2f2e76 100644 --- a/Makefile +++ b/Makefile @@ -2,14 +2,10 @@ BUCKET ?= zero-downtime BUCKET_PREFIX ?= /cloudbender/distfiles FILES ?= distfiles.txt -.PHONY: clean update docs +.PHONY: update docs all: update - -clean: - rm -f kube*.tgz - update: ./script/update_helm.sh diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index a13400a..9cb2ee5 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.3.4 -appVersion: 0.8.0 +version: 0.3.5 +appVersion: 0.8.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png sources: @@ -18,7 +18,7 @@ maintainers: - name: Quarky9 dependencies: - name: aws-ebs-csi-driver - version: 0.7.1 + version: 0.8.2 repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: kubezero-lib version: ">= 0.1.3" diff --git a/charts/kubezero-aws-ebs-csi-driver/README.md b/charts/kubezero-aws-ebs-csi-driver/README.md index fc14ab9..4cce78b 100644 --- a/charts/kubezero-aws-ebs-csi-driver/README.md +++ b/charts/kubezero-aws-ebs-csi-driver/README.md @@ -1,6 +1,6 @@ # kubezero-aws-ebs-csi-driver -![Version: 0.3.4](https://img.shields.io/badge/Version-0.3.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.8.0](https://img.shields.io/badge/AppVersion-0.8.0-informational?style=flat-square) +![Version: 0.3.5](https://img.shields.io/badge/Version-0.3.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.8.1](https://img.shields.io/badge/AppVersion-0.8.1-informational?style=flat-square) KubeZero Umbrella Chart for aws-ebs-csi-driver @@ -23,7 +23,7 @@ Kubernetes: `>= 1.16.0` | Repository | Name | Version | |------------|------|---------| -| https://kubernetes-sigs.github.io/aws-ebs-csi-driver | aws-ebs-csi-driver | 0.7.1 | +| https://kubernetes-sigs.github.io/aws-ebs-csi-driver | aws-ebs-csi-driver | 0.8.2 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## IAM Role diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index 8503fc5..119efbf 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-kiam description: KubeZero Umbrella Chart for Kiam type: application -version: 0.2.12 -appVersion: 3.6 +version: 0.3.0 +appVersion: "4.0" home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -16,7 +16,7 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: kiam - version: 5.9.0 + version: 6.0.0 repository: https://uswitch.github.io/kiam-helm-charts/charts/ condition: kiam.enabled kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-kiam/README.md b/charts/kubezero-kiam/README.md index a71327a..991ef10 100644 --- a/charts/kubezero-kiam/README.md +++ b/charts/kubezero-kiam/README.md @@ -1,6 +1,6 @@ # kubezero-kiam -![Version: 0.2.12](https://img.shields.io/badge/Version-0.2.12-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.6](https://img.shields.io/badge/AppVersion-3.6-informational?style=flat-square) +![Version: 0.3.0](https://img.shields.io/badge/Version-0.3.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 4.0](https://img.shields.io/badge/AppVersion-4.0-informational?style=flat-square) KubeZero Umbrella Chart for Kiam @@ -18,7 +18,7 @@ Kubernetes: `>= 1.16.0` | Repository | Name | Version | |------------|------|---------| -| https://uswitch.github.io/kiam-helm-charts/charts/ | kiam | 5.9.0 | +| https://uswitch.github.io/kiam-helm-charts/charts/ | kiam | 6.0.0 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## KubeZero default configuration @@ -75,7 +75,7 @@ By default all access to the meta-data service is blocked, expect for: | kiam.server.prometheus.servicemonitor.enabled | bool | `false` | | | kiam.server.prometheus.servicemonitor.interval | string | `"30s"` | | | kiam.server.prometheus.servicemonitor.labels.release | string | `"metrics"` | | -| kiam.server.resources.limits.cpu | string | `"100m"` | | +| kiam.server.resources.limits.cpu | string | `"300m"` | | | kiam.server.resources.limits.memory | string | `"50Mi"` | | | kiam.server.resources.requests.cpu | string | `"100m"` | | | kiam.server.resources.requests.memory | string | `"50Mi"` | | diff --git a/charts/kubezero-kiam/values.yaml b/charts/kubezero-kiam/values.yaml index 3c5b38d..e07d367 100644 --- a/charts/kubezero-kiam/values.yaml +++ b/charts/kubezero-kiam/values.yaml @@ -25,7 +25,7 @@ kiam: cpu: "100m" limits: memory: "50Mi" - cpu: "100m" + cpu: "300m" tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index 3776ccd..a78f85d 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application -version: 0.5.2 +version: 0.5.3 appVersion: 1.3.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md index 9f47d63..daacbb0 100644 --- a/charts/kubezero-logging/README.md +++ b/charts/kubezero-logging/README.md @@ -1,6 +1,6 @@ # kubezero-logging -![Version: 0.5.1](https://img.shields.io/badge/Version-0.5.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.3.1](https://img.shields.io/badge/AppVersion-1.3.1-informational?style=flat-square) +![Version: 0.5.3](https://img.shields.io/badge/Version-0.5.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.3.1](https://img.shields.io/badge/AppVersion-1.3.1-informational?style=flat-square) KubeZero Umbrella Chart for complete EFK stack @@ -18,8 +18,9 @@ Kubernetes: `>= 1.16.0` | Repository | Name | Version | |------------|------|---------| +| | fluent-bit | 0.7.14 | +| | fluentd | 2.5.3 | | https://helm.elastic.co | eck-operator | 1.3.1 | -| https://kubernetes-charts.storage.googleapis.com/ | fluentd | 2.5.1 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Changes from upstream @@ -82,7 +83,7 @@ Kubernetes: `>= 1.16.0` | fluentd.configMaps."filter.conf" | string | `"\n @type parser\n key_name message\n remove_key_name_field true\n reserve_data true\n reserve_time true\n # inject_key_prefix message_json.\n emit_invalid_record_to_error false\n \n @type json\n \n\n"` | | | fluentd.configMaps."forward-input.conf" | string | `"\n @type forward\n port 24224\n bind 0.0.0.0\n skip_invalid_event true\n send_keepalive_packet true\n \n self_hostname \"#{ENV['HOSTNAME']}\"\n shared_key \"#{ENV['FLUENTD_SHARED_KEY']}\"\n \n\n"` | | | fluentd.configMaps."general.conf" | string | `"\n\n @type http\n port 9880\n bind 0.0.0.0\n keepalive_timeout 30\n\n\n @type monitor_agent\n bind 0.0.0.0\n port 24220\n tag fluentd.monitor.metrics\n\n"` | | -| fluentd.configMaps."output.conf" | string | `"\n @id elasticsearch\n @type elasticsearch\n @log_level info\n include_tag_key true\n id_key id\n remove_keys id\n\n # KubeZero pipeline incl. GeoIP etc.\n pipeline fluentd\n\n host \"#{ENV['OUTPUT_HOST']}\"\n port \"#{ENV['OUTPUT_PORT']}\"\n scheme \"#{ENV['OUTPUT_SCHEME']}\"\n ssl_version \"#{ENV['OUTPUT_SSL_VERSION']}\"\n ssl_verify \"#{ENV['OUTPUT_SSL_VERIFY']}\"\n user \"#{ENV['OUTPUT_USER']}\"\n password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n log_es_400_reason\n logstash_format true\n reconnect_on_error true\n # reload_on_failure true\n request_timeout 15s\n suppress_type_name true\n\n \n @type file_single\n path /var/log/fluentd-buffers/kubernetes.system.buffer\n flush_mode interval\n flush_thread_count 2\n flush_interval 30s\n flush_at_shutdown true\n retry_type exponential_backoff\n retry_timeout 60m\n overflow_action drop_oldest_chunk\n \n\n"` | | +| fluentd.configMaps."output.conf" | string | `"\n @id elasticsearch\n @type elasticsearch\n @log_level info\n include_tag_key true\n id_key id\n remove_keys id\n\n # KubeZero pipeline incl. GeoIP etc.\n pipeline fluentd\n\n hosts \"#{ENV['OUTPUT_HOST']}\"\n port \"#{ENV['OUTPUT_PORT']}\"\n scheme \"#{ENV['OUTPUT_SCHEME']}\"\n ssl_version \"#{ENV['OUTPUT_SSL_VERSION']}\"\n ssl_verify \"#{ENV['OUTPUT_SSL_VERIFY']}\"\n user \"#{ENV['OUTPUT_USER']}\"\n password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n log_es_400_reason\n logstash_format true\n reconnect_on_error true\n reload_on_failure true\n request_timeout 60s\n suppress_type_name true\n\n \n @type file_single\n path /var/log/fluentd-buffers/kubernetes.system.buffer\n chunk_limit_records 8192\n total_limit_size 4GB\n flush_mode interval\n flush_thread_count 2\n flush_interval 30s\n flush_at_shutdown true\n retry_type exponential_backoff\n retry_timeout 60m\n overflow_action drop_oldest_chunk\n disable_chunk_backup true\n \n\n"` | | | fluentd.enabled | bool | `false` | | | fluentd.env.OUTPUT_SSL_VERIFY | string | `"false"` | | | fluentd.env.OUTPUT_USER | string | `"elastic"` | | @@ -113,7 +114,7 @@ Kubernetes: `>= 1.16.0` | kibana.istio.enabled | bool | `false` | | | kibana.istio.gateway | string | `"istio-system/ingressgateway"` | | | kibana.istio.url | string | `""` | | -| version | string | `"7.10.0"` | | +| version | string | `"7.10.2"` | | ## Resources: diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 7a883ac..24f58d7 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -11,7 +11,7 @@ eck-operator: node-role.kubernetes.io/master: "" # Version for ElasticSearch and Kibana have to match so we define it at top-level -version: 7.10.0 +version: 7.10.1 elastic_password: "" # super_secret_elastic_password @@ -145,7 +145,7 @@ fluentd: # KubeZero pipeline incl. GeoIP etc. pipeline fluentd - host "#{ENV['OUTPUT_HOST']}" + hosts "#{ENV['OUTPUT_HOST']}" port "#{ENV['OUTPUT_PORT']}" scheme "#{ENV['OUTPUT_SCHEME']}" ssl_version "#{ENV['OUTPUT_SSL_VERSION']}" @@ -156,13 +156,15 @@ fluentd: log_es_400_reason logstash_format true reconnect_on_error true - # reload_on_failure true - request_timeout 15s + reload_on_failure true + request_timeout 60s suppress_type_name true @type file_single path /var/log/fluentd-buffers/kubernetes.system.buffer + chunk_limit_records 8192 + total_limit_size 4GB flush_mode interval flush_thread_count 2 flush_interval 30s @@ -170,6 +172,7 @@ fluentd: retry_type exponential_backoff retry_timeout 60m overflow_action drop_oldest_chunk + disable_chunk_backup true diff --git a/charts/kubezero-metrics/README.md b/charts/kubezero-metrics/README.md index bdec431..1a7328e 100644 --- a/charts/kubezero-metrics/README.md +++ b/charts/kubezero-metrics/README.md @@ -1,6 +1,6 @@ # kubezero-metrics -![Version: 0.3.1](https://img.shields.io/badge/Version-0.3.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.3.2](https://img.shields.io/badge/Version-0.3.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero Umbrella Chart for prometheus-operator @@ -18,8 +18,8 @@ Kubernetes: `>= 1.16.0` | Repository | Name | Version | |------------|------|---------| -| https://prometheus-community.github.io/helm-charts | kube-prometheus-stack | 12.3.0 | -| https://prometheus-community.github.io/helm-charts | prometheus-adapter | 2.7.1 | +| | kube-prometheus-stack | 12.8.0 | +| https://prometheus-community.github.io/helm-charts | prometheus-adapter | 2.10.1 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Values @@ -104,6 +104,7 @@ Kubernetes: `>= 1.16.0` | kube-prometheus-stack.prometheusOperator.tolerations[0].effect | string | `"NoSchedule"` | | | kube-prometheus-stack.prometheusOperator.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | prometheus-adapter.enabled | bool | `true` | | +| prometheus-adapter.logLevel | int | `1` | | | prometheus-adapter.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | prometheus-adapter.prometheus.url | string | `"http://metrics-kube-prometheus-st-prometheus"` | | | prometheus-adapter.rules.default | bool | `false` | | diff --git a/charts/kubezero/templates/metrics.yaml b/charts/kubezero/templates/metrics.yaml index 6c09bf7..906bad9 100644 --- a/charts/kubezero/templates/metrics.yaml +++ b/charts/kubezero/templates/metrics.yaml @@ -10,6 +10,12 @@ kube-prometheus-stack: {{- toYaml . | nindent 2 }} {{- end }} {{- end }} +{{- if index .Values "metrics" "prometheus-adapter" }} +prometheus-adapter: + {{- with index .Values "metrics" "prometheus-adapter" }} + {{- toYaml . | nindent 2 }} + {{- end }} +{{- end }} {{- end }} -- 2.40.1 From 978670654268022604e44f6cbf7b27b5a3858ff6 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 21 Jan 2021 13:35:20 +0000 Subject: [PATCH 02/65] Fix changed key for kiam --- charts/kubezero-kiam/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-kiam/values.yaml b/charts/kubezero-kiam/values.yaml index e07d367..00e2530 100644 --- a/charts/kubezero-kiam/values.yaml +++ b/charts/kubezero-kiam/values.yaml @@ -48,7 +48,7 @@ kiam: host: iptables: false interface: "cali+" - whiteListRouteRegexp: '^/latest/(meta-data/instance-id|dynamic)' + allowRouteRegexp: '^/latest/(meta-data/instance-id|dynamic)' sslCertHostPath: /etc/ssl/certs tlsSecret: kiam-agent-tls tlsCerts: -- 2.40.1 From ff3ae1f44f572d72980b5bca721d074946b1d97e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 26 Jan 2021 12:54:56 +0000 Subject: [PATCH 03/65] Istio minor version bump --- charts/kubezero-istio-ingress/Chart.yaml | 4 +-- charts/kubezero-istio-ingress/values.yaml | 2 +- charts/kubezero-istio/Chart.yaml | 4 +-- .../charts/base/crds/crd-all.gen.yaml | 1 - .../charts/base/files/gen-istio-cluster.yaml | 6 +++-- .../charts/base/templates/clusterrole.yaml | 7 +++-- .../templates/telemetryv2_1.8.yaml | 27 ++++++++++++++++--- charts/kubezero-istio/update.sh | 2 +- charts/kubezero-istio/values.yaml | 2 +- 9 files changed, 40 insertions(+), 15 deletions(-) diff --git a/charts/kubezero-istio-ingress/Chart.yaml b/charts/kubezero-istio-ingress/Chart.yaml index e66651e..15b3889 100644 --- a/charts/kubezero-istio-ingress/Chart.yaml +++ b/charts/kubezero-istio-ingress/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-istio-ingress description: KubeZero Umbrella Chart for Istio based Ingress type: application -version: 0.1.1 -appVersion: 1.8.1 +version: 0.1.2 +appVersion: 1.8.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-istio-ingress/values.yaml b/charts/kubezero-istio-ingress/values.yaml index b989ba7..4603747 100644 --- a/charts/kubezero-istio-ingress/values.yaml +++ b/charts/kubezero-istio-ingress/values.yaml @@ -1,7 +1,7 @@ # Make sure these values match kuberzero-istio !!! global: hub: docker.io/istio - tag: 1.8.1 + tag: 1.8.2 logAsJson: true jwtPolicy: first-party-jwt diff --git a/charts/kubezero-istio/Chart.yaml b/charts/kubezero-istio/Chart.yaml index 903bd5a..a349c7c 100644 --- a/charts/kubezero-istio/Chart.yaml +++ b/charts/kubezero-istio/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-istio description: KubeZero Umbrella Chart for Istio type: application -version: 0.4.1 -appVersion: 1.8.1 +version: 0.4.2 +appVersion: 1.8.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-istio/charts/base/crds/crd-all.gen.yaml b/charts/kubezero-istio/charts/base/crds/crd-all.gen.yaml index deea651..35b4db8 100644 --- a/charts/kubezero-istio/charts/base/crds/crd-all.gen.yaml +++ b/charts/kubezero-istio/charts/base/crds/crd-all.gen.yaml @@ -3048,7 +3048,6 @@ spec: - CUSTOM type: string provider: - description: Specifies detailed configuration of the CUSTOM action. properties: name: description: Specifies the name of the extension provider. diff --git a/charts/kubezero-istio/charts/base/files/gen-istio-cluster.yaml b/charts/kubezero-istio/charts/base/files/gen-istio-cluster.yaml index 0a30433..ac70215 100644 --- a/charts/kubezero-istio/charts/base/files/gen-istio-cluster.yaml +++ b/charts/kubezero-istio/charts/base/files/gen-istio-cluster.yaml @@ -3050,7 +3050,6 @@ spec: - CUSTOM type: string provider: - description: Specifies detailed configuration of the CUSTOM action. properties: name: description: Specifies the name of the extension provider. @@ -3537,7 +3536,9 @@ rules: verbs: ["get", "list", "watch", "update"] # istio configuration - - apiGroups: ["config.istio.io", "security.istio.io", "networking.istio.io", "authentication.istio.io"] + # removing CRD permissions can break older versions of Istio running alongside this control plane (https://github.com/istio/istio/issues/29382) + # please proceed with caution + - apiGroups: ["config.istio.io", "security.istio.io", "networking.istio.io", "authentication.istio.io", "rbac.istio.io"] verbs: ["get", "watch", "list"] resources: ["*"] - apiGroups: ["networking.istio.io"] @@ -3621,6 +3622,7 @@ rules: - "security.istio.io" - "networking.istio.io" - "authentication.istio.io" + - "rbac.istio.io" resources: ["*"] verbs: ["get", "list", "watch"] - apiGroups: [""] diff --git a/charts/kubezero-istio/charts/base/templates/clusterrole.yaml b/charts/kubezero-istio/charts/base/templates/clusterrole.yaml index 521c24b..f53b830 100644 --- a/charts/kubezero-istio/charts/base/templates/clusterrole.yaml +++ b/charts/kubezero-istio/charts/base/templates/clusterrole.yaml @@ -17,11 +17,13 @@ rules: verbs: ["get", "list", "watch", "update"] # istio configuration - - apiGroups: ["config.istio.io", "security.istio.io", "networking.istio.io", "authentication.istio.io"] + # removing CRD permissions can break older versions of Istio running alongside this control plane (https://github.com/istio/istio/issues/29382) + # please proceed with caution + - apiGroups: ["config.istio.io", "security.istio.io", "networking.istio.io", "authentication.istio.io", "rbac.istio.io"] verbs: ["get", "watch", "list"] resources: ["*"] {{- if .Values.global.istiod.enableAnalysis }} - - apiGroups: ["config.istio.io", "security.istio.io", "networking.istio.io", "authentication.istio.io"] + - apiGroups: ["config.istio.io", "security.istio.io", "networking.istio.io", "authentication.istio.io", "rbac.istio.io"] verbs: ["update"] # TODO: should be on just */status but wildcard is not supported resources: ["*"] @@ -115,6 +117,7 @@ rules: - "security.istio.io" - "networking.istio.io" - "authentication.istio.io" + - "rbac.istio.io" resources: ["*"] verbs: ["get", "list", "watch"] - apiGroups: [""] diff --git a/charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.8.yaml b/charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.8.yaml index f878321..6985a68 100644 --- a/charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.8.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.8.yaml @@ -108,10 +108,18 @@ spec: value: | {} vm_config: + {{- if .Values.telemetry.v2.metadataExchange.wasmEnabled }} + runtime: envoy.wasm.runtime.v8 + allow_precompiled: true + code: + local: + filename: /etc/istio/extensions/metadata-exchange-filter.compiled.wasm + {{- else }} runtime: envoy.wasm.runtime.null code: local: inline_string: envoy.wasm.metadata_exchange + {{- end }} --- apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter @@ -339,10 +347,18 @@ spec: {{- end }} vm_config: vm_id: stats_outbound + {{- if .Values.telemetry.v2.prometheus.wasmEnabled }} + runtime: envoy.wasm.runtime.v8 + allow_precompiled: true + code: + local: + filename: /etc/istio/extensions/stats-filter.compiled.wasm + {{- else }} runtime: envoy.wasm.runtime.null code: local: inline_string: envoy.wasm.stats + {{- end }} --- # Note: tcp stats filter is wasm enabled only in sidecars. apiVersion: networking.istio.io/v1alpha3 @@ -502,14 +518,20 @@ spec: {{- end }} vm_config: vm_id: tcp_stats_outbound + {{- if .Values.telemetry.v2.prometheus.wasmEnabled }} + runtime: envoy.wasm.runtime.v8 + allow_precompiled: true + code: + local: + filename: /etc/istio/extensions/stats-filter.compiled.wasm + {{- else }} runtime: envoy.wasm.runtime.null code: local: inline_string: "envoy.wasm.stats" + {{- end }} --- - {{- end }} - {{- if .Values.telemetry.v2.stackdriver.enabled }} apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter @@ -629,7 +651,6 @@ spec: code: local: { inline_string: envoy.wasm.null.stackdriver } --- - apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: diff --git a/charts/kubezero-istio/update.sh b/charts/kubezero-istio/update.sh index 3993833..5d6a364 100755 --- a/charts/kubezero-istio/update.sh +++ b/charts/kubezero-istio/update.sh @@ -1,7 +1,7 @@ #!/bin/bash set -ex -export ISTIO_VERSION=1.8.1 +export ISTIO_VERSION=1.8.2 if [ ! -d istio-$ISTIO_VERSION ]; then NAME="istio-$ISTIO_VERSION" diff --git a/charts/kubezero-istio/values.yaml b/charts/kubezero-istio/values.yaml index 2b58c92..a855e1a 100644 --- a/charts/kubezero-istio/values.yaml +++ b/charts/kubezero-istio/values.yaml @@ -1,6 +1,6 @@ global: hub: docker.io/istio - tag: 1.8.0 + tag: 1.8.2 logAsJson: true jwtPolicy: first-party-jwt -- 2.40.1 From 0e0a9feb123ab981d96ec365b1a919c8f6b2ac83 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 26 Jan 2021 13:47:33 +0000 Subject: [PATCH 04/65] README updates --- README.md | 27 ++++++++++--------- .../aws-iam-authenticator/deployment.yaml | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ec6247b..1fe73d1 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,16 @@ KubeZero - Zero Down Time Kubernetes platform ======================== -KubeZero is a pre-configured collection of components deployed onto a bare Kubernetes cluster. -All chosen components are 100% organic OpenSource. +KubeZero is a Kubernetes distribution providing an opinionated, pre-configured container platform +incl. various addons and services. # Design goals - Cloud provider agnostic, bare-metal / self-hosted possible -- No vendor lock in -- No closed source solutions +- No vendor lock in, most components are optional and could be exchanged +- Organic OpenSource / open and permissive licenses over closed-source solutions - No premium services / subscriptions required -- Staying to upstream projects as close as possible -- Minimal custom code -- Work within each community / give back +- Staying and contributing back to upstream projects as much as possible + # Version / Support Matrix @@ -31,13 +30,15 @@ All chosen components are 100% organic OpenSource. - support for single node control plane for small clusters / test environments to reduce costs - access to control plane from within the VPC only by default ( VPN access required for Admin tasks ) - controller nodes are used for various platform admin controllers / operators to reduce costs and noise on worker nodes -- integrated ArgoCD Gitops controller + +## GitOps +- full ArgoCD support and integration (optional) ## AWS IAM access control - Kiam allowing IAM roles per pod - IAM roles are assumed / requested and cached on controller nodes for improved security -- blocking access to meta-data service on all nodes -- IAM roles are maintained/ automated and tracked via CFN templates +- access to meta-data services is blocked / controlled on all nodes +- core IAM roles are maintained via CFN templates ## Network - Calico using VxLAN incl. increased MTU @@ -66,7 +67,7 @@ All chosen components are 100% organic OpenSource. ## Logging - all container logs are enhanced with Kubernetes metadata to provide context for each message -- flexible ElasticSearch setup via ECK operator to ease maintenance and reduce required admin knowledge, incl automated backups to S3 -- Kibana allowing easy search and dashboards for all logs, incl. pre configured index templates and index management to reduce costs -- fluentd central log ingress service allowing additional parsing and queuing to improved reliability +- flexible ElasticSearch setup, leveraging the ECK operator, for easy maintenance & minimal admin knowledge required, incl. automated backups to S3 +- Kibana allowing easy search and dashboards for all logs, incl. pre configured index templates and index management +- central fluentd service providing queuing during highload as well as additional parsing options - lightweight fluent-bit agents on each node requiring minimal resources forwarding logs secure via SSL to fluentd diff --git a/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml b/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml index 6d6d235..6e3cdc7 100644 --- a/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml +++ b/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml @@ -116,7 +116,7 @@ spec: containers: - name: aws-iam-authenticator - image: public.ecr.aws/x8h8t2o1/aws-iam-authenticator:v0.5.2 + image: public.ecr.aws/zero-downtime/aws-iam-authenticator:v0.5.2 args: - server - --backend-mode=CRD,MountedFile -- 2.40.1 From 9001aaef5715a652340c7ef552d24c1a3a434f6b Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 26 Jan 2021 14:04:47 +0000 Subject: [PATCH 05/65] More documentation updates --- CHANGELOG.md | 46 ++++++++++++++++++++++++ CHANGES.md | 15 -------- docs/Quickstart.md | 34 +++++------------- docs/{Upgrade.md => Upgrade-2.18.md} | 54 +--------------------------- docs/api-server.md | 15 -------- docs/cluster.md | 9 ----- docs/kubectl.md | 21 ----------- docs/misc.md | 14 ++++++++ docs/notes.md | 49 +++++++++++++++++++++++++ docs/worker.md | 15 -------- 10 files changed, 119 insertions(+), 153 deletions(-) create mode 100644 CHANGELOG.md delete mode 100644 CHANGES.md rename docs/{Upgrade.md => Upgrade-2.18.md} (54%) delete mode 100644 docs/api-server.md delete mode 100644 docs/cluster.md delete mode 100644 docs/kubectl.md create mode 100644 docs/notes.md delete mode 100644 docs/worker.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b0168fd --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,46 @@ +# Changelog + +## KubeZero - 2.18 ( Argoless ) + +### High level / Admin changes +- ArgoCD is now optional and NOT required nor used during initial cluster bootstrap +- the bootstrap process now uses the same config and templates as the optional ArgoCD applications later on +- the bootstrap is can now be restarted at any time and considerably faster +- the top level KubeZero config for the ArgoCD app-of-apps is now also maintained via the gitops workflow. Changes can be applied by a simple git push rather than manual scripts + +### Calico +- version bump + +### Cert-manager +- local issuers are now cluster issuer to allow them being used across namespaces +- all cert-manager resources moved into the cert-manager namespace +- version bump to 1.10 + +### Kiam +- set priorty class to cluster essential +- certificates are now issued by the cluster issuer + +### EBS / EFS +- version bump + +### Istio +- istio operator removed, deployment migrated to helm, various cleanups +- version bump to 1.8 +- all ingress resources are now in the dedicated new namespace istio-ingress ( deployed via separate kubezero chart istio-ingress) +- set priorty class of ingress components to cluster essential + +### Logging +- ES/Kibana version bump to 7.10 +- ECK operator is now installed on demand in logging ns +- Custom event fields configurable via new fluent-bit chart + e.g. clustername could be added to each event allowing easy filtering in case multiple clusters stream events into a single central ES cluster + +### ArgoCD +- version bump, new app of app architecure + +### Metrics +- version bump +- all servicemonitor resources are now in the same namespaces as the respective apps to avoid deployments across multiple namespaces + +### upstream Kubernetes 1.18 +https://sysdig.com/blog/whats-new-kubernetes-1-18/ diff --git a/CHANGES.md b/CHANGES.md deleted file mode 100644 index 02ab072..0000000 --- a/CHANGES.md +++ /dev/null @@ -1,15 +0,0 @@ -# CFN / Platform -- Kube to 1.17 -- Kube-proxy uses ipvs -- metrics support for kube-proxy -- no reliance on custom resource for S3 buckets anymore - - -# Kubezero -- fully automated one command bootstrap incl. all kubezero components -- migrated from kube-prometheuss to prometheus-operator helm charts for metrics -- latest Grafana incl. peristence -- kube-prometheus adapter improvements / customizations -- integrated EFS CSI driver into Kubezero -- prometheus itself can be exposed via istio ingress on demand to ease development of custom metrics -- backup script to export all cert-manager items between clusters diff --git a/docs/Quickstart.md b/docs/Quickstart.md index 6623ce9..485dd68 100644 --- a/docs/Quickstart.md +++ b/docs/Quickstart.md @@ -9,7 +9,6 @@ ## Deploy Cluster - cloudbender sync config/kube --multi - The latest versions now support waiting for the control plane to bootstrap allowing deployments in one step ! ## Get kubectl config - get admin.conf from S3 and store in your local `~/.kube` folder @@ -22,36 +21,21 @@ --- # KubeZero All configs and scriptss are normally under: -`artifacts///kubezero` +`kubezero/clusters//` ## Prepare Config check values.yaml for your cluster ## Get CloudBender kubezero config Cloudbender creates a kubezero config file, which incl. all outputs from the Cloudformation stacks in `outputs/kube/kubezero.yaml`. -- copy kubezero.yaml *next* to the values.yaml named as `cloudbender.yaml`. +Place kubezero.yaml *next* to the values.yaml -## Deploy KubeZero Helm chart -`./deploy.sh` - -The deploy script will handle the initial bootstrap process as well as the roll out of advanced components like Prometheus, Istio and ElasticSearch/Kibana in various phases. - -It will take about 10 to 15 minutes for ArgoCD to roll out all the services... - - -# Own apps -- Add your own application to ArgoCD via the cli - -# Troubleshooting - -## Verify ArgoCD -To reach the Argo API port forward from localhost via: -`kubectl port-forward svc/kubezero-argocd-server -n argocd 8080:443` - -Next download the argo-cd cli, details for different OS see https://argoproj.github.io/argo-cd/cli_installation/ - -Finally login into argo-cd via `argocd login localhost:8080` using the *admin* user and the password set in values.yaml earlier. - -List all Argo applications via: `argocd app list`. +## Bootstrap +The first step will install all CRDs of enabled components only to prevent any dependency issues during the actual install. +`./bootstrap.sh crds all clusters//` +The second step will install all enabled components incl. various checks along the way. +`./bootstrap.sh deploy all clusters//` +## Success ! +Access your brand new container platform via kubectl / k9s / lens or the tool of your choosing. diff --git a/docs/Upgrade.md b/docs/Upgrade-2.18.md similarity index 54% rename from docs/Upgrade.md rename to docs/Upgrade-2.18.md index bc7442d..9032d9f 100644 --- a/docs/Upgrade.md +++ b/docs/Upgrade-2.18.md @@ -1,4 +1,4 @@ -# Upgrade to KubeZero V2(Argoless) +# Upgrade to KubeZero V2.18.0 (Argoless) ## (optional) Upgrade control plane nodes / worker nodes - Set kube version in the controller config to eg. `1.18` @@ -53,56 +53,4 @@ Ingress service interruption ends. ## Verification / Tests - verify argocd incl. kubezero app - verify all argo apps status - - verify all the things - - - -# Changelog - -## Kubernetes 1.18 -https://sysdig.com/blog/whats-new-kubernetes-1-18/ - -## High level / Admin changes -- ArgoCD is now optional and NOT required nor used during initial cluster bootstrap -- the bootstrap process now uses the same config and templates as the optional ArgoCD applications later on -- the bootstrap is can now be restarted at any time and considerably faster -- the top level KubeZero config for the ArgoCD app-of-apps is now also maintained via the gitops workflow. Changes can be applied by a simple git push rather than manual scripts - -## Individual changes - -### Calico -- version bump - -### Cert-manager -- local issuers are now cluster issuer to allow them being used across namespaces -- all cert-manager resources moved into the cert-manager namespace -- version bump to 1.10 - -### Kiam -- set priorty class to cluster essential -- certificates are now issued by the cluster issuer - -### EBS / EFS -- version bump - -### Istio -- istio operator removed, deployment migrated to helm, various cleanups -- version bump to 1.8 -- all ingress resources are now in the dedicated new namespace istio-ingress ( deployed via separate kubezero chart istio-ingress) -- set priorty class of ingress components to cluster essential - -### Logging -- ES/Kibana version bump to 7.10 -- ECK operator is now installed on demand in logging ns -- Custom event fields configurable via new fluent-bit chart - e.g. clustername could be added to each event allowing easy filtering in case multiple clusters stream events into a single central ES cluster - -### ArgoCD -- version bump, new app of app architecure - -### Metrics -- version bump -- all servicemonitor resources are now in the same namespaces as the respective apps to avoid namespace spanning deployments - - diff --git a/docs/api-server.md b/docs/api-server.md deleted file mode 100644 index ca66fa6..0000000 --- a/docs/api-server.md +++ /dev/null @@ -1,15 +0,0 @@ -# api-server OAuth configuration - -## Update Api-server config -Add the following extraArgs to the ClusterConfiguration configMap in the kube-system namespace: -`kubectl edit -n kube-system cm kubeadm-config` - -``` - oidc-issuer-url: "https://accounts.google.com" - oidc-client-id: "" - oidc-username-claim: "email" - oidc-groups-claim: "groups" -``` - -## Resources -- https://kubernetes.io/docs/reference/access-authn-authz/authentication/ diff --git a/docs/cluster.md b/docs/cluster.md deleted file mode 100644 index 0f97551..0000000 --- a/docs/cluster.md +++ /dev/null @@ -1,9 +0,0 @@ -# Cluster Operations - -## Clean up -### Delete evicted pods across all namespaces - -`kubectl get pods --all-namespaces -o json | jq '.items[] | select(.status.reason!=null) | select(.status.reason | contains("Evicted")) | "kubectl delete pods \(.metadata.name) -n \(.metadata.namespace)"' | xargs -n 1 bash -c -` -### Cleanup old replicasets -`kubectl get rs --all-namespaces | awk {' if ($3 == 0 && $4 == 0) system("kubectl delete rs "$2" --namespace="$1)'}` diff --git a/docs/kubectl.md b/docs/kubectl.md deleted file mode 100644 index fa8283e..0000000 --- a/docs/kubectl.md +++ /dev/null @@ -1,21 +0,0 @@ -# kubectl -kubectl is the basic cmdline tool to interact with any kubernetes cluster via the kube-api server. - -## Plugins -As there are various very useful plugins for kubectl the first thing should be to install *krew* the plugin manager. -See: https://github.com/kubernetes-sigs/krew for details - -List of awesome plugins: https://github.com/ishantanu/awesome-kubectl-plugins - -### kubelogin -To login / authenticate against an openID provider like Google install the kubelogin plugin. -See: https://github.com/int128/kubelogin - -Make sure to adjust your kubeconfig files accordingly ! - -### kauthproxy -Easiest way to access the Kubernetes dashboard, if installed in the targeted cluster, is to use the kauthproxy plugin. -See: https://github.com/int128/kauthproxy -Once installed simply execute: -`kubectl auth-proxy -n kubernetes-dashboard https://kubernetes-dashboard.svc` -and access the dashboard via the automatically opened browser window. diff --git a/docs/misc.md b/docs/misc.md index a48b318..9b0246a 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -27,3 +27,17 @@ Something along the lines of https://github.com/onfido/k8s-cleanup which doesnt ## Resources - https://docs.google.com/spreadsheets/d/1WPHt0gsb7adVzY3eviMK2W8LejV0I5m_Zpc8tMzl_2w/edit#gid=0 - https://github.com/ishantanu/awesome-kubectl-plugins + +## Update Api-server config +Add the following extraArgs to the ClusterConfiguration configMap in the kube-system namespace: +`kubectl edit -n kube-system cm kubeadm-config` + +``` + oidc-issuer-url: "https://accounts.google.com" + oidc-client-id: "" + oidc-username-claim: "email" + oidc-groups-claim: "groups" +``` + +## Resources +- https://kubernetes.io/docs/reference/access-authn-authz/authentication/ diff --git a/docs/notes.md b/docs/notes.md new file mode 100644 index 0000000..a0d5d04 --- /dev/null +++ b/docs/notes.md @@ -0,0 +1,49 @@ +# Cluster Operations + +## Delete evicted pods across all namespaces + +` +kubectl get pods --all-namespaces -o json | jq '.items[] | select(.status.reason!=null) | select(.status.reason | contains("Evicted")) | "kubectl delete pods \(.metadata.name) -n \(.metadata.namespace)"' | xargs -n 1 bash -c +` + +## Cleanup old replicasets +`kubectl get rs --all-namespaces | awk {' if ($3 == 0 && $4 == 0) system("kubectl delete rs "$2" --namespace="$1)'}` + +## Replace worker nodes +In order to change the instance type or in genernal replace worker nodes do: + +* (optional) Update the launch configuration of the worker group + +* Make sure there is enough capacity in the cluster to handle all pods being evicted for the node + +* `kubectl drain --ignore-daemonsets node_name` +will evict all pods except DaemonSets. In case there are pods with local storage review each affected pod. After being sure no important data will be lost add `--delete-local-data` to the original command above and try again. + +* Terminate instance matching *node_name* + +The new instance should take over the previous node_name assuming only node is being replaced at a time and automatically join and replace the previous node. + +--- + +# kubectl +kubectl is the basic cmdline tool to interact with any kubernetes cluster via the kube-api server + +## Plugins +As there are various very useful plugins for kubectl the first thing should be to install *krew* the plugin manager. +See: https://github.com/kubernetes-sigs/krew for details + +List of awesome plugins: https://github.com/ishantanu/awesome-kubectl-plugins + +### kubelogin +To login / authenticate against an openID provider like Google install the kubelogin plugin. +See: https://github.com/int128/kubelogin + +Make sure to adjust your kubeconfig files accordingly ! + +### kauthproxy +Easiest way to access the Kubernetes dashboard, if installed in the targeted cluster, is to use the kauthproxy plugin. +See: https://github.com/int128/kauthproxy +Once installed simply execute: +`kubectl auth-proxy -n kubernetes-dashboard https://kubernetes-dashboard.svc` +and access the dashboard via the automatically opened browser window. +# api-server OAuth configuration diff --git a/docs/worker.md b/docs/worker.md deleted file mode 100644 index 0c4a767..0000000 --- a/docs/worker.md +++ /dev/null @@ -1,15 +0,0 @@ -# Operational guide for worker nodes - -## Replace worker node -In order to change the instance type or in genernal replace worker nodes do: - -* (optional) Update the launch configuration of the worker group - -* Make sure there is enough capacity in the cluster to handle all pods being evicted for the node - -* `kubectl drain --ignore-daemonsets node_name` -will evict all pods except DaemonSets. In case there are pods with local storage review each affected pod. After being sure no important data will be lost add `--delete-local-data` to the original command above and try again. - -* Terminate instance matching *node_name* - -The new instance should take over the previous node_name assuming only node is being replaced at a time and automatically join and replace the previous node. -- 2.40.1 From da023067c9f6267167d6a16fbe5dea300647954f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 26 Jan 2021 14:09:01 +0000 Subject: [PATCH 06/65] More documentation updates --- docs/Upgrade-2.18.md | 6 +++--- docs/notes.md | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/Upgrade-2.18.md b/docs/Upgrade-2.18.md index 9032d9f..bc9fd1d 100644 --- a/docs/Upgrade-2.18.md +++ b/docs/Upgrade-2.18.md @@ -31,11 +31,11 @@ ## Istio Due to changes of the ingress namespace resource the upgrade will cause a brief (~3-5 min) ingress service interruption ! - - delete istio operators, to remove all pieces, remove operator itself + - delete istio operators, to remove all pieces, remove operator itself `./scripts/delete_istio_17.sh` - - deploy istio and istio-ingress via bootstrap.sh + - deploy istio and istio-ingress via bootstrap.sh `./bootstrap.sh deploy istio,istio-ingress clusters/$CLUSTER ../../../kubezero/charts` - - patch all VirtualServices via script to new namespace + - patch all VirtualServices via script to new namespace `./scripts/patch_vs.sh` Ingress service interruption ends. diff --git a/docs/notes.md b/docs/notes.md index a0d5d04..1a181de 100644 --- a/docs/notes.md +++ b/docs/notes.md @@ -17,7 +17,8 @@ In order to change the instance type or in genernal replace worker nodes do: * Make sure there is enough capacity in the cluster to handle all pods being evicted for the node * `kubectl drain --ignore-daemonsets node_name` -will evict all pods except DaemonSets. In case there are pods with local storage review each affected pod. After being sure no important data will be lost add `--delete-local-data` to the original command above and try again. +will evict all pods except DaemonSets. In case there are pods with local storage review each affected pod. +After being sure no important data will be lost add `--delete-local-data` to the original command above and try again. * Terminate instance matching *node_name* @@ -35,15 +36,15 @@ See: https://github.com/kubernetes-sigs/krew for details List of awesome plugins: https://github.com/ishantanu/awesome-kubectl-plugins ### kubelogin -To login / authenticate against an openID provider like Google install the kubelogin plugin. +To login / authenticate against an openID provider like Google install the kubelogin plugin. See: https://github.com/int128/kubelogin Make sure to adjust your kubeconfig files accordingly ! ### kauthproxy -Easiest way to access the Kubernetes dashboard, if installed in the targeted cluster, is to use the kauthproxy plugin. +Easiest way to access the Kubernetes dashboard, if installed in the targeted cluster, is to use the kauthproxy plugin. See: https://github.com/int128/kauthproxy + Once installed simply execute: `kubectl auth-proxy -n kubernetes-dashboard https://kubernetes-dashboard.svc` and access the dashboard via the automatically opened browser window. -# api-server OAuth configuration -- 2.40.1 From 2d06fa61ff9365a8d386d847f13b91ff349023e1 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 26 Jan 2021 14:27:22 +0000 Subject: [PATCH 07/65] Add updated AWS arch diagram --- README.md | 8 ++++++-- docs/aws_architecture.png | Bin 0 -> 131664 bytes 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 docs/aws_architecture.png diff --git a/README.md b/README.md index 1fe73d1..26c2e08 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,14 @@ incl. various addons and services. | v2.18.0 | no | yes | no | no | 30 Apr 2021 | | v1 | yes | no | no | no | 30 Jan 2021 | +# Architecure +![aws_architecture](docs/aws_architecture.png) -## General -- Container runtime cri-o rather than Docker for improved security and performance +# Components + +## Container runtime +- cri-o rather than Docker for improved security and performance ## Control plane - support for single node control plane for small clusters / test environments to reduce costs diff --git a/docs/aws_architecture.png b/docs/aws_architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..dd3246fb4737f85ad97dc039bea8521476be286f GIT binary patch literal 131664 zcmZsCWmJ@5*DfUtJ%E(-0E0-2(%oIsor0uv_fRS^QX&EZ1Bi4>3@stj-Q7LJ(42X{ zv(As>`>`Hi&CIi&+4sKpz4N-_v@{e6@t)zKp`j5fE6M4gp<(W#ewhK-sCQ;Hw98Qc zusmgybpfcKAb@Qw8X6;-vYfQ8pZP%+P7C?(*~grD`~}rx3>6X*pmybG_flXSR~5@M zbxyvDuZB94a`Eh69n?NmbuV%FbYti|1?o191vW3>H!S;x1ZM27zW;MDyI9k_eYf?v znltU`YyY;5|MuW+%l|F7NSW0Ngigx%U)M>N6p?&@Cyix!UTdqgqm$FAg^re%<^4c^ z|0x`5_+R%kh958{U4ZD;S_%sl>h<;dOB>HN!WHS0xyZ>8U1vk7d^mKBn`Oqz?(S6^ zTU&1x6Ib4RM#D|||F0_rMko0IS=oHV*Xh2kKPuNd z4%mJKxyN2OPCwszkMb?TgLUd73lGn5LyvU_ixkE`SB5&{zMP26T#3_9yJSRK=qtOQ zEiYYu1}(lYKtvO!EKC*Tt<7gXz7|kUMpRA@8!?LOyQaN~K4OcBr)STaZ%EhKG#WOL z$&SNP3j6n3U`ZkeEgHKmjkGKUEf%W3Tpy^|d`QnuCvWzJuB}GO;~`kt%!Nd|1A^fb zLuQ;VZ~vx$OcYa5V0a(qbvUE_0Uw{MYRG%v)kk?t{$IgoRT=Z;MY{bXNw;5nd+Rg? zH@66$GB@szD#CVCx>zcF1^cyWx@MKty)VXUt!gI3uA@zZW}qHNMn$Sd?~0#FmzMDQ zeef22<)J^KI!S@9=EW({0o@) z#n1>$E_Crjoz6?UginVf>fE;-PsYkrDcLzWlf*WmMTVMLT*oZ`*h9E8I0*_0p07Ww zw!QZ7M?_I>F4l3tJXVf$M%o(YfEq40TT@U+A+y#h>Akell(pH9wzkSN!jH9Y?1n>v z3sMo~Z+_UHIsLY0`}fHc0NYPW6$a*KE-zrg4|%^4Q!r&jXF7j@`$p^M>HLx(XX6uX zfayj=tsuK*GkI(I!)tvPgU5Ph8}Im6#V3BhW<4iFE=&)$%%@a5R8zL;oQ6+{ujTN9 zQ`0@>RrB#8-EJ(KXzS+bJrFZh)yOdMf;y4bc$&WjQ+)oE|LW0}Ib7?b7ksP5+Nq#) z>6bKp$d(cH;~@sSi2FByk7KXJ=1l8a@gIPVo z<>%6j%-NjptUAj_~-`)AW&I>(i5$I2wX8SWmT%>aUl`suBtS8g>JMzo*9VCnSp z+Lo9c?x8U`b_E!HPtZVVagR`k|B+@^A~1WxZ6R++v!bp^Q9CDlZ0kz4;G*xI^E5jj z@t{#!VwzH^NEI~nEvnkhrTkmKPOisDNLPv>PpV8^O%T1`^7hI`2l6Gka^SUhd3F5lZ#PbGoKVI((wQ{A4cAiE3doIyQ zwXiTa#xD3A(}XHYKUYw4r`+ZQ7{5*5Qab&GImzd!V%g_P!eL4nMIK+ON>5BD&2ql~BRH5yez>7W1grPW`hEK%<6TAy{ODCn~B z?s_%GZt(=p;-+Q6!6Kf~?y75k{y|V8#rrX63=t+P)bRMgK6bg$l_|chF`l4x<4r8a zO70f&W2#}=ShU^uc2G}8#H)gi=`0TaUAKt;X1trYRnvmsZh^Cm&5_}Gw<8_~So~%0VqrN{?R-hukA_o@J^!INhpcT>@5=WXg>cT^?@IS3U3Ae5fN|DSW+= ztcRwTEQVb0^cYiSn+^E%MnhQMJmlIAy{6tpuQp9}7xmCI)a_jFTw@qQezn{lH7i#P zq~SX{I>MZ$%d7p*-Zq{MiEcLSm(TBxw%uQv%*r0#z6`p0pCRBf$8BhCpt`Ohw2HNl zL@ott5{VvEnw9U47bS8UC^qB`niRY7jTu-3v-&OvH?%xH+*UhIzF^l=ox~=fVzmF} z9(}jLCJ9}$eobGMarA4Q==wKL2-6`d_@#0f7~f9#K_70W+x`rPT=M&0&f1sVfK@^t zZ3 zJ^Gkp`dV*~IQZZGRq}fjM=mg<}Ou6p$cQ;~>e(%1&!O2eJ za#O1fd9gDp-}Lz4$iBDOgp z&YQ&01a!-d4fKb=#U}5mFf?3-o745)Kmo&gJu4c@NaV$|McX(XzFzm}2XYSEV>mKR zBTMA)T#x_*wdKf9l)DT;C*icd3+_{v2O0L+&^Mamlh~76s&LX>DqY#DZdHrW`?^xI z=Bb~O5Y{*R=zzhRA&~*ne(}J|Ps06JxnZqsb^{db?Aq&5v0j6(r~BZ@r4kKQms;%E z(5o&6pBqsuCFIc}muPsu0?feNUtUA<-m627RZi=830_cLtwX_S$XFQ|i9?)5EP1Cj zSLbA3lJS07a5;#1zTTw<(l5OI&1OGj{A1RKa~ijn(KX2S?~UwqTp7RJv4SZUsGRkcDT`zdu0huihqj_d>PA!>?Kbf==B_I4uK=-%HkF9o!3W@YCxW?y3 zDZwa%9YC#iQ$$7$oJ-qc@nvPsJ|~!y{>oLRH1uY-Xsj*tG0j7LKL&aFBR`=UJcLd7 z%r^KV9|`jAY#1NB#rqC|l+b@^C;0X+)xcY^hpTmcfkN+(oMbl9HVMKCV2y9jeHXnc z<(u}!KT-EE0}Y|AhH>X-N59PK6BjiRO>KQx!RW0 zN;@W=8`BN3oWHz$6l(x5HJ41xG2P53qvfO%-=m*3y)Tkj!eK!zC-^dB^DrhIxJEFH z7QQB5%{<0u6=p9#5t~4CwMrc7;O{ujG{%REwk7>^_oREtV}J`Wzx<#da#3RO+xZ2+ zQ><&I%H~Njq@T2w69$7` z&UF)sp6?RTi5T8ftu1IQp7UE{5GQ+A_A5Qzb-m==O8q4DT~35x(8v4LoY*GePpz8O zI!El=)1vIuCt1d-gTU9xB|cVyMw&~y20Y})nrU@NvA2ia3BYR~N4Q@ze<(D2CeZN4oc@)+RP=y`0FIO*>o4yT8J}v}M0A9$oF-|NU+A6|&Uu1&d8F*kH zU^wKM7VKlg3JJ1_z)6+^mc@~Rq3sk){;|6a%OUe=Y;HKt*vrzPLUz zK{Bv6AN}uJvc>7!%R{GGF1d6@j0uf^{JLD0dqNU&(pQpYzAz;GaFtatZkdQO>&50e zkxuTJkmF%<%|x=a2wt2CLEyFvrc}C*wKyei;EHOG(S*sU)MXd}SqOc+p9-T}jihQ>X{_>D z(og}9@y&cNO(x>Qs{l<&uFqCoQg};Wv3Eu?+43rN+&DuR=>1m0(~9UZr}`l1Y;zRX z?SbJwHyy+wjML_UWvAKLKH2=nu&y%<9tyAtW2bc#klB|@=r374NO4+#tsfsE#W`w~5{Edd%|?cS4+*&B$=;hH&h*3a0pHCX z++r_wg7Ho?^3+$}$$J67Str~PtaAjx*!z7eI-dLg^qj^v;6#rSqIq!=Q5SlxVw z7%20l9`#W2*^Aykb@t;9%H(?pgBK|Ime}(#kYR+Q5)wI7F|8eP^Y_tqa|!q++3T<`*c;Cn zw9n*Ve$!&W@Xm9=(EI7URGm*yZ{yBMP>EO`4k6Xb51!Agh}A5&A9IB=3-(IC=NVKF zNiWdfg!V?e+wCbe5N&6{hE#ZFCqK5X0|c~@+b`TL&W3EM$8$@*eeZl`P;Sh{6M}%2 zE|e5)94yIxoiM)q+^D*dn>?JBoh4Ed4qKh`Nlx2(&9Xv@P9V1u+#@AC^;IR7-Y@rL zwWpmV0uK`p5_7pXIe_uk|Rj&^vJ3*x&x;?MyMx1knSzsYC^_)2x(_#co5%? zhJcU&$?i$E^ol%D%8Gcn)^90tYy9Dl9awUld#9(p3??#9R}-nIWG#2=ZmD);0XDoJ z+hl+2LgMXtXV7W4943vC zwG2iCw%Ws0ucWb6`;WFIEnp|f8ad6m^aALBEJ)&dgX|+aSXfOwzR>KI4k}8Z-J{== zbmFWgSAEWsZll;eNwa>#Mlb}t{Ae$�z<|aidF6f65(3EpR5g!wq4?_S@;*u{iyl z%@W}!-8!1Ad9r0$8Rwx*e8+Qx0}+4(ZnR?U9RJ{NaXcAYHIvWG4f4wyo!W^6(xDuY z|A;129mB%xfA5}D+|DZp!$@%)75eeFdW~{ z^*&B*C-B5W_R}8Z0c|t^N%}aFv)X@$YJh@bqO0TQnP*o|SbDOM3^_0R6fuHS{fEvm550obijpC0Vw ze0eqVqsM=z%g>E(7M&(enP5hz#lR0NdhC>U97}VFD|T`hOB?wsK`j7Rwsa@K@E%ib z)D`$CAYw!p7D;2+GjB-hLv<`vUejz4k=0sS@w=NsYz0_zmEEW(>447L8Hi3(m~KC7 z7voEnoEBTIT;TeJ_IGoPTh`^DH?X3yH-4Hwu_D;UCVR0#!IMeiFp_K!T9bOX!dD9J zRE_0wD09F?YIxvfIy=98f44UW%rSQE6mDjghP~_{bfmsKsWbkeK?<^G&IEMF#Xfko=NFugIo`kb7{kNvp9V1yB0_ zw8PdZybl@?e>sl*wcE_cwjf=wi*Q*kP55i0?_}MLBFo}M=98g|3jMbSPSi$A(mh8F zZJZX3DoYv2RL;Y^7F$eYBGuA4Z^Hr<0^p2&BggG{5wba;h?ji0QI85naUempW!wo1e7OmH zdR_5Tv<_SzuBAi2+%D)au@xQ(8akoAl7o$!gIsIH4L#h4Vi?2y0|bc!k~vOGe89WS z6Nd(5Ii9c3IP$>-g&Dd6uJ-iH3v#D-?o7uHl=fW%> z9H=rog9-DxTG<-8?t8tHhJ-77=q{<3OevqHf3$N@Px$7wz*nssceX8=#Zq#)^Wxg^ zu<7Y~De^d*PtpH?eeAj#I|HrzxtJ#)Fd9eXrccW6J8xuSk`KgCAyA9yU}ZLJ8z(}B zxSTiku{J-}1!2 zZ}8;^AdFbNGmxG3@AS$Hc37j);ZL#2RF zGB?mkx!e2DMuGs0-TV326kA z%q$7sr`D%VlEk8sAzWkb3tH!VzQ7pZhaEOR!xY)GQ75@m+JG2mTHF|=s6GfTm7LD^ z2cY3|ev#b{$5QV-U|Ao(M}x)tATb94D$jLm)E-*L&;)Ah+}QfaX9a#UXt(n+%#oVeVFte&9;nq{zPr%7wcGl z8?);i8=OOE4qJc_SB5;9*l!v%|JtZ+DpV8vDj(G@)_1UgoA__MzM>;z_iGy=Zc+Ji z*KYf|_1%|B887UcTAGV4&+yma-<1no9>=kL+sZve!Ouk7yCM_fIE%dVm5@IL`IhCd zI+@2V-dHMnk&AHu=~}$nlM@c^;}u^0l5clsa@@zCESdH#xmVdWaR?OU82O71qvc6^ z!r7pELcrjin{5f6Sw%*(67FT0es-Ee%yUJVtL7VEmreF`y{QYJp{Q2G_RCYGcJ8ht z(nr-)F?vS!+j`Ddpo4MIp1=dMaH*4~F!F%k?&eblV#H*<;K-rQGq9q3>b`2K><#S9 zj_@TqfFjlrz!N04$`q$@GG+-(n0+wZk67jFgrDHa?4bPY1?=kiK1?=jb+Ct#QN#)k z##X)3QY=@EZyl0@?Nj>%Q*dMdK1p#(^!3XAB6KyL^I#j*){z&+Bpxf zf)woy27~rvE8qHYtzLWczeD+arl~n=t(fH%>gY;&Lq{(ad6?{hLIvON0CR}(zE6}= z7r(-h{r7=rB?ww?HwE@55$OZzu#S=s@4X>zy(&BWJN7%wqxO0>hwNg0e%y40@l6VW zx4?UhL{5l&9)I_!Z<0l+Ei!8agKx@%N)5KsShi`4ONs>quk*%*7Z5;?h2;dhC6{i^l)4fNOPA7!D^hZc1{qmLS+X(_GkMB0D`q-vk`~;!%&+P zdcD)nb&h`iBK$7jW*|t$qtpBA7Yvi?flV4rc}R0B7w4S;@YRUR!~rV&3=8 z(P{*ajiy@%=NwM)OkGs6RVsdQ2p{ad{q6!Azv)P`A*CWaxw>TC6YQU{dWrA-``45H zcZ<)*7I(z|c5#QF#ChFAP*c@NJXW47+d^7XU2e*p(DJ`u&kF7?GKvaL`yyFGzt8dM z#a;ZOnP4Pl{X;UBaB-Y=6*En{MgyYD9A}r#qES6j{Jg~UK!X}UeobFWR!tOBNTA|Z z$qwDs^AFqw6+e5ei;Nd_oeV1!*9NlqFbVoFTzZ45t&r@Oukq&qjs9U{BjH@c+XCP_Zb2xjxrTi&Jaf zBX#9PAX*z$&P{ha>!o?}B4SA=a$wcb%sP4V#8}Mpiasp&2>ZD7S|uF86Iq!Z?v@2$ zXspyop4zsR0W4Bntaua`(?bo*;aDh7{xj~7gL^bCBS1-x#}dS{q)ZRUQTKt7ftz`E zqaY7N9g(dCWZCG)Y36w(_hMgOVK&J10_TF+NDnz(UyRi~SD9!yF*OXO_vaP?_c4My zuv`)xr~r<1lbG~=l-@g%h<5MMJe)vIOUVdBTr!qlI2d%j4IMPu&v)%Lywo>(QjK9Y z{@{hMz&+`UCwFH6aaj9rJ$f+F@9`_Q*Lb{H__ut_@_pqI{rq8gBa6C zD>S&rzPiOg@!hit1CL6~z=^Nk^FTiK?=oANTOrI@6cwNIZ;se+e+)W+e`*Z~wrSJk z(!SL7X!cD5mlii@Q^+?RrmYaZWBfk-zB4d^*dZ^giEo@}_$fAwN=_~&yt9h{3K^VX zbY3GYM>Tp%j}`j7Y{xRMELVt#74lNlU^d@B#=};qBemrGxLRC+i&VT)N{WhWj5=tT zSA9Tt$h}o%e22K$ejg^%%8WA)yE25B3GrN$K{_25WaidH?xcgg2J9rp?AiI6GueWlck?Bmglt_${wPA}egwJ)(g; z;)ymn&>0xD!vRQDuiY2@BR3-$V_A%gAzlqe5`xU!idjr$;5e{Oz85CnlNa0%B`A_`kKq zJX?$G4<3-!$Z=)5ROxGE;UyP2v|M&BpZO^;9?N9&xA1O2zRHZSb4rYxQ42t?`C^>r zxP!u4_}gw(e%#DzbL8=RFG$*AlYBk1eABy%EpJ!SJ_2ZY+|K&Ydy2+6l2+GFU24;k zAs^lS&U7v_aAL966rMMQfBsx592Zr1b1c#p)$!k5I`$hi85;+Sp$5vfDF=eiKU#zy z)H5nQ;-1p;8$X+`*IzS@>hBez7<$o<@9#Np%`zUl8Y+4}yFU_2F0+Q(0PE=H^XO_7dadIMJ zVnPEn!{qQc@N$E5ZEcYSQDZ7gw1gW=_;WE%K&E0u4AK5uKBn*sZz#p;!+=sB`TxGdd6xve*+MQf=(&>v5s-O#gF7? z1A5=Sk4dM4q6f@I9ib)_M^|)_j+~8Qeo@;-3^_3pKhle&X5_hr<6LP(k0EcM2_kEY zYPHS$&RF0*Mir*)bEqHiB0$mCnVXWU{nT`BJbDKMPA`V=hcbW#RyJk5u$>c#_J6xM z*iR6zuB}8X!~K*MKXlLSaNU@V(nPS5?tUelBPHVq0O>NE7HT@a-&2lRjJW@>@O-eg zqKhbRIA8sQSXJN^Gl1>jF&yxOPC7D9r-dyxWF)!WXKC7RnS|$ntYzrti?$kIRa_Aw z{enG1sTWD?L(r)gSa!kAK8L+3`avQNR{-z^p6UPMBF;CFVhmirpmBd*2WaSR;G42F zZ4qI$#<@-G*D7^SRP))RFyoeQ$`t?LpF!G(ny6n-%n;IB!yTJ9{u;4}Z}Fsa@H8sb z>pG{lNuFEGiaBfcFS@yT3;2h72btVyU2P4n$VJr++_CDMEIn=1_3y<#-{*J?*$740 zIc;)DI>f$8E3R__-V0%K>vy`pe~}nHXkxeD&939K6aIO*_+DdE2bSq$VxG>80R#Z3 zyQKI%Y#IzVWq(*E#Yw+x$uMk)K__hCxgv~UaB`~}C4dm=v~-X^V24LBe@Ln{r*#_U z`xT9R;@?U3&O<|_1+7VJBkCmBnYX|LsN;lXu3~JRE=MYLc@*Dj;h{pxg=Ems!dgeLh8tpyF@e?1wwA0+W z{}>avDuLW3%Jqg6YVl!F#B4f&?a^9b}kI-ZU#DIo>7)}lj-Zjji} zC~h!0h(zYG6QAd)GlVJ&_%pt3)p&-w0|-k#G($srletE6Nw;zjK|7(h$~MZ{HGTCy5Cg&EnPTyVLXAC%N0laSGdBBpYL3Gc z#e>(cM>6zEFTcE5JsYbk6EO+y)j*7Lg%d9G8GXlAs@p5Q;Hn5_oPQf+S7vj3IWPe8 zb0ZH)eCf;b!Hdl-0~sKVmK%KXA_LZElK`a0*L)%2Oyy-g0$3$-i8{*~H^Af{p z7P4JnHr0&PS!LgPnU>_abnTO=0-L1xtV#C-0fDa{QKRz*t6~f}>>H-?7%D|7w{sY# zn2*pD{*rgBzv)LYrlE`LX<%@bhFI)H5HJcVx@Ln$a4foqo(n;u(Z7M5E4JdC%8C35 zn~{qZ=Viil?J@y?G8*(#;SFx9xPi4S2Xl3O}gC?u@WW$vjX9m zCe)g+bqTd5OH9>s!CKZiIvGj8w%LC>m%yau79H6q@*fetvp=$cb@%h_m zKVQDtf+r&7zi$DepL`V=j^mcqm#1x)NdyncTJ9@%fn_rbz+{%uPpO4Adi-ur(t%en*BKQ&ex=tX{ebtko7ajcdDuEC>WrKI^=_s-=$E%N;}wT0@V z`uyfj@MWsT-uihoB3rGvEa}^7Bnzead%r>EL@OtD{54a$--TEqgMtIL`w;fMpv&PD zN_eNp7FUHSU9#tZCd^hVbU#)w>-iXbXYFtDzmU03Sw>G(6rK_t5^9i{t^$i`R1e8k zl&8+U^Mp;0U3j$4#TH~4pPikI5BIH=yTGb`DK(W4uC`Wm)~M(zjX$PPu76wuv(pEL z8E+W>Y*~KMEw2ATXspa<_r>hppIe>R{?V`WU7X726FjZXCgjlM1N_WMr&9hEV@#gW zMlL|sF4fuZ>Pf%Tw8C6=(65c3bv9{5qS_HZ^qXr1IgPRuOLT^gF7#om$hm^XXGmo`5t7e9D}*J0CJ1_z#S8 zpVQ~VxtF(Fn(V)j*Husw9|&$v2e03ox`o~B%j=B5KYBGCIKT65m*_Wi3tYgBc7jZV)F+Cf&I+mzu4R(dS1416W%c25*c7Bp>T_hu_H1inf>3ZL!KCKi3!0 zvHuRdfD$IQvY_yVEb%WsNwfvi>|XBYsGNq0Psnxpn)^pLv$=7bOaCiWh8km}+|65~ z>*i3Ui*HY+z8-6&=P++7f|sOyi=y?IPU1_Z`f)!B4jjP){)m@Qs&t%m4BBV)s=NrI zTe|(Z>j_)S(uCcVM5U#R74cfA6A{$?;ijKaEE6_p?&Wa1|0*fTqW7XjqTz>>$io#a zJ56KTM%vLqR*};1hC%$W!oTwL<3dzn%wlGAF;kUwe?($vtxa=Yo44`Fz)IFfIH4iB zXMCCp#ktzg^ed9dC|y1;slb+-75q1t#`0yh$`F%Vp7Yy((+4YBd0ykE8#mW&zeo(+ zt%MgGD#s|V9yPP`nLA9DY5PwKO58U&`+G-hnjOFIP)-XQ)^0{B#H_!8(bPTV`Im1H z{)rX%0!q|ja^>jm>3`s-8JfHSYrOO zJLvXISP+l$QlAj+d7C>xzpu(DJWnPR;vMu1TqF_stVr7R>&RnDyL}ZH6jHYDPlx)M^A$bP5&- z9-~QB%3c-;>Hp#UQ}FPW<-h1e; z9PN?ypO*_FYxOe+HtX2ucg1`^c;!nBeEgR=!D#azL(HeHED4wtFbZ5=*SUTuN&|39 z2Bg%Au2UJ6`DketDfPm_Q-N#cEo+0TM|+Y!&hMvD1^Q9`1Ce_|T-0aj`>*2rqNMTx zJ+1aZy#E5rc|kx4XgPg}a+bPz+gsVQ#+^M=M2n9L?7oq$Pk+;@;=xhdS+l92X!4QL zUi{aWgQN0p--`&=YQ*%51v3}CY1mOmaNb^|2vaC+W3<^4ev@xn0Qpyoh5-0P#j>sc zJ;@Wa)$HlQI@mj%|HjLwR1Cc&<+Kbz<~ z`WG;LG8t0tMWGhN1b?V6{f8Owlifn8_v3KVU4ly{d=57aaFveQu6cv}bNtyaD#ZUq;IRA#;NIV4opis0rIT3nae2%pt+}FINF3$5@MM zQtm6%&EH1%T4*bN3%1PCb7l4WIeBxE;5dz(iXlk;qs$LmgZJyy_+@Nf^pH5vrA&)A z=)F?`nWdu_1=+Zi>1t#bPxVUL#a&c)5q&0-6w*`2sC{!2)T?oStq4k9a)6~g^QJ@U&xr$Q^Qp3RfyFfIDn~^;c zae&RHz^sICm9yVDYw;qVc3Av+)6$g<`;w>c{+(CWFEC0mg8!`;78al*)YWCAu>188 zw}mG%w7_*x#A&vX{B8MkwoRE(g4mnflSY-6^x~SyRpkaB{Y29oRO?gkLH*)Fr#Yxj zgy-HZ4F_)9mpyj9$DkBq!a6ef%oA``xMw^oVBcCd5P1L7+fhc^|0b1@hfYLG#MG;4 z?hp?5t`D%Kdp>qu|1S?p%7+ZlptL;db5iIQvTHVbbyYHGZOFzoq~{QrN!Fu0JF}`Y zQdYA;L2=-r&^X%T{=&lVxPhYr6%NE%P)LSbH-yo|9t|vH z8C_?jE}6VMo6~C1KOSLP(R2~QvC(ocf|-iE({dtfOgkGOv$JL#%bd_S1e7U$5%uvI zW`B_~WRUHKix?bkWA)w_Fv<>sz#DWjr|QJ70sGGd20R&#cLXPtalO=Dz8cEr!Eqgc zE}I9ZdO0p@;7z!@{x1}fSy@Ic5Zw&Zgp;GA-C<$8$F%rk60tx9G{Qzf)%;cN7&oL8NyD54&WHo<~1)nhY#>EV4~()=c&vSmkK zoVNp5K{S(V?!sxnZ}pR)%sI)zJhuT%4Xa0xx#2by&;zW>Jt?RI}jZgtAqIt4_#Ys&}ug-=EFyL`4lc4cUA+d#f4R_2Ko{t9OOp zYPT7MQ8!SJG$f|{tlI{-6bWfgDcy1@E)!Qo@yD(pv*oNsS&y&c+-crN?fP?pZ2aG7#TTIrUmp9Ww#L zb|a4&YuOeTbks$3K$tE2IlRd?d2Ef8U3Y)dQ%x3;CC+pV{x=CB1qg+Y#CpA#qY`V` z*PE|bj=Djv(X4M~uqWHiOUudy{9qpWkjf^iSLMnwmYHbiq(g>1!%lBs(Ysh?=JGOT z|Lc@X${M?;t?6{O$1X8NDCeB(Qa1OZh5-e(bAPwueD9l;Ev3+lV=eNeCUjguqPmhK}bmICN z5_j)+7pfvtUW5-3u6~x@SXTar9myDo3(g5J z6e;(2_)AFy9W0ALX8uD3x#oKl*SjuToM|-*v@9`nL6vn(rtjY2!nd&1+g1N7ftbg& zg(xX9@6oLay3Db-segGsteM4mCYquWN4nowrji^u`G$Pg(6c<@Q%QD>R1=R8M{hCV zs)po);qF*tvawrvCkn)PRs$DAT>qEk1Gp%T;q>9m53#w63vZp`sZz~s5woMv2VtW* zO&8t*cZn?xRAt`i(){;kPu2e-KXLQ^f`jEajOp`yW2Cp2 zN!OmY$6B8HShImC!<>nbRBGah?I^<}7>Ew2kdcFj2xH}(Hh50tP>Kme}ZhNKX%*90if72W^ zTshJ{W7Gl5MRBM1be{*oMYd^~4wO(%Cas}GE31yk{eOvMQW)lB-<6PGdC&eOz5Yi; zS&XN<(ck^3|8-y}2DP7h7--R{^KZ_-g*j^D2ynGvG!bid|EB*a^5V*C^bcMYxcs62 zi^_Rfq@b>V59$BCvZSzSEL1xJ{P*tPV}m;X?MZ0zO-#?t|EusseZ4u^i8^dq*b2IE z(P6!RT`pRiXewD=aS?jv>tPxC9tm%6eo{DfTd>sFJkgxK1NZa_{)U`$lI^*RRA;PS zKAdO+-F(z;`?%9~THbJbt8v*Ad3QJ!Gc7pJ<^kY|u{6m)pXM&V&xg<2nRX&2T8_nU z{m+Ev2RAa6Vx*=!I|gyA&Ht|!IzUo^45gQrE-3wsDTZD01eY3-;@Xm;FUtiKS+p_l zhXmS!t2Sy-s+A*IS%H{gpho4h#7c~3xFbuwt~PURe~)j2WJpgsAGBdL7APJ0DTGCm z4Ua^>u(^!XdQHipios5@5~u;q*6CeUXGucGYN-rujq@rs`xVkFGE3k zPQ?{L#DG3`ds0X3q=w=1pXY-WZtWY`P6J2*4Cp)uIc^~FperLAn;1=@|Az%Y?N>8a zB?)Zl>DUpxCHsGNoly)WW7oKTDLZabSo43r@9FMQJ&EUkmAs^k;VHOR;4!$NWzNed zOG^LaJXWaD=K_aEUfQ5O4I}%0kMD@w|c`CEg=R#{m}iQ=)9?s9YwcsF?dR@7{Mf<`+A&pK3s#x7Nt8)UxjOj z=CH^%rK0$A)sGUMN`EXJ_h{@crk?FCmUmYvSr}<^*~wo|Npg*ukZ;v(gx&_?Kk2+p zW4F6eLJPLa<^!F>X(=2T;q4W4OpI0{h{fZ?c9fp!aB9<-{^{4E! zzjxk!{yn~R#pQnt4yUTXWokFsO|ZY<&+YwEBJtJ?>z~>P9E_e>6}Zy)J5uFq+}8u+fQE1 z8kKC+RZRV1-X@4wUEfP@O^N#XmolWX?dj&W@!DgPL0cMYX8p%M`a?5CnE{21Ww*Bv zRuLDUEXs;g>@rI>J6D_gS#d6FA1>DpCJ~9ssxWQ6LR0nU3qA*E)Dv}o-*TM%4{7`) z8|_g15RQa@Voj3b#T|JVeRxu1AMl1_S+0qapJ<5@FQhzO>@4b_Q103-ap}#_^)dnV zOhMh9ihmJ@)1KY3WGG9r?(=|c6t%wDmqIDTHNn)^!9sXda-1_UkY|(6qEKc;@$Qi9 zdaK9%rJLC#_v%qE!456*h9`AL{HUWCu@=>3w{dj1am}MYkRxfz$?&)V6h4mYUOj?l zp9_~X{6Xq2@`Z;ALc+^NAAll=yKb{}NOK!XPg6`4rAdc$AoxpSY?r4bWz!V`jYqb6sYa}}6 z-*|G0m2n`qp+18S0b7yF7e-x4!0&M1id=`|cgKtS>VMhkupAy2^3`q?Z@10e@Io~H z?+K1foWFloURGKKxe83DvyD4TZ!=|nENbbmNtxl&Ge0b`cu=`Km{IvZw7my3oNN0& z8X`dwors7|5H&|XP%(js(bQXRXdDxelwxGnRHzpS9WIOS|x(fbXxYH-?DBOvat+Z2J zq4s;x*5u3f!xrp)(tu1!scBCsA9wMP{$+LZqwEc5ha)kuvc~~JMXs+YhO!jh2gDMp zMro=CTW!E^kc6nVJOXm3zdp`~pa!nKPIHf(Wd0h3l3cjU_cY3UzS0~VA3RlX)KHj? z$Qt-k`kQ#Y-=cG%$KjlWcM);(eTTP5{UmLn#`mnPtN`RMoapl{mBbqonbha-=L&Hu zH6Bnh)z8v`=cpfj8n58{C}qkiwV_>(;Q`4ImjZiNc(}k5`(ADm2Lmy?95RcoWPt&A z&0zXeZ{U`IboHdy10Jyn@DCR%aj>@Tvh=*ac(Q^2#=5$^LEGQEsj}Xow`h}t;(X^! zI$`n3c6XmQCg`Ar{;DN5I-*Uta=+7$t}MgVK9xh_19`>y7$)3G2h# zbu|zO)5m#Tf<6@MgZz7Xmg@!hTf)sql%kn_q9xcA%X!;uLOw{sQM3op30NPgMMj?{@7 z*@^d3W_p)UA~o%qdNW4l*~$0g-K01EMUeYuShd{Ry_U^2XSCvwtrWqHv))@4Td7Fa z3dhfNkxFrgeeNom05l!pVYNL!Z=#ly<&d{ux-5u#Zwgk;9u%6K@MS`|xhfXF@Uy8L zd;1)}qJCl&e6Pw()=&0WrE^{sb*MR)4$acTI#=^hPou$#)(lkB(C?@KgyN6aOyBUK9A*HJgO7w(pR3C z{C2@%+neb8l5ba~?1zriiT@|hqqU%ExgXygojmqayt2{(0L&g#6{}m{!~W?LDtNwe zV(-PM8FCIz-J~7lnC8c!hFa}E{%$mGO=VtZLVZo$O-dn|nQ0b~ToT6wmblOr`&>t` zB=i(Q)9um`pxuv#$vPT{;RVZpKYUe7y}`N+Z{)qbYYO-8eyuI7fDe(+&;;NMBVa_G zrbbzS)9KKPqn0t7+1r2dM{l-#ki1>0*(y305JpE^RmK%C=3(^uCXXa{qm2=k2*XJTYI?jjT^|?3#!zC zd3kWMh~m5nUkT8aM(fP@<7S>cT#GH<65V5UMnZWsgYo%%HV=`wd>yr%C)annG)Dp! zX=nFK^zvpxX+3h*tYhiy;<`Er}^{Si$WIKSZ~ z3V=u&vtdj$ytQ^0<;R68H)?{ze(T<=cifl(ThqkaJTdqIsu4|Lu~|JG9dJd=`!{^@ zjTnF!j%}M?+;e&We}+j%H&cGbON=m)UL5INzbZsf&i#F?STQaA463O6xZ@6`t1G`u z%T0OOOdLDA7}cJn-KHWM=!XdY7;$r)ZIK)~0wpsOY57JIOqo;DAdH)gn}F9=ATc6va*w1&n!)dBlV#i_KuAP(IhyzZmY8QQfNjkpLM zzqS3f!LuKgAnRqN6iCZ{{+_?!gS$0-^!d*ri-9sD+3~zSY9W}=#iXqXtW}A}bDP$E z@eDU6Ioaw=MD~pB!GjnP2r}Wk23hV6+Hf())11+^n(T?IpR_yrx?=TUH65S5&$Ks< z^#6-C^rIz_V;P{!Cz_+KI@AejXy8V%vJFv5cpWEXjqt+lr|KN~+FUzBLrrRg9oeG` z$j7+!>$FY#^l957DC!2iY`yn?+@gH)d?F!Hi~cOGi6HUaJI}RGJ!E7&_7hG5=F?vW zvLvyS9z!D3t*5iG&I~McwI2+ZmGL-_kgKWburs`WRG9UbXx$F-ouvGBv`5Mwk0$-= zfT@@CeQ=EuFDD6R8;D%-$uHBI!YkBg)h|pP{DtsYu4PPY*!HTLKY}clyV}C#qtZne z?g=<=kMC~pV=ut2&AYxyHlKgu_EZ1DAcLt=sNBj7R$((QbG+VFxbU$*6Dotd3C`kL ziKd9cb{k#ShUDSr#xW^&E3a4|u)k$>g*cbx>ow>dj#*c{wKWqJ!#zR^q3Pe)-wY~7 zEw8Q4zM=qPB@E>EdW1NtjXVI0C*61QkSiNVGO^+ddxuE~|5oYkv+O3@_`&y{D)oB2 zZM&b1QcvVBs48wcyA93yMNX-6j&E-2bLy~6mfLFTZP96XzlbBR^_qM$0>CqGoxP=&~S{#cuT6z;{?K(U2o-S#Q}D_ zfMS0YDt^EirGd_ekMz2bt{=U^JhRBXbm6SUoRte@Cu>m4Dn^IeUq7g&f^W+({c7_$ zioVD1@_}?=xgyx}D#heLZ49qe;_ack3I(HSDU1i8UZ z`I53@5M8gjd!K*7C*$HNLY#CS9fXY+*JV#kJSli7A7UVKGBjS21^xO8e=J8J;J@Az z?VTcRwp^QsQ@%pOL0z&7(dvDIJ^$InXYQ($Rmj*HZBCWa8i;Y4t21>XFWA(- zcO7|D1UaRGPc@&-FznYA6u@^=kn1B#*OALaZ|_^TTA{(TQ32D&_UoL9ZW2Gb2T}>J zyHDv;cNkW&zxWgkMmJP=&pB0H-S&qPrcjMh5VAe)?fjF;!0qb*Rh|=q5FqVYaA$() z3@54y}=ob&nQz13*fgR=!)Dy^gIn zU;<|(yoKr6$2#EOPoi~`SWXO-f^;GsY0u?+4G%9Tcl>YdAS2BBP^Sv=F-kE$u0MBqk^>b6RXS&C2*9J|5lr}vEO%_2`?Ajp8EciMtbDA4*D5Zg_jmj&-%k<`>Rif#}zVgfP| z170v^;)o*0`acwTn@KT!X%AZ55;d^&0^-U&{}zX{FJ3x5u}OdWPgs$_hT9VXd;p*2 z2retIhsQ*C76^lX{wuEbmtmN_wDtbn3vlKpXm02il{^Q5aD@;wH^do=ZZHz^T)xo= zDcPDf%;c9(3z{1mj~oYrpmi#t!Q@fmE0+aMVxr->LoO&1oxE{fJ-5)Dfq^K@_22+pU#dKh>F3sPfWlT zSYhC+!Km)lM%TnYcMdqlLkxZCY6K=KBy2hw(Us#iq1faq^!)vnOH_^twW^D_G>(*OQeRQB->$KX%-`#)w&)}MiHMcq5BUkbj5aIMMybAkD`8u>;Q zOz&~A(0!`!%F&x_aeQW-Z1S|aV1KF6*=LTek*=}6 z=~$=5J|MOSVVoQI6YrgAY|0it#1Do~#?iy>kbzha^VOFLJ8D8xx0H=mZIE7dd!m4{ zrqDUkVl?0!@u6**{Od6p!VFJDrFGr@RoIiDR~OsK$i*bBNM?>udJ$H75E?>6+lu7(hasiP8GapiQBY{QEpBh*yCQ_yMehf?=TWu5(>Jy!s9KQ~VuR^;gBs#DI9a>-_k^ zfD(V)Gt*M-JiPpKx2frc94=kr66Xu=PM-yGTOE%0vUvN~>GUEWxvpE0>0o+Oe4imU z>>&n=GP|Jl+a?A&f5fHKr}t>bhIl*HS0C-C%@LWqI94<|OS;_4$f44X(LP#4NVd&fJD{hi& z95PSBSN_%R>-?D6V1gh#(EWDMQ^%z-0fDq zcGnm#qB=KfnXlq}yo-+H>$I~BmFTq!NKHVNZzVPLqSJ1W!Tch{5yWP1lmJ%Kc_8?) zl4PY@a!bpa!OrGGB82VPHi?zQ6^rxgDu>}NvAAg(ufz{4?sik>Z|MgVZag#g8Wu$8 z)hC;61y0IXO%$svPQTcU_eqUDbrb5N4y8anYjpa(WW}WAwt~}42_1kZ^D}MK39#u| z$bhAMh#kxW*C!?gt^x~yg5KlOQl1OWnVWd_J#2{`KkH;W8hia0dkMUmFNBt* z^StU`@kwr;Z@aiJtebHdJ#um_6i}ThbaeGtS)i<)U+$t@ZPloc;$vbNid9ApcS{~Nly)j*QOjPS2+ z*nA$dE6-UCoS=`LSGg4O!TM&8RG==ZT`$iz;pWajy@j zn_rvTCaYi;N&4J4cE4|baKTaFFyC?wPsykwP}njN0TTWI1KqhW6A5-j$v1ML1(|Dg zv89k3K(IzKFNzV(!vyyU@km^Q^Kz?OXUN5*gccLIvBsSLkyV zqo}?6Wy+;H{hEKR(EU>xPLY)dYg!~B`)&tM=Jc^|^iqjhYz=-Fu16lp?=CUXbPV(; zEQ}y+Vo=HuDct>$5#H(D5$D}0WLww1$%_?vbBW6#~YbtZikx55k<( z-=Vs_dP{11JA}t5zm0V9W!e^KJ$tQU#OqKeF%Fq4osZE_8GA*TIB$jyW6;}x*&*~$)Ei5)b_;)_E+|#lza$%= zQ)Q`7O{i0ZYw7_jJXI8p|LA)3Ud`pY^_Zfv?JaVJC?O(5wABDZjZCRrdc9}p6sjX# z`xlJuTSdDHe$_~|3B0;+f#3E*B29QJ{v-X_a-lAr?3R1&ZW@iPZiC^Z$C{5&oXb=$ z|H$K^cC)S=?F)Dc^4sasS0QlN!V?fHvTY;OX(FF-K<#u8G!uPY-FJ{6{8(I zir?p3PY?tR?)t5LUM4wjY6%g9#pEK6eLjp&!2|L`Iwzk?r!m>MP;1Q zAGblp6eNxOZ*VnmX3P0lE@kpqj~k;!852LVW`WPyGbih~LP|BqS=dfL}+7y?4dOZeqhZ=H5a*XOu9m#8RYY)@A^oJcV^J~mH zl`*qrx}5A{Qb~`8`T=}dY#nP{awm_801#$eA|l~gV{|i#XI)RC@mDgDcg%iVtR&;s zl=_j=Qw2_m9Tz6ZQp%Q4!MErcpk=swgz9UXVqv@4WNJK|N@;@#SbCFRXdBonin;OZb> zJ8s*IpQMx`$0v3@HxO9 zJU#ZnA;{N4tcX%T8nUH~;wC4SlR}rki+`5DLAKug74Daa1sy(F(lhv=1heYg*fzp;?tJjSd+pu1@7&6(oz$8*he7-SHiqR z4>ZN_Ya>CR4(=F!YEd5bs(*7A9K>62Bf<5Q{ZrK26VL@Y?aUYJ9hiKIidaja5~+LH z1F_Z#0sMgaHi?n4uI(`pOEfnvZ(s%jDr`!w5#P_vUtP$Zu_obAl!ER&6c#vcB(wXg@Sn#>8Oy!xu5U6g9e-FqaO`bxpR! zoN|Z8#(%>hp@~Umw1C%}g+Kc{8IhnH$g`|?_R=~leWR%u2#}0F@mQtE3XNX z(dVKF?*V$!JMr$H#^aC6kVBSO6~848R%+zRY&pbX#&CqFL{sHvRA1)XW$D8b1RR@lUuctNU++;u-@w?35 zKPN#yAYi^%;t0%OM?av1%?wQ$$5KRLG>P!m&}<`q$u~MhS)QZ?#Y(J}JzfL>BxnSe z$J8VGACjBVzx!?Sp+2!(Dtf60clVUV!0J%)=_awhylBfG{f9^Zp~GqiD4|+y1enMF ziPHg1#{^Pi@ZP63R3jX(!N+rQ7XQO4;J4x5o2`J0wb3r7X8?TtiCo#GqVfP~o)g5o zq3`jegp|1G-!KtcQ!|Npc4=nd-uSc}BUT#{?h53Tf3x*rcO+H{x+Kd){Ua=T_y=Y- zH@#hy6O=H2{1s2EAfC{Xkv1;kr$R1+-jsIu-#-^91$6|Rco}fw87XKozOtM^3GT0_MOWE9=g{LoVmm+E}bw`ag*=ef;xH)%kBx`J@4JCh6vz za>x?`d-LkPvSNg5cy`|A#!VxsgH-iz6*$@Z0~Rc4GI3 zQ<$!SK=z7y9pv1l0oIMMaGz+{w=-pBGystW)Fn)rZATku<=g1wZwJYbfv>jld#;Qz ziW>d)S;fDH@*h<*x4G$3Ox6x%y%}8&Qe{2&*~EKE&qY|{HgYfpL_ZW zI*R3KA7eCpxIM{`0+!e`Zj;Z4>x)ivZ=fjuQ{yyN-T$wqWH#}i63l;H4njvorEBo- z;*k{Lq8%~{W(lQ)Kx<0LV7lPgMOg;jzBCkgJAv8XEsvJn)n7j}Zn6rd5W?T){r^|qL=QsQY-3juz8vpaO>w&Q_53JO@ zOC4=95p45ao}hZ=;UZD#TKjPFZCpXkOCqrIp576U5<|K3dQHna(T_JC*BzDJ6x5tR z?eDXry{B9j_qib_S2S@Nc@6z#pgZH|Dv~rRoZt%EirADreo7we&269A{n?R^Pur%* zfO=fg*MG>M!udD$i=Wwv$Mhokr~YcT6AhnBrPe)upSur=K3!fNbsN152la1h`_^1A zIy|=Jvi+~EN|qcjnX-g}AZ-QBS|bkKchfG~{1x-n#tmPJYB#S!_v0vQ5gi!q@xV|4 zMfR75N(Y7lIh9k6Ek6dQDRi%i_km4CX4$X3Pgws{ZcBFB#j!@K_T2a^ZM%YsZbwKr zae-ea?DXd0-Ng|jEY9>I`z*li1A!3T*<0Hzs>|`VFMe$^R(W!%{XvC+me-(0P@FOE z$s9RU(+Z$nY&CNYhzC3yi&1&}?MnIR~hkBWf-gq?BXO zD*g~=l8wsRpQkuuiPa$?xoqdeI_;UdBbY$fLJbeO-z0OSYU=S?3Co`q>U6_Ytdmi5 z0>Xv5vXP$XRM$00@OnWu2~PAOwS=Z`1*+K4T7zypQE1{%44j=Xa`r{~<JwGdZ_Y{wqGDc<=lrH2NJ!qPEnlGYK!(3Njn!|^9604iuMXl}g5_U5~dSN{|xc22pzGkKWCM+6hOtldx&KIikI&?9-dFrE2&Z;XFFBtz$|rb^>x~` zt<>`$Oy4!fss`Z9)^9X90oA=`)R`1;CZcrwQ>6aky3X03rXF6FVz@?$*B15SO#4eFRgUiU(^CCxdEb^jb zQDGEj=%X^&(qSWDxV(BvCmk8^YGA^5MuoK6QdUWT>;LVVKf>q%Kb#yN6tP+JoE2?B zt+vEflK*f}_F-CA@*5=1M1%S?sn3??2RjZU*92y;v) z>7RFaDcsc?X||uK*mvdn*R3bBmX{q)#8S`hfI?#_%|{t!=~1}6zH)p3h$HP$^({x9 zBHR~Axg&hTs9&w%bk}f#O+C>66k-jV_>+)Ta^{-F_u+p*lHk0hp!vLvgo{2&$y&$o zhgeu1hy<=olSJM^g6lvD7?esDYX^w1KOEmXK0y94jpP4fn&ijDh|SF@x%RlPbpYR} zxd!u^{t7F_s-5cY`v!unlxPoszjKJ^-OeY9;Z~>yi;4tfR3r>2$CUYa3Dh#)MG*jB zbQWsf2=FK-Jq)jm5lPqgVb-*54@%9;?caS*?bxFZ`kzBZpfXrD6hVaB`F_YHl17$i zg7Vmxai&19?3M*ku1V}agTiTa*Qo9@jfuzJvzET$OQSj{DWX~0ri-EbCi;|9kWsZG z?&QQzr#<i%kM$XYzp!PgOt-p( zI*>F$UJRT_{j07HjGC@bWR{6t{#6i7{wj!%93G%EbAqXw+K@njar6i)-|t7;>}K|s z`}4_;JdY%YFK2t-u66Hbo$<~aBfkM-TGg(BhF+>Y$TZ-bO)Dc8k*ZJj#gLciOf6k^ zSUUI##3#)M7defvBHQiQ_2bhM2s+P%)&$TvlLZjDAqO#MT#<6=8Z;lH99IY1r+blc zSiqnOsZ}h*7Xr1+c^Nxw@kA2!@y)EWmNEdbX?#MERt@`eiPV3MP_!ZuYFqe@)phn!Q9C_j5@tEDgK1o;!jjDn$LoO7CB>hYlS^1+2YP> zI1egrZI6<}=QG|farIWe4%4zDKa|A$Y8}4p>jBvwD$_OIHFF444$%GPyz}o1#PXc+ zXy@)OdYuC|ArpPRWd&Jt$FDkOqw4=uBgy}z8coSH*;E`Zd)WM5zylcptO`2(wk*7( zpFsc-JX2sSz-|($v&aE2l_9gO3hTbyo6@nkcsKbtn(rnGa#y|c5=**RqNzb_zdLb= z&g17G`9C^(dL~F`p2QdV&3?W!m6K&fNM{!nE*BbpD9V}2B`~jDCzLyRmTQ@0z-&?k2ZBG=km!5$$ z&k1-SeGK};^ph{s=TAPG4XOdxd;$z+G!TtFKmjbKtp@#eI3|iQ1*iu~oEjmg{>&x* z4VhPc2pKnd0yKgDaHAom;yeQXhD!EXbHYV;CD26U)+H^+bvM zh41!sI)(+BUI_#Mnk3VeJYrA)&b!wkKNSfb2Z^xvQ{C7^rxXxUCs^n<0is%HeaoWy z$v6k|zmUQPYFLqsaune=-+d2~;4+&-_~VkSo&%PToy9~v%CJs(iWTnwUi(8Ju7Wk4 zm~QG6f+TsGbr0=(a_z6)z6T6E~w$ zVt7~qm9c;rvy=M;eFo%8yR+MRRM%k=DfgOcij!rXnxMkHGG6~rLgM}}2?>L5@Kg_i zEV8P&y1E9?Xt+AtBOQskJ@IX_!3-|D{BGT(Jjq-!g z-d_eK52A7gl6~ru4P7St@KA|Fw9T`jac6WEG5u#flL?!R%smWVuLVHSZbje>t<=A7@{PUNJI zFKGxJ8d=_IKLdz5xV($1{-4!@f!MIQVK}{1iE=ol=Ck?i_=Mor32CTQ!y=NZzN@ao zy9w_jX|PV_3bJd)<#=KBasSdOdYzezcwS4Zd_evd;zDBohtPH>mur4?21AS2BTu29 z+fPJhZ8vJRJZxj8?XK=}jakL<6Oj4pG=b6!*UCW;!2V0+sHuyVDOS#K9ptQ;qwClb z`c*C5Z@aOX28+q6=Nz}~Sd9un7d&mg6&)q?aIKj#w409w$VdMH;@;ISu86=yA=jWW z<^9(k{fQN%Ukgz_hXt6z-!iRegrEA?yr`{wy)$>H`plk#I^~V@gqqGxQdid?wa>3u z^=d02+q4*%o{snH5ckJ>dN=DJ%kUbr^Yk2=n#`^4`fj1+ZVI#X+_f(P3Y1ius`WW* zN{6E~JLv*GiL(&v{pnmS)yIMb#Yd4+C$=x>%u$8iA|e@fuJ_wFa9vBz#fSDMSKg@g zj2S2J2^M5PnW_b%L-@)z+_9ua{c6%)%*q<{JZ62_HUO0S|sO3p!c)4m`WYyVvYPSwf_`chbATDV*h87HXWAEQ}Npu z@??mN+mGovG!uv~-ywe60>LY2DQyLeT_zK^%7C>m2|h(Nf)2dQfdF4cEaRPY^=M_k zN5-52mdS+AgM3*mRGnO%pS;d`v{U!eG^TYn94C0U?(3a{$G1W*YOPx>YBeOlrL>fx z7w$7p_K$5IH3w6Rbf3t@J;BJ=;v0WvB{T8rvF-TP*X8T)S!1!AXngLy^YgzSFhdo1 z<`c&fCF=EHW$epCQr1&}t@-!iH3l66N`!>lcix~*BoG6EpB-~r35CiFOCR3(GMoPX z?8c<`BzAwM^gJwts*U`C1`EFKS@P3TA2G+WbY9Z}U(aI)yLHF`1#bLGHEr1beeh%)fu^ z5t>h)uvG*)qwxK|NQZem&o|f8)4zBBbSILT*m^?P5Bx}(5rpj2`)Me=&Yr-5ypuxk z&guC{R&rFK8#RsxRddUm!&)sm7ae^ZZ<<@9oKwE${h3d**D1PjeP!xE)1>>=Lpr5a z^;#L>=A-3iX}MUs_{m=vfB!TeY49MMtqW@X7p5T+tO9{ zxgAebwpaRJeB`ze?M~(u9&$L}o;9kpo4wn%H{TY??KpDd$@VC&=>9Fl3&}oe4m$7C z%I@8BESZQj+zMMUUvt$=iL(=Z37BAf_m$}9G#erNz2{=$)1ua2#Qk*a7CV(ba@&hy zU8@U}jZA;qo02peM0Y+#Z(@74F>yeyc;?K?sPu`@!^sx6OY<#m88ltgi^w~vmJnW4 zZP!z7J$U6=Tzj9$z24NCTi}UaL5+=?gsIls*S)4ghG(blGE`fTz*2DhWRln;0Z{zN z9vi<;27wrF*|)!>6(R}Unq69a9hLVy;(3=qXuqakVU_e5cx;s2@`sWQ}UEg~;UNHL`=f>QM}@pdpOnY@w+Om=ePOq=RlKQM zr?89U;8)GzG}t`L-R(%qibA+wPGq_x!TUQ8jkd2wG83f#;y82%{(tfwX)AWC4BFs% zPHdxBg-0`1f2|8@ijCpjolGclPp{6cW%7vd)NeGERaQnXs6@(h>lU4LJ@fhCb=mKh zbl{KtoEXxs6e@R7u(g6yFvRyQ@!ADv&lQ-TExh)o*sX)*xg4YAFWtYA&&8st`5)h+ zmmxk36wj89**Cok~!nST~wAXo+;d@dJLl${&sMixcRd9${PolG~j&l*? z>dgP7@hm!IAKC3tY_Uhf9XnfVdSD#f@tBn5k6Vo-nd$ z+W7C07yR-)-xxQgf55I?i&3Cg*An=qPmjS(lr)Y@uQtGTlHgs)JW6tRL7p9h8Jx+w zMLtD4Yr5K#Ov|FjPV?ao{N|e?rL}fd-Y*6}r>3q^+glIhlkcyL@C-PcTFGlp!NBwP zaPTOdgAB!7f@(m>eqSVQ6zCo`?+l`dAaR92*z!>Ww$eA65O-t!P(j()@dD(w(z8$qZxMs@a+S=n_TG z9HUX(H$SW9$v@^`HYFOqPrGG*r`%kmq72HmxyjP1LM3`r%faNG8}6LL*7fehSgH(R zul-w?o7crRe+e8nRE)k#e^cb+GhnQ^L3(Wtl0#?_<9DP&a)&Q303Zj;zmTJ>sOp(^ zbX85o+hKx<3M-sYJH0AP@N|I(3Hvt|!=8q(E{@6XF|+9Zpn%H9zI$!|@LHYM&6dzb z?2BHd&yqP`)|Qi*aCQ(y$xesk}x4~oXev-V!m(CD>e=GNkBx<$z5T5-plukqeeW$#TBxYe0pdxI@eWsY-J zLvfY^p^`2dgfXo$1E;F;l{r~ev$!X{VNo4c@iz&Wm%=b&I) z@J^tTPt>-5YKmzpl(m)#aaLnc>A86$Qu^JSdx5C19KWfI^8~|v#uWGMUoxEPE^4)l%(2VRQ3_XRQ z*=WZDrPCa|-DaQ7S(cYj$b%>uWBR(DimO#!g7EQ{Qp*EdyYtiG=h1q3GVUyz1uPfX zKXd|3i;VkU$)ySk^o5P+IAOQ?njH~5sK%uc@wi@YJ&hgzAR$15<6?Q+5t-fgF-vL+ z7N1u5s)~%bxCdh{6UwZTCK;~1VX(H8#J(~m^;IXAaVTFk!Y3S#$Fg$vQTmHS*6kus z#L@H7waJPt$v38p2L}fa?{jk=KB$#zn@i^3y54+i>j=3j5>w-sz@>SM$3*d}QCnZ& z^@F_K1BCh6I0*um1jCD5>cy&1Wb@`}h+#l+a0HIkZF(6vTm0OAtAv~@M-vv^0XVw~RMG?Nm=9A4mv zF=@oZS4o+u^Y_Bi&!It;dYO|#H+s$cCGb0X(tg>h*$dx^+QO*m4Rk&2Cyq$im0GWk z;Ud}O9}~G)u!z}dF!Jb_8qyc``sv@t^yW@>Z`42m6V;9^@`#)Mteb38?IL>P^;_vG zW+}m)y0iFar)ItS+&k%y_he3&sU!552bQ~ad)Q<7XWPE$9*azIo{KqjacL9pC|A=D z5GgxVC?~K#B&&qFkb69mftWP#_bsoNc zr!^3Rc4LfV6Bz&Io>6^MZJ4d-?7f4{0;IX`PW>a zkz1)}DYU{skM``O+z3w9K~+$qynfO;iudhl5>KFHW=$fy)|I=D*72Ky4#3FymF8Oc zRfCD4FLqJiU#o;U0V++yai)+0}&g$ZEWI?l(||yB4ZK-z2HC6W*<^Nf$UFFz$*Cys5zk-A-1& zhm3a|t*jByE{@zhqZBo>S-HbOqdU}h4~5NR(skvBr_qlY$S9rJ`8qqjr@EWnGM%u` zE@rWJS|E0s!p{IF}O=$VKVTk4Jz@WQ97ZR9$?o-234eX?;9^D@;57#57vievT%%X)~SKRHN0nBZ=kB zX3MO!;B}FvTDKtN)sp;}PjM|MB36W1hvyUnTmn0Had$SZig7L>UFVsXr4 z+n;`Z_qp0;NLa7FKK-(GjT-K-iK`&EBeIij;$8Zdw%KQ*`~FTk8Bml)AHJ^oUsA6jK;x%#jJ zoo*z{go`l_Bye6b2D*5tT1bfB$3%Brs=Kvv%5gDB7-*kqXC#x5NIumi7Z-P;`u!#U z%E!1s!rv6Y>CV0+Kww2aBf;b$y6hteR$a8~I1D|e`;3pF1>Ur?n)y~%Vc=g}zPMl6SH|!xZWbwDiPR2W5P8ysBxC2cQ zx$d>qceEUzSNhI%J?W#$`USkiRmZ3`_@`K#S<~%q8YaPe3X91iW@IS>5ZuNT;!yDkX<1Bon{CnBpzP_dG1H zuJJ4Kn!KklYm)UdIOevrYsEl5#2Z*Ft!)f;d%LPIkxQ=_*(G_dq&|?>IBGXL#MG`Y zOyDz3Iepxk(-5`2ll#!KO{mEXL3;z@y}lpqxQ-}$bxLK+Wa#bLRb!Crx#<$#>3#>; zkl^a6?E$*|u&&GoiuPyIaGv6$J~nBOlqdDuYjJc@ObQDJ)$@*^d^7x!xNKH;cz;+oI$oH0BhZA>Tu(( z0};GnWAu5pq9J^s#Ek2+*;e%PQaS3Q1h0tc>8i6U)h^c8o!0_{17+!TTc9iI>T#t` ztN46IL|CP8z;&%ExZw>D0%n3VP$L~Bfd-Les|U&kLdPTs?TGsnw*83Wg}q?}>qjbc z{U^>8#EhYRVr^T~#iK6=OyqBNs4*?NgSHR9Y}&VwBlwH<_umuIDFU+yJnEwYY(eI- zi{epp!IIzvgor^?Ybv+vpP_K5y_7%YSuS!#75b0F_C3 zbcWTQB0yF8RyEScuWxd=U=sIsFAZ)-EOQ9$z{9@NYil!8RSMh=Qnu$_W#9B0Ymcoy zF8@-AqLZ{FM%J9IcUh>aj34 z?}t!4g&L*R@NYKdHEK?IM9_=Khc??KYArh6?#8eG;Ux!6Hn2b^662qq;G>;%Ra1#+ z@Af${&~?}nzZ)ud;W(|fGH!(hm5U}l*|#Bi3e8-RvLDZKBw1-;uTSF13XGt-8(4q? zTKYcs+`jJY^$i*S@uhZpGU z%px`Hmb#_Ks`j`(^6fEKLil~BUUwNNvDy4_jQi zdZQAFlP2O#y;@HVk2!z5LZu_vc@7 zuKrvEAoEG2PIK_ZGUM)_^L8q(OcH3FF2~bT!G_X>-SL%@k?*%WDgxV7(gNY~;Cy<8 z9XzHCdKZzRIiiGDy2At-@* zDW;1sEr`#l{f!~RciWxA$-;J1Q|?xklg@L^bVvlcp@I>aR!nQg@mUYn$v8lDO07{i zj-R8VKa;TYRPz4MftGm*$M+S`phAfsQ1bS45=_iA@siuqPxnln4ST0vl0H$UIk(Rv z(}FH)nB_(}0f5g^@iV1zvt#zfj*{#;0yJl4kG@J`4c}ItVEXGD`N%EuNbENvj{El< zQ&lkT#D!qCSv;2hq$%ZDh_7}Tn7JS@qfI5#>;D8MskX{1jkxkQl@x-t(GA~>mfc_5 zh9Yi+rH{WiyaSK(eD7krlutH=2SLQxPYH@A><9BAPYx)LstsLPUVfvk2pGMXPP7Se zVlOq@BBCatHO;@nI3VrANW2?@#?3@-f%`TwNARQAAdHtPRDw6XLrClt+;)dLrofXc zRuGwHiYxji8i!vQ2k?Xsnvo}p_&e#MPx1gPsNi`lv9pC;)apS&xKmG*QsJh%ONRB@-IK7en;s1-qAVFgkhGrlauALjmUwB`A3&Z`HrLMgTC$ zDSL|ZksMtk9c9s(@<0ivUFuJ(*aLeYO5i6bk_LQOQu1LLHLtCgV3Y(qb&yI`R*6QO z%W{0}|5-UUU)*H<&D1G@Lx}kfeC{UZA}!@V2tX_WD=6%W^Mn80G!NY@m`-{1g=r4L)=eM&(-K^0jW_fE zbIW|;e7=+TOKpJf5A?!XCHy=XLcwxa)Z!uuxv9|;fF;m=_>TzyYt;LH?d5XdP=Ifi z7k9tDqEm6;Ni7kEYCSnwCj33!PI^Vplkm}J=x{&}tcchd&_x|>c>d^b?1B&f`#hQn{If-re)q$I-M+(&A{ z)l4U%^U0d#(Yg8f<6#plthwe7%cYcnU*(XgUN01v1P4C}L;Un;XtwxM~ z!|I2+)QFL4SqUCa4%79nE_Rw$m`=2{l7@!2Nkzo59JcIpZAe^8pJ0cvoz*O5B zSiu*y&Px_k0lp}-rL@5#zX`6!GXhJh(24iML;~}xR+;MgM?F<-aWPy8X&+kB@Xd+2 zYLLX!VRZ-kj-c&-j%ty$TUp|b>{ah~@_x=C@z}53FQeZ!C zaGJv@@QkApUTv`?_5&mFPkeapgM(oYJlXQ2IPtPZ{e4!3!5qyQO1g+1J&DJaO7SzF z9HeOT@PLcTq*1LVSI`56(br7^5RJD?0;lpF&=)08Ij*?J7;Wb(-{G8gsRsiXm4Q_%Y|kCOaVZW>1L>brs*T57vUtCC0Sbt80k_Om-aavb^p9p z?YO;bUDPyA7rhDpm8Av`1B18_8|7MZ`K>!uD}r_|EDD}mRBM4U7VMYu3i}`E8kSZm zI9dATHDtEbdwb1~(q)l@$T4}&o_$9W#Hj=AQX zYwhHYk|N)crdY6Hu-`yntIKVGO&k%shWWkM%#}5c0xsis^B9Hm!?YY#%?<%Rg3E@c!4=cA8qyx6Hr6;m<~TgBj0dRqus^-8PO~3M8k}Tv zFaN5qYrfA8!X2NWY%{%0$^Ph|At#dqnaxEI$8K|cTvax&T*Lodl0@g1=tXvq1830e z{|wb*K}U2jAwZI7hsp3Ia>tF^FUvSt}l$I zW4=GRqkptos1#^l*(Rfqo9>0ZD-rcYbuX2ys4LGf3>61=3oADEA@0 zK+gTG47{gHNMqSR`IDv+<2Uf`B}f13yr2WAB>{6JcF-ygq|1(nxnAuBl;%6Xp>03r{>=Ht&H?%2y*5 zU~ptl#j#Q&B%*4C-==H|pelSHm_IP=b+oFxkn8cb+RAu}l@e`36$h*3S9(|!iGGNe zX)xlDjGN`wx5%~ot#N@7^V8+BENxhhwPZHa^^fYz*CsBi-ZcTEG0NznVV>S{*Abg{+If z6@#0I=|?$O^2f(|(wNBDpPV_omY-gk?%E8nbz_|62E0}&1J9jcS=nlR#rk!nDulLW zOZb2xJV)Tqmv`JwVw(F{t`yQHSMI)dgkE=ffKizt{=w$lYrrjy{Az!SkuO?=6#MV@wV1|^o2CS? z$su>O7Tqb};Spnv(6L7Zd?IGgVd@# zE5H*xH0&8>G)NJ_3Q~e_jo0^g;UiDX_|7TNN{K?Al)WmSgKxi*LQfJOc@>XHS)r2a&|$f@ z_nWYTwa0@G>z)iJ-Q6=edoS`|F4$ozeNaJgN=oc$bDA`x@Ktnj;I{OnE)w;mBIMD! zOkk@JrG&vZnR4D#y?wbQXP8O>hrmU%Y&UBEb}9!3@6whc?4tMf9Jrtsa;f&ezlYjq z;xA*Ww@^ZlLywE=E1FF-pnyB+I_g?&5iM)%FQM0v2lw%-$r|A})c~T1#}}G0|5^I0 zuyIT&Rzj6x?aXIQ%I3S2zm5j73!i&Mfp$+(>yr*5@9wi}jY;vEJp0Xj_nybLKfNNy z(V5fM6Y6b)>%q9;u?%RlPvEj!uuVvmfV7F6WlJQfgl0JWcybSualCbI=PR47Mj}q= zk;Ws2g_{?0p~GfJE|=A_y*o$vqKB?8yRn5%S?Fp~-8*vG3sZmg+?gP;@FBWS`@>^e zzBR>Pn-8Y`cZR0`$Wim)w7*dwX90{tTFBJUt=T{zlluW`4`O2;LoXNT=B8-A)WeJ6 za4>!eP^9WmxC1cT>OG1NV@QBDu@bh8fA#s=Ao(U!2gSD+z&%>EVn7P`pBau^U8S}V`G zHoGB-(!nDmtt3}LDSi)Da>z?s0LpSh)1M;#K3S9i>J{@?B@Z=L;E4bf+$=WpUnmf2 zw1QW9PFvjZ@j8JM7o=e#j z*#HjrhtlOw(2$+_QGtk6C4&Nc4n@E=NB>$$ef*PzV=;-9d85$-AP=QRaFqB1pM@Ng zzHz|G;w7n%;>B^yD8E}0#} z<2bUJMIaPR|F+OY-L9iuFel71tIFu8(quge@Hv8FphaXQ3=8OPVc>d2AK-%zec5)^ zfAI(aJXXl**ZT%#5~u(XA#3NT$`2bDMh7{-#}x;A(_Nk0!rF76esTaF#O9(H;ShkAZKQc@ZS1Wu^-E6`d{Xha#Yt?Hf-vk66yPNz zd>mh4{MP~aojk?;f44D)FTJ4$PXAw9;crG}98C(C@{bhQa9hAqLh}jN7`xJKsHpze z*$6c;k{Mt9zzXm|{Tne*X)-t(eSCI=GNiSMByx)ad{m*E93F6A{TGSxpTvTYa>LG6 zfL#qgFym8chNP+0IMLOL)o6fmOEXE4G+5xTjd3)Tzfqyp?UMAF?Gol(tu00bnKy;d z!D1u0JwsAl@?vL{M(Hclz1pqg$4t*){+D%juyx)LSdYs$uV-p)8<{lX`?sHDpTHY5 zIMOdPxuUn+l?3a)9r(tFb+$Fwc{(l3XE{Y^v)F*wKnQ5c?L>xOgN_!3r9TsoQaGE9 zW^Gy(F9aJjH^}vl7te!U=BZ*Z>DD~cD9}KUXG?-tjAa;oZ~ohXJDT%aBXk#sOL#fi z3(g!Sz=r~u$4hO(|4SCxY*7e1<2HFckUl=%;TP%EV`s^RcUwxq3VA=iOB4p_m=EgK zK4Px-s0B?H5_J}8l`7;2w=)5*Vq??Iu5bwyOw1seP}~MSUEu$iA+tne0(!kooYrd; zS-1Hb_DsiJ+@^14!)em#V(_!Rl9E@h=##V$Q^oVqEN}WoF7v*Z0=#F{M2+-jGtg2i z$8FkU+8jv>ri7FpV$J`!nF(!)dFa{F2QClF7@*FU%8ch2y zbJmGA`?dT5l3KFKM@Tk}^wV+^uc z6p{qy`o{N{wdvwxTVDc)KDkb{MK}9mngSpw*Oc3>z7hl-i9Dea`>H)zSSH31jDU)J zvUPf#3{)!sU;ok>L)UX%XVByVo5b_MCrz}m;u0?*f-Z}Gad;T(yGjylFeZhmpQzBr zaC&f*)^ibDie$Di+(W8OUP8E~HgLC8ikmcwIrP2|V*B?Ec2QE0BzrqoO_mfrhg50$ z0@ktXaXqS8r3~^;3m|W)?roQD$}A>nO%*-mXhl;Lu%bo_-s~sZEe?p_em}Q_zXo+J zHal#my=?WOINI|(S;2W*W`wGeDnuz4m4_%Z|5{k#T zw%+N0;OJe;IOYxUB*MGwF`me$s^_BOT0!1IGmNCb^jsaQ`%M(Cy;}b|>xkk4znpCI zD!((phZ-)+Lv1?w0AA^yv2KKkjK)X;5O=9U9kDo(n!*hVHTg`tiU*5pF8BD}X%5&& z3xiaTj>KF}xI!rA!OI3RzE4QTBpaQJsSR6192rwH*PhTdMR@|dide7=t7qg&_13oa z&?i+GY{wl2f`k)sqP;29T+_VFmXGR;W1U{PI&3V8XdWnTB5Q>dhd4TNmumcA0Gw86 zS#lA5`C4EE*TP2_*h%ycr|UhL z54YRt=pOpTZ_C?C6sW{Jo@o4dh03B;l3_vq`yi`NlCW60K&|viSRXLNQ70s&+Fk%> zc@g1g`Q5)M3%50ksTcFVV$Cv?6VjreDz3fMTFX;Qmkh;~T@9u}nrn`=X+&YROL+hw zE^(#;77EHr`(#J>1?YIGTso%v;N^R^4tDe za|R`Pd@~gv_&^|UoNG8CXY>9_!#car;!>P4ms6`oTF;66(*u|+IK16W*GIs{bOvc~ zi!vnq_%U2)@;ecn=C*sYU5+yfED1D_OQ-%jki%kmkUe3V7>p0Xq!^N2^nq=bZuGjp zMrB~S`^3tqBhrZ+SxG?`>gYsEQ7!`;32LbP(BF6T5wyBi<8iY$aOugAqm~{o`??jQ zSQoT5x-$Ae(dFTfbm?+Zv($_%QSts);(VQWljj|#5ayV;M>x9p|-6 zIeg2V)?TXO(-XGYc~{|ir|LySC3{-@N&w>@;2WCUWTPGkC5e^QxQ6aXV0-Q}<=c+!3-h=3j%_r zk4T<~7{LU<%|j_#4r#*gF#nfn9Q*!aYb*!AOm|R__zlYefUvehuGt27a3MfoZg7(L zH$?P_RUncLJHvsy{hLobxT&&}kN`?a6d7-Wa;FQ?J!%alrr%)jvczX|m^k!XiAmZP zY~BV-SUi1;uRM5g3=%7w!rQ*2p}@>3{4;Ziuyp4@;M2(iaNPbn#kwrgQlWwz#u~*_%k1JSyF8h|W(DOz$F_|#60-4bae!Tta z#J6f&+j`ezlvaDxG}IwJqXIwtI=MXfN$&H=RdFFCUal{Z>1m7{p(QGVgbF50OcMO=b97v$x+oJ+f$1vl@kUQI}4er#y}oa+aT6C%3mmmoVs*W)OW}OM{4J&>tMhZ3;Y1eZ!83b0C6$>A9vE{A@A?lyS}#m zNy3^N@z`o=hXAz7RL|Bdd;g0(1dp*sKPJ+X$*_e~C0z^^lR^k7Pc_xoa=Jvi#wuO1 zMDKYWeY1Kt(nzMXz}rS=lAj?s6k=ZYy)xb=iCo4{J+4n+8C35H{QUe@PS+Wu>E!u| zd2+-7_X6Vg7s(g^Q2ZYtW88MA3tE9^HEeLSH=isZsogT!n>rMp8qxwT_zl0M@x&j+ z;!PpNI@KH#KVIXN=+z^EkC%hzsx438V{2`fJL>FP?V8rR!jVaMO??30FHdRk(nzJ5 zx0lf;AOxF_<0^O_F$ibF(xHq+rUs{RyVaFg^yKy4e{UJ?^ij{&k6^>D^G z0e+}wV8Q|LqqaJ3ixO>_KlgsPik0San^uGIwH52A8uZ0H{~1XTs5_-z<{d8-RPLL* zgagNdJ>nM2ax1Jg`r$PpK^Nd<`8uwg%FhfZxBW-UnM9%W98P-xz>m=NW&*bdGobbZ zfapoGfUUoFtwZjU z=;`rvEaFU=TV7n|W-0Z>UVsy)(P>G-$!Ix07UB=jOP70Zd4{pL0rlB>vsB!2V+m<- z2OpK@rwDp7Z9B;tM8PIA2S2@oQc3p`iRk{EB-!ae9GTlJ+5RN#reA^Sv}4Fvlt|?! zcaOrKnSiGfGM_FX*yv9PM(x?O`-`Ciubb=e}x8Rhz`x&J>Al`aJGjLU|0kgb&Qmp5~*OdWN!^2O@TwNsn9XV zNh*6FxF<~}(;{Y+0#aQ@pL%}26p?2uANUGfAUJS3wtzoy@`czxsWKXLpW*KMRo1(l zeyVfWrX(6nJXln$Ek=`dlaPR^3;=CW&4)o@mSnSA#BB3v`ADuZ=iBmkG}qTZ-#KAz zXo5d&_Sq#IqT+oAuoZ!_5K?a8+j1l}JwdWpd8)tmyYG(Psy^qMsXd0lD=joW+bH+T z<~8MarzMoF|5R#-vkO2Izp3Kn;H7CS&N0iWqP}Lr6YQrv2V&(Ecl8e2t25=T#&=h# zY_`wMN3&4pd*>l~>=vd}>G=v;;dpUi4rn38&;lWGH%p}CiWNNlDtm@sdY9ifS$uvo(-AU3=Ur>!_B0{;XLX$|TCW-{PFLm&Xb4@Dpw>p;iny-6VHpzKOkryDC5L9@dTn63Em0sOtuF5LR& zY%*#8PDLSIj2)z93naoIZFw*V{~UvK!UR05%+-GiqY%$Va9n-{&ru+%g9d(0#rsFlz6{xOphYJ@!va5hMuN3oUo+@5|%AmM>(M6M}(_X z)U~%+)agg*0jMWN#0@(nKTU=UFg_k28*ZGgE*riFPpSVjX|Q7?Ml%il1p$BWEHysZ zBf{?W`-Smw4Vf8FJ{U8WirWA)iMA>$n=cg?YCieg0x%6rf0>3jY%U?F)~v5B5jxHQJq#Os4HR<`x!=n{;vBoEU?<| z)ECLhoweE*>cii@tcyG#sqxd>$i|l6xBKy`YqoiJD${q}?dkw|+2fo5p9TMvSy1g= zj?-cmEx*lzAbAuXqbmHPZB-Scay-7<#SX4yj=0;!%{s7_3M1-!~kZF&*Ck^*>h3#l_M z8Qy4o)wa12p{&mCv^Aw{1GG-r0t_?x0n_0-Ro(Lek#-Z!hUPI+q|9tAzT3U+r zl}Y{UvM`mP9aCevLA1PItrXARwTvvl_WVS1kTn6&;f@gYWr|dc*Uc5OIq#}^fZS+S zi~D=DkbW7k`c~i4YC?}le#+TR_;h;4)ClrImndvyO_W6}HX3(=i=(fW7?$Dg#87Ca zIsfp2vSG6^Yth?E)cE0K+_{6FhTCloFIwAEewa|xotJ$2`ibxdjC)N{K#n4Aee?ZE zTy#CUR$Jd?^6F`99`<|>k;R)r6n){UccsyFM2nQAsC*AK!UJg{VH85aE^Hw=+1}}X z$KIsqx?01*%)CSrFv4#s#!4F%)Pr3KIdnU*%9B0x1*9mSwCFS_dG@v&7$@<5aNGeS zN8Mu-3tghcLe_($q61c&AhnYPw(VM$Ni659$#Y2Yt%7EgoC8cWXYDeM93!h+F`UjUGT+%cP_^BfZQYU!d_{adO*10>_~6vv%D@+VIHLn8%>@aA(Pb;y?`s__8G zEQ##NzX^qq4?j6w_MUNBi~e#wd^*Sb`;=zAx4@6#m^18?2M7wdEI^iXSi?vhB~GOS zYOa~ag4tX!8`T0AG9Z)<6dl&H!drB*9BXez0*Uz2!z^(NEa zU&wBC>`k2~SnSzt0Y6dsL(u1dxP5-Mt6@gl7Ny$a zURVqs!*?uD^dJ!=I6rwZZSyXK)no8oxiJ?ywHYUd{{0hvN)HUCBO>sNzT+PWcSl|U z+p{MIw`rp^oEGM(FLSb$ohxz_ji%GYjl1rDyri~08Zk`?y|_{-yREfFowi*Hg7AGr zCW{{q0!o%M!QjqBCl@rl9s8AyU5AxrGn9j+3Xx}CsQhKd{l{Dmk32KIeS%7_&Df3J zU6ZSj+|kK|6Gk*WJ^kAuOr=jrFnMj0= zlkda4mOoWEdiGOtGhIjvcCfxv@-qts;)5y!WS+foot%DR|G2rQfZk5`IDSlI{W~E8 z$abJ8z4c>mJk%E_BUB8uJQ(?Qd@Od4u&nBWn~BaeHk#`>o2JF(HtE1I$DMT-MDxpa z>3cT(wKvg~Y9FGP`_@vV&54V6vw*vFv+=Cp0>RP}*`9;@Q(qC1u4Aq6HGkR%VN*%l z(K@jlwPYmI0bRKLgdsl}RB22?U{P9~i~JEzrPD$z$Lp~)AQf;6-q+Y$W$s%$6(vub zY+Kw$FSu-$$!d>$NE{N{x(OV}ox#bPshQU2wi)+C^sbau-vlhDDKHOphGlQ3_|U1H z`ta|C@9#=p^wVS--lNby)bgRzSbx(vnOEz>12&#|G!Ry3nxbG~N z-y~D_;xaw=88Wq7As?SqRv7vtj`uWf)Nl%cE6zzgV_C?kisyi``p-{KsQ|JO!p_=aW242L8GES-Er``y-Z5_=Wf{w3T_#u~ebl|l!mMXckj(f`a$BfBIK@=T zw`?nXfyq#c^%*9Tc5JkoVp+MTcJy^QXgE$I8S!&xRa(6>i=r#9XquYF5hjob-%Uqi z4YGDmdPv#DHki-qyls4sRNA-?l70(QYsTd*)L<`q^vx4uguIZ($dbp;O;|D5Xs}=9{CJ4+c1~tNNQjfbK(R__ORPt+247@2wmyYD= zT=ntGL|)ftE2(n$C7ba7 zQ0@9DqZ{Fju#d|*1B#MHK|k9ucYAR2cRy79-O~?o&AM84L>?k++V#%7Mej?_YNnh9 ziF|&dRWId)g{g;1$BzN#ZWBOYh6#b>+_SasK%I${*ZjmI-)y=A-t7%M%G+)Je9bVB zj)q0!@E8#NoGM&UkB|PT5>CZU)&?GyyY$sZ|6s0^$ykOrXjiT9b%h?EOarOT$6*phOvF-z&H)ls{#3|(=B>-n!T@6g z>E6p6M#U8D3PPs-K#zMD`>Grgz113w{wtM=pskOHXUr8Xh!UaEOT|bHG=& z$eWg4@bGREUekHz|sQE5cMcoN?a^+jrMvxB!11O9Pr)wg*k^-M*MV-`V@1OyGa?;2q%2q35T; z{6%_c*rCG6j!H__t@+y12C#0QV$8j;-snc+vbSF|CFGL zW==u$!rV!`lG8_f6@F#+A$q4P^wqW0rY4DE5ieU&WF<*Azz+O+P`}VpyTOK4k9j=L zO0Zj5@Q!!H&k9S_Of%|q-!++Oc2rUN3AVSCeq747fusE8dD0rQ;67;v9@WNonP;4_ z+W*PpQtP=s0$JGj*o^{PNe5&yae#3#?4SNPdXfOe3MKYuKR@Nv8Ayn@W}NsGNqFAJ z&RhRJ3pF0JI*wb@RdqUhwoM~4w|K?sr3{lX2c)0AgEx`W)e!b>K{zl!#+{JM>k zj0f?g;-p#(c<`R7*h)z3zbFte+WN(f3vtEOFG$8l3HSMZ1^`C~y^>A^;qq1ebX`dI zR~BWx{6+dghBc9UmzPDQkWJc!=KC++Z|22Wzm}wHPL(;?PVWgOTLUS869hG}L9+OU za#_*vA*To;8ZNbS@_K5+Ljv>v->=Wuq$(@B@L{++c}i!|)A&6tVH>23DnZNo$u z7CXyFs%JwJu}Y}|qHuD{Io~K(ypbjD&Ek@B?BAloW}gG?drF+BF`9{nk`vm?d_%YB zbdOKJTXNe+f_g+#PfCPq!hdj~vjrdCc1O6_4vjyI6nX*Sb?Q%e4K>&gos04JfhH)T z21Vv0$=WTT)#gvgQl2Q4(b=*TFdoA_ zIFKbR0M3IF%JOa;*?32WK&lKIHRxzE2!W>q$aw)X;=svL{U`^pGu4%qPLCp@qmy`# zbKWggF!famSwkx*@on1|sYPJ6T!DKSnn7%-#e^jRE9*qP_dFLUJob7p##)_wFlto5 zHASUJq~JGdixR8;Zfaf-#Z`@=rB zI!wELd|b$?BaVW+<9<}OdPL2@!G?4G%QXNV(5#%?X|yT=&7Gws<5!Z6a3tr(Fl7S1 zR{~L45A1_Q=T~f-O;nFPb$7yOmz0&^A(|h^W)Ic@%gt__S@CW4mb-3@ zS7?2K06EU5X%P?7d48M~@OW)m0{@J@(|iDEIQeT7{0(o9Um9=v414QM@J2MP@eE^s z5g+d}Qd>Z7NY*GE0PG}C^NKM&0zi^O53&r|iyTfWugJzad4>^tt3>4Uf){bRrW=ww z2lTqIp(V$L;Qk*q&pW>ob!kUoy2F`OMga#T+(z{xQQo5S8rs0ASfNMVTI>u7iQR?N z)aTk+2}Fplfdp(z)Z#hlE4~^MyM0Xul$Dsf$vj&M^l41ikonlrKvZV8 z?d`k4)h8Tghrbg^TVs-T-J-sV=-^#D$eR;NHNi$Y-`JKn>P7)&B%3iMB9K+?^SCoK zC}^-z|A+d9X7A0ord+bh{9_}`?Dt*?-WrgUa+V)V0J!35h z?n5C(Jh9I(S^ugp)tEV}yMg|26o~Qp)pG82|g_BZGdA9J)oF$2uP-Rad=CkZ+mdBPnGP)J(0BY@ChVSb6B^?95rR=LCT?yX1LRu z74WT^HMdAuFx(NpcsXQn_5mX6!=XV>|Lh=roIU1-lnoIiyJjP{fR2`4|EvV6X+dtjvGB0D!=d zM4yUH4_F$vQOu#KHz?Be{quipd|J@Aj!_^=e6fjH&Z_BHq#3W!b2ojCl)|$(zk9Ss zK8ycO=4x46#tXkUk&8uKuDvG*0fOEfJG8LEyRA&DvlxfR)*yy=83_jzwp30+uVX~7 zcR;#&UGDys1rTDKDw9)rr&cue7Ej4?Q0<%w3mU z!>&v}7Yj-?XL)#|!%-!0{|Tpzjp50YU}V;s8D>B7 z)2WV5iN6m61Xr2uZ)Zt(e{LyR+QpQ8A2(|@Xq9@{<4{DP-?lql82V}Y_O_o6H3I(M zm6;ml5G_A&i<|vbNKskqw>4SVTzOO^op#VzS)>0T=3z)+C9^M?a4~VZ>mFy1I3*2Ia-7XI-`ph1-Ce)*+^^RCIj%7e%NKQe>H0k zg3(yn(Oa2wW^jn;hlr~x1+AqsO}L!UNZn(PD?7~QVXDvs*sIs4axY5|GIB+DO5v+& zvI#_+oJ4sFJ!!UnVJA&4k_>55{5b*5-s)R6YF~kYlikyPfV-hki3QO5UFrYYGyftv z(J!+adPxp(N@(ofL+gD_=S@R;HF0frOr+M?VZpX2`t|xwpdO*EEAg&nM9}n`a7|C=750l{G(xa3UdTvU6k z+45n;Toeeh0zcA7svoOyA-hq}s})cLrTE;;b_rfEa$)$da_E1}H1kSquU zPB1S1q;uY!uW-!cRa@n0OdZ&FniD#E2WRHq%EQQF%I+{u`&4YrW6bP zz4Y`szYfm9N1$ez{&(S3@)VG*=r4M_{CG~6k*6^I4U(fXCYxsfn3(qQ z2DdF7K~J6qY0mY2$zOT>I{4L;_JqAUxk##7>CQIO5ES#usD!Y>QS@xFwNbUS8wAPD zfWH2-&DBb&3YK-CqlP+K4hl!e=_0%jb+ndW^{oCl2)uo>Eyid_CoObXQup5zE(w&@ zA@xjx#^})T8A%aT!-eq4yQ{?pio3siF)sc(cMPlb{dDasxE|{qiMtlVm1`TocvzN_ zjj!$bwwq)qrX|*9?KB2>3;QB{Xn3bxoC8V>^Dtp?clS=l8>-kI>0(MQRbF|TO8dw( zB=a)3wHpBUnDHe`Wo(qqz!I9OJW0}gC2xXx?kD!WmwTUbk5!SkSD&oj?F}?gvT^n2862UXFp!9Hv;94vC1G<10%w4t*+ zZHsNcJ~A?BPpfG`oHo*w#F{UrK}-E&f!P8PUuSrNay)$VU12(9;lM2Cqc{E zD^(Aetgl++CEXn7MT4D;mOKxvZMEP=i_;x(%Jj`ZEBfsnv97Xqgw|9h@J60$Nz__D z8{U1aomRc}@m`$|WqX;0H7SPQi+Dtd zRseHt@yH747lVYamy(l{b(~V#tJiA&{>qickZ5y+VY)|8DVoI|p<*Z~d z^nrkzf(U4S!P}$J?e=FI?+=PbCkL zF1LDv0wt3)b?!IxRmAbg07pUky^gjFkB8-r<&DZ1luz6rV*G>57?kW|q z!N+h-F>O50Ts7xF)0alF&z+ZF*clfu&Xclus1Xu(&v2k`&#Fcosp9H!7Ce9%Y! zjuQW7XLSXrBl(M|7E^^;wo{=Q(xl8S;1$hqeNf_Z_{9)H(0`3ws8cjF`?{xNvY*Ik zrjEq8z~Q2v(ArIzqC%#jc4hyEy~UunDgN%yK*%7eedevVLMw)C$uj zvsz)MPj};P&99}@_w(+XVG6YbZmY+stWJV`*&!Fbdu4}sI!EQIj;A#Qsc9YZg>H16 z+SX)DQ=5Uh!}OXcTJBTq^t=((Od9e%Mr zoYr;+I;K{vz+bwg8?~!?I>v|a+P?tk`$QruWWVZNDW{kJ2;6FmFP_M9&h!)R3N{%J zY*HKx24hj1J&UoioTX~hTO5ohUoA2v-?4ZUZp@C1x5eLAdR2l}M>n?EHLJ9aN3e^( zs6V(@t*Tvovt&;DFq0{6Jd;6ra&)L0g2RjS`jWXDNC~-#6*}Gl{!b?RFa=Ct-w-A_d73MU)wG> zA*FA#J`s;dBFtRW;)KrxB@lWP2~r`VB?L%{t8LuKbjR|lc?mv%)f0uhF*b(qKAA5c z&f8`v{Nh~MXRtP2%cm(b+a`Q`er0`fqUgbKZmI+mS2(%qqH0)2c}9p*Z?pO1sd5fb z46GJ0pMTl6Cax>e@Oc^IF+$0moOVrWO{&t)cx4{z0*3K5~C_yMz zi=1uZ7)MUfU<|Z-2amq9jxH|Z0Qrqo?rA6x4v+N?$zQvg{ATd6K14hnTW-Pduo#wc zt8l_E8b0fVXtKp)L>ih(zTkX>H+;hio9vvvvSz>77g%Q1{^V_qJyq#(NB#H+B)HrI z$8&ob?F+pcs1~T0q1ElWZ3rbD($CK%{YPsWGPh>44(;)mtVUaF5qcvm=~xucyLMfs zjzuuFwQt0VIJG!agEy0}I^~_seb39kUb0JXmBAX?7d3*cd<{*DeE_>*yHDC-hI}j4 zjc?G7%l8-LumG0orexhRVz&gCxHC~T(mJS8uH z+4B%-^tD~5e|!Klq!Io0!T=kqh3HYW;Wxj5yKAA93yK3gGQdty{|Yvqnx#PI3`o+r zCqGUy8~DP0ao)B)oB;^MG;h9Lj8t1o&p6zZLWYvUf&RM1cdF6_6T2&IOQ~Fb+O+}) zBHjI2Mrvs+d{z5r6dAcL!$ZY(4|^Z#JWk22bQ+&vLe71(6H28M_zfL=BT(7*n}zs- zt?z~d?o&hiV|y?=2KTj$1y~I$RAian)ZnTl3zBPEJ@QsT0xKE)om#23rjF{z8Kus? z{*Z2Uzu%!Lnsf6BcNM{xuqV});ubTTbFo29=P@JDDAGwbtF11R=cn`zqOw8Zj-cX3 zF=%!BRdlm{yP`ci}C@&aa}C)3Q&kXk?Az-rn&$DV`GKL64=cNm;V@5POmLu7ohag(_W)cD}N`mWT^ zzKeCVv>LzFiyLO|s0OKVpt?4e(6Vs_`W1soo=Lfg0GfQ*H0XqBDTj6}VJLlweIap= z-yDgKad7np^9Ta}_|AK(R1aa@OaEsxl^%)OzHNbh;dPZFoe{OeIp@n91o7w}>c?c) z1?sFzz?YDhK93BIeYy@*-FtS(C_3Jp0w)nDIsC3=39MoSC2eXQ7lQqol!F~KtFnBI zwNNC=Yj3EW zQp^QpFh_d}uf2}t{w$RG=S*Tb0f(mb2=JhAbpe(xI4y`%dF#E+kV(cAgtR@vG9d}E zhN)8J*bJ0!7=W`&JSym3kZ8Bz0n@5WDHa-X;ei-#54u>CFZ~3vtjYrKj38j9w7`Rp zmz5WJ!RyWMv zW9cQ_%RUO!oWE0`viXsIxj(~2OK}{szUmhhWt(U`t22PiQ+uA^%W2f{IX5%@Lg$hU zJ*p>00q8bo$hA*I$q2)~oY+wH5$1-(25}}6h=*khw#n0N1HZCK3^?lp$?+O$=`h2m z4*Ue|*UJ;(Q)jDv^6p6kiWTGq+&|Mti1AL zpeN8X`v1jlf(5p-!`m)jf&Szs)X$ToC<33$k;w0}iP21zd8;-Xowx{V_WWW0Pt$#J zNda00I2iU>FB7q-7Gky4mdk`^xV2x_go+3fdjapu$*~1^)KLx_uL8df%!Q=`rs;A_ z{44WS*#vw1mKsHwoLZY@K-IMwI+nbIBeJQyJ2=;}3HJ9sGW9)n>tLJ$51$jPl^pDlL^IjHdRyrsHgMx4g4Gsuh+z!s zJv|aaima^8F&PnVf{z4WTEBLwb#7{1PF=*3w}tokhH;z|9x1Ppvo0yIY*U;h{OCGm zMx~G-#(OW~#XvNfwcgH+z?-79tv1{Q<0jpIwRlr@Qd!=K+(9)ja4x(}Qu#hGJ*i*R z4S6Q)!h0wsV2%L2L%i0XTE}si{32FCfY1|wB+UR4rLoUUs*9F!dn&^0)(&9ERCE)F z$^D@tFv0AN*OWqNj+M*OwY4gM3i;ck%-qme&L3Xkmd&>e=pg+ald*?cZ#>WH(%V|6 zsZSy(f^{?I`AcB$)Rxvd^q##i<5)GU2)QER@nq2Xd3JN_WC`ENIB+&22&}m}_g=X# zS=BqB=2<$2%n?n)DZK6_fkzBs{4eO`R}84#^!k=#XA8JtZ!3%2q?CnpCgOzm+%Dh) zpX-d2Q!$MQtFSOlW*z_FhRTq?RhGvg(dwXl0~2WFvJBP-#hiWK(pWjHKx;CsbEd=5 zq)8&Ww)OE(B{`Rcgwobq4XVrtupW2BqRd>ht$AO5XO`fJx(J<8D7^vz{6kFy`%LGz zEPTJVpgR(PQ22`fB>OVR#c#y$-|jQ&ZP`L6eTL?Czn z8lfRm3gACQK_=KI0cj6N>tFTzBV|LgLp~&p4^F=utM-LvuYH%N=HYl6Q__Ax`X+}} zx-udyV(MZpReas{n`J6j-R$kJggSR&hC+l;yslrr<;>p#9c=?9Q+hSDvKwf+Wq|xl z$O)zQcPQI>VF_gnY{c@bZg56k=U4)bw%B*GUULG?TQ6N{snE7R9I!hyg|c>`JqE;1!&ygGfOelo7dpP`O;KBD@B+Li zf{mOoLkI2%YDcZBPV_3P@PMASc3kT2UI9!VF)(?y5P}ANFKA}{|0sLwuqwN6Ynbk? zO{W4PDUzFRR0JdhX^@caPU!|wLZneZke2S222r}ZyX#vUeSYUT=Q`K>zTY4457)KX z_qx}bYm70+oU_K_OvQQ^n6_$4R4F__YT#$Nw8^+i|F+X~X+3w_w zc&BBD@?xX(YE{hzX|e5{Wie_r_N(M+GXaikH9>+^=if$sBgfczB^A^CQs>eSCUUa^dT%{iKdOH0%ti`y4 zX~z5VkTqK_Nk)O^V*<~s#y#Q7mE(u|m5WC02k?_Q`;Ter!Z7ak{Z~V?FRU!#x35DS z7bFH)l@}yVZGZjqf?mbNd;DA-e{uF~V(6Po^bTpHTO`buyc@|n zT%=QtbY1g30C#t`HRo1D-2cmV`T_27x_#c%-r0~Bv+|0E+3znFF-%l}vt8oCbAt^j zcytm-XB){vgQi{lA!HAEDaXpNuTV}6i^RgwW|u$-H%XuUx;!XI^$qn#G`QVi^ZOJ- zhlxeXg}%@hiJKEe6=ZN-dq?kIEXft5?q7W{=jBiU4O}yn`|OpnbdAi$>imBi2pWvQ z&4{pK@=;x2C&THj_F}d>u2h7+rFU)M`*$txb#2;(K_f%&y{q9adM@r%WvA? zxYVsK%^}_@$gp}?>>EvTX^nGJPr^XPn@#q?Dr2VQpIRT7bWZ5i1Y+Kir-$9DhJ(Gj zAR({PwP*;8Ls%(e{^|tlQNqtjpe#b})UMts@0`&zs7#5kfh>_3PPkp5ct05()D-fS z%IywiaC!CxIA1eb%@6t4_Ik|F$lR8~fd@O8-}%XiwYuAPW4yz{ILsh%OKnxvS#g@q zrNpA=p4e!P`ubK7Qt;{#>V=~fI#ZE|6ZuYq?aAvZ*PF)?q@u~;H<~IktxgqooUeNw znd7;6BGMX=Fo+*w=O|M8(Rm3r6+xfd#x^XC3J=z*aGOnh!BqV=-n3p^?S1?RYx`$x zu$OEUCGzh=4Y(dif53-i+*U==_kJDU1SNmiJQS;L(S&d33_2Edccq!&`-WT=Z|VDr zIIO9e+od|#bK`8@(R8qc^$Q-r2ax1nkFItsnjk?{U=44q3XOqJ|Tpx6X3QSoU{IBla=+UN#` z-EZ*w$(JfVk~{SNIaCd_&zgaj^{j~g0?;(3YOqV2`I6YO^jtBIN#}hxC{8aIVpvw8 zD~uTNsh==(`lNOQvUHoqMqfM0VS-w$^hM#f%?y{-71k$TiHKgO*%RkGL|bam*X#Zw z=q;PUe)8d4SETE9W%yvRi6*Il9Zm$r5ajo+V7t#PXk&23;`EH)em+(ucBbmY#dh^K zb_g*P#|H=5=Y6hn#_k~cV3t>w(m->Tob13gB$NQa{7yPh_{WYqk`;XFnzl1tjhqur zW1_p&Xz;V4tk|rd!*;VntwMeX1?AWrw`8%ypWKOpTila+rp^sZZ_ z+TQa%7=YfuS84T3!UY#f*|SwA6u|MHoS>Xz&t>+z=HdGXdVy|TX%a&=FPhsuiX_-* z__+I11iLSbZBEshwF|J0XheqCVjGUcB9%Pm@~y#ek5sMaJei!I88(3&X-qg?V@@;o3wx1(enXTPA zpBr^=GMAqXNx9i*>$4MST*rP>GcSfjQ6+gY$#5p91!K{ z@BT;@H#m#lnZ+y_u_AW z#@$F525z#w7h+gbA5Gq4(;pGSF@^ ztnz3JiPaYysMxzoGZhk|lB_gkpMeuNB=Tk3e0kuoRbdN@rVe&%q|ZaOCbLl}`MrX}9qXpA*$y2?ne}2s!woOiX&TGQ zHe=7-oG%}{_3E?Nm9P>`jjb%+pz2R%gbPjL&Jycid!ZYBVGpS4jnl1sf)0u&1-JHY zBZC)id+D*&Ed{%LDqVMoyuWyKWFFI@eCdm8F#j>dxe4NGRN!clItGjCH*6=0ilvG- zO7)LD+xtJ_lJg*AtjeLccJ$AkqRt7!xjIwXb*=^U{-AegriMPB!mF&E3_mK~%Ie%w zxj$?i?6KkZf!Zl|;cqa(lde&&R;rLo=<|6Zyk08*6gTsOPB4bu{aRcT=7A0V{54oI zA!XHsRCYvW03VMEH8Z(XhGcy$Wv7Qd%A<$<5$KXgN$5_Tt}-A#m9LfaLSN%_EHC^m z_~Y#jRU;*iWG5nmh{OqHwObbt#-h>9Wpyzw1ZS*zyRO)3(Hsa%D3B0lG^PRF9|H-K zAW(=jX^=2Bx^$XQSEMp}$&iXzGjiYto`=BexQ)5JW^A~`PDT^3U&cJpSbU2yP+p`h zg!+4jA0_40NFY{HndGXjzyB{k({1SEqE_UK&S)A94WBnBy_VkNAb+ zatNoKiIRknv^GSk9IQd3?!|iS8HPT2GRl@$QGOGNF`ytjFOXzv+cd#*OKR(5#5gR;5` zh&?Wpu)zLPraMPMn=J3v{d+s?WdJ~2FH^*`5!-hvL%7aT5Fk;d-A+U6G=cR|&1Ypv z*?E0RI#;bR$;3(HyT0%iA&E0MW;|b2_tP+93lZFeS3rJbHU3wtNUAMesN;YY`O&>l zM|9c?S9}(!g4CElZ(HZnS}5d~u{JREfP7Z*;mFhUCy_KfJdCjy^9JzgprPl zqnR*OE=^x^fEv_k3Wlg^qQ^TMCs+CVEPbX+8ZZTIp?Wq2V!#znf7>6` zjpNhC?qncKr{cKY`;+8g~9!Bqp~5NpzvxU@#W+TwUCEa9m=Arh;lP7>? zM)`wAXU-FH=c*wO8IJXeD(-+4AakQ@-(`eLVLa%Qo!xG~LgHh0yojXIn8w2` zEPDRG18fIORFu!|DWPjf_PXlwE;bKuR19;xyy+1iA^n^S z3%O*XIUwD}d;8!xJNzSPp(NGA{Xzu18oBGyd^Yt?P3c=;DAeWs;$GkfI$$L0WDg_> zl0;F?V9vJ&27^vV4vUeW_)!-+p27(Xcledgz6|vDaA}H=-lU_Yg_FVALb1_p3<`;* zm*}MSPnSyCtz`^^so@Ya{JOMw|iUk*$8inQQ0 zR$;WDK*KJ5AZ9;?-N+5Lt1(~K*5Gs?2r$yPCwGQCkihF@GP6=_TjM#;inTk7tlQw(-(xeL0rBjd zp@_Dk8OvvD^`dY(K z8~uwGrx-dFb;gb~iO7RM{gdIyyum(-Q^Ki5!OBS2gJLvQY>Y zFIjz;u`U5$0L?slHK8Vo6qtZD1}XIF;xkAuERzZVVItlML|jcNK|6!=&mox016l;@ z)P^kXP2lhb!~HZ8N6W=4ac?Qg^Z^D8YnM@nd!&`@iuU*A)z;=#9zff0*Mgo z#nE7++ckzeHYL@O&P;FU*h-W4SfvAIK+w;Cq@HwxNE^ZOsD!QyWxR;Fh>4j3Wpr~@<0^NYc!yKhTcy@S3( zqfnEwtm4RH8kOuwM0AJUJZwfaxai%lx{H4|k@~2Qp%Py7zWO}SsrET17X_}ni#f`WQQMMf$+q-`!wMC<7BlA6BNt*^ zmqv#}@~b9^;RGxGrv8_W#02a7ZwCn*oG+Dke$145--j;FAfA-kp2qd&3l-==#)>ah zL&6ugqy>X27af>C8MQ^U_j^_v87vRzUme|7;+K6w%Il0GI@(}3#JVi(!`VG_qH3(pBN1?+sPGy1+a-NTLOw6H~*X_P5Rgz1JZOtPsQeR zFh)nplZ)EOkHevE@iUL_eUUbIKrT*C+>OG{)fMH*)SJa~WDDgiX@9?hLQB2nCDZCn zRhACaHW(}q_4bIp*>6Q9g=TVWO_wC;IVQkBcE7~>{zBC^ikJSgbGlcSUhDQB^jklE zzA6x2+5DCF0F$t>Wg`MZ@5>WZkl9}Fa(o7=k^7Qb`iFfC@sZHS+N}xc94d;=+wBbI z10Zelf$l}R4NC*xH^wkoxNJmkW@h4U)|z#^S>f&=`Iy_~aIyV9z~odxk!RyC+5%_#=sI(b|2gp^SRgz8 zDmC^X5AUVSMx(A(iC{m=1#nazmva3w!UhNyTy59v9Uh zzC5OIkoVJMFn4u^IR8nu$EdmcD_+=(qOgmiuzE}YaW}ySNY9iF>+-?>%N+`0d&xFi zzLShj;sW-On)wxTe)4b+T`W7*iy{nFm}|2BQE~<~Bl}S=C76NHiY@iO!qA`FJMK|P zi@)zD@Ak0|M_N2EPa?(x$T4(Rl1BLJ*#H*(G{#}%g}ycnHU1O!$Xr4KrXK;&6~XT| zX>x~$VF`j%4+|srnK87GD8wNJO@j4~3cl~p*$}=;)vO7<1TMPo;jr^Am%V9Y@pVW8 zW(5QIlvx76(5*e1q30A2(lG3dvu`$kk^F_&$K>_Db7k)!c1KmZu9-j3-mPqp%T?82 zwPPybLk{ZV0Cji76HRN5RFQoI5L5fjFup&;{I4nnKsJ_0K(3o09q3B{O0piIsTn;% zQia*lc7wt#0m$A-5nlV_>HI@UGrz!WX$KMS6$7gty|_I#55Lo3lw%7MBJ-#%U^C@S zAnP#bxE;(P{x17L09J4j@#c65x}~V2Jac1h5@Q58oEs)lU`u-sznALcwA$m^?&u;# z3ynps#P5!Tp!XJ!l(WmeqgvW{i6g#=cV~rMo>{VO2z_;I%*wa`;Nz zq@XYCUVR#tYdgQwqa=Nik~~Ul{Iwj9&QNMxExA+(yq!nMFqW}%!;iPjKjgVn*>q0W z7Pv$AgnnAM?yY?|bum-y4qelc1|9;8tnK}qtbGexTfRg14sBG}|RooOn9?d5MLpgl5Tk@FBgGU-Qwhll4JvZ39Y ztq&B913l+RISmQgLMcQ8GCFZ~F)y91VK}@668f$sSKh&(CrFLcWNjdZU*3LA)RW#5 zL@8M86{_<=O})N`SUg2YB!pEo;WvW*IXku!zj-I z)QqN{^7mDQ=X4lC387+Q*9P95pwnFh4n2`z+xZsp9-R6)=!wWd8k8^iSIHj zs&Wj3O_OA+Ucw@NAUHpPqiMju^I+A(*$g3+fGM!hD+sx#5@6yC_Yy3uH`@aGL?aes z7{yo3_0XmrqfuIe%zJ}hl%1G$YlKuycBzUq7AO)MY^h#%z5if^ClpA&jM=>PnPR?2 zbsZq?G9NNTcU*Gp@0)zVdj$gmKRbahf)DN$Uja(v4nI!lyLB1;RpSMu!$P(7x#&kn`b!qo{7(RtOBqQyt6zQun9 z#R`rf;9+@qs!`nR-&pkWI}&Ih{4w}DghPe;fS9%8kUz*eb4}D`gL$gfA>3mS6Y{WC zc;Mt5pxx)Wy2*00?t5S|;?4atnlWq!)MHz&m0FYyIjUDGxP zLWz7v2=7_6@L!d0Hb=hukWwq+xFgaA1Xjo-AVdCyFK?FScO)+y19&E*P{?rWJE7M; z{y;+(Yu|9nFsWy}s^zoMZQXpyp?P+xzV@r+!L80;rPmiyoC5WtjG~;v27e9=`?Vof zTj=Es4?x zqNOwilWy;VD{V9z-zE5&7NMS)qu$;eVs$9$5m+jvyHE}|7mYc>I#lBTyo)aQGt!vP zdB;1`{*@rJBJGVT0SwTR&R%>twr6?T-R_WRdQQ4E(|`}MsS6Utj@3_! z4f_$0eoyWLqi}j6VxA`Qb&$dwq#Jv4G*Fl(A!YUkPP25`7xh^>yT*40 zjfU=Do9Lqt>4&u{cG1t=KgS~0mh|~0z(z)z{VIW|8Y&|sPVnY63(x0QUPyj^S1m!j z9>S!{S*gEr@8;R?(gTCGfblDtGu@PrEqxX*Y0b-XuOcq@_RG$`py-+hoBqNien@3W zFb#x{hKKK{`e&|CGr8+9ygqyi9!j{XKyWnI#C><~DSSQ38JYb8ObE~fLO>D_0_In< zM8W#5aDevEB}BsdzzZVjyIw+zq#B|jJpXb?6xMr^e~6Id>r)@3*N64v(ls|E>Ex{v z`W@M^N>S6aX}~4j<9l}~f<*$rr;3i%%ae$a;u|Dzv#|Xd*KvwK`=tG59ODFongP+pt>8%oF(!Fsi5C z@+zAnv#JX=$&LPZh7d23XC$UEaHP7i3og6|xfsL~aPTJH{Ew6)06@A~qvXeT0O&U< zWCYU+k2batyD&BWYZ$;Q805h~Kll>2z3ZO3s zg^KWRFsWe|n4o`)1-Xl(RRuRK_21VPp@mNUG!A^<`~U!^0p)DMs{mgPdOQ*cdCD#9 z;V@-DhuH{37}#WZK2tINcgNevezp|4GJ89<=~Sb z!)V7mNJmH6Jss+Ka*)VrFmY{itKnrT27W$M_rITTJ=)h2JYiuYcAm^3EqKBnoYTj+ z3;kMmE2U+Oq%0~V{pTrR3+E(10O-xTkJeueffwr^y*3Ypi08M}A2h-m>R72DhI@wh z2iu?0sq7Wrae2@cXF@LPMi5r41f%iluyg%?O)Y?SXL}#sK?CduVQXpu57(^a(C$9sVlu}3F)nO&;X?N;Ae}#?3 zR+GUyZdLg=7jrWg`v7}iESLFTt^bRP<&#sW7K!|&Vm8tjz^!gz*4S^m1RT1c^*_+F z$pjr~;M$)itolA9<$*gOVa5myPZ&a_<+E_R-nj1>$IdPlMR~t1F8L(YPcD{0@<-je ze*-Rrof05B;tQMo{2C8SU1|VX#LYlzHZ`M>Oa`7h` zic0j088s;I-ePLLkXqhG@4PAdOj&)<{Laav`2!Ut#Q_MIKr4q>jyaIG1mn^iIf1+{ zD7{Hi%GG$o7wgF35x=C|blW0%XH!9yY+W6nRiyLzeXn^dW-lC+jYy-Kjbvv1?B1QO zMh7Omw_GYDEx-bk{x?%0fvtWA7?qe<-^&w00KdmIxZJ;_EkDh!{pjBoNQX*qN9Rji zwbxz?=X!pV_4JGK_Zgr-U7t+9rM^F0gCzUDT$|)|6Bf|3;SC1sO zB#^<`{U&FlUW5R=T2nX{5RsZa423WyhbSQssXIhK-$_ayXi;uV=U)A4A z)zf!erK4rT`0(uX1Yu0wHS5)~KDb3<9&2{v(F+$iBNb!g5v_YDPWb{SPOw?dLnyz9 z?Z%jaMy(+{Wa;Dw8olsiPfPmirC&LBa^MoMqe!%VoNWmp>-l7ja1Nj;P<8#)w64Gq z6sQ4_Xi?a$lvGW=fSmT38+T0A-eHJ6<8WUKu$a2k>L!b+5Vnnh6{M2B z-o5$?=2OZYEml)vI^7yWN-Fb4_HeQCeJ{9+h4Nl+4rEy_$P)wG19~|8eRjp*QmB!jrNBIl@lhv zJnH}P0?a?jhL%Z|66Hx?H7WahJlYBq2fiM(@FTUSt1WMn;%F%k>zF`>rt*OdCMuu(?#lAVh4!%A;sxOW2`h%FBGsZ7 z&7W-3qY?S9-Eq&sKuJ>&+VJnl!CF2n4g5DovVdXT|7Y!EChW)|Ct9*7NlI&#D&PUq z9{hav2pi&NjPd#{upKpz$H|^M0D#0DZ7y_k8=P1y8~bnLmCe7|YwJ%@?_i(`h)=nq zOAvRC>0*gj-j6`gDBET(q7Ys$G3=bzt9LsDQ3Mm08)Acofi7EgUN!GE-!t-t&U8jX zDcu&G`9&tHjla=EvB`s}Mh}V!=k-N(s3pCyOs__B-Qk#wWj;7F6~EeYRMt`debYf8w;K(W z@H$U7WN9GXSFb`JWmZYEVLv@Xs)k2HPKN^!lMp!<9@J1w|7g?&iA~!KOMpuR?oh;M zki6&RcYhC(_v=F(S-%YaYD7aa)g_Y*$-DK$>%8UG(^Uve(bR7l0QmvVApM)}?HI17 zC;IeMfi7jd)9I??VOJ#In_qKSxyyVd1j6PEbWppT0RnBC* z*EN(;w7`dB2GnJGP}HjAWQ$l4ip3V+GCuPC%W$@gvk zt;6K}(-_wCggT9aO$6w!=f#=9*mp0)I7ZkdqiztgK9LMJX<~N+S0;6Syc^`_>=ZY+e$ldz$##&E4JKt@*O`KCi5#_Og5Vh~G%?q=4m9Uq$ zg0xeR!exID-a1tx_6cQU|6~~5At);IOOY)M5EKD6fyh3IjaNBgw_6^RzmptaIt=Op z2!?N)pADUt8i%~e&C(mNP^T*tviatgR{mER1}JldN@v6#9|p7e=&uKuq75_} z*{>wnk6ug;H|cICR`MZ0j7~6r3VsMXh;h;i;c?dl^cb0`@al*3MRS?UxwIi5gM6xN zagc7c3qOw)?vg>bqimY`qr|jRndnM3%WLw(_mu^DN#RF9JPm&W^QsQ-}`BDR?Iu|GiLGlt^z4NraAt*{1RRVYj{k?B*uA#Zg z4S1N5qQsfPg%k77vct!k8(P*0WyRjTeo$5H{LCQq$d2IxzM>K2r=YPn$gH}xs4}>l z*dOvo{kug+7H17c5WYC|O4b7pTJCHW4-_MY_+)3QcY7#}N*qlg8`{cLVC$k~w0F>O z(q~atd>XZtD~;oNyn+1efC335lrZUrw>hFNw-*Gwi4b;i(b#}~R@y2oUDb}|28f6- zZwBZB1Fj=EZ8J6Y_iX>s(HbhwUl_46rCyT zL~1!Lm=3@yG+|j`6i|J%jnbFl{r$z&$@Zh={NlYe1KXQkJL;V%(*)EbOU!{=Hxxn> zc1*_M==0sxj9-_}XBOxd*Q9rcYeX8cDSLLO^T;rzMOy|w;;#Qh-@_8p%?+C0*7{wv z)Pmp6ZA}7l7Ag zT;)Vk!z=H5;qEZ1P(EU|=9u<-pck?#1+%Q9xG%FQfB= zT$+#x=Tx0{AWf_o2Q=#2Ujv!%B0683twYkPZqrM0_EceHEu2rNp4M9-4aLCPL>uHsr^{KSoQ&9;c!flxN#_EsqJFH%qqXuSu)>*remXoR=S%ID2DTtub5v%(PV2R+7(~e zA}Cub?O7HPi?iH3qRbM8;^`ZaRQdO;`6ew@!y$ua<^m0F`CRvcE$3NY^_+$sWkv!`2N8;E8E&uw2fsN^&MVHmn zNDSj77bHi>fV~I4^BX{!i%yjOQ06|yVJ^!W#RL*q73WT#;zyzTSXA6p1WXFn*saPO zSb@GQvDVCZ&&EgrZkGa}#jM5e%>pA#KyS?kGOzAkhrZ^aERfT1PcQg_1IkTpb6!P; z0@BNuEXVgnEmut)X3i>M%@4bcUQ3E|puZ%RVt<&8^&CH@ z&nJpLWsV8S6W_16YH;l;2fUgcSZ@q6RO(Pi?pF#0efNgjZ8NnS{J@KW5U;aoRRjR( zaDO!Qx^pH}P81#0rqhqi#4?&%Sid8Jj6K=>Ncw9JPU1a(mYXs#mwIquAlfzN2Ip*K zR@_wE%<%?hTZH8_`F`lpnVWEer$=oOudMyq2Iit2){(* zhgUZ##+?iUxtQnzI>n;IILu21x>E>w|A;FbgCkd0q!-51R<`?+@gUa>k(O2vM~crF z$S{(@I6QC&BS;crqDU7f+xb$k*L5AkDWhVT=uoPm!uY!plRO277@Tt$J-i$^}t5FbFn+ z9(?07*^m91KQ$c{+soAl-Uz_k89ixFf+ao+fY}jzqAxt6TsAN!<~0WQSN+G5o2kn9 zN8_RakmogawH3%HLljy1_+Yk47b(G@T7%M1Hi~ygyJFE?^Y|gYGmGLqWRV7aEK)V` zUI&3-WJ6k(T%}z`*nZ*ZRpv#=EEoaEeRoYeyng{i$3-w$w)B5LtuC^_pw0z?jGhD> z2O3x}%JBRJ(YvMn#3jx)gbJ2cj($4yPbz!Vlvah!9`@)VOw&8;F4{@o9|IyDCD-|~ zSg8scSZhICvGb<-Rll+4b~*;^?SJ2@#qUvX2OG5>OlBn7c|N}*a(p^S2I(gGuhe#I zqR#hFq4h$%N4p?njA+FKyebxN%&XC00K!q_H&yJ+cZ-(_B=?C~RDHo{pjM8?D+1M< z?a9gvlfI9(8zT>L!sjnfcU#*+2@ST!N`@hQ0`^P3;O~+PZJ~je*Fft?9}HT9uqhclX4(Wz22%T~ zF{;6|&Z|M-6Fgf8=g6K9J#kuUq47S0e^qJSw|TXi=Hpu56)1Xrq@q;clfV&-h>Fux z{^ln?PRN4_;%dC}qGw5ue3RF@!b@EzODO+2x5_O4%@7TAV7cEhlOzP|_lB3Fm>m)r z?{rya%I|VjwzV$1)cR;|QUsk`i7TzA7jAv7&g^p3s0Hki7@sA=f#EDMGv#3XW^+87 z7Oniddo9L`3mAMjdsj0Fdfq=EIbe}ijls~&sW$@SWu~i{M1j{dK3DJ$RK5^_hnAUI z)2Otj!XkTgPlK0eJ0vA7jb^DY88Xx0)x-D1d3UA{QUVCryBDGp5Xc1cSFVh2cH&8N zk|;A@zy=r3kO(|Oa2U#zjuCJ$NyUYH0W&ritiX~9>2>v|Kp~HeyW1dTuI!{~WCHft z@129xyUz9(f!G>Uc!~V?t| zk44EJ;}N3tsUsW`UCHEju;iP1>2-OszdNWB>~*s@kC={%*ba4nRU9dLd&SKGE%4?i zM^Ecq1rt80kBHr`8m~{ZbmJ9-78A}qjx-WD^r*pEMg(KYmRrb#pRd7pUcDL4p4a{K zGe@yCl7bJeUYGQTuOazlD7uw@xFaj#cK8hUr90lk^7EVP%W8#i61MI@sC$U|w^!{} zLEBgAX__MyqEl-u6t2gb~5U*9t>5@5F1JebnqcIA1f!FxN-N5mi&0$_J2 zrsx0$xVcW;U&3m#mjA(8iZ!tXbd_vipDxIMK1Ts}J_v_4lNH~}i$yOvC5bAk<^)`J zC~-OT>Rh=XUqx@u7$85vvr#~%4p#?)!DJ@8A{GomqJ_0Lop=NHpT;+VNmcW|@?V0v zO$oGqQPfsM+stx9+~L&51*pQ=q~4LUM*RE@r~HbkLfVvBq_XgO#ronq)zW@oG71BA zMb!4q;ecqhtt`CxEH2RzCWL)S_qw=cV93L3dxB@in<(e{;@Cd`ozPy<0R$fdkeU^m zqT<6T7TV&rew+^1!D}!JX5D)1MB?PrL>Yv+TEcO)ntsv9xc*a9YQqIBU%R!gr%-H-x658t#q`A z(gIE#d?#tp2>lB4uw5%s{q-1=5Md||L$|;9Nvw9;X<&m;TUS=Fk2Bw0baL z_9H98U~MEOEm0onaD>5N6~El zG!wS}_=#*-c0?Cr}m6l&cnQ$=0F!PF)b|SD|^(ViI zc)FEv2WLO~8Q>_f{r0Dx6vyX`CQPaBG`KQOckQe#6gR2-8V!VYtj-W;h4{`es$Q} ziH}y<88vXW2Y`>zlP>prhsFxlO7RfX0T zNMcQ-U%Tg_U0@)`Z!Iym0It-ku89&O)KlsVKJ5CNHMN#Ohbw&vyGZnUMT6tkhRCX1 z*-uJKP)6-K2ALCsy+qXd<0^FDu|}WU)WtVXtE!}i!Z2mZi7O}w*`GCD?m($qqR`u~ z0mQ2wMc)p`RM8HZAP7U73TW=bFF>sKaOEOlBwAUMmu2OmvF4n+Z?BBy91SA1!!X->tIrT!zK$*mU72fr9AyKJMcEwrCO%LB^&ayy8j+L3`}MU5e=F&41vy2 zMpj7dN1YFrZ`D$v!+}?VNKmJ zl<`KD+X)(mP+B~#DluP_@Tl9~4)!sUSd^H2$d4agT!okQIfjCZ&V|l&&ZGnc^+ydpTQ|}w7k>fDdXn9vy9*4$g>Tq6 zQ9+BFZ059B;Fh2^!*2AdOqUHUmwAN!y}b#Rh6P8tHST*|qD+37qw0=Nd_{8nPdwx( zj5`%b4G?xO3#IHuGP?z0i$jcYzACSjJ0%Zh)%NJj(G^ulbu6NLYzZ&wsTW^q)fX=o z{qfg}-Y$Gv_4grPc7IN)*vm*w*)lSgbAxHS><->v6mAD}7P zbjBV1FnUQlBbzRfZ86TI$E*b-k&YA)0v~)Iw;nwcx0ydgT;NHb9LORwP*4hYgE~ai zZu1RV#TxRd>2FR{gzKCAea@#`t;5l90=186qzlX9v8ax|*69W+_ z&XgLfFIC$IQUnvb9H zV0%)od!%%3PjWuY=);UWFYN@7=g_(C)83R1Y+>QKDo-5ahLy7yH{svhbSE_iWC_?h zcKlV}m+Ahg7?6TmWy4C{|0p-sII#`!=#KdzM;p&_DL`i-k0EK6up_|F$cahJA#~~J z2(DS&p^gga$GJ*sSF6mRBcSjj8sXQ5NT2LL!7h3bC?0{AVv%q`T3e_R>oT5oelQus zJUp73wC+?g#L)ScW6^YOIXf&3->}#R`lZ%K&;RAFUQ_ZZ2mGy`$=B2}WKzxS@DsH= zK`PTpMMR?M2`ENems$15ntXY;0kTfhCS=jf{KPc=m57!Ex*@EBS2op=K~vINOD?;o zT_u0`H>aO9;d;g7w3Sofu?^9Z=DVf0tnv<$8)r$hYHuTa8y)zr&kyjYt~bdcTRA&B z{?M(})k;R62CEr0k3A2fbBzj=E4m2!5G$o52B6upq5>$vCQ zKVE>Z3ra$g_q)2=s@XD{ZVuax7|V+e(wOVGGCC=dtJuaxFwz8!#fQ@#C(K# z!S8ly4{ks2pFR6eJ+sL@JA!qS@{bzH;0rF?6C7w)`jCbhxc60sTY`<>>Fu+@#)W6> z{mL;m)m87&kLnL73|`Xz{=nxXpBr0y-xHtv!PNOC%d<(*m|Mqq^U^DDp6CN0o!B%Y zLTi|lY`YbDgN=aL!*t3n*J1Q7Apvb&L(8_gWjw=q9<3OD*KIBr*N@hgH$ja9n;gW z{i1^IWu}~{fUs1F)l-}x08X41E?Af#1cxwo947sL!{taV%`xGe;dhEjv4+XE*R zx&S!(!A2jS`}EUmo;F8Abbn4(#*i%B|Hsx_hDF(K@5543(nv^1gOqf42uOEIDxDHT zcOxANNOyM*9fAVV%}~-YbjSQ}e4f33`#p~L+kBas`@Ukud7f*n24x=j|HYg`pH>*C zxrr6peODX0)EFoGHL6IfpyDIE&!(Jy^WFB{5(yy`bLZ#f;xbaqN@GZK)H2N8rsC?x zsVRLyy+5rH{SVU}MO`N+rgr4Envy~&lQas8Y zI-=us^``x3P~BHtmT^P(mA~?iMw~AVhgt+2SH)A&9wj2xT= zdrS%YNT-?IR6b^~=#m%rFT`a$^iBEsa5fK0KzmDHpUv?63q=%mrf zCXH`k(yb4CgIR_6=ZvD6S<*7yUv}pXg&j}UTz8tz2cLhRM-_8;9_!3HPA}F z6=VN(YbV~7f)?TJYsfOJ!)e?$V_OBnos?H0KSI6@q~Gp(^k5Cd{=-rHTav?pV}#6| zEY(U}K{sO#+#5bX%gnB^xvdvN_0W8%aBE>C5V@J7x!siyc}Qgwg>yY^0EG8P$F5f2 zRM=L0Jwu}06bP?JVb|aZQS|G$HK-M?x%rWFY=2hwDQ>^g4%gn^L0G2_EM^HvsN7ik`zh+5Ld~U#aUbTyV)eX2H^Y~V(a&(*j@T7AlW^uyhPet4#`(rYwrRM?OI2q_2M#GWBPAvb^>`H00Pn@`MuXRL-tMUWd@}Jeeojr$t)`=!x!6{smwl!~k0Irt{}hWcb_+Xz%KA&-0P4n&YoJ5?GTfG^^Ogb| zYu_JTMh`g$(uT^>|DnE;uK#~ftP;k3(GdIV4a>v4mMa! z2Ex^WahnbYj`2-ci@VHA?e>o|DzX2yy`bF*=_ct>Wzh#j0Qz+SizsJJda*ZIj#EAE zHLvxCNnRhXJ&{h102q9Hb{5p?x-S8si|);Wj{qj$YrtUI0T2pe{gE%%ZZGy7|NOxC z#^*%U=6#bO8f2mEWKw!zI%2tL!>`T0q!set##%tm4IG?lyUckWa!3X!Q)Js-czH`E zQ--wbpvyO#`0|j;0pQdi->idCvs4UV#eyQ>y}8Lk zAt|gM(qQBvG@)SF{n==11hNC+c5UWY30xEWKy*LcT|KG;!++8i5Ohhm$-{in-38R) ze)}DK;hx@UHCgx_-R#GgpEu;f`-34&+Lee)mSU{Tdi4y(A^^;|b?F^Vc`_JM@IHCH zQw9fiFweO&UG{a+PxSmz(6|tge*hT+y%3$J7GOBN{9Z6LvH#6Sk@Hrd9l@yQ1u{j! z0+znaV+1PqTHoSE^2Ddx+?Bgjt|4D%!T9qznksA`th%BD4a-QnLttTpVbj>Q^o_H{ za({#2@u)IwppQAuaF*!G;hvDs@59J8?`MSHpIBG`fImxk;>6m;PW}vpJ;!);EKMKj za!zF;a-M1Cd3*i_7GW_1T*u$i0);Kk+daS~>Lx5q+Mx6|X$G>aVxr_(b?2mfK-@Rp z-6Che;LwIm_^VfKDB)i$`>FT?C~`@8H&beR3V^ibX`e?k_%ZlDZ>=Z8j(v0=CoeP4 z&kn#s{s25U$wIePuiX9rb4vwK$&K&NAu=n9f zijLbnE;@9cU@Hq{Ed%i9^;WurOZ(EO{)^#bq2;tHi6>11r!hla^j}TGok%v2JttXc zu1k36SnDSI^^ictwxwV4|A|z7d2`Y!2ggf37t0Ee%+XDEGsfEm+JD3MK0gmJd_Wc- zx+K%&iQl`#4xo>HE!lBEOzC;P!)6@(x?Ha{_^U?Uax7al2+O^y9@VNjo3qNh$! z4S8LptdTdOwC7Idd6oJWLTB2U$#LlAs-Ibt{&R+;4zBYmA!0)gC$hwdbNU0H?Wl*M zE+s;f{mW#R2YWv6^ae(6Ue>aq)_p=-^lNtOgeReeix7RZ+?H2yFC$iO+lf#k_##K! z{PJL4Xm~v3+_DOMVd5@`25dZLmSTngcPA<83VJLs{FidM5cY7j;!GJ+mEG%$&#Z%e zZ2!!g9OA3Ph>fzcp&0;bV&D*_VvW%$=?7BEcY?z zQ&WPh-_1JGNBN8wR8rw6z{Cx3F?K3^7=3GSnHN71H?{FWowXc`q9DH*T2MJ&R?^NojBNSq;UC^t7<+6T(1i`1Jar zr?ve#^%r=cuLBuCEr{sXvT)Sj^0EdW_QzZLG;T|s*-ZPEimH6skEp6(RTh@c5soF1 z226@p+bYv0AhP#biZgTM1O_=rU##)DG>O>e*&X1%-W`>JE)4D5{EM>{{SQ^9wVA^! z?3ISEkl%@3O7(ez!SUo0;WXCna@jSQll zhm_QWKDpoho-I|SlIyUrp4S~6=rfrHgJr4XmShTxoZn$E<&ecXn?H|1T4!ZFM=+5W z<6p8=^6P}266mL0_?ozzd^}8_8Q{zdxJmGZitmcpv$qXue_0O)oi*pT0PZLJduJYK zebEpv^WgdyCHpVp@GA0h`Fv3I8{q?i$N0!cQEj`1t3v(pC}_-6#aboX^xhp3!vt2y zp`3$GzA#?rVfJ~kf*yCkSA9N7V%8aS3lwkeWEK`auGFypfiJiu=3IsYi1+Z{Y6VrA z7mKnY>w`i86k;_HUETYr($He}n83MBX9RNl&-w1WuiMPbw3|*)Az;?YmdYo|IBMuy z(w)z34i_kp;(bFV?FmR+AR^Sz-D~^BMixMz(tr5$-AT6z=NXZ7=$C(MBY#Q$50$`h zGnOvmGa+R;okN`=%^Rso)6lz|ZZG||wLnS$ulsDCzFdRaf!b34*1}Sx3T??9*R!}K z-=sGxf|#2J-z1Q1UvWU_8tBgLWqD}V$Ajl6+5O~pqgd4dz|p4r9Ji?1#t#U;9z}py z77tv01>|A=M^1`aBEnancc+y~W!iOSKUcEq)rSKruxDW1M#d-V1LGB!qdu1xqv;8E z^k7IcTxwGL^bbw`*!c{R=I#iC@G57Q&?37E+(!ROt#4=vIX~@x9DYirw%dx+t`5Vv!na3{L z5TFIP0rxqL6GydIAP{E6>|U+-g#kR=S-(y6EADG>&l?>0T5O$GmlE7~z?OXtsnXg6 zYF3}bX~oH##Rhp)fVBq7R{s%%T8}N}G4SLWjXXfhy)1`=TjvuG zq!~S*pz*vHDS%|+ zI@9qxefl~0lhkb+!teg801;l}i<28Rt0LbjS%Z1|*Js~F12Loe+Pb2nT-oYn2{*mw zqcjY@&6H{m?WjirIIyg=F%ZzwOcYSf2#f8*KpINlnMUFl+> z{^bSkZXsKuOf*gWBQeiXxvWYJI8K)L_7LivDwYK@&YfhhDSd#ue#k-|GgM*o;_4zL z7nr-Evzev&wx4U)IPi*aA4Tz-3a>TprWXsqx+R(=+>l!{#vY%$30d)LYgZXo07B15 zRhB#ZzJKThfbcmXJo(ym|0!krKQzfd(IHNFs#NQp7F*OnhLoq{Vq~TqU{9a`qN1FQ zk_lefl7q#*eD%9>waX87I;r+pdHSP{G4%Ff8(G3{8FIu)^Aio|-4!e+7~1p8>`bDb zzWy`~|ENpKQ-(j=QYCVSZ%iUzyG$#Z^&@)GV=k92A)IajDMq2LN<1J#(xiH7rSq4T z_*af<2q>Rg;|@hz9Y?;hV*X}f98PV0wnAjtR89W9#aW}cn66(?LjCfe*vo)ayWl<0bc<#JgI=lrd)fX>1wicl zH$L(NL|YiVSmm|5?;hB;w!;1^Z-qpzL3skmMGVod&kx|*QU?rqPcB=q^~|yB=WtX{ zSkjbHNHI8Ds3%mjWmsoOdj{lD9p>qV!xGDa^~?!?7COyfKog5Xfx(d@n^ork|CXOw z)O`{+qxo5-NDTv)R}(G2@7ZWZZL&q#Q4#UW)gMP$yvN_CExg_#guAe){S|vw+8`Hm zNOV!40HVm=Px3(f(dge-tZ=H|#s9Jw{gAvpR5)6c;Qs$KAC=byI$Jr8n>xc07W|@s z%E{zBu8^$Y?`pjO$za|39dw2tgx&#HKgj+H#Jx2_HBsT>v}8*+W^v>Ul^g9BT2v=t z3F0~1@39geZDi6Hyz4jvJ><9r{7^AAezj(rmb`;9GG`t*pO_M*QD-K3NRRlDn!+od z5$w~UMShZ1rThK14EJe3{9m!HwRF5aNz(W{(=S71^tLHkuGIcSdx8^|NU&h03p1XP zco-EVgJexEKT1#15a_^Mclq3NqW3++L=m?k=AR~v&`p`GUu8mH5(>Eu;eM_c`ZNAP zN1vT;`$#Vt<)YeABwbql^9?oSDmVN)qkK<@co04?q#j#Rg%br%@M-q#laqq;VhRP{ z1VxV!B zkO3FcstOP`0Pa|dRPsSfN(^Wdukl3>N8ZAR>hX*_O5G2r68Sp;`-984c`)4C?|!E_ zLnf~Y=|dBOSiy+%Rf2|J{Z0(67EmXs+(72hp)-yWL5K)x-=zQKAm7YpH# zZT_-VARvxTHx->`8E>sIqyqLPh6AkAxChkc8_>9_b@^p`V(+YD;Ug|Ql*0HQ`8miy zT5gD}*E0DlFz)N4bbr&!w=t6V<)qM<2Ox#XYC)E6wNbtuE@Zk{wP!C%(K_!QQvAW1 zxHLAM)g+hyq47sO!!+@+*%VFskrnW#^6KE>VE=K#67!x3U43A>xT)M;Sn|_I={JCq zJhV}*Sum6T@%-u`d$EI>X6T1ur(1+%0B>MeY}Ujkv3cEhIl`AKxi?Y+QCc?!9up@V zU6#>X&mmK>K!Gs}W>eE4kDqqab^4}4tKb?G=$v11uvNN5%n%BSbxy{3{=kOGxK)aZ zFzXT!UDR*C57I4`4AnVk`0|12h5!2&Yw@W0hURTUt_B7OlHMyFfY+0BQBRR-Wynp6 z#YQ1{S}Xux93Ox`$r@*oI;{IU8(KZwr49fvdzigXZM%PP656=Z1-r8lqPg5Z+#%FS zv_&5FMkL=~g7jeYG@uYQm(-HC{yoo&Qs>m7)#iM zH001}5@hsEI$wX+?TQ$1hI3>)a-hpdPW-XDw5q@_a&R=hj}+v>esbBn4z z4kdAsBn|4NM4Y7!CuSym6*hJyEIM?ubr_n+$f3pCGgq#cM9ERD$(}#p_3Q(Tv`nv z=^BEUyeCs-vpR+v=*euPa~{p&%~fbYEV94sjRwwl=6zTfUe`p|GsV8`MmJYim?bv~ zC4>Hq4ZvNV;6M(~&**jPJc7`7W_77-%h`HuFY0z}`yIEkc34D8maFpCp& z>YLEb3Ws@*tWOznKHW0T5Lv*dBueJv@Lr@-YIHrc;%TY9#R1& z8kk)yC=~%~>y6E%L)4%G##{hNpK4bHJ`_4n8rXM8$yypy>j z8Bu?1B_1HRaNWzvht|gWiuwB7OJpIQQkUchyi5XM4y&;%;GgDwL-wzz|}*sU2eA4oG+KB0#6!Nr*j%FC{49Sg;S~05;he$r^!jmH6K^ z3i$ci9@=+jTp0s*!((+5&|Gg}qn zQ|xT%QX%FN8VNB1_}nNB-HRJ*{yx7BYmdFAcpR1fzg|D44Cq6I#$Tb1e_2Y)NmCc@ zpvsKkV0A?}0#)rm*+n}cS&PmD6mSrUA2wMGE0a{y?u~f_I8X1WC7yXb%9xVxdrkH4 z`?bsVDy%^}_}oiIJHq?{=aqK_353#4+>*;q{V@hKPQd8wSA4<#C)mK@jf_S~yI!h@wFleCfs+3vk0vFzd zTFmjemDMSIT`<542D&FC~IU=&1poW!?^d4y$pw@9~nRyaDRjY_51cFO0o1> zE&>#|Ka>=$vb4Ytxc%n`{<9w=E>7PGf_lq8MMgTP#)5U7s!hSW>&;^ow;QoVY<&*6 z*RKVUUL(L$_@rr!<+z$LsOCTOTiuD^Wb;12cZWS9^o5gS++OXW@Yh<@tYe=v1McpE zNWe1=d+Y_=xOAyG#&LXL1kAf{Y)Vm`@*m?gPNofHSFeV|qiIj`?jAd zE?p${?x9h3q4jVWJZw%iB};<>aK z&^uQme==8brMnnpx8XEP;>3UPG|_flN~vrrfL{M3iRHz^3{_(#yqz_Wz4)!NO!CnEwX))+Vclo_t zVRQ05WjQd0e|UTi*R7Yzp^C^b6nxv{kt;E5Ni)-4$0KaVjR=&x{|)2cNAWmgf`F9A z(+%>SL8ocDRCee^_%bkjNhy;bOC~1Ee0xl}H*oh4g?i}fQ6%he52JXt9-&;H2VJc| z!9LX9WBT2?SJi4>7%E%=@EBgmo$nv-_HXFvey48VPw8L(nWrxzlOugAbXF?!$y6AP zCY)>;tTZ_p>@SV|&xI~N943N3dWt)vGi2ng3l=nvzpWtG0#1dfT%sZh+=(Q7{;qncjf3Mpw!Q9I6w`LkahPO4 zs~rUF3d(~P816x#N&&54HA3J6RR0QU1j=cCkTVa%?C0PFZ7mZg__!}{j3*AyxAD`{ zcW-M^InL_vT|&^yhNb zK4T_L>@Ku4o6~&#Oh-?Cyj^F1><7RsI-8DRlH{DmW_23()ZcvLwt7)s;Wy;Bls{ix zU*G3%m@!Cp!BnhWX8qA2t^P2g@(`j@rbkPCKE{={DxBn6J8d70j7K;cqzzvKd;iQf zJjw#27N_FF1YBcJ|0EYd=uSa5%`p=n*K(@|ZwadO$^a7^8K-Os%+L?B7cTb*Ga<{N zGq=}cU5XI}w)^^_?xRvlvE(`RbUd*dZpfR2IRDi5Vn1(3X(%95@B`BK_x?tlW!;SG zI6B)En2Q{C)L$E6r##gP$z*P>?|K5o%y#nBQFIo`VuE72I6i3wi-<^4)X*Q0C>giE|GLK8-qbnZu~ zW$Nobf>(&1ai*zMoJPLF&JSsZ#9F&&iM`)c`08(bc=>lWB!ZVkW26pKdV2U@_J|4x z!6MzN%NDSBX!ZPKu4WHjF*;U_rAD*gpUETI?v2i5_#j{jdl7(~aQIkTy?bQ2uF|3x zx(g(%T&x#Kt<;KoqQ1f;J59(fXKPUn_BCW~(iCexUP>h~XrX|PL?yxuC956I-sdYO z{>nc4^gh|9PR_M4%I{2;Hyix+#%xsDPss1}%XhT}4ex?DB3AVTaHdGK)Dx0-%HsSx zaE~$2f|AlTeC`s*hU9Z#nQf>h0uH?69*^QUHIXislQhQ|C<6%(9W`5WA1&Z!VJ;tk zpYx#lc=lb7D8Rp;^{bT-IEpAM5%(O6Pep-U(C87>(u2omh6`Q?LLe z0fn)dUovfmzw_$y$;N81PAN;dadHjz!3SXzlcO(Ly&T;hODv}CO1~FOVV7<4do>?7 zsXIDMo$Zqg5cwTbvH!41L=+*_Eno>rVG{j{LkM+_M}84-_k%i+lz^{3O?gi>!)=m1 zPL&-6Yw1tUZQEf2gR8nq(f-dyQ|;hRzhM6gNx5!pHD+M6v>CPwq7>)Ei{@G_YqJ>VuUrAHP*>R?y@%50a zI=-4FzL!2e@glmsZ&LNSVh zCxK2R4;z`kz#~z*mvBF@C4ZAc?wT>b@+Z*6)m*+d#7>6?=N*_j7Lka&4!+1q)JHIc@kvN4{W#t^cQhKZym&Tg79Rg+}_+1@X(~}Nmnam zqOKuk`zgLb-bRhZeuTo^6L9%YuKpQLr2KNPe+VUau_aG^yDqWf%O7;zbx4gZPO*~H zU#>$bd=3?#*mB~l*Z;A~XS|O+H%$&VMB^u&j_+A)9M2_uun*>BYWcLVR9z*Qwi$JO zK@2DIfPMK$Szeb+S8epY0?#u}GI8bmLSNuA5>J0xpTPWZX=V+Z?U1X~b<^xY*I>@3 zkQ_l64-|?DOlPI(;huc&e$n0$_t>eNGiT+TNM}>;0QIs254=79guYQ~TB(wbd_M4tnuP7`^3gVsX605f$X=Lzj zF7XV-#|4hOWRpqO>fC>q!|7# z^p8nK9_g80br~nx4F6CfLY&cyxn9vuvFoq_l$I9s--e(Fk{fgtokC=4E~{m^D}7lg zNE`aIl6egO5lVTV(&%3|a0sk^4~*oVfIC!kkF=WVZ~?!#x03fhmC8H9ON4*e>6K%+ zyXt*(+@hEGhqH{q)|zTX{M3vsblO?6on&NB(Tvs_b+)VY5AsO6{pq!B_+3Gx6{dc% zC>X(YcNki9#^@0sv0IbmJa}8QnTYy(`I=DoBZgCZrTm@5V#yZV)ep#Rx0%awvo0V) z*34fSUgMmb@5S~M8IP_ET5#DFM%_1a%O3OEFGV&;`WcmkgSpznizO{;zPlA0{enwI zo9mqy-z>Pglqa$mLho4~f@^-HVL+=cQZ@SeG#$Q=y`MYeh_|gnbNg zS;o^gEGP)>U+>K4?=gfSpd(MCev;B*1B)JTZ6JbqNo50nsD8E9d1#zq1KF%ADME60 z$v!QfX8aVs8|Oz6;6YA_m@M{H#5C!V zV@B}oGS+Vt4zMu|pl98mYOPel->~8@q8#|SHp;-mx<`iXnuEBu9d6rq@ra3V8g69t z4J;DvDuysTUTaP0L+_O9S*9A{V&kNNxvBvD&t^+kZkFmc4}#atY3i)WfIwj9j1pnVL?A zJY4P~Y>#VyTNG%>MYi1;fhMWXUd>Ttz&p*ArNdglWo=YMM2TLj9fs1YW46!JhgRkH zQ(rE5+#|qm>Zfi8Nk+|=gW+tz@x_PzgWez*#0R$qHk&_DUB*xIi_Ga&iSWh@d82PJ zlF<4h(`RH~A?C>q<7%?z*{Esp9MSs$!L3t!^)_O`sT!K`m0cZ)$9b9Zy-DJHWHly% z7IPEziD)v?YbDkwDKhPDMoDpU)gP%cBp}f8NW^jm?#pi>i0)XQrl0m|#B`yb;$wpU zRJuma(M7tf-9=Vw1NNz>^o8nc9ST0z+?3DH7hv7)&=))#uIPTu!mCjr8l?eT_!F*+ zZ&#_C#7lgrLDw=>lHZau0^5+`yhj8w4pE9NxLR5HxD{z9}?AzG`sw^ZBypnA7LiX*$7H zzC#eTS@uI2c9=9(^lGPSvYU-REoBR}Sh&vZWDUm!;rFCqZ8K z&5Z7Uo(&q>O%9%;Aj|3Q^D^o28{G`SZKe(Q6W$aA41nzu&geS!#l?=9vXF#EhAqvB z!%vY}I1`CM`@<50UZUH0ebz}Nz8taW>G}Pp_AciNliL4$++&J%UG+pz&swSr>po2< zQoQ-4(=3(%A5zZ7tyJ?7y?IU(W&Fe8;2H)Ap4|CMI7Y!&8f@Q}Oo|9T?=P|TNp})D z*Azi-;1*?9o3TIFDg};?BOrnGdfoepj+P43uQHoSqP)NfC>k2LZ{X~BOM3d%naL^P zAhZ-0bedP0;Rzb>3wxpsQ}!b0@4oV9<)OywO(`oXqSM1eX2Y4hMsJLCRQNBB{BC8k z1GbU3)-!{37K>cvLf89fGh4gL{9qOZQFr-bKbO`GTXkv*-@+B_i|SHlT=h+BP(aSV zmeo_oz3M19%qUiTjT*iKy_J1ylCn8{wP3Fn5@pe2plmudAuG0)0Dr0^d1+`{@a}9uXxh1<|E?XWp%&cq{@0_%_Mwvh z!R-P0=iaRe)xY?lXmru?(#6KpR*Agh!w#nQ53H@HB)0r@r#$rADk&_&l; z{b7fTA19>yjnhl(bXb=c|IN2pRBlksuVxo-!US5){x4&6WTLL*^^fEkc|W4Y3S(%Q zcvmqz3hnN{buZvYaXWN#a65gR^-OfP$k1M~LD;?2N&%KPA(@6zt{6AYGw~OC9;)JU z=$Kzx04K7SUw?_8#?|g76r2m55w`GF!i!;6RWx{afkkKi&T z;lyJ3d)6zi1LZ5%l|04d=C7G!T2(hO+PhDgNCs%qyklsvO@@vcTFBq|9-Z{+puf{{ zJO|6mt4%IQue!FjCxDus-TeNAt$q{He4Y6W`h*3Gj#A)uaLui&BK(BrTiU*4_Y2|n z#K&{jZ{r3+ggAtwYIMM`gEUh3mMN&GXY&2H9jNy#+dYJW-RLXU~SHJS4# zUpmBRRu3YTwPTAwcmoxGLy-p<2ty*xuCeF^B*PZVJQx=x?i-k?OCrgO4gG z0-zJdp73PY-Bs3LGBs1%J1oL>vD*J`i4Xt3bXmXuM1R>cY$HS(1N_t)Z4h}BG)8z>fif@#J&W)r=T z!@rOf3N)I1-P-6Nx3uG@KQVi4I{-nBW$SlFED!H@oZpLfzdz?US0Eap5U#1iGgl;+ zHzf$|o&g3Xi=m0z@It#Pqle-IKB1X!Hc<%D?>#EP3=2Q1PF3h4QBqPCL-G;eV2=#C zB2XYKbOox+KoPQY{Eboj^`1D445V6~%wbnaJ=G}77JHlM{XM9VssoMng3$bYHHUg_LP0`sydT}Bl53J<+%yhq;EW)|To zCnzv)H>AUi4}w$qntc>EI)&*+yO&v*PVTVZDEz^;ViT0Qy*(5Zq8t4&1Zy|fmfN{1 z_ZwBReXuSJpl(-T^8->I$cwL4kiPZJ{rR*GsYem#bt3QU>xlBh%Y(6*YZl#W@8;rFa% z(U0nqt)p*5Pi+@sEqm1@1zhcljtVB5#wX)Jqtic=5x?YFT{Xip18M!VJ5!;y4nLRH z%!1B+DIjl6VG8gQ(yM%SDc?Vzp7!uo^?%@}#uHXGU2Wd0 zv&yKq;`TH z^Yk-Y+wqiK4EW*#*K_#0XW~MYjLc`0I<%lWH|f#ecGS-O$h$VY_Vi)0%df&@jiE>F z&!Yq6Zze32nAdZ1a)#@I13l|PUGvV3XFGcNT<*D2hBJfZI0>m9jI31ByA)gI!0IjzZlj~~5qU0}AEGtSywybvl6PDkG7p7<$;R4H9hFrv3j z9-{(N6)vUE&YYZV8Xw?`N@)eW*3h1#;j>O1x5n!viu&FvEJ2WKZMgsp(=WeFDCFZ_ zitk3Bt~s~CkuW`4J|Gyt#<12lgKl!E>KtETiA8SwB+751=q#FP6U7TTe=X|G?w!5o z<#+W+4GeV(&_S8ak2?@FaPr@`{`o&6eyB~<_}V-?ThZo<%hbA-0f(< zZeA2Y_i>uVGYy#DWvVl!^%4jZ+!K~$G}nUhYX`ots=|2TC&?1`6jX;(5+!3 z?AN92ln-b5=# zZPfONC@?7p~x2XB?(;L_}LoDzbr z9mMsS{tRKru9jaUxMtCE$hRUEVM@IWE$CJi^d@cL4jkZ6pR;WnethGz zXE=MF%{n@j{P<)3;l;$P-9i8)J%y8S3+GQEl`0kTmX^a#A8@oNAI3I~ndc5kDi%@4 z`_lV7`#??T0&-hR4a^ zb2$0V^#jOCJfs9uqqqJRmB;g~&-0eAN{IsNcjXzSepuE?!P1{!Um4L{Q7ZoQalu70 zJVWIf5wIfmZ1SnvcHs5+uu0|0p92eioHSQ_^8uEMn15!!P@hWVx;%zF5K#ay2nbj* zdcS)+JEgDF1*?C{KtZ8llsm^S#=N<73xEDBpCyP$kigglaMGI5iiypj_A=!1+T@r6 zG_3rv_7-{O6LK*Mh@)OtqqWf-zJ~5jAf}=uKu(0CP&#OaKJV#RbJu}=Kw{UOp!W5Z zT5VjbR!B!bKLWsR9p8nl@HjC!>{_t;kQFAmb)XdO?$VtoO zH!O~%WPC_-^EBCuK{CoXN;9C!gL$@MWkOSF=B|z0P#1mE8gkjype^~YJQ(N`?^w3* zyR5SLJ+;%9eF|n;GdF%Vq-ip7Q0(wKa#_c9dPTnqCq8R;Ivqy(Ok^BktIJXA1=;Iz za?+iU7U_giFM5sP5nm@E-x%x46>nM;CI?{YoVuO1%s zwb^AwJx@t((p#Ss4qx%28x+N~+70$oU(mTmj=ud4zPrO%2|ida;dfw{<}JMHGfFHz zB+(uoDC0dEgi|5>>|=_TKr0_;(uc6geln~gdAl4W^OJI70)`n+P$!wMDtVjjoMm#& zq-aeb8bFTLa4_{1F{msyMUg~GpNFe>y+(aoB}d_>>}rv=;5$6a#=557#)c?eVr!(g z+NFG6GeG;+*A?JP3+P!zI`0>piQNpB=>QGht=r5uJ^~sgE4X=buvc+GEq+Gr;*uPV zcKaz^UrCDrS^!4EazU4TiwUVok@#&eH-hC|AToQLm1mgAo zk9P*LE-Oyn8Kk=X)cjvFAdIXCLR|D|&S3I8M>v`-)76 z9Ceql$^Zt07s(|v zD0X^W5?S_#x?fk?Vc||RL=c)L1*MC}4iy37C2ai=Bsn zJU`HWZ|fPWlh=298S*>^FOD3>tFfCtbuDB3ia&qGFD*`qw%Tz;xm@U;>75CU(#ME` z2Q@bAqVYxN#JjyTt$}&)bDtdb5|&& zttjC_0MP0$DI6E@CvcQCEJXeN9KID%%SS42xjy639>>epDE|hCiF!Sb?fb@$Su%*o z*gdq$?QtBc(EI@txmdgjPD1gn+hxaBUWcH&K21muJ8{5){;|;_hTBVUH^&DxQqN{N<-=lndb_@Gx84-1LL@OzC!>kk;(s9gg|k@2HnFXXM1)(OnyR&iDS z$}=XoYrG@#pZaT+0-sf0CW6;1>Ac1&+pb@#L{m?VNK2FE0d1wKvuX?w=ci zf?-&E^|~~u`HM+bK}Lx~-5uS0pNYo4=^PsMI4IezPjnztHeFHN|52YHXz7$OK(boQ z8D-@da<>IdOsgn$g3{v1U0is2Rsj8}bnx?DKRM1V+@mqm#_o<>_zny@y?G{rIMzG- zZwAN!vBr}=)gbc_998tQ0&(1gR_-Z?r2Y9mtAj`aR+0y{x!gx}(gdaFw%R;eq|I~l z?`+Fs#a+v6RJNMuf<_lLoTTCp+_IHh`;#uc3d9#acCP5o%8=)l$NA&_56)N*_Bsw} z7_KEfx$rw!Zqb|p%+=9viWCUCC?9doRV+?{IqHzc{LhDTVpaHoeh|HI2mM^nV|zeX zduBKa)m21zVSQe}pGYnVfFnfFk;$~qUEpL!)Z!na4i+NZmdf-&7is5Oz_aDjF|dt` z8KXdik_L@!e+}RFHT?_{dC8nu807Ou?)`OUxX8C}i1kh&UM&|89-c|yK5JFS?DL6T zI?ZNxy5<6!=gc{X`OEpIJ;lM2Y!Rg|3Sg$eR=BdiA7aHmb5+w}5NZ1@A-6r;-nBcA zobH7ma`}JXO5focjf8un$_K8H8`N`All9mB7lm(2=UJAM3%`*F=k*V)#4Uz#KDC1xqn*cOlBvynO-sL0^!O~5@vGMmw=04FruPpleB)+2;2aP{pZYQH3 z`SS9BKn=!Q2e%Qf1AfQ}dD{LA!RlhMET)1ZiF4h!q7ipsbMjaoIn_=Ls>H;Wo$5F- z@BcfA9?4qXiScowSg>OvIQ_7iyK|gAMu&`cW)IfiQBoPaON)DL$1LyzsawCec=E$S zLvsiG$G@>Vad}`K^4ww)4`ec_)4!T;u`MzEwjq)voi{@}V%QugoyTg_%GY{H7U0=X z`u_-f>!>KhuI*b=LO|(mq(P(D z*`30f9-W{3xm88R*un1Px*U|s|ZP@2Wk;vBThnKYp_ki7LbG*_uZZ7senU2`pkAlZI z=*RFdfN2PL>|XJ_<248fn$v$teswrbLHYzCsJY<<=_-DB@p_mtxJn%F{Pnl=~&ct8@i?aWiL% zV3NYSH>F%~K(hJ%Uh_`J%(|}SgqQbW)Fo%##oKZ>zZm|U3lmW1D>guSwJ)Zuna_*gwBCU^Pu%*@Q|!O?aOt=iX#i0HC5=8fH4*W(cPllA9by>+@Z z*guRGQrie5Jf_}cNYMn&1KL?Y>4;VzLPGVUAY3A38d`C82DqGfLynWNTmOR;C@rs6vsP}h)qE!eI zdZTz}AlNB>j$V8>FF74w~=EVUk@?9 z8jr7ZgI4JdVQtupbZ3^VX+}4WFE#p5Wb16cIafU3cp6^O21sYQuhY^g)jHd;5Ca)d zhQ0tfdlRWPqFs@LAD1fAcZy<@#~Xznq2#C)zx7Uw%= zKo9)({yo9Rks-#tlEu0USb$P)6^ce+?AmSeR8JT--pYUL)4emnez@c1^Vby1F(kGk zNykiVR}xOSe3E=4NZ}FYgmc%o|D%q;j07u4$H)S%*@wi-DgFSwp)C(JuXe`B?zmv6 zaW7*s1&|WO^7X|s?=O}I7Uk#JQ4=NFPX|2E(0kGZ3)wYGJ)b^ijBO<_ZYNbFKoT^l z`#gI1@KZSEpea|PqSBjhg`AH;BdJVuA(3exNxXdCkZGFiN6}5i$e_N_*@qQ~5tIep zT>WTQqkQU#PCZ#gwZAA%cL0b9RPy5|>VnbF&BX-lgis)Kd1k z!}wnvQ%2u%_i(f3 zmfax@fDr2;>d>qn&%s`W`LWy7>~eRuSpcJ#m+hhYLN$9ck;OvP&xN+FPJ!PJE{7R^wsdhfo>4B63r)Ei6$d$vRqr*PYxMMmldHCb&S6Twn*pG@fYLpw8fdyv$ zGxJf9Vsx()0jS7v7l3gs^lZ~t?PFvw58G(sxU8I$&94t0(d#T56^T*DGNeoT=(TCI z_>9PGx*=su<|l1yryr~qX5)Gh(gyVw2XG`x5e1z9KUd(~E8TIaDaIF&cTvm99I=b_ zbJFkfz*EJ1122hq{#;|k_I(SGK}FSE{q*r8>x3c&A2_P*yp$D7I#pgpb;R9kll(c8 z@oHtYWaNEVHLe1>Hdx!EV+&WIeaGpjKN#9UV-7k0_}E;)j0w&3zr{m#;}I9H>0?l$ zKqe+&ahxNYMI@qdF2Y8=LLc3nCW&;j4%ugQJT8ZCB-E<2>Aq7173DU%tq}rPYDO62 zV^EQ%alQqOD&yD!XRP+jAE332WDukbkdd+ZRl?vR$}f#{k2y&V;kBoDN^$6#;kLXN zTxK8wern{3jEQzge=4op(lT9a^hkHN?hW6v(9uB*>5Ix3p9lbzqHrqUiegwG@NcG+_c|_FmgH3s*}WwA}D-u{LQ3-t3BB zB{qH1Ib2JX8dW&-A*K5-TZ6?szu>`EQ0@)vLK!hohz0oIo1c@8re=!xFVWxr4&9Z` zP`RVCIw$DXLgtT2CA&-4lC>U_Va~Tbn26dJt|aDwdW4mlpZvxXnu%V>1$dq0h)nG0 zjHXVS&`)#EDh9qSMhAFDzc>8yX8+xwaWgft>+0*sq{bJmCprK0hu8M@U6TB-7{Q5r zet_FFb@rwP#?GSArVzTBY~J{`RC^dIv46^$c3vI$`yg6rlH#;5#a>m zlrvstrZ%||rfubW*^9lRimMWdWU-EszaBcIw*1}7^A3hgH=UwJVo@=&o-vPmaPOtZ zZhiQ9#{DqXk(&EocVmhfEg02(W6kfek}B$8qjK=7DNR_#C^NJOZF8=?pp8k-MyIO=u}uLE%rqMFaf z1q1lv`uvM;;y3ZV+yWFDia51K9KH;jDZ)`3m8ygwB0@r$bF};?_Has8m2`Q>N0fEf z&m;8SjGsw7=$Ulf?(v6scC-Q3SYNhq!r~f$BLM64u0qr4H4XSe?-ZddaA$7nM&_RE zeKJ95)XaU4uiRpFCZ~uU=H$S7tMjk%!LY@FLb1xLv zl0VQyA5|}tQ(NyY^tro7m>$Rv>I+IFmT@V&k0{mN7?mPNQ!mMd0LFLvq zdU~e$i_P%*GoNd$OcM6;U0Yd-{!`xZcGG_rjeh3$g)DvT{{CM4rPfRLm+XwdpPIME z|90u0y}nXKm~=Oj;eWAiA^OhpUF-8JtOqVR;W<+6mIg~_tRtcZ z21ai>pJ*hOXkyKYR0fWuvwf+rR;;IO`gvS3@Y>43p%#k;k~bz_0hRmjODOY)v`y#n zkbwx6T7w?O4s&xZ9OHtfP2F(oSOFFnDF5Dr01Oe=H8^9l5r{`&wGzdHIv^f7V=FKIdufP3DgLo#tQn6!UimLiOe{Q# z-hG;sug-_>2hG1?>0!1?D4xjN$aX&@rSy+sShN`J!)^3A#v7)U`q(ZvuayUSOe+Q@ zijE#mec^~RKkF(twbv9`p9n;$+JvVl?7xJ~LnP7@xNuT8z#r13(E!v%K+Lafb&(n^ zPoAU0)htBv&$ZsGs<|xw>p5+eRnE-@3T_k85xe@lKjMX3450I&r?Nk*lU~b}&-KRa zb=7*5%^bwg&j;)D5|cPo??8Au0Y6DJy_V0m`W(UUgh` zOsn(C!og9`x+WeB{Ob$tajWQU$jMVO3u|x;%Y8Qs)St;Q#)Sd{J(Jd}=xg8MSJ0W+ zU-{GiT6;?!-hRaAZusvwi~OWqR-dTTE`+9x&|A)Zi{Kf#78i9on|QQ0QybFQqrud< zDBX{Utl>jLD3a7I5&-tV9g@gi*{ZB^@QauX#DxaT4Yaik&prRU1o(6OA=m3Eqsl53 zccz}REbzruZOZ0Pddg1JN)bvzoAh^2`~l+R#ytW>_QiIDY7;40Y)$j*xwz%tkEYR& zClS^fum7Z?@#L^Lr2scD;+DRu3T0SV_XL5S1VNW_q%`Xwe6%eWIF1BdZ`P3GgRZ`t zu(2-X5@!ld!vb)WgsbkO$l;gFTfMyE7k( z%l?33s4kSydk6puuif|h0k;Y_WLxr{t7(SiRpF$;nncVuk68%S6JHcD_Fj)NIy{Qa z_ImDL08j+1otS@ep6CQp6ghNJfCCNAg6_+_w=XeeRR*~Pe2>(ItH*H~z#bT8`}C)} za+^}pcs!88)tqr37ptqZDXD3G7wmjpFy-eg>*>9EiP|9X*5j9{3dsMp;Nqz-0bNA5 zS;({YRv^1>@`osBXe18*&zl5RhV?i5vsZQ(n@NGhj9Cp;h3Oj7%_YN`-q)Ld_oF0i z1OSt@3v(Dn^WCxbLsrAwu_ienK7@iN(?6g%fXJp$B#Hj0%aWvW z%Si|hiSUr|AtflGxAWRJA->x%?vX-!QM6X zU5m4Wc9ers4A|{sgta#@aSn~ne`?DB$H`FRPRut8_Jsd? zL}p|7P^g&_CpW{Lw6aE*B?%A(Jv0GcFWfVpx=UVR(|@U$nOxqtnY(NCS~Ex-t{9Ip zitaF1H8S{+%hWw!>m5o<@=V}5Y4hEcw!4#)&+T<(Itatzhsv+BM+|?=r5y)gvHppUw#_u^o7S>Ky~sLkOS^-LgA+w;13Oxk?06AP%;PXAVG+*oj+$!jH!2VgY~TK6RK zgIt0*4QWqE2|yy<#j^*=-R6FM1Z?gazPJ^&Ko=w3T&)-TxA4J)aobA*D;vm79fTkG zgxUwOU;`X3Iv=1EvGhIxzrTxuB^TK^{(lRo^*{&!TpQm9Vu6!Mf=7r{!KI*ce@p7~ z1LreR0I|-*%<{TE`!k_|*TdOnc})F`)meTYrN-uIT>`c0c|glvcaty~Kg{i+ zgF_2cU{KIjUE^^j#eeE79NMCmJ^7!Fcj#~u;4>hRA?B`cEbZK>?8gI5Vk^}c6c)FM zV~uX9FyMaPw|MyYDy-`96R^Sw9|L|;ZbQOKHDi@oa2`GzSK+4G-`&ne zs2p_Cxv=)|=FFE=OA+&r^vs;s-cLRt4d;84qxR6R8eGq+wR{=Q&iJ-njP(#lan4H2 zQuw!LctRjrg2&crdr;}7#@XCp-tmp#wz8rLd-*LCECjq5jRV?BN<>~_!gl#uJ{raA zF9CTQXoND!*hLC^;sn9mfLy_5mFew_kdl)@b=R;j3sm##IY@pQ+D(3G1oxTZ`F0jn z_K5P~JsM-0!Q;h9A-{r`L)9Ky&E8hPt=s(X_xCg_lDXCC7h?2p+dE;u7CB7HU)6?v zDydX8thO9+W8ldIke+WOOyC=49DMw)2@mc=lpnwJx2xS0jW2a|T^v=0<%{oJ`{Sil z*1C^--ETLA72Q=sv)Uw3uC{0|*{$8xj6YVBl_}Rh{K%K6ni&6zmC^L2z>(P({q&X= zvE_{DE)j#Tsew~e_UrR0$2tUCMfYSvR1Ndopy0{8`(pLQ!DKWIK<;xJtN`{_~kP={QEE!Ugl)Tbt5@Gva7NGE%-H;uq0lbVhaH3VQ*|QV$PJ9K7 zWd0EIGBR#sV>_naLHY0U-hZ&2Y>pW{uI7kampFl1G=Rb_gCHxbRIY6JApI#(FCh%=J$43x#y5LVyVN&g8YbLBzt)XzM~(+ui;HENZBSb%Ssd^lNl95iap-jHhb zyRS~W76Mt+$>w!?hZwV6Ljp6 zjRgt-T)o#YH|1ZxK0x30kkDw!`>RA&A-JtMIc8U~wbE5tE1IHRVYzV|KDCuA6W+8| zA}1{tlPFJLhY5hd&eFinAL%1;_bWqznzwpWj_=wU2V~xK`4Icmh9Jk3J&w(*5(KK6 z?jzvc(Ul%JThPq?fcv7+3;(v=yN74f3Ln;c+H20tlx0+JKWR^li(q+Cn5XBFG`4%- z9lWfTD^b|ouUi@-=ha|&7EhqXxL}WBCLKsaZ^-rut&1YK+Usf&1!alQ)xf;ej3ZxM z;tPPl!{$tlfg zZH3fQ-IjxSxHm-IR=fLIAG<1Jf1kVvb-wO1hBII zZPo0;u(JFXut6JT=-KW%d5PI!G$*SY4Q)u$=94N`kfWNSffFKB3YhmJ(CfXqZMrGW?ntM z3?wkGsJ-6KHpAUvl-evqK`>w9t4vRQK)?gafS-0sTS={O^j=&Qv8BPoJ7hJhO1G01 zfcwjxH%J23*5g|*8@PdF)ll zMzev@@u2GVJRU1L9||wAzdWxpe@OpmxEyiI0_{q1m?;B9Z*hdcWL?O7s8pjh_omu8 zGurn1Us)HgO2)ri_M^KI`Rv8gUntHt)PD`Rf?e(rz968bAb~Ob>L9TUJ^R5iLiNF6 zO0jx_6_WS%b0Un{Qns0uZmL)wK;$HxVoBh_yk&lP=C;WvspO%iQ8Hi3m`3K3fV$&g z6_8E-uilo642DlJ(x_hvY=XNPyo{!X$4n_V3B>Ru#xedJsW;dC3ftm8=#rC@DJ6<& zoMQMt)n{FC% z&6>H}8%yer$jCG~ASVQCS@z4~I{YlS9VCFWFf%{#J-L@D0Ce~@sTPBx!c_{%z=d6 zB-89$^EwkQOF{i|xU~X=h5+`ie<=Yv-Xm^-+62lCcA0-2;&dq&1=eY9%|KK53^|yu ziOrO5FvhL(sUfxj>qy@uMTubWHHCjPc8&`zL>|5~#MXR;^z*Gsz+q;>C*VX}ft9d^ zwbJar@zFM||75Of7u8#S<^dzC(*-o{kA_bwBtZm-&CdrL=?cI%yQ}US@2dM3mE;Bf z5a4fs8dN;Wxw)lx;w2iwTkx+MbYV2fpYkdhq6SJ4bPMpT&@KT{Rha!7Ci4|6@mzBlfQ1KdPjU)8IvGVl+jd%n;o2cWTkXg&-gB z@t;Z%@E|sTYKGDDoSz1$?~UeN4{-gIfNxO!w5ZdGbQHRHwn@_%=pxKDK}I^ECZW`( zG%5Navt(f0;F^L)$5JW1imr({Zw({Vf= z{Tym7!o{Bi)~s3l^4|0yiS0GQnt!KsNcHfVe9Y?AVB~m4dFlMNHxn*Jg;lNN zh#4!u^d#&|*79+`M6)ykO7*+P1eEIRuuzkQz*u|Z{62}jebdq0*Um;DVR2afZ0X{} zFhfCAy382N?8V_^5$i>CRn^W27VKClT-FnH$FDY>ui?g3ea;%)+qa#hP6s|9}> zIcaL(MMrESoqa0SZS|$aa^?XARNoVFPKP->CO9y zGeOJE_l@pnnBHz#KkN=X8fA8BtE=VP6dkwW_BhjlI_Z4Cr8ay)$u`&#u!}-OO?Q=;sss8(fwDN6e-{+-p_EHW^X|@vzvUc3m0I zwMu@9#k-b8#AY}TgxxT8Dp^)GUr&m=DEr~0W$_3n?4YrUNBK1DeU=t?m2sq+@0nMq zpvD27C*5uM*DSen(Qi^SqW#uU5s)DNlkL@~AK8Evt(Z~+7b&a_{od_j z-Rjvd%JHBhP;PC=nhzzp7(mT|ad3OzIWT6pxp5|L5%DFV>*3&u913heDgFedqM`X|d!H;cJ%Di_L& zFTmPm4~FNx!|J2T-gJHd`V(Ek+jjSGbw}3^dvF#5gFl2DJHpirnF*}josY~KAvdJ6 z!X+UC=Sza14(JwkGI%ER_oolFY#l|hH^?FGyP z)|MYn8W^Nj7m74eXF3=v=bM6ptVb+Ns&Uj7yk43?s9jg}IYr*{5>>APE)zU4tTA%kNGX({-J7|>&KZYIqj(FVX1EwSY z8jp)IyBUDQwU)s6mw?=|*X=MthhJ`3^JUcUV~$S7Rz`0wenjWL4C9n=_ro9|ES4M> zMR}QQ^iCLtys(YvnYlk}J_{Z?s_m#eZKMR>P$IaR?b~E1gVr-I?e)Hb(u_i|xs;_i+WLg;xWY}PV}WdH z+gl9h4K34M2xl^Zc#N4vEOLAOV>X83*QK%p*w(P|y`-N^T`MlnB@ubRthu13t42Fn2D+&@r;IJ09~*?mkiRob9;C)wYdo_r4;`Z1kp*953u?Yy}39 z_NW9un;l4UXqe2|C-ILUbHyLwH$R4@KEmRrk=7 z_2y|0qsm`-`%G2aDOhbuBqk99FQf>NhCzJU>E`ZcDq;s&ss+1V3Y;yJVk`jG{yy-$ z)A_b?XmBG#)^p`^!SZZ5!Hzy(pdn@~FqptQ;BidRqw?moF!|)MBc0|~8$IlKAqSv} z9-)``1c9EX*Hd~NQG~BzYaDBViujJYoeMiFtd}etEW7&0lCU75iX!=mh7>k^RIc#N?#-oXW$(Wc*Qd7rW?$CPKLNj*-GRwF%7=) z+x)?#nf6{B^2o0MIGMl{800*?I!|l4>S}9AgMJhn6Gymmy^?-HwUxf$^~E-%f1&O4 z_an@{vV5ny(u>1giYDz3^jGRxJ!^jEt4E4C9I$kd7gO)C8 zmvqMmh!g@sSEG9=lLS+vyPa>I2UFIxOMh|y_XjiyMz^-%pI%bH{{#e}ePByI{{8#I zo}?Q=7;P3Xjme1ZZU;SA6jpvSxhXeO??e`KqDSZX*`Sh?XI2dB*KoxSvsZLS6XW&S zF~f&4^8mv#+eCLyB5&!2lV zW#MK9TDq*S0rP^^4;fU9PtNT?9d4Jj$IQQhBjHbS{n7s{k0od}XR5T8soTb?i<{&d zL)S;*f)=HkM(S*{jMQ6;_e-U3VFs}Qyx6<|d#P3!P2kl~P zp=G6|ir;ynGD4I++TvpGIfwk`@6h)X59Qtseq@V_nO$gu?262E*r18*&4Kk|`G6+j zPG>XO_z0nXE)n_1rVyx@qz_8D_tnMPynsUtJQCZDtoSkPk62!66MG*=n**f?Y!Dxw z@@Chyj?HvKda*ne4{q#Ep-`o4h8erNLW*@dzBaSfk-$W^Xv_U*+wdm|AL%H=LD%PT zrD>Z^Kf#fhc?n=}p2R$7kll52#>JOD>Za!4d2P$%zk5UI3DyIT~Q( zLINO0@7DeKa0m(CB-^KNx4qw4RmbSnazB83J!uK8^f0;kT2$FPScV$pABd7 zAaIG{9{e){2qfR<7MfeXch2>IgsFkrBwfh*UWw=K$Bu_MDY|?*ne}^-FMBZFXXlWf zM{MI8J1eg9sq-PD|DwO)5k>ar` zZw2+RzOf)Dh+Q`-8U3Mc(i9=aqOa6TK)fdMvay&rV)P9;MArX6a<|87k3 z)JB8#$+3#cRp!a-PBtjUyKm<1aEa=`iUsgf>7}EWIT>-1qU=6;cJW3_Djw`v{i==b zE(e-o)#F$Jo6qc2ZQN#XA1DFZjXqjQ&Wh4=7nc2ZZl}Thc78Q{dWZX*})PI18nz zIbzDd&RLuTOWvPZSU4`^CcpH!fpo7>tVd9o(^sQzuK!T_?%~#@4xCr+{hXCLwc#hX zxhFPsF!>tGKIg_J8ElqmaD!XSEv!^=0Q8_-qARPMCND?=BZ`2w!d~e=O<&zz(`O>x z9Qib5d_khn0w1(x{NOKgQ>AxIsaUg_T++*x-?moE^uhpR)|Lj*P7wtC3}zJ z&m3VZ$9I>*vNteIdUMqF6*+Xtq$@z2=J)DxKF9v~bybuHj9`)G_mIkoG(MJ`r}vSj z->?^lJs|Jy=l4;VqQ9{$E!TaNa@rRgTM6;0DmM&YqyaQT;ERnkjt2LnG-)&b>&>&E z)PH-!ENJrQyVF?AB3h6VEBqz6h~YJc&hJbDeil)gC+Qz=R}1$+smW9kKH!%WBPLE3#d;Rg=q}Os`i3eIB}GX1RPlCfYnm=RgvK?sId} z9d{s=+HtkXRY*(SSH9o;xPe$=EYQEXgdK2OR_P0!^gaTJ#a$QQ2WYa_B$scJ{U-q4 zge(s~s%8f;1EMoMj0Fl?#KAf4$A2wgLT6dAs8xCyWR88|onU@;mR^6lT4Wtj1F97`9regIaHYNwlgZX z2@xdJm5iOH=J@McISdiOpfPn+2q;w20v{@YRBVIc~i z?y!)-xp04onGzLyjtOdHCU&u-jAl4nEO3g>Yjxw8J+;H+RHU9XOwmQE7A%e(^~$5h z_VG~PUU~;G&z&u?wRERCZ+e2`KXW@xF@^G#kJnIm3W1}slS~bk6l1O7tw3SiqQn0B zYH98`t!(;7ns8a{bJJ#bs}HM3WP|62--er9KCko6HS@MgNW}8LJagau#NAbz)$-x$ z&~lJTqm4=o@LaUMw?cLAWTLSJ*Gv-XcT_wAz+i zJ%6Lfzg>$6w6w6r;Qi;4979BO{3mg?n3I_5`$pmujcF__S9+)l9%B7vNtpf;ApGn+ z=vm-@E>c*?$+>o7^j9)k1IawE08BF>w%6Mj1OE5?5{EO$_33?o1Pk_l1*{h(rmXf( zncb{sD%Wp!4#v8%5e$oMGS#GyC=WyhI^WKJS z!HCHto|ST}fW$;Ou`|K%HZdXnRff>)RhghM!wNQ2{i)y!mt1ZdQPu$%K4CgtE@8_JpoBH4oUsyi0 z5U;-Ry{y)~V{5k0sK1qTRMgB2=91)vTC}h#DMJ-(_5K{eb(?;{H|=UC1Kbc?P@^j| z`GSCD&hbfy|M0bqVJQU%kTVV|G0+goS>GD3*m`>=gaDhxYygP!C6~Vs=y$&YfaGwZ z$;o*(h!A{;HW)i5<56B@V~Adu6MX^#o8+?=4jKoW_-$E&6VG=>F|Rb1dUkKy{I9;4 zT>xG?Kn6Vd*SH5$ktnVJo!`aLCCg%mPn2iW!f3q?xKWRLc+vh~-u_|p89lX?GbaF! zCX)cD3me$exXWcnmXyOJ)89iOl1hq(n|$Q^7)Fa`MLqW>&#A!n_gFa!vlcHR;DV?h zh)M24BY+zvU`qz|$L5nrf9}J8%vL)-;S8oSm(Il^kVw;m$!V*ZH!jvLv*CpKSl+2qf z*~!6+==R*)f3kSF%W`f^OW&#jW&L?3^!?wH5;3aat8}&Pm-5TcBys<_k?nd(aVpZN z1CLlETP2AuNlf77>b0~Se~gZx?soupv^4(t!@8?i@;-^B<$Ps}=AA`R*rJ-oerEP{ zmeJritKM~1;Tn-!ja1pDlft#f<Na<=-qwa&T6_{9-yk6+g!&MRnn zYDa5Qor>nVc7rkYSx-%$GPv7=JmFJ+ZPA>T;a$ginkZlYs_Ey*(uZo5!Q2;hE@G1;?{5{H zZnsX!UfiDGK>Kn|&MTD9-}sw?`GO$}@y&z_%G^by0Cxv$hCs(L6-|4mcqiRQC{LgF zk3j^^0WG!Y6==!h1P{+zBb>JBNYU+ZZhBu3?f! zw@Z`ESV`M6=k>vg(yBLcJTsolFHlf$8Y5#?bQ7XehEo2X*I^;=_Sk2telpI$KLj*u zd$W@qOZ5_bsPbOUH*8Cg-;6bm<0pp$`sn)(hbPOv*a)5b1oCO$gQXW1=Ax*hs-%2h zpZ{qpHYikMr1AY26&>6UY(F0MQr|LK}=7i#`67hyv6aIv7;kOVgTji z9blk&@FKk{xL8sEQs%h=6u1~`VGAysMKOGXc22nt9?g%QD)0iXBLj|vaV2vQn+%^A zOP{dVQ3Ax)yo#4VH&?g1?tf=JT{V~j6j{kGL(>&M&|lNWkMceb42E{2kL4h4`BSifAy754K#pz7fi*|f?Jl> zIUjgVKje^Xa>jDYdiL$LApwG+-P_`P`c}d-RR;A3P!c(yeAoz#LiOO6Rc3?K>J^7> z)X67g;%0lE|7Z7HR*4dJfj-(JZghgD}=vM)BnC*M94-Ye} z)vb?f4m;6I+FeASewdHCPlPjS-nnDiBH-{_x>4_DKdJZt3Qvu#1e@RT@ZV`r&u~C7 zUb8IuE!O-Pz1edxGZwLF>*(J@+z;-kgLo;1;9UX=-Q*LTqnzN7mggfKM1-vx_p+o> z-N9l>Mf-Jp`RIGIHOF=f?adt(11HR`C&}iyUr+r18vRKQmed0hsw=C%aQO0jK-KC% z&)^SHB9DK>6$ju>Xz-TW(g?B2p|M5Wjk}kI3)>yT;Q#N9_t~Md(2fvMhz^(QJup zR#OZAaZk_|{};!8m<^!3As0O_aQt+cPlMW~=GB4F)^oFi~5ifgS`)L?t=zbrrl_y^e) zLl7T6;M@>$>6}tiPS&Zwg0Fh3%mIU&E1LIQ5g}VTZMjW%jH{Bul-eig)e~v3Dv4bZ z&mKbtU-G+&T*ZXN@S2CHTh0TA1*oyQzGwfo!4WylXHK6GMkeuu1BW-{LHG256yU5z zzwtO0P(kGtL?|&fukCeqj(rPQc&V|vH^_q!DqfgBrJ>BU24!Fm^h8;9^65T({J$M2 zEM0TR69B7ZOSN`|UPiPu$%$pjxP&3qS3DPjvBdaKx`G&<$*DunQ$rpzW`V!|HSf*K z`unVqv6cT^`pPt86{hP}Wb#SA9K%y-^Rkz8`~9aD{-!>tC9#M+-je*RK&C9{Yrw`= z4h2HYmKIc0fkQE}T}=+~c+@WqFM(1QV5X!xfDe9`0_o<7D^o*WMC&*QFkZ{cS%4k= zPoX>T)RL8S35DDVbSs|wmzQGv8J1o20(TGCZ37h;=vimmCWdhDh^)^pi}{cIFs>^J>1S8~`bK`Bz4N#z6+(2TpuWQa@); zfTa(9X*hq_=tWyD4}q4*mqI1b;iu$zV_XJu_k8xB1C1|x`c=-@lEwqSK}MgUrb{4= zo!0SJ1)gx}xFBd3+DtDcZ8Tn+-?bP2WtEuzFIMSp7QlQ+??1d4jr1na;sC^i}3<2<&Ve3|)SaLTuvl=*j*&N~m=0i)wu;AlvhS7~m~*caTnnryA*VYjKgP06FHD z6O~J^j&5*GIhsJHrK71(H~_=`>pwh)K9dKROr{%E<-`0L)}iMX7CDs3h%(x z3Mb$ZNhmRUubL#R>nsDT1`|X0Q*W^Q2J$4wm2(JRdFFcx3EiTw>9` zq8Y##%G1bcbgYH|P8Rq89=krM2%O}1bAo0r>G$KRp-QXjD^2q`9U*Fcef!(O;l(#o z10%z{Y5%z5gwmx$F~?;jfQHfy|^8E?95c-v7)N)W~QH;@g!GH?{@XQs*xEZz=R zR?2PRk!r`DYwCpoBG7K!hr?HW%%ZZQ02&F{94w#AsvZML10{{>x_{g0b!~QWeWb8K z{L5)))@3lG@@R?{@jOVzyJptvtLLu8i$*97jUM0k9p2Ex>nr_za<=p-$$cPGn?+t_%|SQTx{EC;j&!Co@{qvkfZE>}?wkuEi<`OpvhxefUn2pH|tX z=uUn50)%a{)QhAJF(VfNMPS77DfDMJn}tIfE=P*lO~oX9RcP zPiVPXS)5y@Ty{}fDGb}4a^CvzH^0+}t#0Wk#zZAnWvf$WTwW*J2i$xP5-~CzPcwLH ze!WeFJLi;Rm!D+EGRe>&T7q-cYB`q}HSszb9|*;;GhN_=POdW z{HHzkUAttGI+vOD?M-LE^{cXB0;Bbpk{Z#J4{PR|g}LjtJJ#CICcHY=nyDjB;w+1_ zg9YJ7jbD7mCoE*Hyl7t5fhUiw`d!RoXQ7Dv0b>*QAmZDt9IZsItRxY7-+J(ztl$<; zE@4IbC`*tcLJm!wNWPh_Hr{+nB@TIJ6B!Ocgki{g&eq-+o?qBk*iShl^c9Xf>k#d0 ze7>i^KkC3|6*CB49_1oO6h$qR%Ps0$TR2^>ri6LHJCGeFxDYmYhpNeTLuuOMFICj( zjUcdYO@dv14Y)|I|J)I*z5faF(Vh9nc!N1(Mo1Gc9Dm~{6;68tZoYlD+M)USy*+1( zums?a4Ttd?HH$}oj^=W-GQ_FjsT5{P6bK%-5y>rYlna8HjL<9$>6mpE64$;k3!kjp z9se9GrGsqsK;$8>PY8{bo@R11a;qB%XmX8=G{`Wa2bd5i)2toyM<-*_TcJ?-vCr| z*rW+!oFQ6Sl%U%u=>-pcari=4qNusT^iJEQ?EAtm7tifp zQQhi>>cld|ngkO@hRZp>e9MPFmJT`rJ% zqSo?oEnl?$!iT6Oc|tZ~$4MN6f^V*OV$2yhmomr^IEV6|ip)FY-1eM}VJa@Mpzh+V zub6q|oHr3ah6&;%J?1HSt1CX7E1vS@ae}?&%k+8j!`2kXR0JjSe6G&mL(^uUmAQt{O1~0f!>=ax`e*PlX!szyVqeyGO=}m+2j+v@uwqIILG_|bo z?xHkwan3B_gxlV%==;R|aZz)h!mjD9!P}^i(*yF!cIaYewhWBO1C~I(q;}AnlP=nt zPAz1o4z1gfmMRn9HJz=`9q}`%^dpMPMlDH-$~SY~sNlx&Ez}m=Xya|+1X3dDL(&rC zlOZl|@C3;b3GMD&E6A%Qv2e%o#ngAVOAkEd!cMI+$Rp;O3y)kOS5_6<-|=yFG7u>ESnrR3PbB-5{U$w6POyoEBUAG= zjL)bk&qrLR(xj>6U{s*iV!_VtGqSVe`IY_7EK;4z$BxQ1)Vm4wf`WiXBB1n@xQ5qG z{;ysw?YQidrkbU~8;Lm`1~NClo9+}u%h6negYM!JXdi5@NnrDQiME2)UtBpzd<0r_vy^t{LJgjs=+gE+|s(&dL$l6;YFzplO# zg3v(A^QDqBt-rRbw7rSDb(Ems6c{FoaK7mpL0-3PpMuRH5Y5j0+R918xs~Z&hxQal z?fvk7>$z%)N!(2aHYdY2?uCx=ZkpGV;;;9fQyEOv)vzzZVbHnR@fW3y!%Z5>K^Z zEI$Nq_AD%^8CdvL8iButKD{13Hyf5=_GudOdx!`d46G4bmbJcUGMyuca}2+Y^n@3X z#Nmal!7Oog{_PoEAG4NvsbPCao>6w+o443~G$x~oj$$1Gbvf`CT?hAts1^QDh1L^g zPf8$#OkW9DhTNK7J@uTj6n^r%eysHNv2tgJg52V;JLT^`K2>Em);lB2IU>_;Y~(v7 zH;rvdb!!s+w8XY-Dzcxf|L6us*?LE|@~qT@QWVuap7$`GZHU}r>B6o&kl%uD(&_#N zz91L8fW7h|;>PH=wmoBe0jR*Ni81U1S26?&G$ z5^(?$qGwnv+L(*mHX`dY<&<~RH99$aEM;NV6BH~`*P?06HgSytT`umcmD0r__4jyM z9wQs1&jcsQBtl?$eQaUJc~biSQ1;eQQMFT?tAYCH=K)|^4dtg4@qt4P#4JMkkv z$5=b$;Xo4KnPeE85q|yTQq0{g4pX=btu%ko3pmOGOtR%SXfRa>uCrHQkRl8W4!g}< zGzvaTFb>j2f5tZmWq%giFsEksje31WL7SA>>_g7exAhvp-IxWU%Oy~2aoNs#N1Y7w zaE0dAb^W~O$nQEoob=iqC}szI)sJ9DU*|)D=hn_&a3vlH_*Oon7caZY=8pCb9g4bl zH6RGTjMVa|k!ry-wnRW&XAzzQg<&!zW1*Yd6=d7e9`AfvWtp03jLxk~IOK%+v9yXU zklXh{(^G*ElL}1`o|~Au;g{j2`-J`MEj2_WC>wB+&u$X0jtdqY!WXbY2P_*!It61& zc?JIBIbHCCkkY8Hlqk2tZ5u{P)Q%lWP1`Nb+j*I*;{eJy#Zyn#d@2H;AZGCoQ%leS z$=p!rGv+dFivD^3;+E{&7M4KF`s0`9S5R{p_!#qRU>-DhJ=i^osh6$A%Y8bi+lj7xq*<{vZHD*uIV zD~I%(2W2{zhUkTjkiA~#J2kc)DQ&*nIzF2=+OD%@&{^r3gZ%;b-(@;E73xXDF}`^o z(Ao|gMFF$VWgmqezNFV{d7L3YbkCEC6oXV?&ciJ_@`2Eh9y-C21n7QW(qy;9ug47O zxXDBa%U|wSTuwDgp?5JEUBltR$7kd1BM4)S+}%~=e&5W15v$hwPbzU+?GMwWbwCA0;Zpgiy^ht9YPfbCGX9} z?F_+sxsvILOj1hQGxH+MW#y|+!3E;J_3Lt|{JS3&9R+_EV7W?oq5ADz-jgP-#Z66i z=N5s-eG)Y5ziiSih27N3=>!-X;RCjMTp@Xue@3s`+T@5>_r+AiiyK2yyMlpB24ChGH)>q-x6!nX@pw!1@ax|qx9ko#{Qf^waQ|7mD z8fN2f_m$r$x6K=+n9iR^)yOm}YS$Hq>A(h;&kEUGaBa*s6Vu)^?Abp2GjJrBk}u(I z6N#dfixP9ONq2J@%AOnrp`j^>#8_F-(XB}I>$9b(f`+FHtzNu&A%TQqlX-_$ec3cW z&e}HJ%3MCTerU&Dsr$T=Gae6k=LT(L7qV?P1g*KZmg0)J1(7nFcu6_KADtb|q8o)) z2Qo&RYOEoNXUD8ndOkVgt|~)05-}E^J(iRnxi4hA5`yXiGu)G2v#kda#*MF5!7zN* z&NQbfZ7#Ta$d|W)6Vh@haqgM#2Lo#Yv2cv>;$Kjw1tu(&RJzLF~ivzt8DP(Tm!VB4yVM6uzJH-U^b{o6T zB-&PnUwW^mX9$S+SzeyU(hktXodGT}@-wA-y}I_Q{N2)P?yEi*Pci!+GH1w2qdud2 z*#M`L-4Tl?UXs3u_6XGefJhM$nOk9Z6om_}-?)j{Zyky8at*z}PjBdLo>~XIkQx1C zUTX+M0w!E}yvlKeJL9DLJpXl)OdMvW(WPpyLmioCreke;C*Q8}1^4xBtKay8#8|~M zoi^xh4K1O~hcvfmBn!y>4SWLf6vuExvFZ$T^@Rdl zMjAe(`E@&CT%Hu;>6L^SedrK18p8T%GJ@ruh61h{9g0&eUQv>>OUjzm(dW$iusFn& zJwGP;y7tgdsc$Nup$ok`a-3SuJHzp<&so@HWiywS9_LJck06$zLzSJtu3?^}hUUPH z)2|wi6w)AJJ(+D|p-dh<7IoFQvVfeG7>uP`t0qYAG+?{2*=f8=Oe<}^P&NlzVb!{F zJne^GC?W82nQ8G%<*&CO{kq~~D)4=R^AbtMc(Q|0?teI&+@B_8LMjz9Wf?l{Al5KB zMvW4CWp%rriS$PzGi0}P=N_`-(&H9;N_vS`S6W6OonZ~N_l=@?B~AxL)0j*Aqrj! z4-HvKaK@~cByFz8;St_kN7VP_Q#kSH28l;#peMGMGFN}dDWu3}$>oebv0sPhV-iHfc6UWFGt#AT)ao zJ1iEGOUMRDW+x{Z?ytbQI;}?!IH}Puf^F+>?!|8z-Q8ii-L6Nsus0*t8=B%PN`JID zNrCX&wXQh18c}DB)JRg-*Ofo!pUac05llW|c;s})6)|kL#@Qelf+{t09{5-@Rx_)7 z=A}E;&gBGdx3;+dA#c z!Mx!OK%yG>)zDKTK#PFC@sx=-2vZGgc4KZ#Ufm?7sM3g9>LxDb{ZMVPNklty)h3^8 zR}{3}DDg^9A_{_@YfNQ%UEnK`1)T<-9r^uMrdrA_}QS`O$XC^WNB(lql;qOssk0=AOm*)p>!px0;1~BrkQOmK9951>)_+ zu`{>&RuEorE;*FC!#U(Sn@@fSI!N_iX-&joM1NodFC)yq-;qW zdsWm{evv{RfhgI@sF!>REA`=p8drW8t2Q&+{t~=i#4=+9T8czgpZ6T|iTKJ?nbkH? z8!=YscD8Bg}gaWRsXqZv&7@I}W2KpH;qdpirWj0-aDNXEn%d9(bb*=C~|Lk}rbEpMQ{~ zz+-KRXOS~q<(w)dk3Qq$$k&DVp(i%uN~DX^SNP)BfSEaB&5I*k@^p7ZJ83XxWR}6E z`U9Vv`8cWJa*Jc5V)LH>!L0mKFUYFUqj`CvpOV_l>H2zyecL+x?oZ6?*gzs*>Cc7Y zA7&xOfjvvT2TzqMyWg-aj&-0z$$^fZU`>is*ihJP!V6lrR306}^D0{Y{v%Pj7dLE> zxnAVKd%g6)_V*ujXs`T87!B+^^W(qUzNNT~ABG6U%2|W8XDJ4Pdb{lZ{7lJ@@otvt z|4VGMDN(mD`>O)$;l$FLl-@@l>XBa4UssK*#(sGpO`CWx*RI5@+~l+Cu)UnF9QH-@q4j2bCu^wW^uHAJ6%MsVgg}kd*m+|kt zsn}Jv%!+7{+8-Y|$t2Bi;=B`O7eN;nt0FevG82P-W{B8+^x~jJ<(?!US99FkO}w(K zZR6JGAvv)Pd@7{$SGvJxx5jSlNM>`Gw!##!T0>ax{`80R4g)*4_!p#9MUD~nhni_rzVDOhI3pJJ!bDCJR2m37<^P@!it5SfV=A`m@m=?n^3U z(V!u-Ee+W1=wcG}rrr@Yk?*n4N6&Apkx^^WSVf021!-ppk4NEwj@B$4}HsR)9XJG2Vd;SDB2m=W}bN{qz{; zsXjA}=5@?z${$>vrs>V+5$G{`rT6LR!c?J1L=J=Q!t%6j2*Um^NV{UaM@A%`1UOJ zw=zYg#Kl-SfHgOpw9>V-*#;?^_xc5DD_lOJQ5^`6ZnBL8U00Jd7s5-aNu)rsHN@8> zAZ7ztJh}tRvb2L;;`4rM$eO`LKC%8dmLZR{Pu9eJH&4mV5cLkREwa5y%*cs!7{W=#jBG zd;ZoEGQmB0O*CsT*(91*CGbH?(n8Z;Ha!bG&T`+`E)%0I0#yNau~O@6P;KD&HVr@S zF5^tuH*h)76YoWI^cXTGa|=8GN!7g_7-C&I?GYwVtRjh-kk&vb-cJVB354&4pN30U7F&%ai!9 z5g`VsIFDjH{BG(JiT=+lz&*n&e%|YfeRbEmrREFGj_`c$J4S=}1trD*sgCXKl*{+} z`eEDvIC)a0L5z^Gq;)$0Vtlx0%Krt)w~PG&i>LJ-YfKbDdb#Mhj?1H2%+)#C#dD~` z{)q?_W0c1hBdRF&{Kq*-FGBkMF^b+=X4mf~fU=l_5q^kZyReaT(KlRFao%gPM`?%vWFEh> zm&~DfX?093O3a&tSl=v z_W>#eaOCD;jlrBg!0~KADgGtDa`f9%hij`|O)kdI`qD9a75I z+{b86cku1TTAjc4yL#f-dU`MB_LbIW-|^;~M(fl&#`V@Dq^l|QqA@QxSfQ>)*T$)L zoN!0rPo;hoAduuDCYJYpub~%q0H)v4c@L?zYPIMvIzL9xSEqipH~#WGpmE7X6{y(0 zc$~V``-hXHV=m05)sqfl&Eskr!q9PO2a$30#P~+TPT*>c281;t3L~&hCt_2qltSbz znYrZ#*(Sea@eXN*9?togxvZ`RD7x~yL_f1^!{{9@?Dvg92P=#B_ilQ=hY^uwZEH=SwC`loOBsr&kwvFV+vfrAQer6f}7 zo$f!5E@u)@$QT4@EquqCIPv#RMAQnoY9Ox>zuJoLojSa-6A@}P`A;w0@1v7G^-)Ki zNRi<#ruaUk1Rr)%QP}_(@}7g@ol?2f|&dqdD5RJ(@b*qA_jv;xL-Z_I^LjmTGDzpqoa4K-nC5 zSE%35!DdbTixLII%I~hyR8bc4z6K@PX`_%yYro!Gdw)XvBn9PwiZR=#nfyG`|BZ%R z$9v$wJKjl07?gJTS1tRpPQxLW{e?|a!JvrO|5T5F2YowzO{Fa@L!nu!XWPw)9d-X_ zG?W^5@tDZg(ls(dV5Je)If|Fu5dMMO!ouxmG8O-y&(TMLYVrl4SLsI;W)=E2ps9jj zUs<5K7Q9&-tfbbMtNAKpqXpC*x_<>4eW;RM^VKKY4jX=}UDgdmXoY;`b3J|#GW7D$ zZ`Itkbf+v=GlKZI6q%n>xU+Z(CMW!Al+K^-gB9E;j`42CmjpzT^u0ZhD+eo+_x6lI zBrl!evE^BX%v(YfvH+{++K_Ej|FT&#YDeo@v0t!Gax^Yc6EY?mRv(uUg6S8fjhYY* z<~F<cB3B@EH{)Vqc@*(p%m#sZeaJ(5+Iq5PTlVl0);omtE=y~<^@qiW{?H(eygT zL+PE-nQ-o-EW`=`)!E^xF+rH7e*hv_GZkZ7dd5z_m4wyO5usqdlxDri4+J_kZ}KZW zj_HJUOm0NYRJ~3ZQI44&@Bm~Hr=Q7J77aBvP}6jD6?E$W8Cs$;^*Vjr^BbLAcT`f@ za<87@jW;_+O9y>&D8coXxd!Kz#$?BlW~ULu!|eCCft(7j#ztp0uYPjQm&5{0=F|$5 zaC)#cT`!Q-o3n8!RgDQ(%+?T5#3zz1o?V<2_dnI>ZEmoVodrZDI5}@KMCJGBTh*Ue1{BlFz64Yk@RD1(^E%Md_LoR|2Di8gt=o{h}jn6OI%+%vz2} z2{6 zn`s7Enjl)~_1;e}09pF*Y!lJpCo5ThkxTJR>(N#z6t%~KTH~^j2|3SIJzIdDJ2`di z80gri*t8jIZcK;@w3JCr%Q>lN>mk7V3zSMrxVJCr<;{IL|A=!UP^~V zLyun?r^x&7dN9L>lu}sfR)YBMdoI^ofG0m&r@FQMYMnHeHcMFrFI#SSCOLM~+YXi| zDO^2)5|xy+m+ekijy6Vpbv7dREOUqKeJ$o(Jk>Db~Q={Tr=`Xz-Z}e-T`e-~6k8oB3=rTAv}g?^ROppIo&W z*BH#{0diya9E1*5j5!qMPu*|A$e_MV;IQIY`OObuM%3NQVG^w3bH1}fZ5>aKcG-uc zsY*;f{vpK_01EY++yR`Jm1t@zlwZ@%lWw946-zPULMlCZy&RJlL+f^z7+gjL(ooy3 z8?+T?tue@N(Z_Oo=SW~G;*^C)N%m5HvkvA;@by}98rTM=d6PI_Zv$P*loL_Rf0Wu2OvNaD~j@3Sd4P?~nUmpK{`@o__2ILM1@il~83g^eH z`osk16;HnuTYx2A`$rso%l4n6Mi|c4{3IgBuE}i7LlnT(U)LQ(_Qxt~*P{vZa{%3{ z>UEcUc^rn};X;vZhiRcEc3XP979m*PVWV*E6Jr+KoxSfIohtEySG;Tid z2TQnLSvm*_SIVJMyzV-Of;Etc(DQAIe*CL<`#T8mT41^^6TJ)BVD=%vGQHLYi-C!Q zjh6t;)RsUD`k4J_h?0wv@x7?7)xS7d(c8Gc$Sc;qm-tRP4m0}0jV{0xe30=9XpeMY zW(50lclU`&gWr@ow#@*$bW`H7K{Ny^UgMIev++Z9?M&ud6k-B9jOZ|;VlR< z4ftJP{Pxqk{Xj&y%7h~G&Tw&!>ueOp5kNo6Pf~`DOQ*XBY|P31l5TqoUY!WbL_v^) zS;~lhWQ@Hr1e|efNf3p5?W445IeuixuTJbpWFVG9wtZ7yPD}$r7z>c)bSJA=F+728l?9vLD7K*#>lpYMr zsQWd0ETK1S_6b(MFVw1D%=1e|N)Z9guUzFYBLe%|k2$mnBp4y#WOr9fim z!mUi?C;!@c`fI{)*>Kdo3lF>wQTre89|<{Wj+ghxD~*h9EI`I_f59_3@-{ueDk&V1 zm1eb{`%C1^xYKfqLUIy-A*Pbr2f03`r8ML=9G*Xch=%zOpL&#gqu>@`$3~(lw{}Zd z40_51WA*6C1WS zd-2MhmMBvwc7~Gl07l*r-@AjY|NDN zw_RS&LG-xEJn$>ud@lH|sjuppE-lsQVG6tP*`L&LD24^I+u7%w*Kah^Va8kK9T>oT zHSD^7@Mm|$oj^m(>iP7hI6_-BP=B{uGEDIS@Y;97?%`wIFT_|WRm#Kguyc}=JS;HD zc5kk%YrgcT$jX}lC@2=>lIO&@FZ}dK{Tyrkcjg!n+4{Or+QkS7+c7^lBIn@Vs%b6W zU>SAys?i8thV7doavzam4*R3(pYKOtPrylICNV;m%|`5pcRSi9ZB&EX6f`c{n(E}? z%I49Sy3dsHvpNLe9OFt}pp!Pwq?|m1hb43t7p3dqT=ctU#%}zcxd{3U#5T~Buk4#g zuaKn%|J-3In?CaWGzAWflL#c00N#aLHI~Za_9!oj{&zK~7y*?w@7B|5 zrk>`GwQe1kN8KYI)6icu$@&62`7VJsS*P+Jg3s-Zuj=tGkR4Lts*3;a0J@4} zj@g!rZBc{(_@3=S%FPLlQx*<6+LtuqM1;7&8B2g79y?OF$jeN8q8EF+9~T!*xhsP$ zeW#J(HuEjV6!V7%Z?^>!Q|9NYDJK9Qit+nc!b?MV3mNd&IM z`uaZ?MyCAjr7V8f_*);@ATlH0YUzIC=dl)9ZP2=;=>1ye9mxP*`i%`frkOubY^Jdp znOMuG8*A*oY&b;=J;KB|t}~kK?51!US3VEe7`u|M@)>t1vUxTIejj)y>YqgA0ZQHY zavHu#enH=IuQx0x{(r5Y1U~nVbhNwM zk&|9EIdmO!)1gwmylKOFqt)q3LgBZv9Pw|E`;)SptNj{f?4|v{pw2LlSSe1}7sbwU zk}Y|E$M!Tj&{* z+nDcy7v26T)aGRN`G%OsyDLxv>2OKhcmWW8mHqS@Vtgt`T9ddZHUzvkadGviNPy%+ z$asOg{TRqSYn1nUL;vb8v^nBlq%?B)!12VH96}v~(MEc0Vu`B)lfRQXF0@G7v03ex zOODw`U3aR2UhZ;GbRLJ8x7Oj4OsTVN1-Y|+i5Ke()umDiW_w1CYxFmrExp&s;D~&q z*eOE*{yVdV7W6CU<>k0U3t6H@cz?h%`_2r8bNgy<-3r>$?9Z2zAq#4$mr*NwCa`#Z zJNHp0lRrThFovdc{{$ug9-*L2<^+ty24D4h_dyYdM1_rwyQc8%nhKcJV0vy-qAll@ zbudHYtLb>HJgX0;pDmR00?CRYmXFHT5*UeSn_5=7c*KNN_;3hhuWuPmySS)Oo2;gw zsF)ZAw7LKW&mx}uF^Ij0i*olSuJcKf*^&Qz(&5dfpn!wi--qRBsM3Jxr{M>X^ER>^ zZ{HL5(2K~Ut!edA?ewwbZ`)8IKFg-pHWa+5%W!lmxu`S_o~H8kb#6c1EifW5U?s^^ z?))q0Yk=+Cz85*NE#Ahx=S(ODff!wMnX(44HZk=1N4yJ5Lze;-dP?I5-c@Shy9bJ$ z1tgiv3c!fdXX2-X1VKq*k1o{Pr1mbu>1dT2I)acHc1%T$ZO%580E9;Z%p9+n?9a!0M@;(2a6DL zrbLT0v@45@?-K;#2?jfIcddy!V6LhA>*H5RV64gy+G<@{$b;FHmk6f;R+9Uky&!N- zs{i)lfd^Z17MlATT^zPvNsuM9x5{OPMHB1x>GnPOFI%47L=8(Kz(s!*Q0&@H+=;+` z=pY(V_9l3;d!2Oz@37$zgyB%)-~qdfgZn>!*eP~aXhcN`-O|2gPjLNTl;P+vH-XQQ z2;ePt)D6r2AS0kw2R@U*1J?-huebrknDce|#@sq73EhP%tl^sCgumdx5xvW7y=(ix zjCqC#?@F=L^(sQ+xbIunO*YD>1a(q3w#H`e-kAs&YSF(#0_MUy$yWxK|LKYxQS2lH z#CRY<%x}{W)91v{@`Y3uWd|bWj?*mzItefW$T;Od1yk_bm-rI9H zr`g&Pm^V5kT^VLq3ABpV$02#4(n;xTch`^xR}?`ihgv)zvm&mOR;9v0$Ye|I7I2>E zGmvDec2fS^xuZ_8*@&-3{l3cMC3!=Kkb1LLG&F#8l=@DIdO4{OsP~xQ9XN7p=bmJ< z=6&z4Ji+Ly2@b2yV@l zkLj40!PU1}r|o5m_4TC)(aBpXfrah5cjDuFh^_!;^{uh% za)ea(RV?mB-+p3B5`8K^TqJnHGH`?Bjw0a7B;Nm7xnU1Gyi}uziE@_`m)%)Yf|tc2 z@pDkTB5(?Fb+fT(i@LpiygQNHaIqi#~;33W0ppQ_j4yvEcq5e zou&>^l@kq#q>5sTrq7i2dUXU=se=F~jFswtwv28d)uwDlCyi5`WmzURmuctsm9;^+ zxY>e|_n%$dVBX80DfkU;BU?26nkr|27KCbMPq>G0c7$iSs-wh@KG~>*>TIcwN44vv z2|=#r`%w{2=e4#5hw2K2Oac3wh?*=Z3+!bB5zw9JM>t)cuQ;K9tX64}^FQ5_<@Y*> zNIU+sZ?KI$-^@zWxis6anuRN1y{vfotxl==t&yK^`G|+^NpIV*-HU#Px z6oAIS2Sgp}Qlz&%XN{Z*S;Mc7xmL}VFQ-O0F|&)ae2B{Q ziXI2Dil>JzjrZcz<}_Dxj*2fdL$(SMR!aM7S4C`syk#7rS!k%$+*ig z&zsYvPymXKwbqY1a<$Ox=hj(xQCdaRpG4?*rpTeCU?JeWfMJlmgjZ5X@G zWF9RKisHBBW|IZm7FVGSYeT#ZW~aLgdA!ly!u79%B>1gXAMn39T5ULq0j+-ve|XRF z`*W0QeGOLlGEvKFX7Szlw^azmI=Xhjkod+lUq+SDQIqh@Id&3kqxGjii@51^p|VW# zx<%Cs4>UEaz+vq&nv(%$y~Qy+3faOSs%P2*MI zUFEmRfXBoibMsqO@p>KX z*Xojl>y4$hhaT`-$X3zjNvyj{7k-0%3zzW=qi)esbi3TMpZWf_e%!<2c&9$?91Ne) zZubXMf7dE>I}i9wosW3+c!(mdldPX~xT{IA*Q-*$l+<|rtkh!cWysUPH=*TYkmGkS z@t_M1=D8-#8`JV<_30uMr|7!Ed&~<5?rJHYWv@V{A`@@1gTz1h9zzo)AGNn>SvK&jR}r(3UBw*KUaLQI5W1a8hgpLde8H&EUK8c=q3Y) zP-OrgQiPx<$bChhE(jLYy$ITc5+i(ep`6Lz{~vabo~1$wc1;T9*T}*HD#jSh3Oul~ z1}W{n6Gqdc6D)>*yl zR}$JD5Op&zAcfynA^7Lk4P;$FdqMlkacyH1^i7}RA$0erV{ZelMPa?8^EPYVunls8JKJlo#DL zJKG(DM$0XqHtH+7HWO(&xcEJ1^Tp!C)Mi6+tEs2Q$D?is>0Nw#jCQ(pDA$!g2|rB= zspy`c91hH`^3L;^`>eiU)F@wz`hDo+yDDs`WNRSl*ZBz}9p&q0B3}91H~oUb4<0$k z%_U^#!be2#2+vo%a`C)DzzlHSU^ogtsK7h=U+4WbIT*tG{_pT_;MA5sJT9$Di!e67 zj1co#NB}rOP5Z$UH4cv8fw#vEldakN7Z&WOt$<5cCz@bv*2Cg@HpJd}kWl8pqagj` zR>vV~3!s%>0s4cx^JIYn zANo*Pl>RmXgsA9RNZLXu)9$Qi?06i5@wbvgJ@Ocx< zud)Q?KN2CbkxiHTm2{^;nVtEYMz!vFR~t<~<;|IZaUT^wDN?$kHwkU)f=6F^?t>;C zMs0rr=ubaKM;`xz7~6)&lzoT&5`CIAvrU~oQt%}ASyKJy`_K0$ zB4R3SLpqjM9ObAifZUwXm@0B`$VGlksnuC^!4H+4eXLG(o6mM=vZ+fF7oa{r^(DK@ANASd;*9*;c_kt*te%frhCn%H!{ z&>QK31xF!1vH&|K0qps)koz_+9*U%g8np-#q-J{tusbO4Vgyz;oFNYf7xXWq+^~yV z)r8f4ZQ3zAwzl2WtKeT39 zMG5%Ge6DD;x5S8wWs#zyh6SKa$Bt9_B|mRZVu!EzJ}Uv3DbZ8Dn*?`o*%8{kPE(46 z*htqOq6>=uavV$5sRqK0&3E7Z!^4srz&SJ`Pt;h~D3d=&JYC9n1Lhw5XmlEAZvtvU zVi^NqQ+Z<|yqEv79ZZi8dJX0bnFZI=(yTo`Q0Oxc*r6r$A7Xw@i zYq@txw<|Y6L%Bhj1K9$wc7ZaTqA;B5}kkn--Pq*cNX97t(t+L4J?P zFfc+4_=X|Dt-lh~QQh}cVQV!$VuEkVz)c27C|;}N&)fTW#&xDf=qNpifeqti`W~9N zsO-S@j1KoNDbc2JUN=&G2g$K6TxTYEq4w9*iKX2D`DEc>CM6;{p#TgV*IR*A%QTMb z4AqRJPrwiVb4_R2;Hfe-PRt|(^uWSmBzV88-sFtX=(~%9SD65;Y2+dX%iP0W%9(4x zK*B}%&*kFutd=zO`m>cw6g#Q#-fIk`3{||o1(1Au;F!+D-=dV!G1p&R#=%OxrAQAC zGyivSYD^r~w`gzT;li$6(cf!1Yia%GB#9#UJTUN?l?LEOpoe>l3ho|iBG)3s`|D>l zhq;Ob0jZ~t3P=WQ_;^i}ojl`oR(!VzWFKE6fx(|UJ7k0Fgd|hflpmM?eonzCi)xyn zzWG^h?1|LAgh|< z;4vlY>HX_&6v?TLpxUKG7ubT}N}K9ko)lEgVmg`XcCUryD@D78D z^aKv2hU>LcuIXK%*~srPFU9vOvfcYPx`LK(>IE83Fx@4@OQZyJg2dNq4))hum1Au0 z^)4}7@yYd9>4^WMI*2Y)qJUHCC`tU^1OC*LLj${|uHLSO!!bWOioc>1d+mWSfXeg# z#{<=X^R1p;TbmBBHqcg~2Td(%HaC8p6AW746`lFEO1V8BIyUsu+F2lUVavEmMHm* zj+E*q4lec|pS^@|#hd@Y3;r$waYpWNTAa{{WT)PwAG90Hf681RZ0X0u)0D9dxp@1Q za>Hi$!(VfWxT@RcbF$eGQ=;-=WYPq9o`A#kMeHY<4=xhnaZT^qHl*fk$k!D5_ju}9)XGvbVlXC52 z-Cx-5_m3$7uIbd?`F{p!995JCz4B@U8>mzvA6t8*VQXEITuay{W=gkl5IR*Vw{ci2 zjOX4qs>)U)!X>n~rZOYaPsoTHjd=zWQOmfNqM=z5y9E-GCG^+rr)%$XbNryB@4NjY zqQL`Y!(T)F^$We63CmHLyWx$UDzjgD4tl@x_+OScn!$0ZmJdNhqQj&6ih0-Rik4MXqXF zu7o;W@ZWnb)mXkz<-(9BqUXC^)NF(OaX3&>Ace_iJ=*np014hGd^V~O)lgn=K3?EJ zg}Zp?_9IH>e==gH)NhjLNnAiHoWv!{DE_MmGX2NH_hJNOQTal{PoHNXGTPT`gJ&X! zYaLh4Hw5!*_60vmO*6&q1!<+x8$8d(x;a=O$OjcW|2F<$gF$9Cjvy{=LTWYo8YzKPjEF?h2pzy#Jff@^BMo6xkTA z@2KeJe-c`}q`2$zid?#QZn%^vV^mxHv$ZYs$CIXh%O$kn;BTS*$E*#m$4R<9z7>P? z@CljHcH=G^5S*DF7$(L`S@6=%1e&z3ac^o4IGT1db#R91R3RKmjv}dmN+(fIesw_P zhyhejwtuu&z&*hZj`yVa8Fj#xU0LV|uNEoLnAGb$<)PDjsW|jtK1Kr=Ti^S54E6qH zg#revn-0#0ysE^dF7jBl8!T%L<8H4}q7us{F8hJXbs1i$Dsqje!XKOgO{nX@B>{R+ z|A#aU^q_o)R5#_aHJZCd%r$`X#dgZQG}_Yt2nnRvKwpIf!*aVPNa_2F|#`#(i88uGp|-bno*=1~P^X zEGHtWyyP$Oan*oI_84q`>Y&K>Q3JWP^N|wSQSX7g?@c$F+f$R>zdTHVT?kB)CUvGe zyvX9loC?cKA|G5lWmq~zo0=cZ)iZC^Am5nIX3fSm+?bvx6MXTvvJ<@z6NV})ej}%p zTAc?M^0Kk1;slAP^1qV$&w7_>%wEsPTIN>r`x=eA!Fa?Ot|R@$EkaiwR-hTMFRDhN z$mV?Qb&pB^A6}=@c89;5(8y$~tZH5tIp7rh_u6X25yQdP7)iniS2;{k8h}6UbU>M4 z%pD=6tcw8hZiGhjLrr$RoT}#%Cfm9wZf>rcG#1WypkMUb5ZcEihZpV-Tycf{&oFXP z!OufQK(SNje=$>d!<^MC^CnuItv7EZ+p(^4xw{GIqyu}0z-IaS2c4D*2gnJi*HwIK z5BEyAh8|Is=gcD&3nH~xVe4y29xvJnk0kT&-Us~07XQ2VQ;jxpbmsXFAqN6upY~Jm zu3jISn;**i(5+v@iqW0sK1-zeF4ofIWJpAol7IVYy_@kh+;w~A7Bt0uvJjPuml_s! z`yUY2fCx*&#}dBa5RLcvr<`@S&OJ|*7DySc)mb?%2rYdlLArBhMi0E7DEBl^ZuP=k zd|bLidHz+L!pf6f!HH>7R`k-=PSE{2uSu%y@5sOuy%6pba}2%6N2B9k3#W}6!P{G3 znjZ}GZQ>U|!u~-E^iPy%u2lcoeuU;bhb3V5(M~)6L6QKp`{I1?+EYvAEU9CgoesW5 z3lr?AF{}AlU+!6<^d*r#h%bkQJ}4FOTBZiQPF+?$EdK z#xkyZVE{Ru_ks^|`|XE|c5~#!e9J?I&*`8#a;_aRo>4XN+~-zg=zA$eXd%&VX?x%2o&*n57(PLY3s z@B`tbEr9G0JZi(m`=$6dU-8;MFlrg{=g3oi6!p>T1gOoE_b9}9qv~1c{+N~>57s)` z!=yt$N_B(48qG6H`hmj>ZVqpf-laVpIKU0_;g#Ee80}3#U+@>JQRt|fEnH)@=-mB~ zH@`V4qoV-0!lGmzLjIqO6SN$wlLKIjYz;?uF~srOWd0YY#EUF4`w^ylpKCh2T6M5Y zKnH0a&h4dom<+5pwcT^NaRZ0#>(D16*#FhbwTDBQw(&W|XdEVNLyQJRU7|5W489r6 zMr{rym9iytatO(^8O%5oWhduj$eL>Hmb9#rXeLR?DHSOqq|ziE37PMHXWH*7SNrcb zf6aBx%rnn>-_LX3zu)iqy>Dz$?IZ`q4Pm!sIeV>welY0%8$BEPK@DzE9vWi+@96gA zS#U-pG%?)N!-epp+6$LQ$r}+ZZ|Yw4mCV`#+hNa6_MhbWL!3?9YYhux+Sm`P$p@#RjBbnXUhtD9NBZ!z&^ytpuL3!tW<+^mh>aV3HDN<~0@-GOVhvpo?(Z z^jYB}Cz&^wRqw9NSAXGRf4|~TL%zgKIgy1BpmA>V#d;qo@|eUChHw#_u#%3sr~8_= zTTZm0oKBq~xB%E@=G|BWmh~=n zU(M?!L7qQ{-Zs`d(`^QmaQ75Rb*&qemLfIk&c%o`Y~}&^>(W~Di^Bdmk1+uGi_GgOm(Uk=N;>$J6@Iy zU%xnC!I{wA8%2*RTp(9Bt{xmNhwEW6ZjSFT9Vt6dw@+#M+0@cl9)n^brPJhNG%Ut| zb?Aj^_3uzuU+9WEN}se(B&_*35-E1lHe|IVgvk~vRbefdAl_IrhB9Id=Vc`zXV4|p z;D`K%(f^aJb%J-4PyAnU*N3SQJ{~;pGbkPnup|o(XsMju^$Vc-z*0(0%@H^GVxzGr z;?8-#cw@|Ai&CaYYnqgidwr#S(ii5Xrj%-OxM>eF9mSX7u3g(-xO}iF4$fEjvYpn$h!z z#i7eL&`7Ty#i*a({_ao|V9$Vx_H9Q2d+t5z3iqonLIBEXQ0bIf5p99_KW)M3cI&z@ zkIqV`Htm7BjY<3F_ou25K249f4#kN_`KN6!0H`!F^>x2(4kO$(u{PuWBpxBU`&{j?*!ehH#WsrxRQ0C~H(aE?2f5)J zN4#TSN3C2+RlqyVBfhQOl)QRM$;oPL8^b`g9&aeku2mVsvB>7C%LcCQw z9C!Qxt>3FMaN)VFZ0^L272&|>qgJ)t-r7T%(;l9S3+o3FB^0WY4OP=SM}`pngVWm0 zh=~cX0Pj4Ut>Z|uez!q$Xp^fx$!rPNh%LzYR%caY(L)m>OeBf?8r%yNk;vz zr%S5bPlx9{%kMi-Xth~;A#sfS{-&W<1UtCoVeY5E$S3I*zf;YZA$@<5VvbMp8-&<5 z;umwdU4R*Dnc83H5uPZQDoz+NtyHw_b{+B#|2(??!16*#(S``Noz3>P>dao- z?#DBJzA0(12+l@}8|6w`bb*#wiSK-gN&esH@AHysudIeigAc1Kn>9Oo#e(F^4mxZ6 z8V?y>&i`C;4ma9Z*X8`+CJXfFf(a@xU zC&Kfln{soAXq=b3T5(Y_0`^<1XOZ526Qnp`Sm1@~BKWf9dYt5nNV67&rXyFC7*0=* z{|jVP?NuOAauq?fL*;vH2`Q!$$8b$3;G$@^_rZ8Ii=Sb*AiuX%TME9!X{-ak2muR6)95m zL8H>4)m=xl#@$J?z!s_K(vG`3BSL>;Q;14v2SLIuCj@AgTJ0-efZvTQ@%fnlL&)@N zYAmUUBt-bq_-q5^yBr&7a2KEHhsIlylE0gS4BlP}I^rL95B!Bm$wr<92P6Ee8e8h_ zWc`PD)4O$Hgm7-3ehRg3%|YYNtJuxOs>iSHxW6LkxFE*TGfur|)5l9b*oR{z#LD&z zJ57?eKEqMF<)F~E&XINIpU>o^`k`URSeLnI0Z8J36s~yGLO3_XT~?&A7>O=reu=Jw z;xy?mg~kwvBryHSr|LBM$1}0u3>OYxr=R`Yqt?kQ+9h9wMze{+aT{yCu%X~Xf+v?- zG%sW-GT1{<6E)EPEGaGsxfkQoS}&gl)BBA|b*>*Y<3BSfb}+-|8a&Csd2$8_*nF9- z@XVy2FdAkwpwDe_<|Mme5W^%92MJ>4u#(Yu6wi?uk}v6C9n!mTQu#Z+Lil>W38tG5 z7JCE#bJbA;l$Sr80(T(_yov!Z%+LD#8N_Tz044EiGfoVDoxn9s9gPr2?-eNr@E9wJ zjEM1&=KWXPV}ZiPxA7Pw`e3n)QWnLOUad04P>3o#zcEm_Z0^0Wf?*YqnOijIwnVcr3k-m@n4C^_>~26zxXVD|jqJc#=S4eRVm^(j*2A1b^Gb*_dQG zOeqg#VOVPFdaFwHC8Zl*0#<1qG^b6sqvgE)CKRrjlA<_cX3(MUsy*>q96ESr*SWi; zU4QR_M;VU`(Jw^_uM;MNfA#1yjh?7!$l3YnbC{dmJ5E%^UV82RzJ{F|YPFMcJVnmJ z3w5-!B+=#D<bxV}?0zlcIy5~QYiSDVpIUNO>zG7n>HZOj{F$C{ iyO9}M{JR{*nW>oA6|2ey#d1UxyqsAsj%D`SWB&`%wwGuC literal 0 HcmV?d00001 -- 2.40.1 From 257d6f5e9c08ab54411914994ce3de338ed382c3 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 27 Jan 2021 12:20:11 +0000 Subject: [PATCH 08/65] Minor cleanup --- Makefile | 5 ++++- README.md | 9 ++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index b2f2e76..ceadf7e 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ FILES ?= distfiles.txt all: update update: - ./script/update_helm.sh + ./scripts/update_helm.sh docs: for c in charts/*; do \ @@ -15,3 +15,6 @@ docs: [[ $$c =~ "kubeadm" ]] && continue ; \ helm-docs -c $$c ; \ done + +publish: + ./scripts/publish.sh diff --git a/README.md b/README.md index 26c2e08..1952453 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,12 @@ KubeZero - Zero Down Time Kubernetes platform ======================== -KubeZero is a Kubernetes distribution providing an opinionated, pre-configured container platform -incl. various addons and services. +KubeZero is a Kubernetes distribution providing an integrated container platform so you can focus on your applications. -# Design goals +# Design philosophy -- Cloud provider agnostic, bare-metal / self-hosted possible +- Cloud provider agnostic, bare-metal/self-hosted - No vendor lock in, most components are optional and could be exchanged -- Organic OpenSource / open and permissive licenses over closed-source solutions +- Organic Open Source / open and permissive licenses over closed-source solutions - No premium services / subscriptions required - Staying and contributing back to upstream projects as much as possible -- 2.40.1 From 4fded1b66827f7c48755f07c092a96f4e757c89e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 12 Feb 2021 11:04:16 +0000 Subject: [PATCH 09/65] Kubeadm chart for 1.19, improved tooling --- charts/kubeadm/Chart.yaml | 2 +- charts/kubeadm/README.md.gotmpl | 8 +++++ .../templates/ClusterConfiguration.yaml | 9 +++-- .../kubeadm/templates/JoinConfiguration.yaml | 19 +++++------ .../templates/KubeletConfiguration.yaml | 6 ++-- charts/kubeadm/templates/admin-kubectl.yaml | 25 ++++++++++++++ .../aws-iam-authenticator/deployment.yaml | 13 +------ .../aws-iam-authenticator/mappings.yaml | 34 +++++++++++++++++++ .../device-plugins/fuse-device-plugin.yaml | 30 ++++++++++++++++ charts/kubeadm/values.yaml | 6 +--- scripts/publish.sh | 9 +++-- 11 files changed, 124 insertions(+), 37 deletions(-) create mode 100644 charts/kubeadm/templates/admin-kubectl.yaml create mode 100644 charts/kubeadm/templates/aws-iam-authenticator/mappings.yaml create mode 100644 charts/kubeadm/templates/device-plugins/fuse-device-plugin.yaml diff --git a/charts/kubeadm/Chart.yaml b/charts/kubeadm/Chart.yaml index 99424f1..524cfba 100644 --- a/charts/kubeadm/Chart.yaml +++ b/charts/kubeadm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubeadm description: KubeZero Kubeadm golden config type: application -version: 1.18.14 +version: 1.19.7 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubeadm/README.md.gotmpl b/charts/kubeadm/README.md.gotmpl index d29076a..fce4c94 100644 --- a/charts/kubeadm/README.md.gotmpl +++ b/charts/kubeadm/README.md.gotmpl @@ -17,6 +17,14 @@ Installs the Istio control plane {{ template "chart.valuesSection" . }} +## Changes for 1.19 + +### Logging to json of control plane components +- https://github.com/kubernetes/website/blob/dev-1.19/content/en/docs/concepts/cluster-administration/system-logs.md + +### PodTopologySpread +- https://kubernetes.io/blog/2020/05/introducing-podtopologyspread/#podtopologyspread-defaults + ## Resources - https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm/ diff --git a/charts/kubeadm/templates/ClusterConfiguration.yaml b/charts/kubeadm/templates/ClusterConfiguration.yaml index f7c4238..a5d736a 100644 --- a/charts/kubeadm/templates/ClusterConfiguration.yaml +++ b/charts/kubeadm/templates/ClusterConfiguration.yaml @@ -11,6 +11,8 @@ etcd: local: extraArgs: listen-metrics-urls: "http://0.0.0.0:2381" + unsafe-no-fsync: "true" + logger: "zap" {{- with .Values.etcdExtraArgs }} {{- toYaml . | nindent 6 }} {{- end }} @@ -20,20 +22,20 @@ controllerManager: bind-address: 0.0.0.0 terminated-pod-gc-threshold: "300" leader-elect: {{ .Values.clusterHighAvailable | quote }} - # Default anyways but make kube-bench happy - feature-gates: "RotateKubeletServerCertificate=true" + logging-format: json scheduler: extraArgs: profiling: "false" bind-address: 0.0.0.0 leader-elect: {{ .Values.clusterHighAvailable | quote }} + logging-format: json apiServer: certSANs: - {{ regexSplit ":" .Values.apiEndpoint -1 | first }} extraArgs: etcd-servers: {{ .Values.allEtcdEndpoints }} profiling: "false" - feature-gates: "CSIMigration=true,CSIMigrationAWS=true,CSIMigrationAWSComplete=true" + feature-gates: "CSIMigrationAWS=true,CSIMigrationAWSComplete=true,DefaultPodTopologySpread=true" audit-log-path: "/var/log/kubernetes/audit.log" audit-policy-file: /etc/kubernetes/apiserver/audit-policy.yaml audit-log-maxage: "7" @@ -46,6 +48,7 @@ apiServer: {{- if .Values.clusterHighAvailable }} goaway-chance: ".001" {{- end }} + logging-format: json {{- with .Values.apiExtraArgs }} {{- toYaml . | nindent 4 }} {{- end }} diff --git a/charts/kubeadm/templates/JoinConfiguration.yaml b/charts/kubeadm/templates/JoinConfiguration.yaml index 6ad03cc..17c48bf 100644 --- a/charts/kubeadm/templates/JoinConfiguration.yaml +++ b/charts/kubeadm/templates/JoinConfiguration.yaml @@ -1,18 +1,15 @@ apiVersion: kubeadm.k8s.io/v1beta2 kind: JoinConfiguration -metadata: - name: kubezero-joinconfiguration discovery: - bootstrapToken: - apiServerEndpoint: {{ .Values.apiEndpoint }} - token: {{ .Values.joinToken }} - caCertHashes: - - "{{ .Values.caCertHash }}" + file: + kubeConfigPath: /root/.kube/config +controlPlane: + localAPIEndpoint: + advertiseAddress: IP_ADDRESS + bindPort: {{ regexSplit ":" .Values.apiEndpoint -1 | last }} nodeRegistration: ignorePreflightErrors: - DirAvailable--var-lib-etcd - Swap -controlPlane: - localAPIEndpoint: - advertiseAddress: {{ .Values.ipAddress }} - bindPort: {{ regexSplit ":" .Values.apiEndpoint -1 | last }} + kubeletExtraArgs: + node-labels: {{ .Values.nodeLabels | quote }} diff --git a/charts/kubeadm/templates/KubeletConfiguration.yaml b/charts/kubeadm/templates/KubeletConfiguration.yaml index 3568211..9577961 100644 --- a/charts/kubeadm/templates/KubeletConfiguration.yaml +++ b/charts/kubeadm/templates/KubeletConfiguration.yaml @@ -3,6 +3,9 @@ kind: KubeletConfiguration metadata: name: kubezero-kubeletconfiguration failSwapOn: false +cgroupDriver: cgroupfs +logging: + format: json hairpinMode: hairpin-veth resolvConf: /run/systemd/resolve/resolv.conf protectKernelDefaults: true @@ -12,8 +15,5 @@ eventRecordQPS: 0 # tlsPrivateKeyFile: /var/lib/kubelet/pki/kubelet.key tlsCipherSuites: [TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256] featureGates: - # Default anyways but make kube-bench happy - RotateKubeletServerCertificate: true - CSIMigration: true CSIMigrationAWS: true CSIMigrationAWSComplete: true diff --git a/charts/kubeadm/templates/admin-kubectl.yaml b/charts/kubeadm/templates/admin-kubectl.yaml new file mode 100644 index 0000000..4460518 --- /dev/null +++ b/charts/kubeadm/templates/admin-kubectl.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Config +clusters: +- cluster: + server: https://{{ .Values.apiEndpoint }} + name: {{ .Values.clusterName }} +contexts: +- context: + cluster: {{ .Values.clusterName }} + user: kubernetes-admin + name: kubernetes-admin@{{ .Values.clusterName }} +current-context: kubernetes-admin@{{ .Values.clusterName }} +preferences: {} +users: +- name: kubernetes-admin + user: + exec: + apiVersion: client.authentication.k8s.io/v1alpha1 + command: aws-iam-authenticator + args: + - "token" + - "-i" + - "{{ .Values.clusterName }}" + - "-r" + - "{{ .Values.kubeAdminRole }}" diff --git a/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml b/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml index 6e3cdc7..102e215 100644 --- a/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml +++ b/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml @@ -51,7 +51,7 @@ metadata: --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: aws-iam-authenticator namespace: kube-system @@ -151,14 +151,3 @@ spec: - name: state hostPath: path: /var/aws-iam-authenticator/ ---- -apiVersion: iamauthenticator.k8s.aws/v1alpha1 -kind: IAMIdentityMapping -metadata: - name: kubernetes-admin -spec: - # Arn of the User or Role to be allowed to authenticate - arn: {{ .Values.kubeAdminRole }} - username: kubernetes-admin - groups: - - system:masters diff --git a/charts/kubeadm/templates/aws-iam-authenticator/mappings.yaml b/charts/kubeadm/templates/aws-iam-authenticator/mappings.yaml new file mode 100644 index 0000000..0785bc1 --- /dev/null +++ b/charts/kubeadm/templates/aws-iam-authenticator/mappings.yaml @@ -0,0 +1,34 @@ +# Controller role which is more or less cluster-admin once enrolled +apiVersion: iamauthenticator.k8s.aws/v1alpha1 +kind: IAMIdentityMapping +metadata: + name: kubezero-controllers +spec: + arn: {{ .Values.ControllerIamRole }} + username: kubezero-controller + groups: + - system:masters + +--- +# Worker role to eg. delete former self etc. +apiVersion: iamauthenticator.k8s.aws/v1alpha1 +kind: IAMIdentityMapping +metadata: + name: kubezero-workers +spec: + arn: {{ .Values.WorkerIamRole }} + username: kubezero-worker + groups: + - system:masters + +--- +# Admin Role for remote access +apiVersion: iamauthenticator.k8s.aws/v1alpha1 +kind: IAMIdentityMapping +metadata: + name: kubernetes-admin +spec: + arn: {{ .Values.kubeAdminRole }} + username: kubernetes-admin + groups: + - system:masters diff --git a/charts/kubeadm/templates/device-plugins/fuse-device-plugin.yaml b/charts/kubeadm/templates/device-plugins/fuse-device-plugin.yaml new file mode 100644 index 0000000..c5a40e8 --- /dev/null +++ b/charts/kubeadm/templates/device-plugins/fuse-device-plugin.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fuse-device-plugin + namespace: kube-system +spec: + selector: + matchLabels: + name: fuse-device-plugin + template: + metadata: + labels: + name: fuse-device-plugin + spec: + hostNetwork: true + containers: + - image: public.ecr.aws/zero-downtime/fuse-device-plugin:v1.0 + # imagePullPolicy: Always + name: fuse-device-plugin + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins diff --git a/charts/kubeadm/values.yaml b/charts/kubeadm/values.yaml index b4d6148..b3bd790 100644 --- a/charts/kubeadm/values.yaml +++ b/charts/kubeadm/values.yaml @@ -1,11 +1,7 @@ -clusterVersion: 1.18.0 +clusterVersion: 1.19.0 clusterName: pleasechangeme apiEndpoint: kube-api.changeme.org:6443 etcdExtraArgs: {} - # Enable for > 1.18 - # unsafe-no-fsync: "true" apiExtraArgs: {} clusterHighAvailable: false allEtcdEndpoints: "" -joinToken: "" -caCertHash: "" diff --git a/scripts/publish.sh b/scripts/publish.sh index 0607651..28117bc 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -2,12 +2,13 @@ set -eu CHARTS=${1:-'.*'} +FORCE=${2:-''} # all credits go to the argoproj Helm guys https://github.com/argoproj/argo-helm SRCROOT="$(cd "$(dirname "$0")/.." && pwd)" GIT_PUSH=${GIT_PUSH:-true} -[ "$(git branch --show-current)" == "stable" ] || { echo "Helm packages should only be built from stable branch !"; exit 1; } +[[ "$(git branch --show-current)" == "stable" || -n "$FORCE" ]] || { echo "Helm packages should only be built from stable branch !"; exit 1; } TMPDIR=$(mktemp -d kubezero-repo.XXX) mkdir -p $TMPDIR/stage @@ -38,7 +39,11 @@ do done # Do NOT overwrite existing charts -cp -n $TMPDIR/stage/*.tgz $TMPDIR/repo +if [ -n "$FORCE" ]; then + cp $TMPDIR/stage/*.tgz $TMPDIR/repo +else + cp -n $TMPDIR/stage/*.tgz $TMPDIR/repo +fi cd $TMPDIR/repo -- 2.40.1 From 9b823dc082269d0510c2768c77718521c0d072bb Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 22 Feb 2021 14:41:32 +0100 Subject: [PATCH 10/65] Make kubeadm config work on bare-metal, minor tuning --- charts/kubeadm/Chart.yaml | 2 +- charts/kubeadm/templates/ClusterConfiguration.yaml | 12 ++++++++---- charts/kubeadm/templates/KubeProxyConfiguration.yaml | 2 +- charts/kubeadm/templates/KubeletConfiguration.yaml | 9 ++++++++- charts/kubeadm/templates/admin-kubectl.yaml | 2 ++ .../templates/aws-iam-authenticator/crds.yaml | 2 ++ .../templates/aws-iam-authenticator/deployment.yaml | 2 ++ .../templates/aws-iam-authenticator/mappings.yaml | 2 ++ charts/kubeadm/values.yaml | 6 ++++++ 9 files changed, 32 insertions(+), 7 deletions(-) diff --git a/charts/kubeadm/Chart.yaml b/charts/kubeadm/Chart.yaml index 524cfba..c73bea0 100644 --- a/charts/kubeadm/Chart.yaml +++ b/charts/kubeadm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubeadm description: KubeZero Kubeadm golden config type: application -version: 1.19.7 +version: 1.19.8 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubeadm/templates/ClusterConfiguration.yaml b/charts/kubeadm/templates/ClusterConfiguration.yaml index a5d736a..e605021 100644 --- a/charts/kubeadm/templates/ClusterConfiguration.yaml +++ b/charts/kubeadm/templates/ClusterConfiguration.yaml @@ -10,7 +10,7 @@ networking: etcd: local: extraArgs: - listen-metrics-urls: "http://0.0.0.0:2381" + listen-metrics-urls: "http://{{ .Values.listenAddress }}:2381" unsafe-no-fsync: "true" logger: "zap" {{- with .Values.etcdExtraArgs }} @@ -19,14 +19,14 @@ etcd: controllerManager: extraArgs: profiling: "false" - bind-address: 0.0.0.0 + bind-address: {{ .Values.listenAddress }} terminated-pod-gc-threshold: "300" leader-elect: {{ .Values.clusterHighAvailable | quote }} logging-format: json scheduler: extraArgs: profiling: "false" - bind-address: 0.0.0.0 + bind-address: {{ .Values.listenAddress }} leader-elect: {{ .Values.clusterHighAvailable | quote }} logging-format: json apiServer: @@ -35,7 +35,6 @@ apiServer: extraArgs: etcd-servers: {{ .Values.allEtcdEndpoints }} profiling: "false" - feature-gates: "CSIMigrationAWS=true,CSIMigrationAWSComplete=true,DefaultPodTopologySpread=true" audit-log-path: "/var/log/kubernetes/audit.log" audit-policy-file: /etc/kubernetes/apiserver/audit-policy.yaml audit-log-maxage: "7" @@ -43,7 +42,12 @@ apiServer: audit-log-maxbackup: "3" tls-cipher-suites: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" admission-control-config-file: /etc/kubernetes/apiserver/admission-configuration.yaml + {{- if eq .Values.platform "aws" }} authentication-token-webhook-config-file: /etc/kubernetes/apiserver/aws-iam-authenticator.yaml + feature-gates: "CSIMigrationAWS=true,CSIMigrationAWSComplete=true,DefaultPodTopologySpread=true" + {{- else }} + feature-gates: "DefaultPodTopologySpread=true" + {{- end }} enable-admission-plugins: NodeRestriction,EventRateLimit {{- if .Values.clusterHighAvailable }} goaway-chance: ".001" diff --git a/charts/kubeadm/templates/KubeProxyConfiguration.yaml b/charts/kubeadm/templates/KubeProxyConfiguration.yaml index 6ca5945..8d30d72 100644 --- a/charts/kubeadm/templates/KubeProxyConfiguration.yaml +++ b/charts/kubeadm/templates/KubeProxyConfiguration.yaml @@ -2,5 +2,5 @@ apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration metadata: name: kubezero-kubeproxyconfiguration -metricsBindAddress: "0.0.0.0:10249" +metricsBindAddress: "{{ .Values.listenAddress }}:10249" mode: "" diff --git a/charts/kubeadm/templates/KubeletConfiguration.yaml b/charts/kubeadm/templates/KubeletConfiguration.yaml index 9577961..c582d01 100644 --- a/charts/kubeadm/templates/KubeletConfiguration.yaml +++ b/charts/kubeadm/templates/KubeletConfiguration.yaml @@ -7,13 +7,20 @@ cgroupDriver: cgroupfs logging: format: json hairpinMode: hairpin-veth +{{- if .Values.systemd }} resolvConf: /run/systemd/resolve/resolv.conf -protectKernelDefaults: true +{{- end }} +protectKernelDefaults: {{ .Values.protectKernelDefaults }} eventRecordQPS: 0 # Breaks kubelet at boot time # tlsCertFile: /var/lib/kubelet/pki/kubelet.crt # tlsPrivateKeyFile: /var/lib/kubelet/pki/kubelet.key tlsCipherSuites: [TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_GCM_SHA256] +{{- if eq .Values.platform "aws" }} featureGates: CSIMigrationAWS: true CSIMigrationAWSComplete: true +{{- end }} +kubeReserved: + cpu=50m + memory=128m diff --git a/charts/kubeadm/templates/admin-kubectl.yaml b/charts/kubeadm/templates/admin-kubectl.yaml index 4460518..bdbf724 100644 --- a/charts/kubeadm/templates/admin-kubectl.yaml +++ b/charts/kubeadm/templates/admin-kubectl.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.platform "aws" }} apiVersion: v1 kind: Config clusters: @@ -23,3 +24,4 @@ users: - "{{ .Values.clusterName }}" - "-r" - "{{ .Values.kubeAdminRole }}" +{{- end }} diff --git a/charts/kubeadm/templates/aws-iam-authenticator/crds.yaml b/charts/kubeadm/templates/aws-iam-authenticator/crds.yaml index 7ff85f7..c1977a8 100644 --- a/charts/kubeadm/templates/aws-iam-authenticator/crds.yaml +++ b/charts/kubeadm/templates/aws-iam-authenticator/crds.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.platform "aws" }} apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -30,3 +31,4 @@ spec: type: array items: type: string +{{- end }} diff --git a/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml b/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml index 102e215..fe40f3c 100644 --- a/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml +++ b/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.platform "aws" }} kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -151,3 +152,4 @@ spec: - name: state hostPath: path: /var/aws-iam-authenticator/ +{{- end }} diff --git a/charts/kubeadm/templates/aws-iam-authenticator/mappings.yaml b/charts/kubeadm/templates/aws-iam-authenticator/mappings.yaml index 0785bc1..41df122 100644 --- a/charts/kubeadm/templates/aws-iam-authenticator/mappings.yaml +++ b/charts/kubeadm/templates/aws-iam-authenticator/mappings.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.platform "aws" }} # Controller role which is more or less cluster-admin once enrolled apiVersion: iamauthenticator.k8s.aws/v1alpha1 kind: IAMIdentityMapping @@ -32,3 +33,4 @@ spec: username: kubernetes-admin groups: - system:masters +{{- end }} diff --git a/charts/kubeadm/values.yaml b/charts/kubeadm/values.yaml index b3bd790..fb27499 100644 --- a/charts/kubeadm/values.yaml +++ b/charts/kubeadm/values.yaml @@ -1,7 +1,13 @@ clusterVersion: 1.19.0 +listenAddress: 0.0.0.0 clusterName: pleasechangeme apiEndpoint: kube-api.changeme.org:6443 etcdExtraArgs: {} apiExtraArgs: {} clusterHighAvailable: false allEtcdEndpoints: "" +# supported values aws,bare-metal +platform: "aws" +# Set to false for openrc, eg. on Gentoo or Alpine +systemd: true +protectKernelDefaults: true -- 2.40.1 From 3342ead3b504da8189a0cd074e9596ed3c4c6d11 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 22 Feb 2021 21:32:12 +0100 Subject: [PATCH 11/65] Version bump cert-manager --- charts/kubezero-cert-manager/Chart.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/kubezero-cert-manager/Chart.yaml b/charts/kubezero-cert-manager/Chart.yaml index 72e6898..bb8ab08 100644 --- a/charts/kubezero-cert-manager/Chart.yaml +++ b/charts/kubezero-cert-manager/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-cert-manager description: KubeZero Umbrella Chart for cert-manager type: application -version: 0.4.1 +version: 0.5.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -15,7 +15,7 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: cert-manager - version: 1.1.0 + version: 1.2.0 repository: https://charts.jetstack.io condition: cert-manager.enabled -kubeVersion: ">= 1.16.0" +kubeVersion: ">= 1.18.0" -- 2.40.1 From bb6fda041de94bbda92c533a00447d13fa6f0002 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 22 Feb 2021 21:32:41 +0100 Subject: [PATCH 12/65] Fix kubelet config --- charts/kubeadm/templates/KubeletConfiguration.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kubeadm/templates/KubeletConfiguration.yaml b/charts/kubeadm/templates/KubeletConfiguration.yaml index c582d01..7002a7f 100644 --- a/charts/kubeadm/templates/KubeletConfiguration.yaml +++ b/charts/kubeadm/templates/KubeletConfiguration.yaml @@ -22,5 +22,5 @@ featureGates: CSIMigrationAWSComplete: true {{- end }} kubeReserved: - cpu=50m - memory=128m + cpu: 50m + memory: 128m -- 2.40.1 From af010a20339c03f717a57e63839c927ad819a9b0 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 22 Feb 2021 21:34:45 +0100 Subject: [PATCH 13/65] Further tuning of fluentd throughput --- charts/kubezero-logging/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/kubezero-logging/values.yaml b/charts/kubezero-logging/values.yaml index 24f58d7..b43662c 100644 --- a/charts/kubezero-logging/values.yaml +++ b/charts/kubezero-logging/values.yaml @@ -163,11 +163,11 @@ fluentd: @type file_single path /var/log/fluentd-buffers/kubernetes.system.buffer - chunk_limit_records 8192 + chunk_limit_size 16MB total_limit_size 4GB flush_mode interval - flush_thread_count 2 - flush_interval 30s + flush_thread_count 4 + flush_interval 5s flush_at_shutdown true retry_type exponential_backoff retry_timeout 60m -- 2.40.1 From 3383efd764b559a592f0baccb378249f554b5ced Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 24 Feb 2021 00:10:14 +0100 Subject: [PATCH 14/65] Version bump of ArgoCD required for Kube > 1.18 latest charts --- charts/kubezero-argocd/Chart.yaml | 6 +++--- charts/kubezero-argocd/values.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/kubezero-argocd/Chart.yaml b/charts/kubezero-argocd/Chart.yaml index 52e6369..4fa5941 100644 --- a/charts/kubezero-argocd/Chart.yaml +++ b/charts/kubezero-argocd/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application name: kubezero-argocd -version: 0.6.1 +version: 0.7.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -15,6 +15,6 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: argo-cd - version: 2.9.5 + version: 2.14.7 repository: https://argoproj.github.io/argo-helm -kubeVersion: ">= 1.16.0" +kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-argocd/values.yaml b/charts/kubezero-argocd/values.yaml index 481fd85..e7f1536 100644 --- a/charts/kubezero-argocd/values.yaml +++ b/charts/kubezero-argocd/values.yaml @@ -42,7 +42,7 @@ argo-cd: global: image: - tag: v1.7.10 + tag: v1.8.5 controller: args: -- 2.40.1 From af6810ef6a1ebf610954dff4360a947752ee3639 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 24 Feb 2021 20:36:34 +0100 Subject: [PATCH 15/65] Version bump for aws-ebs-csi driver, enable volume resize, snapshot, patch for loglevel and leader election --- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 8 +- .../charts/aws-ebs-csi-driver/.helmignore | 22 + .../charts/aws-ebs-csi-driver/Chart.yaml | 18 + .../aws-ebs-csi-driver/templates/NOTES.txt | 3 + .../aws-ebs-csi-driver/templates/_helpers.tpl | 69 +++ .../templates/clusterrole-attacher.yaml | 23 + .../templates/clusterrole-provisioner.yaml | 38 ++ .../templates/clusterrole-resizer.yaml | 33 ++ .../clusterrole-snapshot-controller.yaml | 35 ++ .../templates/clusterrole-snapshotter.yaml | 25 ++ .../clusterrolebinding-attacher.yaml | 15 + .../clusterrolebinding-provisioner.yaml | 15 + .../templates/clusterrolebinding-resizer.yaml | 18 + ...lusterrolebinding-snapshot-controller.yaml | 18 + .../clusterrolebinding-snapshotter.yaml | 18 + .../templates/controller.yaml | 185 ++++++++ .../templates/csidriver.yaml | 9 + .../aws-ebs-csi-driver/templates/node.yaml | 159 +++++++ ...le-snapshot-controller-leaderelection.yaml | 15 + ...ng-snapshot-controller-leaderelection.yaml | 19 + .../serviceaccount-csi-controller.yaml | 17 + .../templates/serviceaccount-csi-node.yaml | 12 + .../serviceaccount-snapshot-controller.yaml | 15 + .../templates/statefulset.yaml | 52 +++ .../templates/storageclass.yaml | 15 + .../charts/aws-ebs-csi-driver/values.yaml | 130 ++++++ .../crds/crd_snapshotter.yaml | 420 ++++++++++++++++++ .../loglevel_leader.patch | 112 +++++ .../templates/storage-class.yaml | 61 --- charts/kubezero-aws-ebs-csi-driver/update.sh | 8 + .../kubezero-aws-ebs-csi-driver/values.yaml | 25 +- 31 files changed, 1542 insertions(+), 70 deletions(-) create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/.helmignore create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/crds/crd_snapshotter.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch delete mode 100644 charts/kubezero-aws-ebs-csi-driver/templates/storage-class.yaml create mode 100755 charts/kubezero-aws-ebs-csi-driver/update.sh diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index 9cb2ee5..b2ea4b4 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.3.5 -appVersion: 0.8.1 +version: 0.4.0 +appVersion: 0.9.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png sources: @@ -18,9 +18,9 @@ maintainers: - name: Quarky9 dependencies: - name: aws-ebs-csi-driver - version: 0.8.2 + version: 0.9.9 repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: kubezero-lib version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ -kubeVersion: ">= 1.16.0" +kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/.helmignore b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/.helmignore new file mode 100644 index 0000000..50af031 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml new file mode 100644 index 0000000..0910aac --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +appVersion: 0.9.0 +description: A Helm chart for AWS EBS CSI Driver +home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver +keywords: +- aws +- ebs +- csi +kubeVersion: '>=1.17.0-0' +maintainers: +- email: chengpan@amazon.com + name: leakingtapan +- name: krmichel + url: https://github.com/krmichel +name: aws-ebs-csi-driver +sources: +- https://github.com/kubernetes-sigs/aws-ebs-csi-driver +version: 0.9.9 diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt new file mode 100644 index 0000000..34db916 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt @@ -0,0 +1,3 @@ +To verify that aws-ebs-csi-driver has started, run: + + kubectl get pod -n kube-system -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl new file mode 100644 index 0000000..fdc77c4 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl @@ -0,0 +1,69 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "aws-ebs-csi-driver.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "aws-ebs-csi-driver.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "aws-ebs-csi-driver.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "aws-ebs-csi-driver.labels" -}} +{{ include "aws-ebs-csi-driver.selectorLabels" . }} +{{- if ne .Release.Name "kustomize" }} +helm.sh/chart: {{ include "aws-ebs-csi-driver.chart" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} +{{- end -}} + +{{/* +Common selector labels +*/}} +{{- define "aws-ebs-csi-driver.selectorLabels" -}} +app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} +{{- if ne .Release.Name "kustomize" }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +{{- end -}} + +{{/* +Convert the `--extra-volume-tags` command line arg from a map. +*/}} +{{- define "aws-ebs-csi-driver.extra-volume-tags" -}} +{{- $result := dict "pairs" (list) -}} +{{- range $key, $value := .Values.extraVolumeTags -}} +{{- $noop := printf "%s=%s" $key $value | append $result.pairs | set $result "pairs" -}} +{{- end -}} +{{- if gt (len $result.pairs) 0 -}} +{{- printf "%s=%s" "- --extra-volume-tags" (join "," $result.pairs) -}} +{{- end -}} +{{- end -}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml new file mode 100644 index 0000000..e0919ce --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml @@ -0,0 +1,23 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-attacher-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments/status" ] + verbs: [ "patch" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml new file mode 100644 index 0000000..0fb7ded --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml @@ -0,0 +1,38 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-provisioner-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch", "create", "delete" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims" ] + verbs: [ "get", "list", "watch", "update" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "storageclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshots" ] + verbs: [ "get", "list" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents" ] + verbs: [ "get", "list" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "csinodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "nodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "coordination.k8s.io" ] + resources: [ "leases" ] + verbs: [ "get", "watch", "list", "delete", "update", "create" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments" ] + verbs: [ "get", "list", "watch" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml new file mode 100644 index 0000000..9d85b97 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml @@ -0,0 +1,33 @@ +{{- if .Values.enableVolumeResizing }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-resizer-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + # The following rule should be uncommented for plugins that require secrets + # for provisioning. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +{{- end}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml new file mode 100644 index 0000000..ffdb1b7 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml @@ -0,0 +1,35 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-snapshot-controller-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml new file mode 100644 index 0000000..061b565 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml @@ -0,0 +1,25 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-snapshotter-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml new file mode 100644 index 0000000..92a8b40 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-attacher-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller.name }} + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-attacher-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml new file mode 100644 index 0000000..e2478b9 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-provisioner-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller.name }} + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-provisioner-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml new file mode 100644 index 0000000..997dc28 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml @@ -0,0 +1,18 @@ +{{- if .Values.enableVolumeResizing }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-resizer-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller.name }} + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-resizer-role + apiGroup: rbac.authorization.k8s.io + +{{- end}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml new file mode 100644 index 0000000..cb46730 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml @@ -0,0 +1,18 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshot-controller-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.snapshot.name }} + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-snapshot-controller-role + apiGroup: rbac.authorization.k8s.io + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml new file mode 100644 index 0000000..f55c38e --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml @@ -0,0 +1,18 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshotter-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller.name }} + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml new file mode 100644 index 0000000..7feff6f --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -0,0 +1,185 @@ +# Controller Service +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ebs-csi-controller + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: ebs-csi-controller + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: ebs-csi-controller + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.podAnnotations }} + annotations: {{ toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} + spec: + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + serviceAccountName: {{ .Values.serviceAccount.controller.name }} + priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + tolerations: + {{- if .Values.tolerateAllTaints }} + - operator: Exists + {{- end }} + {{- with .Values.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: ebs-plugin + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + {{- if ne .Release.Name "kustomize" }} + - controller + {{- else }} + # - {all,controller,node} # specify the driver mode + {{- end }} + - --endpoint=$(CSI_ENDPOINT) + {{- if .Values.extraVolumeTags }} + {{- include "aws-ebs-csi-driver.extra-volume-tags" . | nindent 12 }} + {{- end }} + {{- if .Values.k8sTagClusterId }} + - --k8s-tag-cluster-id={{ .Values.k8sTagClusterId }} + {{- end }} + - --logtostderr + - --v={{ .Values.logLevel }} + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-secret + key: key_id + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-secret + key: access_key + optional: true + {{- if .Values.region }} + - name: AWS_REGION + value: {{ .Values.region }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: csi-provisioner + image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --v={{ .Values.logLevel }} + {{- if .Values.enableVolumeScheduling }} + - --feature-gates=Topology=true + {{- end}} + {{- if .Values.extraCreateMetadata }} + - --extra-create-metadata + {{- end}} + - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: csi-attacher + image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --v={{ .Values.logLevel }} + - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.enableVolumeSnapshot }} + - name: csi-snapshotter + image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.enableVolumeResizing }} + - name: csi-resizer + image: {{ printf "%s:%s" .Values.sidecars.resizerImage.repository .Values.sidecars.resizerImage.tag }} + imagePullPolicy: Always + args: + - --csi-address=$(ADDRESS) + - --v={{ .Values.logLevel }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: socket-dir + emptyDir: {} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml new file mode 100644 index 0000000..1858e39 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: ebs.csi.aws.com + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + attachRequired: true + podInfoOnMount: false diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml new file mode 100644 index 0000000..6fd1110 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml @@ -0,0 +1,159 @@ +# Node Service +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ebs-csi-node + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.node.podAnnotations }} + annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- end }} + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.node.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + hostNetwork: true + serviceAccountName: {{ .Values.serviceAccount.node.name }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-cluster-critical" }} + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- end }} + {{- with .Values.node.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: ebs-plugin + securityContext: + privileged: true + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + args: + - node + - --endpoint=$(CSI_ENDPOINT) + {{- if .Values.volumeAttachLimit }} + - --volume-attach-limit={{ .Values.volumeAttachLimit }} + {{- end }} + - --logtostderr + - --v={{ .Values.logLevel }} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: node-driver-registrar + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v={{ .Values.logLevel }} + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml new file mode 100644 index 0000000..947d241 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml @@ -0,0 +1,15 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-snapshot-controller-leaderelection + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml new file mode 100644 index 0000000..0670c70 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml @@ -0,0 +1,19 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-snapshot-controller-leaderelection + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.snapshot.name }} + namespace: kube-system +roleRef: + kind: Role + name: ebs-snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml new file mode 100644 index 0000000..8ec4c4e --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.controller.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.controller.name }} + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.controller.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} + {{- if eq .Release.Name "kustomize" }} + #Enable if EKS IAM for SA is used + #annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::586565787010:role/ebs-csi-role + {{- end }} +{{- end -}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml new file mode 100644 index 0000000..afe0218 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.node.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.node.name }} + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.node.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml new file mode 100644 index 0000000..3b5ef2b --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml @@ -0,0 +1,15 @@ +{{- if .Values.enableVolumeSnapshot }} +{{- if .Values.serviceAccount.snapshot.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.snapshot.name }} + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.snapshot.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml new file mode 100644 index 0000000..aeb8351 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml @@ -0,0 +1,52 @@ +{{- if .Values.enableVolumeSnapshot }} +#Snapshot controller +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: ebs-snapshot-controller + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + serviceName: ebs-snapshot-controller + replicas: 1 + selector: + matchLabels: + app: ebs-snapshot-controller + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: ebs-snapshot-controller + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + spec: + serviceAccountName: {{ .Values.serviceAccount.snapshot.name }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + tolerations: + {{- if .Values.tolerateAllTaints }} + - operator: Exists + {{- end }} + {{- with .Values.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: snapshot-controller + image: {{ printf "%s:%s" .Values.snapshotController.repository .Values.snapshotController.tag }} + args: + - --v={{ .Values.logLevel }} + - --leader-election=false + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml new file mode 100644 index 0000000..3da90e3 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml @@ -0,0 +1,15 @@ +{{- range .Values.storageClasses }} +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: {{ .name }} + {{- if .annotations }} + annotations: {{- .annotations | toYaml | trim | nindent 4 }} + {{- end }} + {{- if .labels }} + labels: {{- .labels | toYaml | trim | nindent 4 }} + {{- end }} +provisioner: ebs.csi.aws.com +{{ omit (dict "volumeBindingMode" "WaitForFirstConsumer" | merge .) "name" "annotations" "labels" | toYaml }} +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml new file mode 100644 index 0000000..fd13fad --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml @@ -0,0 +1,130 @@ +# Default values for aws-ebs-csi-driver. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 2 + +image: + repository: k8s.gcr.io/provider-aws/aws-ebs-csi-driver + tag: "v0.9.0" + pullPolicy: IfNotPresent + +logLevel: 5 + +sidecars: + provisionerImage: + repository: k8s.gcr.io/sig-storage/csi-provisioner + tag: "v2.0.2" + attacherImage: + repository: k8s.gcr.io/sig-storage/csi-attacher + tag: "v3.0.0" + snapshotterImage: + repository: k8s.gcr.io/sig-storage/csi-snapshotter + tag: "v3.0.3" + livenessProbeImage: + repository: k8s.gcr.io/sig-storage/livenessprobe + tag: "v2.1.0" + resizerImage: + repository: k8s.gcr.io/sig-storage/csi-resizer + tag: "v1.0.0" + nodeDriverRegistrarImage: + repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar + tag: "v2.0.1" + +snapshotController: + repository: k8s.gcr.io/sig-storage/snapshot-controller + tag: "v3.0.3" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +podAnnotations: {} + +# True if enable volume scheduling for dynamic volume provisioning +enableVolumeScheduling: true + +# True if enable volume resizing +enableVolumeResizing: false + +# True if enable volume snapshot +enableVolumeSnapshot: false + +# The "maximum number of attachable volumes" per node +volumeAttachLimit: "" + +resources: + {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +priorityClassName: "" +nodeSelector: {} +tolerateAllTaints: true +tolerations: [] +affinity: {} + +# Extra volume tags to attach to each dynamically provisioned volume. +# --- +# extraVolumeTags: +# key1: value1 +# key2: value2 +extraVolumeTags: {} + +# If set, add pv/pvc metadata to plugin create requests as parameters. +extraCreateMetadata: false + +# ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional). +k8sTagClusterId: "" + +# AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata +# service. +# --- +# region: us-east-1 +region: "" + +node: + priorityClassName: "" + nodeSelector: {} + podAnnotations: {} + tolerateAllTaints: true + tolerations: [] + resources: {} + +serviceAccount: + controller: + create: true # A service account will be created for you if set to true. Set to false if you want to use your own. + name: ebs-csi-controller-sa # Name of the service-account to be used/created. + annotations: {} + snapshot: + create: true + name: ebs-snapshot-controller + annotations: {} + node: + create: true + name: ebs-csi-node-sa + annotations: {} + +storageClasses: [] +# Add StorageClass resources like: +# - name: ebs-sc +# # annotation metadata +# annotations: +# storageclass.kubernetes.io/is-default-class: "true" +# # label metadata +# labels: +# my-label-is: supercool +# # defaults to WaitForFirstConsumer +# volumeBindingMode: WaitForFirstConsumer +# # defaults to Delete +# reclaimPolicy: Retain +# parameters: +# encrypted: "true" diff --git a/charts/kubezero-aws-ebs-csi-driver/crds/crd_snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/crds/crd_snapshotter.yaml new file mode 100644 index 0000000..aff837a --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/crds/crd_snapshotter.yaml @@ -0,0 +1,420 @@ +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + preserveUnknownFields: false + validation: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created + through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. Supported values are "Retain" and "Delete". "Retain" means + that the VolumeSnapshotContent and its physical snapshot on underlying + storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + subresources: + status: {} + preserveUnknownFields: false + validation: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. "Delete" + means that the VolumeSnapshotContent and its physical snapshot on + underlying storage system are deleted. In dynamic snapshot creation + case, this field will be filled in with the "DeletionPolicy" field + defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For + pre-existing snapshots, users MUST specify this field when creating + the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be the + same as the name returned by the CSI GetPluginName() call for that + driver. Required. + type: string + source: + description: source specifies from where a snapshot will be created. + This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a + pre-existing snapshot on the underlying storage system. This field + is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume + from which a snapshot should be dynamically taken from. This field + is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass to which this snapshot + belongs. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to + which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be provided + for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an + entire object, this string should contain a valid JSON/Go field + access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part of an object. + TODO: this design is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is + made, if any. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an int64. + On Unix, the command `date +%s%N` returns the current time in nanoseconds + since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the latest observed error during snapshot creation, + if any. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on + the underlying storage system. If not specified, it indicates that + dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + subresources: + status: {} + preserveUnknownFields: false + validation: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested + by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object in the same namespace as the VolumeSnapshot + object where the snapshot should be dynamically taken from. This + field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing + VolumeSnapshotContent object. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. If not specified, the default snapshot + class will be used if one exists. If not specified, and there is no + default snapshot class, dynamic snapshot creation will fail. Empty + string is not allowed for this field. TODO(xiangqian): a webhook validation + on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes' + type: string + required: + - source + type: object + status: + description: 'status represents the current information of a snapshot. NOTE: + status can be modified by sources other than system controllers, and must + not be depended upon for accuracy. Controllers should only use information + from the VolumeSnapshotContent object after verifying that the binding + is accurate and complete.' + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName represents the name of + the VolumeSnapshotContent object to which the VolumeSnapshot object + is bound. If not specified, it indicates that the VolumeSnapshot object + has not been successfully bound to a VolumeSnapshotContent object + yet. NOTE: Specified boundVolumeSnapshotContentName alone does not + mean binding is valid. Controllers MUST always verify bidirectional + binding between VolumeSnapshot and VolumeSnapshotContent to + avoid possible security issues.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates that the creation time of the snapshot + is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch b/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch new file mode 100644 index 0000000..39563e9 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch @@ -0,0 +1,112 @@ +diff -rtub aws-ebs-csi-driver/templates/controller.yaml /tmp/aws-ebs-csi-driver/templates/controller.yaml +--- aws-ebs-csi-driver/templates/controller.yaml 2021-02-23 18:54:24.000000000 +0100 ++++ /tmp/aws-ebs-csi-driver/templates/controller.yaml 2021-02-24 18:40:00.753541864 +0100 +@@ -56,7 +56,7 @@ + - --k8s-tag-cluster-id={{ .Values.k8sTagClusterId }} + {{- end }} + - --logtostderr +- - --v=5 ++ - --v={{ .Values.logLevel }} + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock +@@ -98,14 +98,14 @@ + image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.logLevel }} + {{- if .Values.enableVolumeScheduling }} + - --feature-gates=Topology=true + {{- end}} + {{- if .Values.extraCreateMetadata }} + - --extra-create-metadata + {{- end}} +- - --leader-election=true ++ - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + - --default-fstype=ext4 + env: + - name: ADDRESS +@@ -120,8 +120,8 @@ + image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --v=5 +- - --leader-election=true ++ - --v={{ .Values.logLevel }} ++ - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +@@ -136,7 +136,7 @@ + image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --leader-election=true ++ - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +@@ -153,7 +153,7 @@ + imagePullPolicy: Always + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.logLevel }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +diff -rtub aws-ebs-csi-driver/templates/node.yaml /tmp/aws-ebs-csi-driver/templates/node.yaml +--- aws-ebs-csi-driver/templates/node.yaml 2021-02-23 18:54:24.000000000 +0100 ++++ /tmp/aws-ebs-csi-driver/templates/node.yaml 2021-02-24 18:41:44.630213228 +0100 +@@ -56,7 +56,7 @@ + - --volume-attach-limit={{ .Values.volumeAttachLimit }} + {{- end }} + - --logtostderr +- - --v=5 ++ - --v={{ .Values.logLevel }} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock +@@ -94,7 +94,7 @@ + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) +- - --v=5 ++ - --v={{ .Values.logLevel }} + lifecycle: + preStop: + exec: +diff -rtub aws-ebs-csi-driver/templates/statefulset.yaml /tmp/aws-ebs-csi-driver/templates/statefulset.yaml +--- aws-ebs-csi-driver/templates/statefulset.yaml 2021-02-23 18:54:24.000000000 +0100 ++++ /tmp/aws-ebs-csi-driver/templates/statefulset.yaml 2021-02-24 18:42:07.223547582 +0100 +@@ -41,7 +41,7 @@ + - name: snapshot-controller + image: {{ printf "%s:%s" .Values.snapshotController.repository .Values.snapshotController.tag }} + args: +- - --v=5 ++ - --v={{ .Values.logLevel }} + - --leader-election=false + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +diff -rtub aws-ebs-csi-driver/templates/storageclass.yaml /tmp/aws-ebs-csi-driver/templates/storageclass.yaml +--- aws-ebs-csi-driver/templates/storageclass.yaml 2021-02-23 18:54:24.000000000 +0100 ++++ /tmp/aws-ebs-csi-driver/templates/storageclass.yaml 2021-02-24 17:43:46.866722719 +0100 +@@ -1,4 +1,5 @@ + {{- range .Values.storageClasses }} ++--- + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: +diff -rtub aws-ebs-csi-driver/values.yaml /tmp/aws-ebs-csi-driver/values.yaml +--- aws-ebs-csi-driver/values.yaml 2021-02-23 18:54:24.000000000 +0100 ++++ /tmp/aws-ebs-csi-driver/values.yaml 2021-02-24 18:41:15.513545244 +0100 +@@ -9,6 +9,8 @@ + tag: "v0.9.0" + pullPolicy: IfNotPresent + ++logLevel: 5 ++ + sidecars: + provisionerImage: + repository: k8s.gcr.io/sig-storage/csi-provisioner diff --git a/charts/kubezero-aws-ebs-csi-driver/templates/storage-class.yaml b/charts/kubezero-aws-ebs-csi-driver/templates/storage-class.yaml deleted file mode 100644 index 094d43a..0000000 --- a/charts/kubezero-aws-ebs-csi-driver/templates/storage-class.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.StorageClass.create }} -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: ebs-sc-gp2-xfs - labels: -{{ include "kubezero-lib.labels" . | indent 4 }} -provisioner: ebs.csi.aws.com -volumeBindingMode: WaitForFirstConsumer -parameters: - csi.storage.k8s.io/fstype: xfs - type: gp2 - encrypted: "true" -{{- if index .Values "aws-ebs-csi-driver" "enableVolumeResizing" }} -allowVolumeExpansion: true -{{- end }} ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: ebs-sc-gp3-xfs - labels: -{{ include "kubezero-lib.labels" . | indent 4 }} - {{- if .Values.StorageClass.default }} - annotations: - storageclass.kubernetes.io/is-default-class: "true" - {{- end }} -provisioner: ebs.csi.aws.com -volumeBindingMode: WaitForFirstConsumer -parameters: - csi.storage.k8s.io/fstype: xfs - type: gp3 - encrypted: "true" -{{- if index .Values "aws-ebs-csi-driver" "enableVolumeResizing" }} -allowVolumeExpansion: true -{{- end }} - -{{- range .Values.StorageClass.Zones }} ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: ebs-sc-gp2-xfs-{{ . }} - labels: -{{ include "kubezero-lib.labels" . | indent 4 }} -provisioner: ebs.csi.aws.com -volumeBindingMode: WaitForFirstConsumer -parameters: - csi.storage.k8s.io/fstype: xfs - type: gp2 - encrypted: "true" -{{- if index .Values "aws-ebs-csi-driver" "enableVolumeResizing" }} -allowVolumeExpansion: true -{{- end }} -allowedTopologies: -- matchLabelExpressions: - - key: failure-domain.beta.kubernetes.io/zone - values: - - {{ . }} -{{- end }} -{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/update.sh b/charts/kubezero-aws-ebs-csi-driver/update.sh new file mode 100755 index 0000000..14f4674 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/update.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +VERSION=0.9.9 + +rm -rf charts/aws-ebs-csi-driver +curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts + +patch -d charts -i ../loglevel_leader.patch -p0 diff --git a/charts/kubezero-aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/values.yaml index 9f8bb5b..77bbef0 100644 --- a/charts/kubezero-aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/values.yaml @@ -1,9 +1,10 @@ aws-ebs-csi-driver: replicaCount: 1 + logLevel: 1 enableVolumeScheduling: true - enableVolumeResizing: false - enableVolumeSnapshot: false + enableVolumeResizing: true + enableVolumeSnapshot: true nodeSelector: node-role.kubernetes.io/master: "" @@ -20,6 +21,20 @@ aws-ebs-csi-driver: extraVolumeTags: {} # Name: KubeZero-Cluster -StorageClass: - create: true - default: true + storageClasses: + - name: ebs-sc-gp2-xfs + volumeBindingMode: WaitForFirstConsumer + allowVolumeExpansion: true + parameters: + csi.storage.k8s.io/fstype: xfs + type: gp2 + encrypted: "true" + - name: ebs-sc-gp3-xfs + annotations: + storageclass.kubernetes.io/is-default-class: "true" + volumeBindingMode: WaitForFirstConsumer + allowVolumeExpansion: true + parameters: + csi.storage.k8s.io/fstype: xfs + type: gp3 + encrypted: "true" -- 2.40.1 From ebdf680853532e28817a0e65e06a61e025d1010a Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 25 Feb 2021 00:17:50 +0100 Subject: [PATCH 16/65] Version bump for aws-efs-csi-driver, use upstream helm chart --- charts/kubezero-aws-efs-csi-driver/Chart.yaml | 13 +- .../charts/aws-efs-csi-driver/Chart.yaml | 15 --- .../aws-efs-csi-driver/templates/NOTES.txt | 3 - .../aws-efs-csi-driver/templates/_helpers.tpl | 45 ------- .../templates/csidriver.yaml | 6 - .../templates/daemonset.yaml | 117 ------------------ .../charts/aws-efs-csi-driver/values.yaml | 46 ------- charts/kubezero-aws-efs-csi-driver/update.sh | 14 --- charts/kubezero-kiam/Chart.yaml | 2 +- 9 files changed, 7 insertions(+), 254 deletions(-) delete mode 100644 charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml delete mode 100644 charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/NOTES.txt delete mode 100644 charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl delete mode 100644 charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/csidriver.yaml delete mode 100644 charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/daemonset.yaml delete mode 100644 charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml delete mode 100755 charts/kubezero-aws-efs-csi-driver/update.sh diff --git a/charts/kubezero-aws-efs-csi-driver/Chart.yaml b/charts/kubezero-aws-efs-csi-driver/Chart.yaml index 77435c9..181b7ae 100644 --- a/charts/kubezero-aws-efs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-efs-csi-driver/Chart.yaml @@ -1,9 +1,9 @@ apiVersion: v2 name: kubezero-aws-efs-csi-driver description: KubeZero Umbrella Chart for aws-efs-csi-driver -version: 0.2.0 -appVersion: 1.0.0 -kubeVersion: ">=1.16.0-0" +version: 0.2.1 +appVersion: 1.1.1 +kubeVersion: ">=1.18.0-0" home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png sources: @@ -20,7 +20,6 @@ dependencies: - name: kubezero-lib version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ -# Once they properly update upstream -# - name: aws-ebs-csi-driver -# version: 1.0.0 -# repository: https://kubernetes-sigs.github.io/aws-efs-csi-driver + - name: aws-efs-csi-driver + version: 1.1.2 + repository: https://kubernetes-sigs.github.io/aws-efs-csi-driver/ diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml deleted file mode 100644 index 1a6eaa5..0000000 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -appVersion: "1.0.0" -name: aws-efs-csi-driver -description: A Helm chart for AWS EFS CSI Driver -version: 0.2.0 -kubeVersion: ">=1.14.0-0" -home: https://github.com/kubernetes-sigs/aws-efs-csi-driver -sources: - - https://github.com/kubernetes-sigs/aws-efs-csi-driver -keywords: - - aws - - efs - - csi -maintainers: - - name: leakingtapan diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/NOTES.txt b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/NOTES.txt deleted file mode 100644 index a0b1e08..0000000 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/NOTES.txt +++ /dev/null @@ -1,3 +0,0 @@ -To verify that aws-efs-csi-driver has started, run: - - kubectl get pod -n kube-system -l "app.kubernetes.io/name={{ include "aws-efs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl deleted file mode 100644 index 6e661c3..0000000 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl +++ /dev/null @@ -1,45 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "aws-efs-csi-driver.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "aws-efs-csi-driver.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "aws-efs-csi-driver.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "aws-efs-csi-driver.labels" -}} -app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} -helm.sh/chart: {{ include "aws-efs-csi-driver.chart" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/csidriver.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/csidriver.yaml deleted file mode 100644 index 4dbf6f1..0000000 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/csidriver.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: storage.k8s.io/v1beta1 -kind: CSIDriver -metadata: - name: efs.csi.aws.com -spec: - attachRequired: false diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/daemonset.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/daemonset.yaml deleted file mode 100644 index bfe3496..0000000 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/daemonset.yaml +++ /dev/null @@ -1,117 +0,0 @@ -# Node Service -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: efs-csi-node - namespace: kube-system -spec: - selector: - matchLabels: - app: efs-csi-node - app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - labels: - app: efs-csi-node - app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - {{- if .Values.node.podAnnotations }} - annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} - {{- end }} - spec: - nodeSelector: - beta.kubernetes.io/os: linux - {{- with .Values.nodeSelector }} - {{- . | toYaml | nindent 8 }} - {{- end }} - hostNetwork: true - priorityClassName: system-node-critical - tolerations: - - operator: Exists - {{- with .Values.node.tolerations }} - {{- . | toYaml | nindent 8 }} - {{- end }} - containers: - - name: efs-plugin - securityContext: - privileged: true - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - args: - - --endpoint=$(CSI_ENDPOINT) - - --logtostderr - - --v=5 - env: - - name: CSI_ENDPOINT - value: unix:/csi/csi.sock - volumeMounts: - - name: kubelet-dir - mountPath: /var/lib/kubelet - mountPropagation: "Bidirectional" - - name: plugin-dir - mountPath: /csi - - name: efs-state-dir - mountPath: /var/run/efs - - name: efs-utils-config - mountPath: /etc/amazon/efs - ports: - - name: healthz - containerPort: 9809 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 2 - failureThreshold: 5 - - name: cs-driver-registrar - image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v=5 - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/efs.csi.aws.com/csi.sock - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: liveness-probe - image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} - args: - - --csi-address=/csi/csi.sock - - --health-port=9809 - volumeMounts: - - name: plugin-dir - mountPath: /csi - volumes: - - name: kubelet-dir - hostPath: - path: /var/lib/kubelet - type: Directory - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/efs.csi.aws.com/ - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - - name: efs-state-dir - hostPath: - path: /var/run/efs - type: DirectoryOrCreate - - name: efs-utils-config - hostPath: - path: /etc/amazon/efs - type: DirectoryOrCreate diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml deleted file mode 100644 index e583428..0000000 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# Default values for aws-efs-csi-driver. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 2 - -image: - repository: amazon/aws-efs-csi-driver - tag: "v1.0.0" - pullPolicy: IfNotPresent - -sidecars: - livenessProbeImage: - repository: quay.io/k8scsi/livenessprobe - tag: "v2.0.0" - nodeDriverRegistrarImage: - repository: quay.io/k8scsi/csi-node-driver-registrar - tag: "v1.3.0" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -podAnnotations: {} - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: {} - -tolerations: [] - -affinity: {} - -node: - podAnnotations: {} - tolerations: [] diff --git a/charts/kubezero-aws-efs-csi-driver/update.sh b/charts/kubezero-aws-efs-csi-driver/update.sh deleted file mode 100755 index c35e274..0000000 --- a/charts/kubezero-aws-efs-csi-driver/update.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -ex - -# Upstream doesnt have proper Helm repo yet so we just download latest release and stuff it into charts - -REPO="kubernetes-sigs/aws-efs-csi-driver" -[ -z "$RELEASE" ] && RELEASE=$(curl -sL -s https://api.github.com/repos/${REPO}/releases | grep '"tag_name":' | cut -d'"' -f4 | grep -v -E "(alpha|beta|rc)" | sort -t"." -k 1,1 -k 2,2 -k 3,3 -k 4,4 | tail -n 1) - -rm -rf git -git clone https://github.com/$REPO.git git -cd git && git checkout $RELEASE && cd - - -rm -rf charts/aws-efs-csi-driver && mkdir -p charts/aws-efs-csi-driver -mv git/helm/* charts/aws-efs-csi-driver diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index 119efbf..5b69919 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -19,4 +19,4 @@ dependencies: version: 6.0.0 repository: https://uswitch.github.io/kiam-helm-charts/charts/ condition: kiam.enabled -kubeVersion: ">= 1.16.0" +kubeVersion: ">= 1.18.0" -- 2.40.1 From 01cab288cb5dbe092082ea15b8fdfbb9505f4598 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 25 Feb 2021 00:23:50 +0100 Subject: [PATCH 17/65] Reduce loglevel for efs driver --- charts/kubezero-aws-efs-csi-driver/Chart.yaml | 2 +- charts/kubezero-aws-efs-csi-driver/values.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/kubezero-aws-efs-csi-driver/Chart.yaml b/charts/kubezero-aws-efs-csi-driver/Chart.yaml index 181b7ae..23c2c19 100644 --- a/charts/kubezero-aws-efs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-efs-csi-driver/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: kubezero-aws-efs-csi-driver description: KubeZero Umbrella Chart for aws-efs-csi-driver -version: 0.2.1 +version: 0.2.2 appVersion: 1.1.1 kubeVersion: ">=1.18.0-0" home: https://kubezero.com diff --git a/charts/kubezero-aws-efs-csi-driver/values.yaml b/charts/kubezero-aws-efs-csi-driver/values.yaml index e02877c..760051d 100644 --- a/charts/kubezero-aws-efs-csi-driver/values.yaml +++ b/charts/kubezero-aws-efs-csi-driver/values.yaml @@ -1,6 +1,7 @@ aws-efs-csi-driver: nodeSelector: {} # node.kubernetes.io/csi.efs.fs: EFS-FS_ID + logLevel: 1 StorageClass: create: true -- 2.40.1 From 322b9f6c2ee1b386d5038756065525b215cf6343 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 25 Feb 2021 23:44:33 +0100 Subject: [PATCH 18/65] Upgrade Istio to 1.9 --- charts/kubezero-istio-ingress/Chart.yaml | 10 +- .../charts/istio-ingress/Chart.yaml | 2 +- .../istio-ingress/templates/_affinity.tpl | 12 +- .../istio-ingress/templates/deployment.yaml | 31 +- .../templates/meshexpansion.yaml | 79 - .../istio-ingress/templates/service.yaml | 8 - .../charts/istio-ingress/values.yaml | 41 +- .../charts/istio-private-ingress/Chart.yaml | 2 +- .../templates/_affinity.tpl | 12 +- .../templates/deployment.yaml | 31 +- .../templates/meshexpansion.yaml | 79 - .../templates/service.yaml | 8 - .../charts/istio-private-ingress/values.yaml | 41 +- charts/kubezero-istio/Chart.yaml | 10 +- charts/kubezero-istio/charts/base/Chart.yaml | 2 +- .../charts/base/crds/crd-all.gen.yaml | 33 +- .../charts/base/files/gen-istio-cluster.yaml | 41 +- .../charts/base/templates/clusterrole.yaml | 12 +- charts/kubezero-istio/charts/base/values.yaml | 7 +- .../charts/istio-discovery/Chart.yaml | 3 +- .../istio-discovery/files/gen-istio.yaml | 2157 ++++++++--------- .../files/injection-template.yaml | 88 +- .../istio-discovery/templates/configmap.yaml | 11 +- .../istio-discovery/templates/deployment.yaml | 17 +- .../templates/istiod-injector-configmap.yaml | 24 +- .../templates/mutatingwebhook.yaml | 234 +- ...emetryv2_1.8.yaml => telemetryv2_1.9.yaml} | 129 +- .../charts/istio-discovery/values.yaml | 65 +- charts/kubezero-istio/update.sh | 10 +- charts/kubezero-istio/values.yaml | 4 +- 30 files changed, 1507 insertions(+), 1696 deletions(-) delete mode 100644 charts/kubezero-istio-ingress/charts/istio-ingress/templates/meshexpansion.yaml delete mode 100644 charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/meshexpansion.yaml rename charts/kubezero-istio/charts/istio-discovery/templates/{telemetryv2_1.8.yaml => telemetryv2_1.9.yaml} (90%) diff --git a/charts/kubezero-istio-ingress/Chart.yaml b/charts/kubezero-istio-ingress/Chart.yaml index 15b3889..76d0a00 100644 --- a/charts/kubezero-istio-ingress/Chart.yaml +++ b/charts/kubezero-istio-ingress/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-istio-ingress description: KubeZero Umbrella Chart for Istio based Ingress type: application -version: 0.1.2 -appVersion: 1.8.2 +version: 0.5.0 +appVersion: 1.9.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -16,9 +16,9 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: istio-ingress - version: 1.1.0 + version: 1.9.0 condition: istio-ingress.enabled - name: istio-private-ingress - version: 1.1.0 + version: 1.9.0 condition: istio-private-ingress.enabled -kubeVersion: ">= 1.16.0" +kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml b/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml index 108c8f2..78bd0c4 100644 --- a/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: istio-ingress -version: 1.1.0 +version: 1.9.0 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio gateways keywords: diff --git a/charts/kubezero-istio-ingress/charts/istio-ingress/templates/_affinity.tpl b/charts/kubezero-istio-ingress/charts/istio-ingress/templates/_affinity.tpl index 400ff54..cb6a91c 100644 --- a/charts/kubezero-istio-ingress/charts/istio-ingress/templates/_affinity.tpl +++ b/charts/kubezero-istio-ingress/charts/istio-ingress/templates/_affinity.tpl @@ -1,11 +1,11 @@ {{/* affinity - https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ */}} -{{- define "nodeaffinity" }} - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - {{- include "nodeAffinityRequiredDuringScheduling" . }} - preferredDuringSchedulingIgnoredDuringExecution: - {{- include "nodeAffinityPreferredDuringScheduling" . }} +{{ define "nodeaffinity" }} +nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + {{- include "nodeAffinityRequiredDuringScheduling" . }} + preferredDuringSchedulingIgnoredDuringExecution: + {{- include "nodeAffinityPreferredDuringScheduling" . }} {{- end }} {{- define "nodeAffinityRequiredDuringScheduling" }} diff --git a/charts/kubezero-istio-ingress/charts/istio-ingress/templates/deployment.yaml b/charts/kubezero-istio-ingress/charts/istio-ingress/templates/deployment.yaml index b5137a4..0c32e2f 100644 --- a/charts/kubezero-istio-ingress/charts/istio-ingress/templates/deployment.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-ingress/templates/deployment.yaml @@ -41,6 +41,7 @@ spec: istio.io/rev: {{ .Values.revision | default "default" }} install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }} operator.istio.io/component: "IngressGateways" + sidecar.istio.io/inject: "false" annotations: {{- if .Values.meshConfig.enablePrometheusMerge }} prometheus.io/port: "15020" @@ -101,12 +102,6 @@ spec: - containerPort: {{ $val.targetPort | default $val.port }} protocol: {{ $val.protocol | default "TCP" }} {{- end }} - {{- if $.Values.global.meshExpansion.enabled }} - {{- range $key, $val := $gateway.meshExpansionPorts }} - - containerPort: {{ $val.targetPort | default $val.port }} - protocol: {{ $val.protocol | default "TCP" }} - {{- end }} - {{- end }} - containerPort: 15090 protocol: TCP name: http-envoy-prom @@ -220,6 +215,10 @@ spec: - name: TRUST_DOMAIN value: "{{ .Values.global.trustDomain | default (index .Values.meshConfig "trustDomain") }}" {{- end }} + {{- if not $gateway.runAsRoot }} + - name: ISTIO_META_UNPRIVILEGED_POD + value: "true" + {{- end }} {{- range $key, $val := $gateway.env }} - name: {{ $key }} value: {{ $val }} @@ -228,10 +227,10 @@ spec: - name: {{ $key }} value: "{{ $value }}" {{- end }} - {{ $network_set := index $gateway.env "ISTIO_META_NETWORK" }} + {{- $network_set := index $gateway.env "ISTIO_META_NETWORK" }} {{- if and (not $network_set) .Values.global.network }} - name: ISTIO_META_NETWORK - value: {{ .Values.global.network }} + value: "{{ .Values.global.network }}" {{- end }} {{- if $gateway.podAnnotations }} - name: "ISTIO_METAJSON_ANNOTATIONS" @@ -254,8 +253,6 @@ spec: mountPath: /var/run/secrets/tokens readOnly: true {{- end }} - - name: gatewaysdsudspath - mountPath: /var/run/ingress_gateway {{- if .Values.global.mountMtlsCerts }} # Use the key and cert mounted to /etc/certs/ for the in-cluster mTLS communications. - name: istio-certs @@ -296,10 +293,18 @@ spec: - path: "annotations" fieldRef: fieldPath: metadata.annotations + - path: "cpu-limit" + resourceFieldRef: + containerName: istio-proxy + resource: limits.cpu + divisor: 1m + - path: "cpu-request" + resourceFieldRef: + containerName: istio-proxy + resource: requests.cpu + divisor: 1m - name: istio-envoy emptyDir: {} - - name: gatewaysdsudspath - emptyDir: {} - name: istio-data emptyDir: {} {{- if eq .Values.global.jwtPolicy "third-party-jwt" }} @@ -335,7 +340,7 @@ spec: optional: true {{- end }} affinity: - {{- include "nodeaffinity" (dict "global" .Values.global "nodeSelector" $gateway.nodeSelector) | indent 6 }} +{{ include "nodeaffinity" (dict "global" .Values.global "nodeSelector" $gateway.nodeSelector) | trim | indent 8 }} {{- include "podAntiAffinity" $gateway | indent 6 }} {{- if $gateway.tolerations }} tolerations: diff --git a/charts/kubezero-istio-ingress/charts/istio-ingress/templates/meshexpansion.yaml b/charts/kubezero-istio-ingress/charts/istio-ingress/templates/meshexpansion.yaml deleted file mode 100644 index 67c164b..0000000 --- a/charts/kubezero-istio-ingress/charts/istio-ingress/templates/meshexpansion.yaml +++ /dev/null @@ -1,79 +0,0 @@ -{{- if .Values.global.meshExpansion.enabled }} -apiVersion: networking.istio.io/v1alpha3 -kind: Gateway -metadata: - name: meshexpansion-gateway - namespace: {{ .Release.Namespace }} - labels: - release: {{ .Release.Name }} - istio.io/rev: {{ .Values.revision | default "default" }} - install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }} - operator.istio.io/component: "IngressGateways" -spec: - selector: - istio: ingressgateway - servers: - - port: - number: 15012 - protocol: TCP - name: tcp-istiod - hosts: - - "*" - - port: - number: 15017 - protocol: TCP - name: tcp-istiodwebhook - hosts: - - "*" ---- - -apiVersion: networking.istio.io/v1alpha3 -kind: VirtualService -metadata: - name: meshexpansion-vs-istiod - namespace: {{ .Release.Namespace }} - labels: - release: {{ .Release.Name }} -spec: - hosts: - - istiod.{{ .Values.global.istioNamespace }}.svc.{{ .Values.global.proxy.clusterDomain }} - gateways: - - meshexpansion-gateway - tcp: - - match: - - port: 15012 - route: - - destination: - host: istiod.{{ .Values.global.istioNamespace }}.svc.{{ .Values.global.proxy.clusterDomain }} - port: - number: 15012 - - match: - - port: 15017 - route: - - destination: - host: istiod.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} - port: - number: 443 ---- - -apiVersion: networking.istio.io/v1alpha3 -kind: DestinationRule -metadata: - name: meshexpansion-dr-istiod - namespace: {{ .Release.Namespace }} - labels: - release: {{ .Release.Name }} -spec: - host: istiod.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} - trafficPolicy: - portLevelSettings: - - port: - number: 15012 - tls: - mode: DISABLE - - port: - number: 15017 - tls: - mode: DISABLE - -{{- end }} diff --git a/charts/kubezero-istio-ingress/charts/istio-ingress/templates/service.yaml b/charts/kubezero-istio-ingress/charts/istio-ingress/templates/service.yaml index 237be95..0f9dbf2 100644 --- a/charts/kubezero-istio-ingress/charts/istio-ingress/templates/service.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-ingress/templates/service.yaml @@ -38,14 +38,6 @@ spec: {{- end }} {{- end }} - {{- if $.Values.global.meshExpansion.enabled }} - {{- range $key, $val := $gateway.meshExpansionPorts }} - - - {{- range $pkey, $pval := $val }} - {{ $pkey}}: {{ $pval }} - {{- end }} - {{- end }} - {{- end }} {{ range $app := $gateway.ingressPorts }} - port: {{ $app.port }} diff --git a/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml b/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml index 9f4dee6..cc01ef8 100644 --- a/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml @@ -24,7 +24,11 @@ gateways: targetPort: 8443 name: https protocol: TCP - # This is the port where sni routing happens + - port: 15012 + targetPort: 15012 + name: tcp-istiod + protocol: TCP + # This is the port where sni routing happens - port: 15443 targetPort: 15443 name: tls @@ -66,18 +70,6 @@ gateways: podAnnotations: {} type: LoadBalancer #change to NodePort, ClusterIP or LoadBalancer if need be - #### MESH EXPANSION PORTS ######## - # Pilot and Citadel MTLS ports are enabled in gateway - but will only redirect - # to pilot/citadel if global.meshExpansion settings are enabled. - # Delete these ports if mesh expansion is not enabled, to avoid - # exposing unnecessary ports on the web. - # You can remove these ports if you are not using mesh expansion - meshExpansionPorts: - - port: 15012 - targetPort: 15012 - name: tcp-istiod - ####### end MESH EXPANSION PORTS ###### - ############## secretVolumes: - name: ingressgateway-certs @@ -99,7 +91,7 @@ gateways: # A gateway with this mode ensures that pilot generates an additional # set of clusters for internal services but without Istio mTLS, to # enable cross cluster routing. - ISTIO_META_ROUTER_MODE: "sni-dnat" + ISTIO_META_ROUTER_MODE: "standard" nodeSelector: {} tolerations: [] @@ -142,6 +134,12 @@ global: # . implies these objects are visible to only to sidecars in the same namespace, or if imported as a Sidecar.egress.host defaultConfigVisibilitySettings: [] + # Default node selector to be applied to all deployments so that all pods can be + # constrained to run a particular nodes. Each component can overwrite these default + # values by adding its node selector block in the relevant section below and setting + # the desired values. + defaultNodeSelector: {} + # enable pod disruption budget for the control plane, which is used to # ensure Istio control plane components are gradually upgraded or recovered. defaultPodDisruptionBudget: @@ -170,10 +168,10 @@ global: # Default hub for Istio images. # Releases are published to docker hub under 'istio' project. # Dev builds from prow are on gcr.io - hub: gcr.io/istio-testing + hub: docker.io/istio # Default tag for Istio images. - tag: latest + tag: 1.9.0 # Specify image pull policy if default behavior isn't desired. # Default behavior: latest images will be Always else IfNotPresent. @@ -206,14 +204,6 @@ global: logging: level: "default:info" - # If set to true, the pilot and citadel mtls will be exposed on the - # ingress gateway - meshExpansion: - enabled: false - # If set to true, the pilot and citadel mtls and the plain text pilot ports - # will be exposed on an internal gateway - useILB: false - # Kubernetes >=v1.11.0 will create two PriorityClass, including system-cluster-critical and # system-node-critical, it is better to configure this in order to make sure your Istio pods # will not be killed because of low priority class. @@ -318,11 +308,10 @@ global: servicePort: 0 # Deprecated, use meshConfig.trustDomain -# trustDomain: "" + trustDomain: "" meshConfig: enablePrometheusMerge: true -# trustDomain: "" defaultConfig: proxyMetadata: {} tracing: diff --git a/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml b/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml index 2cd775d..fac8f49 100644 --- a/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: istio-private-ingress -version: 1.1.0 +version: 1.9.0 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio gateways keywords: diff --git a/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/_affinity.tpl b/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/_affinity.tpl index 400ff54..cb6a91c 100644 --- a/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/_affinity.tpl +++ b/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/_affinity.tpl @@ -1,11 +1,11 @@ {{/* affinity - https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ */}} -{{- define "nodeaffinity" }} - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - {{- include "nodeAffinityRequiredDuringScheduling" . }} - preferredDuringSchedulingIgnoredDuringExecution: - {{- include "nodeAffinityPreferredDuringScheduling" . }} +{{ define "nodeaffinity" }} +nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + {{- include "nodeAffinityRequiredDuringScheduling" . }} + preferredDuringSchedulingIgnoredDuringExecution: + {{- include "nodeAffinityPreferredDuringScheduling" . }} {{- end }} {{- define "nodeAffinityRequiredDuringScheduling" }} diff --git a/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/deployment.yaml b/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/deployment.yaml index b5137a4..0c32e2f 100644 --- a/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/deployment.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/deployment.yaml @@ -41,6 +41,7 @@ spec: istio.io/rev: {{ .Values.revision | default "default" }} install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }} operator.istio.io/component: "IngressGateways" + sidecar.istio.io/inject: "false" annotations: {{- if .Values.meshConfig.enablePrometheusMerge }} prometheus.io/port: "15020" @@ -101,12 +102,6 @@ spec: - containerPort: {{ $val.targetPort | default $val.port }} protocol: {{ $val.protocol | default "TCP" }} {{- end }} - {{- if $.Values.global.meshExpansion.enabled }} - {{- range $key, $val := $gateway.meshExpansionPorts }} - - containerPort: {{ $val.targetPort | default $val.port }} - protocol: {{ $val.protocol | default "TCP" }} - {{- end }} - {{- end }} - containerPort: 15090 protocol: TCP name: http-envoy-prom @@ -220,6 +215,10 @@ spec: - name: TRUST_DOMAIN value: "{{ .Values.global.trustDomain | default (index .Values.meshConfig "trustDomain") }}" {{- end }} + {{- if not $gateway.runAsRoot }} + - name: ISTIO_META_UNPRIVILEGED_POD + value: "true" + {{- end }} {{- range $key, $val := $gateway.env }} - name: {{ $key }} value: {{ $val }} @@ -228,10 +227,10 @@ spec: - name: {{ $key }} value: "{{ $value }}" {{- end }} - {{ $network_set := index $gateway.env "ISTIO_META_NETWORK" }} + {{- $network_set := index $gateway.env "ISTIO_META_NETWORK" }} {{- if and (not $network_set) .Values.global.network }} - name: ISTIO_META_NETWORK - value: {{ .Values.global.network }} + value: "{{ .Values.global.network }}" {{- end }} {{- if $gateway.podAnnotations }} - name: "ISTIO_METAJSON_ANNOTATIONS" @@ -254,8 +253,6 @@ spec: mountPath: /var/run/secrets/tokens readOnly: true {{- end }} - - name: gatewaysdsudspath - mountPath: /var/run/ingress_gateway {{- if .Values.global.mountMtlsCerts }} # Use the key and cert mounted to /etc/certs/ for the in-cluster mTLS communications. - name: istio-certs @@ -296,10 +293,18 @@ spec: - path: "annotations" fieldRef: fieldPath: metadata.annotations + - path: "cpu-limit" + resourceFieldRef: + containerName: istio-proxy + resource: limits.cpu + divisor: 1m + - path: "cpu-request" + resourceFieldRef: + containerName: istio-proxy + resource: requests.cpu + divisor: 1m - name: istio-envoy emptyDir: {} - - name: gatewaysdsudspath - emptyDir: {} - name: istio-data emptyDir: {} {{- if eq .Values.global.jwtPolicy "third-party-jwt" }} @@ -335,7 +340,7 @@ spec: optional: true {{- end }} affinity: - {{- include "nodeaffinity" (dict "global" .Values.global "nodeSelector" $gateway.nodeSelector) | indent 6 }} +{{ include "nodeaffinity" (dict "global" .Values.global "nodeSelector" $gateway.nodeSelector) | trim | indent 8 }} {{- include "podAntiAffinity" $gateway | indent 6 }} {{- if $gateway.tolerations }} tolerations: diff --git a/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/meshexpansion.yaml b/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/meshexpansion.yaml deleted file mode 100644 index 67c164b..0000000 --- a/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/meshexpansion.yaml +++ /dev/null @@ -1,79 +0,0 @@ -{{- if .Values.global.meshExpansion.enabled }} -apiVersion: networking.istio.io/v1alpha3 -kind: Gateway -metadata: - name: meshexpansion-gateway - namespace: {{ .Release.Namespace }} - labels: - release: {{ .Release.Name }} - istio.io/rev: {{ .Values.revision | default "default" }} - install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }} - operator.istio.io/component: "IngressGateways" -spec: - selector: - istio: ingressgateway - servers: - - port: - number: 15012 - protocol: TCP - name: tcp-istiod - hosts: - - "*" - - port: - number: 15017 - protocol: TCP - name: tcp-istiodwebhook - hosts: - - "*" ---- - -apiVersion: networking.istio.io/v1alpha3 -kind: VirtualService -metadata: - name: meshexpansion-vs-istiod - namespace: {{ .Release.Namespace }} - labels: - release: {{ .Release.Name }} -spec: - hosts: - - istiod.{{ .Values.global.istioNamespace }}.svc.{{ .Values.global.proxy.clusterDomain }} - gateways: - - meshexpansion-gateway - tcp: - - match: - - port: 15012 - route: - - destination: - host: istiod.{{ .Values.global.istioNamespace }}.svc.{{ .Values.global.proxy.clusterDomain }} - port: - number: 15012 - - match: - - port: 15017 - route: - - destination: - host: istiod.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} - port: - number: 443 ---- - -apiVersion: networking.istio.io/v1alpha3 -kind: DestinationRule -metadata: - name: meshexpansion-dr-istiod - namespace: {{ .Release.Namespace }} - labels: - release: {{ .Release.Name }} -spec: - host: istiod.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} - trafficPolicy: - portLevelSettings: - - port: - number: 15012 - tls: - mode: DISABLE - - port: - number: 15017 - tls: - mode: DISABLE - -{{- end }} diff --git a/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/service.yaml b/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/service.yaml index 237be95..0f9dbf2 100644 --- a/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/service.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-private-ingress/templates/service.yaml @@ -38,14 +38,6 @@ spec: {{- end }} {{- end }} - {{- if $.Values.global.meshExpansion.enabled }} - {{- range $key, $val := $gateway.meshExpansionPorts }} - - - {{- range $pkey, $pval := $val }} - {{ $pkey}}: {{ $pval }} - {{- end }} - {{- end }} - {{- end }} {{ range $app := $gateway.ingressPorts }} - port: {{ $app.port }} diff --git a/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml b/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml index 9f4dee6..cc01ef8 100644 --- a/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml @@ -24,7 +24,11 @@ gateways: targetPort: 8443 name: https protocol: TCP - # This is the port where sni routing happens + - port: 15012 + targetPort: 15012 + name: tcp-istiod + protocol: TCP + # This is the port where sni routing happens - port: 15443 targetPort: 15443 name: tls @@ -66,18 +70,6 @@ gateways: podAnnotations: {} type: LoadBalancer #change to NodePort, ClusterIP or LoadBalancer if need be - #### MESH EXPANSION PORTS ######## - # Pilot and Citadel MTLS ports are enabled in gateway - but will only redirect - # to pilot/citadel if global.meshExpansion settings are enabled. - # Delete these ports if mesh expansion is not enabled, to avoid - # exposing unnecessary ports on the web. - # You can remove these ports if you are not using mesh expansion - meshExpansionPorts: - - port: 15012 - targetPort: 15012 - name: tcp-istiod - ####### end MESH EXPANSION PORTS ###### - ############## secretVolumes: - name: ingressgateway-certs @@ -99,7 +91,7 @@ gateways: # A gateway with this mode ensures that pilot generates an additional # set of clusters for internal services but without Istio mTLS, to # enable cross cluster routing. - ISTIO_META_ROUTER_MODE: "sni-dnat" + ISTIO_META_ROUTER_MODE: "standard" nodeSelector: {} tolerations: [] @@ -142,6 +134,12 @@ global: # . implies these objects are visible to only to sidecars in the same namespace, or if imported as a Sidecar.egress.host defaultConfigVisibilitySettings: [] + # Default node selector to be applied to all deployments so that all pods can be + # constrained to run a particular nodes. Each component can overwrite these default + # values by adding its node selector block in the relevant section below and setting + # the desired values. + defaultNodeSelector: {} + # enable pod disruption budget for the control plane, which is used to # ensure Istio control plane components are gradually upgraded or recovered. defaultPodDisruptionBudget: @@ -170,10 +168,10 @@ global: # Default hub for Istio images. # Releases are published to docker hub under 'istio' project. # Dev builds from prow are on gcr.io - hub: gcr.io/istio-testing + hub: docker.io/istio # Default tag for Istio images. - tag: latest + tag: 1.9.0 # Specify image pull policy if default behavior isn't desired. # Default behavior: latest images will be Always else IfNotPresent. @@ -206,14 +204,6 @@ global: logging: level: "default:info" - # If set to true, the pilot and citadel mtls will be exposed on the - # ingress gateway - meshExpansion: - enabled: false - # If set to true, the pilot and citadel mtls and the plain text pilot ports - # will be exposed on an internal gateway - useILB: false - # Kubernetes >=v1.11.0 will create two PriorityClass, including system-cluster-critical and # system-node-critical, it is better to configure this in order to make sure your Istio pods # will not be killed because of low priority class. @@ -318,11 +308,10 @@ global: servicePort: 0 # Deprecated, use meshConfig.trustDomain -# trustDomain: "" + trustDomain: "" meshConfig: enablePrometheusMerge: true -# trustDomain: "" defaultConfig: proxyMetadata: {} tracing: diff --git a/charts/kubezero-istio/Chart.yaml b/charts/kubezero-istio/Chart.yaml index a349c7c..708ab9e 100644 --- a/charts/kubezero-istio/Chart.yaml +++ b/charts/kubezero-istio/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-istio description: KubeZero Umbrella Chart for Istio type: application -version: 0.4.2 -appVersion: 1.8.2 +version: 0.5.0 +appVersion: 1.9.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -16,7 +16,7 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: base - version: 1.1.0 + version: 1.9.0 - name: istio-discovery - version: 1.2.0 -kubeVersion: ">= 1.16.0" + version: 1.9.0 +kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-istio/charts/base/Chart.yaml b/charts/kubezero-istio/charts/base/Chart.yaml index e012727..8cf22dc 100644 --- a/charts/kubezero-istio/charts/base/Chart.yaml +++ b/charts/kubezero-istio/charts/base/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: base -version: 1.1.0 +version: 1.9.0 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio cluster resources and CRDs keywords: diff --git a/charts/kubezero-istio/charts/base/crds/crd-all.gen.yaml b/charts/kubezero-istio/charts/base/crds/crd-all.gen.yaml index 35b4db8..faddc02 100644 --- a/charts/kubezero-istio/charts/base/crds/crd-all.gen.yaml +++ b/charts/kubezero-istio/charts/base/crds/crd-all.gen.yaml @@ -1287,6 +1287,10 @@ spec: description: Applies only to sidecars. format: string type: string + destinationPort: + description: The destination_port value used by a + filter chain's match condition. + type: integer filter: description: The name of a specific filter to apply the patch to. @@ -2817,6 +2821,11 @@ spec: apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio name: workloadgroups.networking.istio.io spec: additionalPrinterColumns: @@ -2884,11 +2893,11 @@ spec: - exec properties: exec: - description: health is determined by how the command that is executed + description: Health is determined by how the command that is executed exited. properties: command: - description: command to run. + description: Command to run. items: format: string type: string @@ -2906,7 +2915,7 @@ spec: format: string type: string httpHeaders: - description: headers the proxy will pass on to make the request. + description: Headers the proxy will pass on to make the request. items: properties: name: @@ -2922,7 +2931,7 @@ spec: format: string type: string port: - description: port on which the endpoint lives. + description: Port on which the endpoint lives. type: integer scheme: format: string @@ -2943,7 +2952,7 @@ spec: format: int32 type: integer tcpSocket: - description: health is determined by if the proxy is able to connect. + description: Health is determined by if the proxy is able to connect. properties: host: format: string @@ -3048,6 +3057,7 @@ spec: - CUSTOM type: string provider: + description: Specifies detailed configuration of the CUSTOM action. properties: name: description: Specifies the name of the extension provider. @@ -3243,6 +3253,19 @@ metadata: release: istio name: peerauthentications.security.istio.io spec: + additionalPrinterColumns: + - JSONPath: .spec.mtls.mode + description: Defines the mTLS mode used for peer authentication. + name: Mode + type: string + - JSONPath: .metadata.creationTimestamp + description: 'CreationTimestamp is a timestamp representing the server time when + this object was created. It is not guaranteed to be set in happens-before order + across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + name: Age + type: date group: security.istio.io names: categories: diff --git a/charts/kubezero-istio/charts/base/files/gen-istio-cluster.yaml b/charts/kubezero-istio/charts/base/files/gen-istio-cluster.yaml index ac70215..2fda970 100644 --- a/charts/kubezero-istio/charts/base/files/gen-istio-cluster.yaml +++ b/charts/kubezero-istio/charts/base/files/gen-istio-cluster.yaml @@ -1289,6 +1289,10 @@ spec: description: Applies only to sidecars. format: string type: string + destinationPort: + description: The destination_port value used by a + filter chain's match condition. + type: integer filter: description: The name of a specific filter to apply the patch to. @@ -2819,6 +2823,11 @@ spec: apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio name: workloadgroups.networking.istio.io spec: additionalPrinterColumns: @@ -2886,11 +2895,11 @@ spec: - exec properties: exec: - description: health is determined by how the command that is executed + description: Health is determined by how the command that is executed exited. properties: command: - description: command to run. + description: Command to run. items: format: string type: string @@ -2908,7 +2917,7 @@ spec: format: string type: string httpHeaders: - description: headers the proxy will pass on to make the request. + description: Headers the proxy will pass on to make the request. items: properties: name: @@ -2924,7 +2933,7 @@ spec: format: string type: string port: - description: port on which the endpoint lives. + description: Port on which the endpoint lives. type: integer scheme: format: string @@ -2945,7 +2954,7 @@ spec: format: int32 type: integer tcpSocket: - description: health is determined by if the proxy is able to connect. + description: Health is determined by if the proxy is able to connect. properties: host: format: string @@ -3050,6 +3059,7 @@ spec: - CUSTOM type: string provider: + description: Specifies detailed configuration of the CUSTOM action. properties: name: description: Specifies the name of the extension provider. @@ -3245,6 +3255,19 @@ metadata: release: istio name: peerauthentications.security.istio.io spec: + additionalPrinterColumns: + - JSONPath: .spec.mtls.mode + description: Defines the mTLS mode used for peer authentication. + name: Mode + type: string + - JSONPath: .metadata.creationTimestamp + description: 'CreationTimestamp is a timestamp representing the server time when + this object was created. It is not guaranteed to be set in happens-before order + across separate operations. Clients may not set this value. It is represented + in RFC3339 form and is in UTC. Populated by the system. Read-only. Null for + lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' + name: Age + type: date group: security.istio.io names: categories: @@ -3528,7 +3551,7 @@ rules: # sidecar injection controller - apiGroups: ["admissionregistration.k8s.io"] resources: ["mutatingwebhookconfigurations"] - verbs: ["get", "list", "watch", "patch"] + verbs: ["get", "list", "watch", "update", "patch"] # configuration validation webhook controller - apiGroups: ["admissionregistration.k8s.io"] @@ -3628,6 +3651,12 @@ rules: - apiGroups: [""] resources: ["endpoints", "pods", "services", "nodes", "replicationcontrollers", "namespaces", "secrets"] verbs: ["get", "list", "watch"] + - apiGroups: ["networking.istio.io"] + verbs: [ "get", "watch", "list" ] + resources: [ "workloadentries" ] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "watch"] - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] verbs: ["get", "list", "watch"] diff --git a/charts/kubezero-istio/charts/base/templates/clusterrole.yaml b/charts/kubezero-istio/charts/base/templates/clusterrole.yaml index f53b830..e4176d5 100644 --- a/charts/kubezero-istio/charts/base/templates/clusterrole.yaml +++ b/charts/kubezero-istio/charts/base/templates/clusterrole.yaml @@ -9,7 +9,7 @@ rules: # sidecar injection controller - apiGroups: ["admissionregistration.k8s.io"] resources: ["mutatingwebhookconfigurations"] - verbs: ["get", "list", "watch", "patch"] + verbs: ["get", "list", "watch", "update", "patch"] # configuration validation webhook controller - apiGroups: ["admissionregistration.k8s.io"] @@ -123,6 +123,12 @@ rules: - apiGroups: [""] resources: ["endpoints", "pods", "services", "nodes", "replicationcontrollers", "namespaces", "secrets"] verbs: ["get", "list", "watch"] + - apiGroups: ["networking.istio.io"] + verbs: [ "get", "watch", "list" ] + resources: [ "workloadentries" ] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "watch"] - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] verbs: ["get", "list", "watch"] @@ -135,13 +141,13 @@ rules: - apiGroups: ["authorization.k8s.io"] resources: ["subjectaccessreviews"] verbs: ["create"] -{{- if or .Values.global.externalIstiod .Values.global.centralIstiod }} +{{- if or .Values.global.externalIstiod }} - apiGroups: [""] resources: ["configmaps"] verbs: ["create", "get", "list", "watch", "update"] - apiGroups: ["admissionregistration.k8s.io"] resources: ["mutatingwebhookconfigurations"] - verbs: ["get", "list", "watch", "patch"] + verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["admissionregistration.k8s.io"] resources: ["validatingwebhookconfigurations"] verbs: ["get", "list", "watch", "update"] diff --git a/charts/kubezero-istio/charts/base/values.yaml b/charts/kubezero-istio/charts/base/values.yaml index af16510..8f86ba0 100644 --- a/charts/kubezero-istio/charts/base/values.yaml +++ b/charts/kubezero-istio/charts/base/values.yaml @@ -12,6 +12,8 @@ global: enableAnalysis: false configValidation: true + externalIstiod: false + remotePilotAddress: "" base: # Used for helm2 to add the CRDs to templates. @@ -19,4 +21,7 @@ base: # Validation webhook configuration url # For example: https://$remotePilotAddress:15017/validate - validationURL: "" \ No newline at end of file + validationURL: "" + + # For istioctl usage to disable istio config crds in base + enableIstioConfigCRDs: true diff --git a/charts/kubezero-istio/charts/istio-discovery/Chart.yaml b/charts/kubezero-istio/charts/istio-discovery/Chart.yaml index daed6bd..60e1a49 100644 --- a/charts/kubezero-istio/charts/istio-discovery/Chart.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/Chart.yaml @@ -1,7 +1,6 @@ apiVersion: v1 name: istio-discovery -version: 1.2.0 -appVersion: 1.2.0 +version: 1.9.0 tillerVersion: ">=2.7.2" description: Helm chart for istio control plane keywords: diff --git a/charts/kubezero-istio/charts/istio-discovery/files/gen-istio.yaml b/charts/kubezero-istio/charts/istio-discovery/files/gen-istio.yaml index da98e33..4a4077a 100644 --- a/charts/kubezero-istio/charts/istio-discovery/files/gen-istio.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/files/gen-istio.yaml @@ -39,8 +39,6 @@ data: mesh: |- defaultConfig: discoveryAddress: istiod.istio-system.svc:15012 - proxyMetadata: - DNS_AGENT: "" tracing: zipkin: address: zipkin.istio-system:9411 @@ -64,16 +62,7 @@ data: values: |- { "global": { - "arch": { - "amd64": 2, - "ppc64le": 2, - "s390x": 2 - }, "caAddress": "", - "centralIstiod": false, - "configValidation": true, - "defaultConfigVisibilitySettings": [], - "defaultNodeSelector": {}, "defaultPodDisruptionBudget": { "enabled": true }, @@ -82,7 +71,6 @@ data: "cpu": "10m" } }, - "defaultTolerations": [], "externalIstiod": false, "hub": "gcr.io/istio-testing", "imagePullPolicy": "", @@ -96,18 +84,12 @@ data: "logging": { "level": "default:info" }, - "meshExpansion": { - "enabled": false, - "useILB": false - }, "meshID": "", "meshNetworks": {}, "mountMtlsCerts": false, "multiCluster": { "clusterName": "", - "enabled": false, - "globalDomainSuffix": "global", - "includeEnvoyFilter": true + "enabled": false }, "network": "", "omitSidecarInjectorConfigMap": false, @@ -191,14 +173,17 @@ data: "revision": "", "sidecarInjectorWebhook": { "alwaysInjectSelector": [], + "defaultTemplates": [], "enableNamespacesByDefault": false, "injectedAnnotations": {}, "neverInjectSelector": [], "objectSelector": { "autoInject": true, - "enabled": false + "enabled": true }, - "rewriteAppHTTPProbe": true + "rewriteAppHTTPProbe": true, + "templates": {}, + "useLegacySelectors": true } } @@ -208,476 +193,508 @@ data: # New fields should not use Values - it is a 'primary' config object, users should be able # to fine tune it or use it with kube-inject. config: |- + # defaultTemplates defines the default template to use for pods that do not explicitly specify a template + defaultTemplates: [sidecar] policy: enabled alwaysInjectSelector: [] neverInjectSelector: [] injectedAnnotations: - - template: | - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} - rewriteAppHTTPProbe: {{ valueOrDefault .Values.sidecarInjectorWebhook.rewriteAppHTTPProbe false }} - initContainers: - {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} - {{ if .Values.istio_cni.enabled -}} - - name: istio-validation - {{ else -}} - - name: istio-init - {{ end -}} - {{- if contains "/" .Values.global.proxy_init.image }} - image: "{{ .Values.global.proxy_init.image }}" - {{- else }} - image: "{{ .Values.global.hub }}/{{ .Values.global.proxy_init.image }}:{{ .Values.global.tag }}" - {{- end }} - args: - - istio-iptables - - "-p" - - "15001" - - "-z" - - "15006" - - "-u" - - "1337" - - "-m" - - "{{ annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode }}" - - "-i" - - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundIPRanges` .Values.global.proxy.includeIPRanges }}" - - "-x" - - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundIPRanges` .Values.global.proxy.excludeIPRanges }}" - - "-b" - - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeInboundPorts` `*` }}" - - "-d" - {{- if excludeInboundPort (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) (annotation .ObjectMeta `traffic.sidecar.istio.io/excludeInboundPorts` .Values.global.proxy.excludeInboundPorts) }} - - "15090,15021,{{ excludeInboundPort (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) (annotation .ObjectMeta `traffic.sidecar.istio.io/excludeInboundPorts` .Values.global.proxy.excludeInboundPorts) }}" - {{- else }} - - "15090,15021" - {{- end }} - {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/includeOutboundPorts`) (ne (valueOrDefault .Values.global.proxy.includeOutboundPorts "") "") -}} - - "-q" - - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundPorts` .Values.global.proxy.includeOutboundPorts }}" - {{ end -}} - {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/excludeOutboundPorts`) (ne (valueOrDefault .Values.global.proxy.excludeOutboundPorts "") "") -}} - - "-o" - - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundPorts` .Values.global.proxy.excludeOutboundPorts }}" - {{ end -}} - {{ if (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/kubevirtInterfaces`) -}} - - "-k" - - "{{ index .ObjectMeta.Annotations `traffic.sidecar.istio.io/kubevirtInterfaces` }}" - {{ end -}} - {{ if .Values.istio_cni.enabled -}} - - "--run-validation" - - "--skip-rule-apply" - {{ end -}} - imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" - {{- if .ProxyConfig.ProxyMetadata }} - env: - {{- range $key, $value := .ProxyConfig.ProxyMetadata }} - - name: {{ $key }} - value: "{{ $value }}" - {{- end }} - {{- end }} - resources: - {{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) }} - {{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) }} - requests: - {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) -}} - cpu: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU` }}" + template: "{{ Template_Version_And_Istio_Version_Mismatched_Check_Installation }}" + templates: + sidecar: | + {{- $containers := list }} + {{- range $index, $container := .Spec.Containers }}{{ if not (eq $container.Name "istio-proxy") }}{{ $containers = append $containers $container.Name }}{{end}}{{- end}} + metadata: + labels: + security.istio.io/tlsMode: {{ index .ObjectMeta.Labels `security.istio.io/tlsMode` | default "istio" | quote }} + service.istio.io/canonical-name: {{ index .ObjectMeta.Labels `service.istio.io/canonical-name` | default (index .ObjectMeta.Labels `app.kubernetes.io/name`) | default (index .ObjectMeta.Labels `app`) | default .DeploymentMeta.Name | quote }} + service.istio.io/canonical-revision: {{ index .ObjectMeta.Labels `service.istio.io/canonical-revision` | default (index .ObjectMeta.Labels `app.kubernetes.io/version`) | default (index .ObjectMeta.Labels `version`) | default "latest" | quote }} + istio.io/rev: {{ .Revision | default "default" | quote }} + annotations: { + {{- if eq (len $containers) 1 }} + kubectl.kubernetes.io/default-logs-container: "{{ index $containers 0 }}", {{ end }} - {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) -}} - memory: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory` }}" - {{ end }} - {{- end }} - {{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) }} - limits: - {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) -}} - cpu: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit` }}" - {{ end }} - {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) -}} - memory: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit` }}" - {{ end }} - {{- end }} - {{- else }} - {{- if .Values.global.proxy.resources }} - {{ toYaml .Values.global.proxy.resources | indent 4 }} - {{- end }} - {{- end }} - securityContext: - allowPrivilegeEscalation: {{ .Values.global.proxy.privileged }} - privileged: {{ .Values.global.proxy.privileged }} - capabilities: - {{- if not .Values.istio_cni.enabled }} - add: - - NET_ADMIN - - NET_RAW - {{- end }} - drop: - - ALL - {{- if not .Values.istio_cni.enabled }} - readOnlyRootFilesystem: false - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 - {{- else }} - readOnlyRootFilesystem: true - runAsGroup: 1337 - runAsUser: 1337 - runAsNonRoot: true - {{- end }} - restartPolicy: Always - {{ end -}} - {{- if eq .Values.global.proxy.enableCoreDump true }} - - name: enable-core-dump - args: - - -c - - sysctl -w kernel.core_pattern=/var/lib/istio/data/core.proxy && ulimit -c unlimited - command: - - /bin/sh - {{- if contains "/" .Values.global.proxy_init.image }} - image: "{{ .Values.global.proxy_init.image }}" - {{- else }} - image: "{{ .Values.global.hub }}/{{ .Values.global.proxy_init.image }}:{{ .Values.global.tag }}" - {{- end }} - imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" - resources: {} - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - drop: - - ALL - privileged: true - readOnlyRootFilesystem: false - runAsGroup: 0 - runAsNonRoot: false - runAsUser: 0 - {{ end }} - containers: - - name: istio-proxy - {{- if contains "/" (annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy.image) }} - image: "{{ annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy.image }}" - {{- else }} - image: "{{ .Values.global.hub }}/{{ .Values.global.proxy.image }}:{{ .Values.global.tag }}" - {{- end }} - ports: - - containerPort: 15090 - protocol: TCP - name: http-envoy-prom - args: - - proxy - - sidecar - - --domain - - $(POD_NAMESPACE).svc.{{ .Values.global.proxy.clusterDomain }} - - --serviceCluster - {{ if ne "" (index .ObjectMeta.Labels "app") -}} - - "{{ index .ObjectMeta.Labels `app` }}.$(POD_NAMESPACE)" - {{ else -}} - - "{{ valueOrDefault .DeploymentMeta.Name `istio-proxy` }}.{{ valueOrDefault .DeploymentMeta.Namespace `default` }}" - {{ end -}} - - --proxyLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/logLevel` .Values.global.proxy.logLevel}} - - --proxyComponentLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/componentLogLevel` .Values.global.proxy.componentLogLevel}} - {{- if .Values.global.sts.servicePort }} - - --stsPort={{ .Values.global.sts.servicePort }} - {{- end }} - {{- if .Values.global.logAsJson }} - - --log_as_json - {{- end }} - {{- if gt .ProxyConfig.Concurrency.GetValue 0 }} - - --concurrency - - "{{ .ProxyConfig.Concurrency.GetValue }}" - {{- end -}} - {{- if .Values.global.proxy.lifecycle }} - lifecycle: - {{ toYaml .Values.global.proxy.lifecycle | indent 4 }} - {{- else if $holdProxy }} - lifecycle: - postStart: - exec: - command: - - pilot-agent - - wait - {{- end }} - env: - - name: JWT_POLICY - value: {{ .Values.global.jwtPolicy }} - - name: PILOT_CERT_PROVIDER - value: {{ .Values.global.pilotCertProvider }} - - name: CA_ADDR - {{- if .Values.global.caAddress }} - value: {{ .Values.global.caAddress }} - {{- else }} - value: istiod{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }}.{{ .Values.global.istioNamespace }}.svc:15012 - {{- end }} - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: INSTANCE_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - - name: CANONICAL_SERVICE - valueFrom: - fieldRef: - fieldPath: metadata.labels['service.istio.io/canonical-name'] - - name: CANONICAL_REVISION - valueFrom: - fieldRef: - fieldPath: metadata.labels['service.istio.io/canonical-revision'] - - name: PROXY_CONFIG - value: | - {{ protoToJSON .ProxyConfig }} - - name: ISTIO_META_POD_PORTS - value: |- - [ - {{- $first := true }} - {{- range $index1, $c := .Spec.Containers }} - {{- range $index2, $p := $c.Ports }} - {{- if (structToJSON $p) }} - {{if not $first}},{{end}}{{ structToJSON $p }} - {{- $first = false }} - {{- end }} - {{- end}} - {{- end}} - ] - - name: ISTIO_META_APP_CONTAINERS - value: "{{- range $index, $container := .Spec.Containers }}{{- if ne $index 0}},{{- end}}{{ $container.Name }}{{- end}}" - - name: ISTIO_META_CLUSTER_ID - value: "{{ valueOrDefault .Values.global.multiCluster.clusterName `Kubernetes` }}" - - name: ISTIO_META_INTERCEPTION_MODE - value: "{{ or (index .ObjectMeta.Annotations `sidecar.istio.io/interceptionMode`) .ProxyConfig.InterceptionMode.String }}" - {{- if .Values.global.network }} - - name: ISTIO_META_NETWORK - value: "{{ .Values.global.network }}" - {{- end }} - {{ if .ObjectMeta.Annotations }} - - name: ISTIO_METAJSON_ANNOTATIONS - value: | - {{ toJSON .ObjectMeta.Annotations }} - {{ end }} - {{- if .DeploymentMeta.Name }} - - name: ISTIO_META_WORKLOAD_NAME - value: "{{ .DeploymentMeta.Name }}" - {{ end }} - {{- if and .TypeMeta.APIVersion .DeploymentMeta.Name }} - - name: ISTIO_META_OWNER - value: kubernetes://apis/{{ .TypeMeta.APIVersion }}/namespaces/{{ valueOrDefault .DeploymentMeta.Namespace `default` }}/{{ toLower .TypeMeta.Kind}}s/{{ .DeploymentMeta.Name }} - {{- end}} - {{- if (isset .ObjectMeta.Annotations `sidecar.istio.io/bootstrapOverride`) }} - - name: ISTIO_BOOTSTRAP_OVERRIDE - value: "/etc/istio/custom-bootstrap/custom_bootstrap.json" - {{- end }} - {{- if .Values.global.meshID }} - - name: ISTIO_META_MESH_ID - value: "{{ .Values.global.meshID }}" - {{- else if (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain) }} - - name: ISTIO_META_MESH_ID - value: "{{ (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain) }}" - {{- end }} - {{- with (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain) }} - - name: TRUST_DOMAIN - value: "{{ . }}" - {{- end }} - {{- if and (eq .Values.global.proxy.tracer "datadog") (isset .ObjectMeta.Annotations `apm.datadoghq.com/env`) }} - {{- range $key, $value := fromJSON (index .ObjectMeta.Annotations `apm.datadoghq.com/env`) }} - - name: {{ $key }} - value: "{{ $value }}" - {{- end }} - {{- end }} - {{- range $key, $value := .ProxyConfig.ProxyMetadata }} - - name: {{ $key }} - value: "{{ $value }}" - {{- end }} - imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" - {{ if ne (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) `0` }} - readinessProbe: - httpGet: - path: /healthz/ready - port: 15021 - initialDelaySeconds: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/initialDelaySeconds` .Values.global.proxy.readinessInitialDelaySeconds }} - periodSeconds: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/periodSeconds` .Values.global.proxy.readinessPeriodSeconds }} - timeoutSeconds: 3 - failureThreshold: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/failureThreshold` .Values.global.proxy.readinessFailureThreshold }} - {{ end -}} - securityContext: - allowPrivilegeEscalation: {{ .Values.global.proxy.privileged }} - capabilities: - {{ if or (eq (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `TPROXY`) (eq (annotation .ObjectMeta `sidecar.istio.io/capNetBindService` .Values.global.proxy.capNetBindService) `true`) -}} - add: - {{ if eq (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `TPROXY` -}} - - NET_ADMIN + {{- if .Values.istio_cni.enabled }} + {{- if not .Values.istio_cni.chained }} + k8s.v1.cni.cncf.io/networks: '{{ appendMultusNetwork (index .ObjectMeta.Annotations `k8s.v1.cni.cncf.io/networks`) `istio-cni` }}', {{- end }} - {{ if eq (annotation .ObjectMeta `sidecar.istio.io/capNetBindService` .Values.global.proxy.capNetBindService) `true` -}} - - NET_BIND_SERVICE + sidecar.istio.io/interceptionMode: "{{ annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode }}", + {{ with annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundIPRanges` .Values.global.proxy.includeIPRanges }}traffic.sidecar.istio.io/includeOutboundIPRanges: "{{.}}",{{ end }} + {{ with annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundIPRanges` .Values.global.proxy.excludeIPRanges }}traffic.sidecar.istio.io/excludeOutboundIPRanges: "{{.}}",{{ end }} + traffic.sidecar.istio.io/includeInboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeInboundPorts` `*` }}", + traffic.sidecar.istio.io/excludeInboundPorts: "{{ excludeInboundPort (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) (annotation .ObjectMeta `traffic.sidecar.istio.io/excludeInboundPorts` .Values.global.proxy.excludeInboundPorts) }}", + {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/includeOutboundPorts`) (ne (valueOrDefault .Values.global.proxy.includeOutboundPorts "") "") }} + traffic.sidecar.istio.io/includeOutboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundPorts` .Values.global.proxy.includeOutboundPorts }}", {{- end }} + {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/excludeOutboundPorts`) (ne .Values.global.proxy.excludeOutboundPorts "") }} + traffic.sidecar.istio.io/excludeOutboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundPorts` .Values.global.proxy.excludeOutboundPorts }}", {{- end }} - drop: - - ALL - privileged: {{ .Values.global.proxy.privileged }} - readOnlyRootFilesystem: {{ not .Values.global.proxy.enableCoreDump }} - runAsGroup: 1337 - fsGroup: 1337 - {{ if or (eq (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `TPROXY`) (eq (annotation .ObjectMeta `sidecar.istio.io/capNetBindService` .Values.global.proxy.capNetBindService) `true`) -}} - runAsNonRoot: false - runAsUser: 0 - {{- else -}} - runAsNonRoot: true - runAsUser: 1337 - {{- end }} - resources: - {{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) }} - {{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) }} - requests: - {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) -}} - cpu: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU` }}" - {{ end }} - {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) -}} - memory: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory` }}" - {{ end }} + {{ with index .ObjectMeta.Annotations `traffic.sidecar.istio.io/kubevirtInterfaces` }}traffic.sidecar.istio.io/kubevirtInterfaces: "{{.}}",{{ end }} {{- end }} - {{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) }} - limits: - {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) -}} - cpu: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit` }}" - {{ end }} - {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) -}} - memory: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit` }}" - {{ end }} - {{- end }} - {{- else }} - {{- if .Values.global.proxy.resources }} - {{ toYaml .Values.global.proxy.resources | indent 4 }} - {{- end }} - {{- end }} - volumeMounts: - {{- if eq .Values.global.pilotCertProvider "istiod" }} - - mountPath: /var/run/secrets/istio - name: istiod-ca-cert - {{- end }} - - mountPath: /var/lib/istio/data - name: istio-data - {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/bootstrapOverride`) }} - - mountPath: /etc/istio/custom-bootstrap - name: custom-bootstrap-volume - {{- end }} - # SDS channel between istioagent and Envoy - - mountPath: /etc/istio/proxy - name: istio-envoy - {{- if eq .Values.global.jwtPolicy "third-party-jwt" }} - - mountPath: /var/run/secrets/tokens - name: istio-token - {{- end }} - {{- if .Values.global.mountMtlsCerts }} - # Use the key and cert mounted to /etc/certs/ for the in-cluster mTLS communications. - - mountPath: /etc/certs/ - name: istio-certs - readOnly: true - {{- end }} - - name: istio-podinfo - mountPath: /etc/istio/pod - {{- if and (eq .Values.global.proxy.tracer "lightstep") .ProxyConfig.GetTracing.GetTlsSettings }} - - mountPath: {{ directory .ProxyConfig.GetTracing.GetTlsSettings.GetCaCertificates }} - name: lightstep-certs - readOnly: true - {{- end }} - {{- if isset .ObjectMeta.Annotations `sidecar.istio.io/userVolumeMount` }} - {{ range $index, $value := fromJSON (index .ObjectMeta.Annotations `sidecar.istio.io/userVolumeMount`) }} - - name: "{{ $index }}" - {{ toYaml $value | indent 4 }} - {{ end }} - {{- end }} - volumes: - {{- if (isset .ObjectMeta.Annotations `sidecar.istio.io/bootstrapOverride`) }} - - name: custom-bootstrap-volume - configMap: - name: {{ annotation .ObjectMeta `sidecar.istio.io/bootstrapOverride` "" }} - {{- end }} - # SDS channel between istioagent and Envoy - - emptyDir: - medium: Memory - name: istio-envoy - - name: istio-data - emptyDir: {} - - name: istio-podinfo - downwardAPI: - items: - - path: "labels" - fieldRef: - fieldPath: metadata.labels - - path: "annotations" - fieldRef: - fieldPath: metadata.annotations - {{- if eq .Values.global.jwtPolicy "third-party-jwt" }} - - name: istio-token - projected: - sources: - - serviceAccountToken: - path: istio-token - expirationSeconds: 43200 - audience: {{ .Values.global.sds.token.aud }} - {{- end }} - {{- if eq .Values.global.pilotCertProvider "istiod" }} - - name: istiod-ca-cert - configMap: - name: istio-ca-root-cert - {{- end }} - {{- if .Values.global.mountMtlsCerts }} - # Use the key and cert mounted to /etc/certs/ for the in-cluster mTLS communications. - - name: istio-certs - secret: - optional: true - {{ if eq .Spec.ServiceAccountName "" }} - secretName: istio.default + } + spec: + {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + initContainers: + {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} + {{ if .Values.istio_cni.enabled -}} + - name: istio-validation {{ else -}} - secretName: {{ printf "istio.%s" .Spec.ServiceAccountName }} - {{ end -}} - {{- end }} - {{- if isset .ObjectMeta.Annotations `sidecar.istio.io/userVolume` }} - {{range $index, $value := fromJSON (index .ObjectMeta.Annotations `sidecar.istio.io/userVolume`) }} - - name: "{{ $index }}" - {{ toYaml $value | indent 2 }} - {{ end }} - {{ end }} - {{- if and (eq .Values.global.proxy.tracer "lightstep") .ProxyConfig.GetTracing.GetTlsSettings }} - - name: lightstep-certs - secret: - optional: true - secretName: lightstep.cacert - {{- end }} - podRedirectAnnot: - {{- if and (.Values.istio_cni.enabled) (not .Values.istio_cni.chained) }} - k8s.v1.cni.cncf.io/networks: '{{ appendMultusNetwork (index .ObjectMeta.Annotations `k8s.v1.cni.cncf.io/networks`) `istio-cni` }}' - {{- end }} - sidecar.istio.io/interceptionMode: "{{ annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode }}" - traffic.sidecar.istio.io/includeOutboundIPRanges: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundIPRanges` .Values.global.proxy.includeIPRanges }}" - traffic.sidecar.istio.io/excludeOutboundIPRanges: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundIPRanges` .Values.global.proxy.excludeIPRanges }}" - traffic.sidecar.istio.io/includeInboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeInboundPorts` (includeInboundPorts .Spec.Containers) }}" - traffic.sidecar.istio.io/excludeInboundPorts: "{{ excludeInboundPort (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) (annotation .ObjectMeta `traffic.sidecar.istio.io/excludeInboundPorts` .Values.global.proxy.excludeInboundPorts) }}" - {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/includeOutboundPorts`) (ne (valueOrDefault .Values.global.proxy.includeOutboundPorts "") "") }} - traffic.sidecar.istio.io/includeOutboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundPorts` .Values.global.proxy.includeOutboundPorts }}" - {{- end }} - {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/excludeOutboundPorts`) (ne .Values.global.proxy.excludeOutboundPorts "") }} - traffic.sidecar.istio.io/excludeOutboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundPorts` .Values.global.proxy.excludeOutboundPorts }}" - {{- end }} - traffic.sidecar.istio.io/kubevirtInterfaces: "{{ index .ObjectMeta.Annotations `traffic.sidecar.istio.io/kubevirtInterfaces` }}" - {{- if .Values.global.imagePullSecrets }} - imagePullSecrets: - {{- range .Values.global.imagePullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} + - name: istio-init + {{ end -}} + {{- if contains "/" (annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy_init.image) }} + image: "{{ annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy_init.image }}" + {{- else }} + image: "{{ .Values.global.hub }}/{{ .Values.global.proxy_init.image }}:{{ .Values.global.tag }}" + {{- end }} + args: + - istio-iptables + - "-p" + - "15001" + - "-z" + - "15006" + - "-u" + - "1337" + - "-m" + - "{{ annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode }}" + - "-i" + - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundIPRanges` .Values.global.proxy.includeIPRanges }}" + - "-x" + - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundIPRanges` .Values.global.proxy.excludeIPRanges }}" + - "-b" + - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeInboundPorts` `*` }}" + - "-d" + {{- if excludeInboundPort (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) (annotation .ObjectMeta `traffic.sidecar.istio.io/excludeInboundPorts` .Values.global.proxy.excludeInboundPorts) }} + - "15090,15021,{{ excludeInboundPort (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) (annotation .ObjectMeta `traffic.sidecar.istio.io/excludeInboundPorts` .Values.global.proxy.excludeInboundPorts) }}" + {{- else }} + - "15090,15021" + {{- end }} + {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/includeOutboundPorts`) (ne (valueOrDefault .Values.global.proxy.includeOutboundPorts "") "") -}} + - "-q" + - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundPorts` .Values.global.proxy.includeOutboundPorts }}" + {{ end -}} + {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/excludeOutboundPorts`) (ne (valueOrDefault .Values.global.proxy.excludeOutboundPorts "") "") -}} + - "-o" + - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundPorts` .Values.global.proxy.excludeOutboundPorts }}" + {{ end -}} + {{ if (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/kubevirtInterfaces`) -}} + - "-k" + - "{{ index .ObjectMeta.Annotations `traffic.sidecar.istio.io/kubevirtInterfaces` }}" + {{ end -}} + {{ if .Values.istio_cni.enabled -}} + - "--run-validation" + - "--skip-rule-apply" + {{ end -}} + imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" + {{- if .ProxyConfig.ProxyMetadata }} + env: + {{- range $key, $value := .ProxyConfig.ProxyMetadata }} + - name: {{ $key }} + value: "{{ $value }}" + {{- end }} + {{- end }} + resources: + {{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) }} + {{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) }} + requests: + {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) -}} + cpu: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU` }}" + {{ end }} + {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) -}} + memory: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory` }}" + {{ end }} + {{- end }} + {{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) }} + limits: + {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) -}} + cpu: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit` }}" + {{ end }} + {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) -}} + memory: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit` }}" + {{ end }} + {{- end }} + {{- else }} + {{- if .Values.global.proxy.resources }} + {{ toYaml .Values.global.proxy.resources | indent 6 }} + {{- end }} + {{- end }} + securityContext: + allowPrivilegeEscalation: {{ .Values.global.proxy.privileged }} + privileged: {{ .Values.global.proxy.privileged }} + capabilities: + {{- if not .Values.istio_cni.enabled }} + add: + - NET_ADMIN + - NET_RAW + {{- end }} + drop: + - ALL + {{- if not .Values.istio_cni.enabled }} + readOnlyRootFilesystem: false + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + {{- else }} + readOnlyRootFilesystem: true + runAsGroup: 1337 + runAsUser: 1337 + runAsNonRoot: true + {{- end }} + restartPolicy: Always + {{ end -}} + {{- if eq .Values.global.proxy.enableCoreDump true }} + - name: enable-core-dump + args: + - -c + - sysctl -w kernel.core_pattern=/var/lib/istio/data/core.proxy && ulimit -c unlimited + command: + - /bin/sh + {{- if contains "/" (annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy_init.image) }} + image: "{{ annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy_init.image }}" + {{- else }} + image: "{{ .Values.global.hub }}/{{ .Values.global.proxy_init.image }}:{{ .Values.global.tag }}" + {{- end }} + imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" + resources: {} + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + drop: + - ALL + privileged: true + readOnlyRootFilesystem: false + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + {{ end }} + containers: + - name: istio-proxy + {{- if contains "/" (annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy.image) }} + image: "{{ annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy.image }}" + {{- else }} + image: "{{ .Values.global.hub }}/{{ .Values.global.proxy.image }}:{{ .Values.global.tag }}" + {{- end }} + ports: + - containerPort: 15090 + protocol: TCP + name: http-envoy-prom + args: + - proxy + - sidecar + - --domain + - $(POD_NAMESPACE).svc.{{ .Values.global.proxy.clusterDomain }} + - --serviceCluster + {{ if ne "" (index .ObjectMeta.Labels "app") -}} + - "{{ index .ObjectMeta.Labels `app` }}.$(POD_NAMESPACE)" + {{ else -}} + - "{{ valueOrDefault .DeploymentMeta.Name `istio-proxy` }}.{{ valueOrDefault .DeploymentMeta.Namespace `default` }}" + {{ end -}} + - --proxyLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/logLevel` .Values.global.proxy.logLevel }} + - --proxyComponentLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/componentLogLevel` .Values.global.proxy.componentLogLevel }} + - --log_output_level={{ annotation .ObjectMeta `sidecar.istio.io/agentLogLevel` .Values.global.logging.level }} + {{- if .Values.global.sts.servicePort }} + - --stsPort={{ .Values.global.sts.servicePort }} + {{- end }} + {{- if .Values.global.logAsJson }} + - --log_as_json + {{- end }} + {{- if gt .ProxyConfig.Concurrency.GetValue 0 }} + - --concurrency + - "{{ .ProxyConfig.Concurrency.GetValue }}" + {{- end -}} + {{- if .Values.global.proxy.lifecycle }} + lifecycle: + {{ toYaml .Values.global.proxy.lifecycle | indent 6 }} + {{- else if $holdProxy }} + lifecycle: + postStart: + exec: + command: + - pilot-agent + - wait + {{- end }} + env: + - name: JWT_POLICY + value: {{ .Values.global.jwtPolicy }} + - name: PILOT_CERT_PROVIDER + value: {{ .Values.global.pilotCertProvider }} + - name: CA_ADDR + {{- if .Values.global.caAddress }} + value: {{ .Values.global.caAddress }} + {{- else }} + value: istiod{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }}.{{ .Values.global.istioNamespace }}.svc:15012 + {{- end }} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: INSTANCE_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: CANONICAL_SERVICE + valueFrom: + fieldRef: + fieldPath: metadata.labels['service.istio.io/canonical-name'] + - name: CANONICAL_REVISION + valueFrom: + fieldRef: + fieldPath: metadata.labels['service.istio.io/canonical-revision'] + - name: PROXY_CONFIG + value: | + {{ protoToJSON .ProxyConfig }} + - name: ISTIO_META_POD_PORTS + value: |- + [ + {{- $first := true }} + {{- range $index1, $c := .Spec.Containers }} + {{- range $index2, $p := $c.Ports }} + {{- if (structToJSON $p) }} + {{if not $first}},{{end}}{{ structToJSON $p }} + {{- $first = false }} + {{- end }} + {{- end}} + {{- end}} + ] + - name: ISTIO_META_APP_CONTAINERS + value: "{{ $containers | join "," }}" + - name: ISTIO_META_CLUSTER_ID + value: "{{ valueOrDefault .Values.global.multiCluster.clusterName `Kubernetes` }}" + - name: ISTIO_META_INTERCEPTION_MODE + value: "{{ or (index .ObjectMeta.Annotations `sidecar.istio.io/interceptionMode`) .ProxyConfig.InterceptionMode.String }}" + {{- if .Values.global.network }} + - name: ISTIO_META_NETWORK + value: "{{ .Values.global.network }}" + {{- end }} + {{ if .ObjectMeta.Annotations }} + - name: ISTIO_METAJSON_ANNOTATIONS + value: | + {{ toJSON .ObjectMeta.Annotations }} + {{ end }} + {{- if .DeploymentMeta.Name }} + - name: ISTIO_META_WORKLOAD_NAME + value: "{{ .DeploymentMeta.Name }}" + {{ end }} + {{- if and .TypeMeta.APIVersion .DeploymentMeta.Name }} + - name: ISTIO_META_OWNER + value: kubernetes://apis/{{ .TypeMeta.APIVersion }}/namespaces/{{ valueOrDefault .DeploymentMeta.Namespace `default` }}/{{ toLower .TypeMeta.Kind}}s/{{ .DeploymentMeta.Name }} + {{- end}} + {{- if (isset .ObjectMeta.Annotations `sidecar.istio.io/bootstrapOverride`) }} + - name: ISTIO_BOOTSTRAP_OVERRIDE + value: "/etc/istio/custom-bootstrap/custom_bootstrap.json" + {{- end }} + {{- if .Values.global.meshID }} + - name: ISTIO_META_MESH_ID + value: "{{ .Values.global.meshID }}" + {{- else if (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain) }} + - name: ISTIO_META_MESH_ID + value: "{{ (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain) }}" + {{- end }} + {{- with (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain) }} + - name: TRUST_DOMAIN + value: "{{ . }}" + {{- end }} + {{- if and (eq .Values.global.proxy.tracer "datadog") (isset .ObjectMeta.Annotations `apm.datadoghq.com/env`) }} + {{- range $key, $value := fromJSON (index .ObjectMeta.Annotations `apm.datadoghq.com/env`) }} + - name: {{ $key }} + value: "{{ $value }}" + {{- end }} + {{- end }} + {{- range $key, $value := .ProxyConfig.ProxyMetadata }} + - name: {{ $key }} + value: "{{ $value }}" + {{- end }} + imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" + {{ if ne (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) `0` }} + readinessProbe: + httpGet: + path: /healthz/ready + port: 15021 + initialDelaySeconds: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/initialDelaySeconds` .Values.global.proxy.readinessInitialDelaySeconds }} + periodSeconds: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/periodSeconds` .Values.global.proxy.readinessPeriodSeconds }} + timeoutSeconds: 3 + failureThreshold: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/failureThreshold` .Values.global.proxy.readinessFailureThreshold }} + {{ end -}} + securityContext: + allowPrivilegeEscalation: {{ .Values.global.proxy.privileged }} + capabilities: + {{ if or (eq (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `TPROXY`) (eq (annotation .ObjectMeta `sidecar.istio.io/capNetBindService` .Values.global.proxy.capNetBindService) `true`) -}} + add: + {{ if eq (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `TPROXY` -}} + - NET_ADMIN + {{- end }} + {{ if eq (annotation .ObjectMeta `sidecar.istio.io/capNetBindService` .Values.global.proxy.capNetBindService) `true` -}} + - NET_BIND_SERVICE + {{- end }} + {{- end }} + drop: + - ALL + privileged: {{ .Values.global.proxy.privileged }} + readOnlyRootFilesystem: {{ not .Values.global.proxy.enableCoreDump }} + runAsGroup: 1337 + fsGroup: 1337 + {{ if or (eq (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `TPROXY`) (eq (annotation .ObjectMeta `sidecar.istio.io/capNetBindService` .Values.global.proxy.capNetBindService) `true`) -}} + runAsNonRoot: false + runAsUser: 0 + {{- else -}} + runAsNonRoot: true + runAsUser: 1337 + {{- end }} + resources: + {{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) }} + {{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) }} + requests: + {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) -}} + cpu: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU` }}" + {{ end }} + {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) -}} + memory: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory` }}" + {{ end }} + {{- end }} + {{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) }} + limits: + {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) -}} + cpu: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit` }}" + {{ end }} + {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) -}} + memory: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit` }}" + {{ end }} + {{- end }} + {{- else }} + {{- if .Values.global.proxy.resources }} + {{ toYaml .Values.global.proxy.resources | indent 6 }} + {{- end }} + {{- end }} + volumeMounts: + {{- if eq .Values.global.pilotCertProvider "istiod" }} + - mountPath: /var/run/secrets/istio + name: istiod-ca-cert + {{- end }} + - mountPath: /var/lib/istio/data + name: istio-data + {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/bootstrapOverride`) }} + - mountPath: /etc/istio/custom-bootstrap + name: custom-bootstrap-volume + {{- end }} + # SDS channel between istioagent and Envoy + - mountPath: /etc/istio/proxy + name: istio-envoy + {{- if eq .Values.global.jwtPolicy "third-party-jwt" }} + - mountPath: /var/run/secrets/tokens + name: istio-token + {{- end }} + {{- if .Values.global.mountMtlsCerts }} + # Use the key and cert mounted to /etc/certs/ for the in-cluster mTLS communications. + - mountPath: /etc/certs/ + name: istio-certs + readOnly: true + {{- end }} + - name: istio-podinfo + mountPath: /etc/istio/pod + {{- if and (eq .Values.global.proxy.tracer "lightstep") .ProxyConfig.GetTracing.GetTlsSettings }} + - mountPath: {{ directory .ProxyConfig.GetTracing.GetTlsSettings.GetCaCertificates }} + name: lightstep-certs + readOnly: true + {{- end }} + {{- if isset .ObjectMeta.Annotations `sidecar.istio.io/userVolumeMount` }} + {{ range $index, $value := fromJSON (index .ObjectMeta.Annotations `sidecar.istio.io/userVolumeMount`) }} + - name: "{{ $index }}" + {{ toYaml $value | indent 6 }} + {{ end }} + {{- end }} + volumes: + {{- if (isset .ObjectMeta.Annotations `sidecar.istio.io/bootstrapOverride`) }} + - name: custom-bootstrap-volume + configMap: + name: {{ annotation .ObjectMeta `sidecar.istio.io/bootstrapOverride` "" }} + {{- end }} + # SDS channel between istioagent and Envoy + - emptyDir: + medium: Memory + name: istio-envoy + - name: istio-data + emptyDir: {} + - name: istio-podinfo + downwardAPI: + items: + - path: "labels" + fieldRef: + fieldPath: metadata.labels + - path: "annotations" + fieldRef: + fieldPath: metadata.annotations + - path: "cpu-limit" + resourceFieldRef: + containerName: istio-proxy + resource: limits.cpu + divisor: 1m + - path: "cpu-request" + resourceFieldRef: + containerName: istio-proxy + resource: requests.cpu + divisor: 1m + {{- if eq .Values.global.jwtPolicy "third-party-jwt" }} + - name: istio-token + projected: + sources: + - serviceAccountToken: + path: istio-token + expirationSeconds: 43200 + audience: {{ .Values.global.sds.token.aud }} + {{- end }} + {{- if eq .Values.global.pilotCertProvider "istiod" }} + - name: istiod-ca-cert + configMap: + name: istio-ca-root-cert + {{- end }} + {{- if .Values.global.mountMtlsCerts }} + # Use the key and cert mounted to /etc/certs/ for the in-cluster mTLS communications. + - name: istio-certs + secret: + optional: true + {{ if eq .Spec.ServiceAccountName "" }} + secretName: istio.default + {{ else -}} + secretName: {{ printf "istio.%s" .Spec.ServiceAccountName }} + {{ end -}} + {{- end }} + {{- if isset .ObjectMeta.Annotations `sidecar.istio.io/userVolume` }} + {{range $index, $value := fromJSON (index .ObjectMeta.Annotations `sidecar.istio.io/userVolume`) }} + - name: "{{ $index }}" + {{ toYaml $value | indent 4 }} + {{ end }} + {{ end }} + {{- if and (eq .Values.global.proxy.tracer "lightstep") .ProxyConfig.GetTracing.GetTlsSettings }} + - name: lightstep-certs + secret: + optional: true + secretName: lightstep.cacert + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + {{- if eq (env "ENABLE_LEGACY_FSGROUP_INJECTION" "true") "true" }} + securityContext: + fsGroup: 1337 + {{- end }} --- # Source: istio-discovery/templates/service.yaml apiVersion: v1 @@ -740,6 +757,7 @@ spec: app: istiod istio.io/rev: default install.operator.istio.io/owning-resource: unknown + sidecar.istio.io/inject: "false" operator.istio.io/component: "Pilot" istio: pilot annotations: @@ -768,8 +786,6 @@ spec: protocol: TCP - containerPort: 15017 protocol: TCP - - containerPort: 15053 - protocol: TCP readinessProbe: httpGet: path: /ready @@ -807,8 +823,6 @@ spec: value: "true" - name: PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND value: "true" - - name: INJECTION_WEBHOOK_CONFIG_NAME - value: istio-sidecar-injector - name: ISTIOD_ADDR value: istiod.istio-system.svc:15012 - name: PILOT_ENABLE_ANALYSIS @@ -817,10 +831,6 @@ spec: value: "Kubernetes" - name: EXTERNAL_ISTIOD value: "false" - - name: CENTRAL_ISTIOD - value: "false" - - name: PILOT_ENDPOINT_TELEMETRY_LABEL - value: "true" resources: requests: cpu: 500m @@ -904,687 +914,6 @@ spec: name: cpu targetAverageUtilization: 80 --- -# Source: istio-discovery/templates/telemetryv2_1.6.yaml -apiVersion: networking.istio.io/v1alpha3 -kind: EnvoyFilter -metadata: - name: metadata-exchange-1.6 - namespace: istio-system - labels: - istio.io/rev: default - install.operator.istio.io/owning-resource: unknown - operator.istio.io/component: "Pilot" -spec: - configPatches: - - applyTo: HTTP_FILTER - match: - context: ANY # inbound, outbound, and gateway - proxy: - proxyVersion: '^1\.6.*' - listener: - filterChain: - filter: - name: "envoy.http_connection_manager" - patch: - operation: INSERT_BEFORE - value: - name: istio.metadata_exchange - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm - value: - config: - configuration: | - {} - vm_config: - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: envoy.wasm.metadata_exchange ---- -# Source: istio-discovery/templates/telemetryv2_1.6.yaml -apiVersion: networking.istio.io/v1alpha3 -kind: EnvoyFilter -metadata: - name: tcp-metadata-exchange-1.6 - namespace: istio-system - labels: - istio.io/rev: default -spec: - configPatches: - - applyTo: NETWORK_FILTER - match: - context: SIDECAR_INBOUND - proxy: - proxyVersion: '^1\.6.*' - listener: {} - patch: - operation: INSERT_BEFORE - value: - name: istio.metadata_exchange - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange - value: - protocol: istio-peer-exchange - - applyTo: CLUSTER - match: - context: SIDECAR_OUTBOUND - proxy: - proxyVersion: '^1\.6.*' - cluster: {} - patch: - operation: MERGE - value: - filters: - - name: istio.metadata_exchange - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange - value: - protocol: istio-peer-exchange - - applyTo: CLUSTER - match: - context: GATEWAY - proxy: - proxyVersion: '^1\.6.*' - cluster: {} - patch: - operation: MERGE - value: - filters: - - name: istio.metadata_exchange - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange - value: - protocol: istio-peer-exchange ---- -# Source: istio-discovery/templates/telemetryv2_1.6.yaml -apiVersion: networking.istio.io/v1alpha3 -kind: EnvoyFilter -metadata: - name: stats-filter-1.6 - namespace: istio-system - labels: - istio.io/rev: default -spec: - configPatches: - - applyTo: HTTP_FILTER - match: - context: SIDECAR_OUTBOUND - proxy: - proxyVersion: '^1\.6.*' - listener: - filterChain: - filter: - name: "envoy.http_connection_manager" - subFilter: - name: "envoy.router" - patch: - operation: INSERT_BEFORE - value: - name: istio.stats - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm - value: - config: - root_id: stats_outbound - configuration: | - { - "debug": "false", - "stat_prefix": "istio" - } - vm_config: - vm_id: stats_outbound - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: envoy.wasm.stats - - applyTo: HTTP_FILTER - match: - context: SIDECAR_INBOUND - proxy: - proxyVersion: '^1\.6.*' - listener: - filterChain: - filter: - name: "envoy.http_connection_manager" - subFilter: - name: "envoy.router" - patch: - operation: INSERT_BEFORE - value: - name: istio.stats - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm - value: - config: - root_id: stats_inbound - configuration: | - { - "debug": "false", - "stat_prefix": "istio" - } - vm_config: - vm_id: stats_inbound - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: envoy.wasm.stats - - applyTo: HTTP_FILTER - match: - context: GATEWAY - proxy: - proxyVersion: '^1\.6.*' - listener: - filterChain: - filter: - name: "envoy.http_connection_manager" - subFilter: - name: "envoy.router" - patch: - operation: INSERT_BEFORE - value: - name: istio.stats - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm - value: - config: - root_id: stats_outbound - configuration: | - { - "debug": "false", - "stat_prefix": "istio", - "disable_host_header_fallback": true - } - vm_config: - vm_id: stats_outbound - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: envoy.wasm.stats ---- -# Source: istio-discovery/templates/telemetryv2_1.6.yaml -apiVersion: networking.istio.io/v1alpha3 -kind: EnvoyFilter -metadata: - name: tcp-stats-filter-1.6 - namespace: istio-system - labels: - istio.io/rev: default -spec: - configPatches: - - applyTo: NETWORK_FILTER - match: - context: SIDECAR_INBOUND - proxy: - proxyVersion: '^1\.6.*' - listener: - filterChain: - filter: - name: "envoy.tcp_proxy" - patch: - operation: INSERT_BEFORE - value: - name: istio.stats - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm - value: - config: - root_id: stats_inbound - configuration: | - { - "debug": "false", - "stat_prefix": "istio" - } - vm_config: - vm_id: tcp_stats_inbound - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: "envoy.wasm.stats" - - applyTo: NETWORK_FILTER - match: - context: SIDECAR_OUTBOUND - proxy: - proxyVersion: '^1\.6.*' - listener: - filterChain: - filter: - name: "envoy.tcp_proxy" - patch: - operation: INSERT_BEFORE - value: - name: istio.stats - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm - value: - config: - root_id: stats_outbound - configuration: | - { - "debug": "false", - "stat_prefix": "istio" - } - vm_config: - vm_id: tcp_stats_outbound - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: "envoy.wasm.stats" - - applyTo: NETWORK_FILTER - match: - context: GATEWAY - proxy: - proxyVersion: '^1\.6.*' - listener: - filterChain: - filter: - name: "envoy.tcp_proxy" - patch: - operation: INSERT_BEFORE - value: - name: istio.stats - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm - value: - config: - root_id: stats_outbound - configuration: | - { - "debug": "false", - "stat_prefix": "istio" - } - vm_config: - vm_id: tcp_stats_outbound - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: "envoy.wasm.stats" ---- -# Source: istio-discovery/templates/telemetryv2_1.7.yaml -# Note: metadata exchange filter is wasm enabled only in sidecars. -apiVersion: networking.istio.io/v1alpha3 -kind: EnvoyFilter -metadata: - name: metadata-exchange-1.7 - namespace: istio-system - labels: - istio.io/rev: default - install.operator.istio.io/owning-resource: unknown - operator.istio.io/component: "Pilot" -spec: - configPatches: - - applyTo: HTTP_FILTER - match: - context: SIDECAR_INBOUND - proxy: - proxyVersion: '^1\.7.*' - listener: - filterChain: - filter: - name: "envoy.http_connection_manager" - patch: - operation: INSERT_BEFORE - value: - name: istio.metadata_exchange - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm - value: - config: - configuration: - "@type": "type.googleapis.com/google.protobuf.StringValue" - value: | - {} - vm_config: - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: envoy.wasm.metadata_exchange - - applyTo: HTTP_FILTER - match: - context: SIDECAR_OUTBOUND - proxy: - proxyVersion: '^1\.7.*' - listener: - filterChain: - filter: - name: "envoy.http_connection_manager" - patch: - operation: INSERT_BEFORE - value: - name: istio.metadata_exchange - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm - value: - config: - configuration: - "@type": "type.googleapis.com/google.protobuf.StringValue" - value: | - {} - vm_config: - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: envoy.wasm.metadata_exchange - - applyTo: HTTP_FILTER - match: - context: GATEWAY - proxy: - proxyVersion: '^1\.7.*' - listener: - filterChain: - filter: - name: "envoy.http_connection_manager" - patch: - operation: INSERT_BEFORE - value: - name: istio.metadata_exchange - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm - value: - config: - configuration: - "@type": "type.googleapis.com/google.protobuf.StringValue" - value: | - {} - vm_config: - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: envoy.wasm.metadata_exchange ---- -# Source: istio-discovery/templates/telemetryv2_1.7.yaml -apiVersion: networking.istio.io/v1alpha3 -kind: EnvoyFilter -metadata: - name: tcp-metadata-exchange-1.7 - namespace: istio-system - labels: - istio.io/rev: default -spec: - configPatches: - - applyTo: NETWORK_FILTER - match: - context: SIDECAR_INBOUND - proxy: - proxyVersion: '^1\.7.*' - listener: {} - patch: - operation: INSERT_BEFORE - value: - name: istio.metadata_exchange - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange - value: - protocol: istio-peer-exchange - - applyTo: CLUSTER - match: - context: SIDECAR_OUTBOUND - proxy: - proxyVersion: '^1\.7.*' - cluster: {} - patch: - operation: MERGE - value: - filters: - - name: istio.metadata_exchange - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange - value: - protocol: istio-peer-exchange - - applyTo: CLUSTER - match: - context: GATEWAY - proxy: - proxyVersion: '^1\.7.*' - cluster: {} - patch: - operation: MERGE - value: - filters: - - name: istio.metadata_exchange - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange - value: - protocol: istio-peer-exchange ---- -# Source: istio-discovery/templates/telemetryv2_1.7.yaml -# Note: http stats filter is wasm enabled only in sidecars. -apiVersion: networking.istio.io/v1alpha3 -kind: EnvoyFilter -metadata: - name: stats-filter-1.7 - namespace: istio-system - labels: - istio.io/rev: default -spec: - configPatches: - - applyTo: HTTP_FILTER - match: - context: SIDECAR_OUTBOUND - proxy: - proxyVersion: '^1\.7.*' - listener: - filterChain: - filter: - name: "envoy.http_connection_manager" - subFilter: - name: "envoy.router" - patch: - operation: INSERT_BEFORE - value: - name: istio.stats - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm - value: - config: - root_id: stats_outbound - configuration: - "@type": "type.googleapis.com/google.protobuf.StringValue" - value: | - { - "debug": "false", - "stat_prefix": "istio" - } - vm_config: - vm_id: stats_outbound - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: envoy.wasm.stats - - applyTo: HTTP_FILTER - match: - context: SIDECAR_INBOUND - proxy: - proxyVersion: '^1\.7.*' - listener: - filterChain: - filter: - name: "envoy.http_connection_manager" - subFilter: - name: "envoy.router" - patch: - operation: INSERT_BEFORE - value: - name: istio.stats - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm - value: - config: - root_id: stats_inbound - configuration: - "@type": "type.googleapis.com/google.protobuf.StringValue" - value: | - { - "debug": "false", - "stat_prefix": "istio" - } - vm_config: - vm_id: stats_inbound - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: envoy.wasm.stats - - applyTo: HTTP_FILTER - match: - context: GATEWAY - proxy: - proxyVersion: '^1\.7.*' - listener: - filterChain: - filter: - name: "envoy.http_connection_manager" - subFilter: - name: "envoy.router" - patch: - operation: INSERT_BEFORE - value: - name: istio.stats - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm - value: - config: - root_id: stats_outbound - configuration: - "@type": "type.googleapis.com/google.protobuf.StringValue" - value: | - { - "debug": "false", - "stat_prefix": "istio", - "disable_host_header_fallback": true - } - vm_config: - vm_id: stats_outbound - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: envoy.wasm.stats ---- -# Source: istio-discovery/templates/telemetryv2_1.7.yaml -# Note: tcp stats filter is wasm enabled only in sidecars. -apiVersion: networking.istio.io/v1alpha3 -kind: EnvoyFilter -metadata: - name: tcp-stats-filter-1.7 - namespace: istio-system - labels: - istio.io/rev: default -spec: - configPatches: - - applyTo: NETWORK_FILTER - match: - context: SIDECAR_INBOUND - proxy: - proxyVersion: '^1\.7.*' - listener: - filterChain: - filter: - name: "envoy.tcp_proxy" - patch: - operation: INSERT_BEFORE - value: - name: istio.stats - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm - value: - config: - root_id: stats_inbound - configuration: - "@type": "type.googleapis.com/google.protobuf.StringValue" - value: | - { - "debug": "false", - "stat_prefix": "istio" - } - vm_config: - vm_id: tcp_stats_inbound - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: "envoy.wasm.stats" - - applyTo: NETWORK_FILTER - match: - context: SIDECAR_OUTBOUND - proxy: - proxyVersion: '^1\.7.*' - listener: - filterChain: - filter: - name: "envoy.tcp_proxy" - patch: - operation: INSERT_BEFORE - value: - name: istio.stats - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm - value: - config: - root_id: stats_outbound - configuration: - "@type": "type.googleapis.com/google.protobuf.StringValue" - value: | - { - "debug": "false", - "stat_prefix": "istio" - } - vm_config: - vm_id: tcp_stats_outbound - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: "envoy.wasm.stats" - - applyTo: NETWORK_FILTER - match: - context: GATEWAY - proxy: - proxyVersion: '^1\.7.*' - listener: - filterChain: - filter: - name: "envoy.tcp_proxy" - patch: - operation: INSERT_BEFORE - value: - name: istio.stats - typed_config: - "@type": type.googleapis.com/udpa.type.v1.TypedStruct - type_url: type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm - value: - config: - root_id: stats_outbound - configuration: - "@type": "type.googleapis.com/google.protobuf.StringValue" - value: | - { - "debug": "false", - "stat_prefix": "istio" - } - vm_config: - vm_id: tcp_stats_outbound - runtime: envoy.wasm.runtime.null - code: - local: - inline_string: "envoy.wasm.stats" ---- # Source: istio-discovery/templates/telemetryv2_1.8.yaml # Note: metadata exchange filter is wasm enabled only in sidecars. apiVersion: networking.istio.io/v1alpha3 @@ -1606,7 +935,7 @@ spec: listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" patch: operation: INSERT_BEFORE value: @@ -1633,7 +962,7 @@ spec: listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" patch: operation: INSERT_BEFORE value: @@ -1660,7 +989,7 @@ spec: listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" patch: operation: INSERT_BEFORE value: @@ -1757,9 +1086,9 @@ spec: listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" subFilter: - name: "envoy.router" + name: "envoy.filters.http.router" patch: operation: INSERT_BEFORE value: @@ -1789,9 +1118,9 @@ spec: listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" subFilter: - name: "envoy.router" + name: "envoy.filters.http.router" patch: operation: INSERT_BEFORE value: @@ -1821,9 +1150,9 @@ spec: listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" subFilter: - name: "envoy.router" + name: "envoy.filters.http.router" patch: operation: INSERT_BEFORE value: @@ -1866,7 +1195,7 @@ spec: listener: filterChain: filter: - name: "envoy.tcp_proxy" + name: "envoy.filters.network.tcp_proxy" patch: operation: INSERT_BEFORE value: @@ -1896,7 +1225,7 @@ spec: listener: filterChain: filter: - name: "envoy.tcp_proxy" + name: "envoy.filters.network.tcp_proxy" patch: operation: INSERT_BEFORE value: @@ -1926,7 +1255,7 @@ spec: listener: filterChain: filter: - name: "envoy.tcp_proxy" + name: "envoy.filters.network.tcp_proxy" patch: operation: INSERT_BEFORE value: @@ -1949,13 +1278,435 @@ spec: local: inline_string: "envoy.wasm.stats" --- +# Source: istio-discovery/templates/telemetryv2_1.9.yaml +# Note: metadata exchange filter is wasm enabled only in sidecars. +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: metadata-exchange-1.9 + namespace: istio-system + labels: + istio.io/rev: default + install.operator.istio.io/owning-resource: unknown + operator.istio.io/component: "Pilot" +spec: + configPatches: + - applyTo: HTTP_FILTER + match: + context: SIDECAR_INBOUND + proxy: + proxyVersion: '^1\.9.*' + listener: + filterChain: + filter: + name: "envoy.filters.network.http_connection_manager" + patch: + operation: INSERT_BEFORE + value: + name: istio.metadata_exchange + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm + value: + config: + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: | + {} + vm_config: + runtime: envoy.wasm.runtime.null + code: + local: + inline_string: envoy.wasm.metadata_exchange + - applyTo: HTTP_FILTER + match: + context: SIDECAR_OUTBOUND + proxy: + proxyVersion: '^1\.9.*' + listener: + filterChain: + filter: + name: "envoy.filters.network.http_connection_manager" + patch: + operation: INSERT_BEFORE + value: + name: istio.metadata_exchange + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm + value: + config: + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: | + {} + vm_config: + runtime: envoy.wasm.runtime.null + code: + local: + inline_string: envoy.wasm.metadata_exchange + - applyTo: HTTP_FILTER + match: + context: GATEWAY + proxy: + proxyVersion: '^1\.9.*' + listener: + filterChain: + filter: + name: "envoy.filters.network.http_connection_manager" + patch: + operation: INSERT_BEFORE + value: + name: istio.metadata_exchange + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm + value: + config: + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: | + {} + vm_config: + runtime: envoy.wasm.runtime.null + code: + local: + inline_string: envoy.wasm.metadata_exchange +--- +# Source: istio-discovery/templates/telemetryv2_1.9.yaml +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: tcp-metadata-exchange-1.9 + namespace: istio-system + labels: + istio.io/rev: default +spec: + configPatches: + - applyTo: NETWORK_FILTER + match: + context: SIDECAR_INBOUND + proxy: + proxyVersion: '^1\.9.*' + listener: {} + patch: + operation: INSERT_BEFORE + value: + name: istio.metadata_exchange + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange + value: + protocol: istio-peer-exchange + - applyTo: CLUSTER + match: + context: SIDECAR_OUTBOUND + proxy: + proxyVersion: '^1\.9.*' + cluster: {} + patch: + operation: MERGE + value: + filters: + - name: istio.metadata_exchange + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange + value: + protocol: istio-peer-exchange + - applyTo: CLUSTER + match: + context: GATEWAY + proxy: + proxyVersion: '^1\.9.*' + cluster: {} + patch: + operation: MERGE + value: + filters: + - name: istio.metadata_exchange + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.tcp.metadataexchange.config.MetadataExchange + value: + protocol: istio-peer-exchange +--- +# Source: istio-discovery/templates/telemetryv2_1.9.yaml +# Note: http stats filter is wasm enabled only in sidecars. +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: stats-filter-1.9 + namespace: istio-system + labels: + istio.io/rev: default +spec: + configPatches: + - applyTo: HTTP_FILTER + match: + context: SIDECAR_OUTBOUND + proxy: + proxyVersion: '^1\.9.*' + listener: + filterChain: + filter: + name: "envoy.filters.network.http_connection_manager" + subFilter: + name: "envoy.filters.http.router" + patch: + operation: INSERT_BEFORE + value: + name: istio.stats + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm + value: + config: + root_id: stats_outbound + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: | + { + "debug": "false", + "stat_prefix": "istio", + "metrics": [ + { + "dimensions": { + "source_cluster": "node.metadata['CLUSTER_ID']", + "destination_cluster": "upstream_peer.cluster_id" + } + } + ] + } + vm_config: + vm_id: stats_outbound + runtime: envoy.wasm.runtime.null + code: + local: + inline_string: envoy.wasm.stats + - applyTo: HTTP_FILTER + match: + context: SIDECAR_INBOUND + proxy: + proxyVersion: '^1\.9.*' + listener: + filterChain: + filter: + name: "envoy.filters.network.http_connection_manager" + subFilter: + name: "envoy.filters.http.router" + patch: + operation: INSERT_BEFORE + value: + name: istio.stats + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm + value: + config: + root_id: stats_inbound + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: | + { + "debug": "false", + "stat_prefix": "istio", + "metrics": [ + { + "dimensions": { + "destination_cluster": "node.metadata['CLUSTER_ID']", + "source_cluster": "downstream_peer.cluster_id" + } + } + ] + } + vm_config: + vm_id: stats_inbound + runtime: envoy.wasm.runtime.null + code: + local: + inline_string: envoy.wasm.stats + - applyTo: HTTP_FILTER + match: + context: GATEWAY + proxy: + proxyVersion: '^1\.9.*' + listener: + filterChain: + filter: + name: "envoy.filters.network.http_connection_manager" + subFilter: + name: "envoy.filters.http.router" + patch: + operation: INSERT_BEFORE + value: + name: istio.stats + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm + value: + config: + root_id: stats_outbound + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: | + { + "debug": "false", + "stat_prefix": "istio", + "disable_host_header_fallback": true, + "metrics": [ + { + "dimensions": { + "source_cluster": "node.metadata['CLUSTER_ID']", + "destination_cluster": "upstream_peer.cluster_id" + } + } + ] + } + vm_config: + vm_id: stats_outbound + runtime: envoy.wasm.runtime.null + code: + local: + inline_string: envoy.wasm.stats +--- +# Source: istio-discovery/templates/telemetryv2_1.9.yaml +# Note: tcp stats filter is wasm enabled only in sidecars. +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: tcp-stats-filter-1.9 + namespace: istio-system + labels: + istio.io/rev: default +spec: + configPatches: + - applyTo: NETWORK_FILTER + match: + context: SIDECAR_INBOUND + proxy: + proxyVersion: '^1\.9.*' + listener: + filterChain: + filter: + name: "envoy.filters.network.tcp_proxy" + patch: + operation: INSERT_BEFORE + value: + name: istio.stats + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm + value: + config: + root_id: stats_inbound + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: | + { + "debug": "false", + "stat_prefix": "istio", + "metrics": [ + { + "dimensions": { + "destination_cluster": "node.metadata['CLUSTER_ID']", + "source_cluster": "downstream_peer.cluster_id" + } + } + ] + } + vm_config: + vm_id: tcp_stats_inbound + runtime: envoy.wasm.runtime.null + code: + local: + inline_string: "envoy.wasm.stats" + - applyTo: NETWORK_FILTER + match: + context: SIDECAR_OUTBOUND + proxy: + proxyVersion: '^1\.9.*' + listener: + filterChain: + filter: + name: "envoy.filters.network.tcp_proxy" + patch: + operation: INSERT_BEFORE + value: + name: istio.stats + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm + value: + config: + root_id: stats_outbound + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: | + { + "debug": "false", + "stat_prefix": "istio", + "metrics": [ + { + "dimensions": { + "source_cluster": "node.metadata['CLUSTER_ID']", + "destination_cluster": "upstream_peer.cluster_id" + } + } + ] + } + vm_config: + vm_id: tcp_stats_outbound + runtime: envoy.wasm.runtime.null + code: + local: + inline_string: "envoy.wasm.stats" + - applyTo: NETWORK_FILTER + match: + context: GATEWAY + proxy: + proxyVersion: '^1\.9.*' + listener: + filterChain: + filter: + name: "envoy.filters.network.tcp_proxy" + patch: + operation: INSERT_BEFORE + value: + name: istio.stats + typed_config: + "@type": type.googleapis.com/udpa.type.v1.TypedStruct + type_url: type.googleapis.com/envoy.extensions.filters.network.wasm.v3.Wasm + value: + config: + root_id: stats_outbound + configuration: + "@type": "type.googleapis.com/google.protobuf.StringValue" + value: | + { + "debug": "false", + "stat_prefix": "istio", + "metrics": [ + { + "dimensions": { + "source_cluster": "node.metadata['CLUSTER_ID']", + "destination_cluster": "upstream_peer.cluster_id" + } + } + ] + } + vm_config: + vm_id: tcp_stats_outbound + runtime: envoy.wasm.runtime.null + code: + local: + inline_string: "envoy.wasm.stats" +--- # Source: istio-discovery/templates/mutatingwebhook.yaml -# Installed for each revision - not installed for cluster resources ( cluster roles, bindings, crds) apiVersion: admissionregistration.k8s.io/v1beta1 kind: MutatingWebhookConfiguration metadata: name: istio-sidecar-injector - labels: istio.io/rev: default install.operator.istio.io/owning-resource: unknown @@ -1963,21 +1714,27 @@ metadata: app: sidecar-injector release: istio webhooks: - - name: sidecar-injector.istio.io - clientConfig: - service: - name: istiod - namespace: istio-system - path: "/inject" - caBundle: "" - sideEffects: None - rules: - - operations: [ "CREATE" ] - apiGroups: [""] - apiVersions: ["v1"] - resources: ["pods"] - failurePolicy: Fail - admissionReviewVersions: ["v1beta1", "v1"] - namespaceSelector: - matchLabels: - istio-injection: enabled +- name: sidecar-injector.istio.io + clientConfig: + service: + name: istiod + namespace: istio-system + path: "/inject" + caBundle: "" + sideEffects: None + rules: + - operations: [ "CREATE" ] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + failurePolicy: Fail + admissionReviewVersions: ["v1beta1", "v1"] + namespaceSelector: + matchLabels: + istio-injection: enabled + objectSelector: + matchExpressions: + - key: "sidecar.istio.io/inject" + operator: NotIn + values: + - "false" diff --git a/charts/kubezero-istio/charts/istio-discovery/files/injection-template.yaml b/charts/kubezero-istio/charts/istio-discovery/files/injection-template.yaml index 10dfbbe..b9b9866 100644 --- a/charts/kubezero-istio/charts/istio-discovery/files/injection-template.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/files/injection-template.yaml @@ -1,6 +1,35 @@ -template: | + {{- $containers := list }} +{{- range $index, $container := .Spec.Containers }}{{ if not (eq $container.Name "istio-proxy") }}{{ $containers = append $containers $container.Name }}{{end}}{{- end}} +metadata: + labels: + security.istio.io/tlsMode: {{ index .ObjectMeta.Labels `security.istio.io/tlsMode` | default "istio" | quote }} + service.istio.io/canonical-name: {{ index .ObjectMeta.Labels `service.istio.io/canonical-name` | default (index .ObjectMeta.Labels `app.kubernetes.io/name`) | default (index .ObjectMeta.Labels `app`) | default .DeploymentMeta.Name | quote }} + service.istio.io/canonical-revision: {{ index .ObjectMeta.Labels `service.istio.io/canonical-revision` | default (index .ObjectMeta.Labels `app.kubernetes.io/version`) | default (index .ObjectMeta.Labels `version`) | default "latest" | quote }} + istio.io/rev: {{ .Revision | default "default" | quote }} + annotations: { + {{- if eq (len $containers) 1 }} + kubectl.kubernetes.io/default-logs-container: "{{ index $containers 0 }}", + {{ end }} +{{- if .Values.istio_cni.enabled }} + {{- if not .Values.istio_cni.chained }} + k8s.v1.cni.cncf.io/networks: '{{ appendMultusNetwork (index .ObjectMeta.Annotations `k8s.v1.cni.cncf.io/networks`) `istio-cni` }}', + {{- end }} + sidecar.istio.io/interceptionMode: "{{ annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode }}", + {{ with annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundIPRanges` .Values.global.proxy.includeIPRanges }}traffic.sidecar.istio.io/includeOutboundIPRanges: "{{.}}",{{ end }} + {{ with annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundIPRanges` .Values.global.proxy.excludeIPRanges }}traffic.sidecar.istio.io/excludeOutboundIPRanges: "{{.}}",{{ end }} + traffic.sidecar.istio.io/includeInboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeInboundPorts` `*` }}", + traffic.sidecar.istio.io/excludeInboundPorts: "{{ excludeInboundPort (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) (annotation .ObjectMeta `traffic.sidecar.istio.io/excludeInboundPorts` .Values.global.proxy.excludeInboundPorts) }}", + {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/includeOutboundPorts`) (ne (valueOrDefault .Values.global.proxy.includeOutboundPorts "") "") }} + traffic.sidecar.istio.io/includeOutboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundPorts` .Values.global.proxy.includeOutboundPorts }}", + {{- end }} + {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/excludeOutboundPorts`) (ne .Values.global.proxy.excludeOutboundPorts "") }} + traffic.sidecar.istio.io/excludeOutboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundPorts` .Values.global.proxy.excludeOutboundPorts }}", + {{- end }} + {{ with index .ObjectMeta.Annotations `traffic.sidecar.istio.io/kubevirtInterfaces` }}traffic.sidecar.istio.io/kubevirtInterfaces: "{{.}}",{{ end }} +{{- end }} + } +spec: {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} - rewriteAppHTTPProbe: {{ valueOrDefault .Values.sidecarInjectorWebhook.rewriteAppHTTPProbe false }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -8,8 +37,8 @@ template: | {{ else -}} - name: istio-init {{ end -}} - {{- if contains "/" .Values.global.proxy_init.image }} - image: "{{ .Values.global.proxy_init.image }}" + {{- if contains "/" (annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy_init.image) }} + image: "{{ annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy_init.image }}" {{- else }} image: "{{ .Values.global.hub }}/{{ .Values.global.proxy_init.image }}:{{ .Values.global.tag }}" {{- end }} @@ -81,7 +110,7 @@ template: | {{- end }} {{- else }} {{- if .Values.global.proxy.resources }} - {{ toYaml .Values.global.proxy.resources | indent 4 }} + {{ toYaml .Values.global.proxy.resources | indent 6 }} {{- end }} {{- end }} securityContext: @@ -115,8 +144,8 @@ template: | - sysctl -w kernel.core_pattern=/var/lib/istio/data/core.proxy && ulimit -c unlimited command: - /bin/sh - {{- if contains "/" .Values.global.proxy_init.image }} - image: "{{ .Values.global.proxy_init.image }}" + {{- if contains "/" (annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy_init.image) }} + image: "{{ annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy_init.image }}" {{- else }} image: "{{ .Values.global.hub }}/{{ .Values.global.proxy_init.image }}:{{ .Values.global.tag }}" {{- end }} @@ -157,8 +186,9 @@ template: | {{ else -}} - "{{ valueOrDefault .DeploymentMeta.Name `istio-proxy` }}.{{ valueOrDefault .DeploymentMeta.Namespace `default` }}" {{ end -}} - - --proxyLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/logLevel` .Values.global.proxy.logLevel}} - - --proxyComponentLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/componentLogLevel` .Values.global.proxy.componentLogLevel}} + - --proxyLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/logLevel` .Values.global.proxy.logLevel }} + - --proxyComponentLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/componentLogLevel` .Values.global.proxy.componentLogLevel }} + - --log_output_level={{ annotation .ObjectMeta `sidecar.istio.io/agentLogLevel` .Values.global.logging.level }} {{- if .Values.global.sts.servicePort }} - --stsPort={{ .Values.global.sts.servicePort }} {{- end }} @@ -171,7 +201,7 @@ template: | {{- end -}} {{- if .Values.global.proxy.lifecycle }} lifecycle: - {{ toYaml .Values.global.proxy.lifecycle | indent 4 }} + {{ toYaml .Values.global.proxy.lifecycle | indent 6 }} {{- else if $holdProxy }} lifecycle: postStart: @@ -236,7 +266,7 @@ template: | {{- end}} ] - name: ISTIO_META_APP_CONTAINERS - value: "{{- range $index, $container := .Spec.Containers }}{{- if ne $index 0}},{{- end}}{{ $container.Name }}{{- end}}" + value: "{{ $containers | join "," }}" - name: ISTIO_META_CLUSTER_ID value: "{{ valueOrDefault .Values.global.multiCluster.clusterName `Kubernetes` }}" - name: ISTIO_META_INTERCEPTION_MODE @@ -341,7 +371,7 @@ template: | {{- end }} {{- else }} {{- if .Values.global.proxy.resources }} - {{ toYaml .Values.global.proxy.resources | indent 4 }} + {{ toYaml .Values.global.proxy.resources | indent 6 }} {{- end }} {{- end }} volumeMounts: @@ -378,7 +408,7 @@ template: | {{- if isset .ObjectMeta.Annotations `sidecar.istio.io/userVolumeMount` }} {{ range $index, $value := fromJSON (index .ObjectMeta.Annotations `sidecar.istio.io/userVolumeMount`) }} - name: "{{ $index }}" - {{ toYaml $value | indent 4 }} + {{ toYaml $value | indent 6 }} {{ end }} {{- end }} volumes: @@ -402,6 +432,16 @@ template: | - path: "annotations" fieldRef: fieldPath: metadata.annotations + - path: "cpu-limit" + resourceFieldRef: + containerName: istio-proxy + resource: limits.cpu + divisor: 1m + - path: "cpu-request" + resourceFieldRef: + containerName: istio-proxy + resource: requests.cpu + divisor: 1m {{- if eq .Values.global.jwtPolicy "third-party-jwt" }} - name: istio-token projected: @@ -430,7 +470,7 @@ template: | {{- if isset .ObjectMeta.Annotations `sidecar.istio.io/userVolume` }} {{range $index, $value := fromJSON (index .ObjectMeta.Annotations `sidecar.istio.io/userVolume`) }} - name: "{{ $index }}" - {{ toYaml $value | indent 2 }} + {{ toYaml $value | indent 4 }} {{ end }} {{ end }} {{- if and (eq .Values.global.proxy.tracer "lightstep") .ProxyConfig.GetTracing.GetTlsSettings }} @@ -439,25 +479,13 @@ template: | optional: true secretName: lightstep.cacert {{- end }} - podRedirectAnnot: - {{- if and (.Values.istio_cni.enabled) (not .Values.istio_cni.chained) }} - k8s.v1.cni.cncf.io/networks: '{{ appendMultusNetwork (index .ObjectMeta.Annotations `k8s.v1.cni.cncf.io/networks`) `istio-cni` }}' - {{- end }} - sidecar.istio.io/interceptionMode: "{{ annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode }}" - traffic.sidecar.istio.io/includeOutboundIPRanges: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundIPRanges` .Values.global.proxy.includeIPRanges }}" - traffic.sidecar.istio.io/excludeOutboundIPRanges: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundIPRanges` .Values.global.proxy.excludeIPRanges }}" - traffic.sidecar.istio.io/includeInboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeInboundPorts` (includeInboundPorts .Spec.Containers) }}" - traffic.sidecar.istio.io/excludeInboundPorts: "{{ excludeInboundPort (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) (annotation .ObjectMeta `traffic.sidecar.istio.io/excludeInboundPorts` .Values.global.proxy.excludeInboundPorts) }}" - {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/includeOutboundPorts`) (ne (valueOrDefault .Values.global.proxy.includeOutboundPorts "") "") }} - traffic.sidecar.istio.io/includeOutboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundPorts` .Values.global.proxy.includeOutboundPorts }}" - {{- end }} - {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/excludeOutboundPorts`) (ne .Values.global.proxy.excludeOutboundPorts "") }} - traffic.sidecar.istio.io/excludeOutboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundPorts` .Values.global.proxy.excludeOutboundPorts }}" - {{- end }} - traffic.sidecar.istio.io/kubevirtInterfaces: "{{ index .ObjectMeta.Annotations `traffic.sidecar.istio.io/kubevirtInterfaces` }}" {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- range .Values.global.imagePullSecrets }} - name: {{ . }} {{- end }} {{- end }} + {{- if eq (env "ENABLE_LEGACY_FSGROUP_INJECTION" "true") "true" }} + securityContext: + fsGroup: 1337 + {{- end }} diff --git a/charts/kubezero-istio/charts/istio-discovery/templates/configmap.yaml b/charts/kubezero-istio/charts/istio-discovery/templates/configmap.yaml index 17174f1..3a6bb68 100644 --- a/charts/kubezero-istio/charts/istio-discovery/templates/configmap.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/templates/configmap.yaml @@ -4,6 +4,12 @@ # Refer to https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md#21-trust-domain trustDomain: {{ .Values.global.trustDomain | default "cluster.local" | quote }} + # The namespace to treat as the administrative root namespace for Istio configuration. + # When processing a leaf namespace Istio will search for declarations in that namespace first + # and if none are found it will search in the root namespace. Any matching declaration found in the root namespace + # is processed as if it were declared in the leaf namespace. + rootNamespace: {{ .Values.meshConfig.rootNamespace | default .Values.global.istioNamespace }} + defaultConfig: {{- if .Values.global.meshID }} meshId: {{ .Values.global.meshID }} @@ -47,10 +53,13 @@ {{- /* Fill in openCensusAgent configuration from meshConfig so it isn't overwritten below */ -}} {{ toYaml $.Values.meshConfig.defaultConfig.tracing }} {{- end }} - {{- if .Values.global.remotePilotAddress }} + {{- if .Values.pilot.enabled }} discoveryAddress: {{ printf "istiod-remote.%s.svc" .Release.Namespace }}:15012 {{- else }} + discoveryAddress: {{ printf "istiod.%s.svc" .Release.Namespace }}:15012 + {{- end }} + {{- else }} discoveryAddress: istiod{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }}.{{.Release.Namespace}}.svc:15012 {{- end }} {{- end }} diff --git a/charts/kubezero-istio/charts/istio-discovery/templates/deployment.yaml b/charts/kubezero-istio/charts/istio-discovery/templates/deployment.yaml index ba586de..c7a42c0 100644 --- a/charts/kubezero-istio/charts/istio-discovery/templates/deployment.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/templates/deployment.yaml @@ -37,6 +37,7 @@ spec: app: istiod istio.io/rev: {{ .Values.revision | default "default" }} install.operator.istio.io/owning-resource: {{ .Values.ownerName | default "unknown" }} + sidecar.istio.io/inject: "false" operator.istio.io/component: "Pilot" {{- if eq .Values.revision ""}} istio: pilot @@ -101,8 +102,6 @@ spec: protocol: TCP - containerPort: 15017 protocol: TCP - - containerPort: 15053 - protocol: TCP readinessProbe: httpGet: path: /ready @@ -148,14 +147,6 @@ spec: value: "{{ .Values.pilot.enableProtocolSniffingForOutbound }}" - name: PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND value: "{{ .Values.pilot.enableProtocolSniffingForInbound }}" -{{- if not (hasKey .Values.pilot.env "INJECTION_WEBHOOK_CONFIG_NAME") }} - - name: INJECTION_WEBHOOK_CONFIG_NAME - {{- if eq .Release.Namespace "istio-system" }} - value: istio-sidecar-injector{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} - {{- else }} - value: istio-sidecar-injector{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }}-{{ .Release.Namespace }} - {{- end }} -{{- end }} - name: ISTIOD_ADDR value: istiod{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }}.{{ .Release.Namespace }}.svc:15012 - name: PILOT_ENABLE_ANALYSIS @@ -164,11 +155,9 @@ spec: value: "{{ $.Values.global.multiCluster.clusterName | default `Kubernetes` }}" - name: EXTERNAL_ISTIOD value: "{{ $.Values.global.externalIstiod | default "false" }}" - - name: CENTRAL_ISTIOD - value: "{{ $.Values.global.centralIstiod | default "false" }}" -{{- if .Values.telemetry.v2.enabled }} +{{- if not .Values.telemetry.v2.enabled }} - name: PILOT_ENDPOINT_TELEMETRY_LABEL - value: "true" + value: "false" {{- end }} resources: {{- if .Values.pilot.resources }} diff --git a/charts/kubezero-istio/charts/istio-discovery/templates/istiod-injector-configmap.yaml b/charts/kubezero-istio/charts/istio-discovery/templates/istiod-injector-configmap.yaml index 8e660e9..15ece14 100644 --- a/charts/kubezero-istio/charts/istio-discovery/templates/istiod-injector-configmap.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/templates/istiod-injector-configmap.yaml @@ -20,6 +20,15 @@ data: # New fields should not use Values - it is a 'primary' config object, users should be able # to fine tune it or use it with kube-inject. config: |- + # defaultTemplates defines the default template to use for pods that do not explicitly specify a template + {{- if .Values.sidecarInjectorWebhook.defaultTemplates }} + defaultTemplates: +{{- range .Values.sidecarInjectorWebhook.defaultTemplates}} + - {{ . }} +{{- end }} + {{- else }} + defaultTemplates: [sidecar] + {{- end }} policy: {{ .Values.global.proxy.autoInject }} alwaysInjectSelector: {{ toYaml .Values.sidecarInjectorWebhook.alwaysInjectSelector | trim | indent 6 }} @@ -29,7 +38,18 @@ data: {{- range $key, $val := .Values.sidecarInjectorWebhook.injectedAnnotations }} "{{ $key }}": "{{ $val }}" {{- end }} - -{{ .Files.Get "files/injection-template.yaml" | trim | indent 4 }} + {{- /* If someone ends up with this new template, but an older Istiod image, they will attempt to render this template + which will fail with "Pod injection failed: template: inject:1: function "Istio_1_9_Required_Template_And_Version_Mismatched" not defined". + This should make it obvious that their installation is broken. + */}} + template: {{ `{{ Template_Version_And_Istio_Version_Mismatched_Check_Installation }}` | quote }} + templates: +{{- if not (hasKey .Values.sidecarInjectorWebhook.templates "sidecar") }} + sidecar: | +{{ .Files.Get "files/injection-template.yaml" | trim | indent 8 }} +{{- end }} +{{- with .Values.sidecarInjectorWebhook.templates }} +{{ toYaml . | trim | indent 6 }} +{{- end }} {{- end }} diff --git a/charts/kubezero-istio/charts/istio-discovery/templates/mutatingwebhook.yaml b/charts/kubezero-istio/charts/istio-discovery/templates/mutatingwebhook.yaml index ca11b4f..f9fd67b 100644 --- a/charts/kubezero-istio/charts/istio-discovery/templates/mutatingwebhook.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/templates/mutatingwebhook.yaml @@ -1,11 +1,35 @@ -# Installed for each revision - not installed for cluster resources ( cluster roles, bindings, crds) +{{- /* Core defines the common configuration used by all webhook segments */}} +{{- define "core" }} +{{- /* Kubernetes unfortunately requires a unique name for the webhook in some newer versions, so we assign +a unique prefix to each. */}} +- name: {{.Prefix}}sidecar-injector.istio.io + clientConfig: + {{- if .Values.istiodRemote.injectionURL }} + url: {{ .Values.istiodRemote.injectionURL }} + {{- else }} + service: + name: istiod{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + namespace: {{ .Release.Namespace }} + path: "/inject" + {{- end }} + caBundle: "" + sideEffects: None + rules: + - operations: [ "CREATE" ] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + failurePolicy: Fail + admissionReviewVersions: ["v1beta1", "v1"] +{{- end }} +{{- /* Installed for each revision - not installed for cluster resources ( cluster roles, bindings, crds) */}} {{- if not .Values.global.operatorManageWebhooks }} apiVersion: admissionregistration.k8s.io/v1beta1 kind: MutatingWebhookConfiguration metadata: {{- if eq .Release.Namespace "istio-system"}} name: istio-sidecar-injector{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} -{{ else }} +{{- else }} name: istio-sidecar-injector{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }}-{{ .Release.Namespace }} {{- end }} labels: @@ -15,71 +39,151 @@ metadata: app: sidecar-injector release: {{ .Release.Name }} webhooks: - - name: sidecar-injector.istio.io - clientConfig: - {{- if .Values.istiodRemote.injectionURL }} - url: {{ .Values.istiodRemote.injectionURL }} - {{- else }} - service: - name: istiod{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} - namespace: {{ .Release.Namespace }} - path: "/inject" - {{- end }} - caBundle: "" - sideEffects: None - rules: - - operations: [ "CREATE" ] - apiGroups: [""] - apiVersions: ["v1"] - resources: ["pods"] - failurePolicy: Fail - admissionReviewVersions: ["v1beta1", "v1"] - namespaceSelector: +{{- if .Values.sidecarInjectorWebhook.useLegacySelectors}} +{{- /* Setup the "legacy" selectors. These are for backwards compatibility, will be removed in the future. */}} +{{- include "core" (mergeOverwrite (deepCopy .) (dict "Prefix" "")) }} + namespaceSelector: + {{- if .Values.sidecarInjectorWebhook.enableNamespacesByDefault }} + matchExpressions: + - key: name + operator: NotIn + values: + - {{ .Release.Namespace }} + - key: istio-injection + operator: NotIn + values: + - disabled + - key: istio-env + operator: DoesNotExist + - key: istio.io/rev + operator: DoesNotExist + {{- else if .Values.revision }} + matchExpressions: + - key: istio-injection + operator: DoesNotExist + - key: istio.io/rev + operator: In + values: + - {{ .Values.revision }} + {{- else }} + matchLabels: + istio-injection: enabled + {{- end }} + {{- if .Values.sidecarInjectorWebhook.objectSelector.enabled }} + objectSelector: + {{- if .Values.sidecarInjectorWebhook.objectSelector.autoInject }} + matchExpressions: + - key: "sidecar.istio.io/inject" + operator: NotIn + values: + - "false" + {{- else if .Values.revision }} + matchExpressions: + - key: "sidecar.istio.io/inject" + operator: DoesNotExist + - key: istio.io/rev + operator: In + values: + - {{ .Values.revision }} + {{- else }} + matchLabels: + "sidecar.istio.io/inject": "true" + {{- end }} + {{- end }} +{{- else }} + +{{- /* Set up the selectors. First section is for revision, rest is for "default" revision */}} +{{- if .Values.revision }} + +{{- /* Case 1: namespace selector matches, and object doesn't disable */}} +{{- /* Note: if both revision and legacy selector, we give precedence to the legacy one */}} +{{- include "core" (mergeOverwrite (deepCopy .) (dict "Prefix" "namespace.") ) }} + namespaceSelector: + matchExpressions: + - key: istio.io/rev + operator: In + values: + - "{{ .Values.revision }}" + - key: istio-injection + operator: DoesNotExist + objectSelector: + matchExpressions: + - key: sidecar.istio.io/inject + operator: NotIn + values: + - "false" + +{{- /* Case 2: No namespace selector, but object selects our revision (and doesn't disable) */}} +{{- include "core" (mergeOverwrite (deepCopy .) (dict "Prefix" "object.") ) }} + namespaceSelector: + matchExpressions: + - key: istio.io/rev + operator: DoesNotExist + - key: istio-injection + operator: DoesNotExist + objectSelector: + matchExpressions: + - key: sidecar.istio.io/inject + operator: NotIn + values: + - "false" + - key: istio.io/rev + operator: In + values: + - "{{ .Values.revision }}" + +{{- else }} +{{- /* "default" revision */}} + +{{- /* Case 1: Namespace selector enabled, and object selector is not injected */}} +{{- include "core" (mergeOverwrite (deepCopy .) (dict "Prefix" "namespace.") ) }} + namespaceSelector: + matchExpressions: + - key: istio-injection + operator: In + values: + - enabled + objectSelector: + matchExpressions: + - key: sidecar.istio.io/inject + operator: NotIn + values: + - "false" + +{{- /* Case 2: no namespace label, but object selector is enabled (and revision label is not, which has priority) */}} +{{- include "core" (mergeOverwrite (deepCopy .) (dict "Prefix" "object.") ) }} + namespaceSelector: + matchExpressions: + - key: istio-injection + operator: DoesNotExist + - key: istio.io/rev + operator: DoesNotExist + objectSelector: + matchExpressions: + - key: sidecar.istio.io/inject + operator: In + values: + - "true" + - key: istio.io/rev + operator: DoesNotExist + {{- if .Values.sidecarInjectorWebhook.enableNamespacesByDefault }} - matchExpressions: - - key: name - operator: NotIn - values: - - {{ .Release.Namespace }} - - key: istio-injection - operator: NotIn - values: - - disabled - - key: istio-env - operator: DoesNotExist - - key: istio.io/rev - operator: DoesNotExist -{{- else if .Values.revision }} - matchExpressions: - - key: istio-injection - operator: DoesNotExist - - key: istio.io/rev - operator: In - values: - - {{ .Values.revision }} -{{- else }} - matchLabels: - istio-injection: enabled +{{- /* Special case 3: no labels at all */}} +{{- include "core" (mergeOverwrite (deepCopy .) (dict "Prefix" "auto.") ) }} + namespaceSelector: + matchExpressions: + - key: istio-injection + operator: DoesNotExist + - key: istio.io/rev + operator: DoesNotExist + objectSelector: + matchExpressions: + - key: sidecar.istio.io/inject + operator: DoesNotExist + - key: istio.io/rev + operator: DoesNotExist {{- end }} -{{- if .Values.sidecarInjectorWebhook.objectSelector.enabled }} - objectSelector: -{{- if .Values.sidecarInjectorWebhook.objectSelector.autoInject }} - matchExpressions: - - key: "sidecar.istio.io/inject" - operator: NotIn - values: - - "false" -{{- else if .Values.revision }} - matchExpressions: - - key: "sidecar.istio.io/inject" - operator: DoesNotExist - - key: istio.io/rev - operator: In - values: - - {{ .Values.revision }} -{{- else }} - matchLabels: - "sidecar.istio.io/inject": "true" + {{- end }} {{- end }} {{- end }} diff --git a/charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.8.yaml b/charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.9.yaml similarity index 90% rename from charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.8.yaml rename to charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.9.yaml index 6985a68..b1db1b9 100644 --- a/charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.8.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/templates/telemetryv2_1.9.yaml @@ -3,7 +3,7 @@ apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: metadata-exchange-1.8{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: metadata-exchange-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -19,11 +19,11 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" patch: operation: INSERT_BEFORE value: @@ -54,11 +54,11 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" patch: operation: INSERT_BEFORE value: @@ -89,11 +89,11 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" patch: operation: INSERT_BEFORE value: @@ -124,7 +124,7 @@ spec: apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: tcp-metadata-exchange-1.8{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: tcp-metadata-exchange-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -138,7 +138,7 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: {} patch: operation: INSERT_BEFORE @@ -153,7 +153,7 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' cluster: {} patch: operation: MERGE @@ -169,7 +169,7 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' cluster: {} patch: operation: MERGE @@ -187,7 +187,7 @@ spec: apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: stats-filter-1.8{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: stats-filter-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -201,13 +201,13 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" subFilter: - name: "envoy.router" + name: "envoy.filters.http.router" patch: operation: INSERT_BEFORE value: @@ -223,7 +223,8 @@ spec: value: | {{- if not .Values.telemetry.v2.prometheus.configOverride.outboundSidecar }} { - {{- if .Values.global.multiCluster.clusterName }} + "debug": "false", + "stat_prefix": "istio", "metrics": [ { "dimensions": { @@ -232,7 +233,6 @@ spec: } } ] - {{- end }} } {{- else }} {{ toJson .Values.telemetry.v2.prometheus.configOverride.outboundSidecar | indent 18 }} @@ -255,13 +255,13 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" subFilter: - name: "envoy.router" + name: "envoy.filters.http.router" patch: operation: INSERT_BEFORE value: @@ -277,16 +277,16 @@ spec: value: | {{- if not .Values.telemetry.v2.prometheus.configOverride.inboundSidecar }} { - {{- if .Values.global.multiCluster.clusterName }} + "debug": "false", + "stat_prefix": "istio", "metrics": [ { "dimensions": { - "source_cluster": "downstream_peer.cluster_id", - "destination_cluster": "node.metadata['CLUSTER_ID']" + "destination_cluster": "node.metadata['CLUSTER_ID']", + "source_cluster": "downstream_peer.cluster_id" } } ] - {{- end }} } {{- else }} {{ toJson .Values.telemetry.v2.prometheus.configOverride.inboundSidecar | indent 18 }} @@ -309,13 +309,13 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" subFilter: - name: "envoy.router" + name: "envoy.filters.http.router" patch: operation: INSERT_BEFORE value: @@ -331,7 +331,9 @@ spec: value: | {{- if not .Values.telemetry.v2.prometheus.configOverride.gateway }} { - "disable_host_header_fallback": true{{- if .Values.global.multiCluster.clusterName }}, + "debug": "false", + "stat_prefix": "istio", + "disable_host_header_fallback": true, "metrics": [ { "dimensions": { @@ -340,7 +342,6 @@ spec: } } ] - {{- end }} } {{- else }} {{ toJson .Values.telemetry.v2.prometheus.configOverride.gateway | indent 18 }} @@ -364,7 +365,7 @@ spec: apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: tcp-stats-filter-1.8{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: tcp-stats-filter-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -378,11 +379,11 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.tcp_proxy" + name: "envoy.filters.network.tcp_proxy" patch: operation: INSERT_BEFORE value: @@ -398,16 +399,16 @@ spec: value: | {{- if not .Values.telemetry.v2.prometheus.configOverride.inboundSidecar }} { - {{- if .Values.global.multiCluster.clusterName }} + "debug": "false", + "stat_prefix": "istio", "metrics": [ { "dimensions": { - "source_cluster": "downstream_peer.cluster_id", - "destination_cluster": "node.metadata['CLUSTER_ID']" + "destination_cluster": "node.metadata['CLUSTER_ID']", + "source_cluster": "downstream_peer.cluster_id" } } ] - {{- end }} } {{- else }} {{ toJson .Values.telemetry.v2.prometheus.configOverride.inboundSidecar | indent 18 }} @@ -430,11 +431,11 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.tcp_proxy" + name: "envoy.filters.network.tcp_proxy" patch: operation: INSERT_BEFORE value: @@ -450,7 +451,8 @@ spec: value: | {{- if not .Values.telemetry.v2.prometheus.configOverride.outboundSidecar }} { - {{- if .Values.global.multiCluster.clusterName }} + "debug": "false", + "stat_prefix": "istio", "metrics": [ { "dimensions": { @@ -459,7 +461,6 @@ spec: } } ] - {{- end }} } {{- else }} {{ toJson .Values.telemetry.v2.prometheus.configOverride.outboundSidecar | indent 18 }} @@ -482,11 +483,11 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.tcp_proxy" + name: "envoy.filters.network.tcp_proxy" patch: operation: INSERT_BEFORE value: @@ -502,7 +503,8 @@ spec: value: | {{- if not .Values.telemetry.v2.prometheus.configOverride.gateway }} { - {{- if .Values.global.multiCluster.clusterName }} + "debug": "false", + "stat_prefix": "istio", "metrics": [ { "dimensions": { @@ -511,7 +513,6 @@ spec: } } ] - {{- end }} } {{- else }} {{ toJson .Values.telemetry.v2.prometheus.configOverride.gateway | indent 18 }} @@ -536,7 +537,7 @@ spec: apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: stackdriver-filter-1.8{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: stackdriver-filter-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -551,13 +552,13 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" subFilter: - name: "envoy.router" + name: "envoy.filters.http.router" patch: operation: INSERT_BEFORE value: @@ -586,13 +587,13 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" subFilter: - name: "envoy.router" + name: "envoy.filters.http.router" patch: operation: INSERT_BEFORE value: @@ -620,13 +621,13 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" subFilter: - name: "envoy.router" + name: "envoy.filters.http.router" patch: operation: INSERT_BEFORE value: @@ -654,7 +655,7 @@ spec: apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: tcp-stackdriver-filter-1.8{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: tcp-stackdriver-filter-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -669,11 +670,11 @@ spec: match: context: SIDECAR_OUTBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.tcp_proxy" + name: "envoy.filters.network.tcp_proxy" patch: operation: INSERT_BEFORE value: @@ -702,11 +703,11 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.tcp_proxy" + name: "envoy.filters.network.tcp_proxy" patch: operation: INSERT_BEFORE value: @@ -734,11 +735,11 @@ spec: match: context: GATEWAY proxy: - proxyVersion: '^1\.8.*' + proxyVersion: '^1\.9.*' listener: filterChain: filter: - name: "envoy.tcp_proxy" + name: "envoy.filters.network.tcp_proxy" patch: operation: INSERT_BEFORE value: @@ -767,7 +768,7 @@ spec: apiVersion: networking.istio.io/v1alpha3 kind: EnvoyFilter metadata: - name: stackdriver-sampling-accesslog-filter-1.8{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} + name: stackdriver-sampling-accesslog-filter-1.9{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }} {{- if .Values.meshConfig.rootNamespace }} namespace: {{ .Values.meshConfig.rootNamespace }} {{- else }} @@ -781,11 +782,11 @@ spec: match: context: SIDECAR_INBOUND proxy: - proxyVersion: '1\.8.*' + proxyVersion: '1\.9.*' listener: filterChain: filter: - name: "envoy.http_connection_manager" + name: "envoy.filters.network.http_connection_manager" subFilter: name: "istio.stackdriver" patch: diff --git a/charts/kubezero-istio/charts/istio-discovery/values.yaml b/charts/kubezero-istio/charts/istio-discovery/values.yaml index ac59dbd..5dab58d 100644 --- a/charts/kubezero-istio/charts/istio-discovery/values.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/values.yaml @@ -65,6 +65,10 @@ pilot: sidecarInjectorWebhook: + # If enabled, the legacy webhook selection logic will be used. This relies on filtering of webhook + # requests in Istiod, rather than at the webhook selection level. + # This is option is intended for migration purposes only and will be removed in Istio 1.10. + useLegacySelectors: true # You can use the field called alwaysInjectSelector and neverInjectSelector which will always inject the sidecar or # always skip the injection on pods that match that label selector, regardless of the global policy. # See https://istio.io/docs/setup/kubernetes/additional-setup/sidecar-injection/#more-control-adding-exceptions @@ -90,13 +94,41 @@ sidecarInjectorWebhook: # Only one environment should have this enabled. enableNamespacesByDefault: false - # Enable objectSelector to filter out pods with no need for sidecar before calling istio-sidecar-injector. - # It is disabled by default since this function will only work after k8s v1.15. + # Enable objectSelector to filter out pods with no need for sidecar before calling istiod. + # It is enabled by default as the minimum supported Kubernetes version is 1.15+ objectSelector: - enabled: false + enabled: true autoInject: true rewriteAppHTTPProbe: true + + # Templates defines a set of custom injection templates that can be used. For example, defining: + # + # templates: + # hello: | + # metadata: + # labels: + # hello: world + # + # Then starting a pod with the `inject.istio.io/templates: hello` annotation, will result in the pod + # being injected with the hello=world labels. + # This is intended for advanced configuration only; most users should use the built in template + templates: {} + + # Default templates specifies a set of default templates that are used in sidecar injection. + # By default, a template `sidecar` is always provided, which contains the template of default sidecar. + # To inject other additional templates, define it using the `templates` option, and add it to + # the default templates list. + # For example: + # + # templates: + # hello: | + # metadata: + # labels: + # hello: world + # + # defaultTemplates: ["sidecar", "hello"] + defaultTemplates: [] istiodRemote: # Sidecar injector mutating webhook configuration url # For example: https://$remotePilotAddress:15017/inject @@ -150,26 +182,20 @@ ownerName: "" # meshConfig defines runtime configuration of components, including Istiod and istio-agent behavior # See https://istio.io/docs/reference/config/istio.mesh.v1alpha1/ for all available options meshConfig: - + enablePrometheusMerge: true # Config for the default ProxyConfig. # Initially using directly the proxy metadata - can also be activated using annotations # on the pod. This is an unsupported low-level API, pending review and decisions on # enabling the feature. Enabling the DNS listener is safe - and allows further testing # and gradual adoption by setting capture only on specific workloads. It also allows # VMs to use other DNS options, like dnsmasq or unbound. - defaultConfig: - proxyMetadata: - # If empty, agent will not start :15013 DNS listener and will not attempt - # to connect to Istiod DNS-TLS. This will also disable the core dns sidecar in - # istiod and the dns-over-tls listener. - # DNS_AGENT: DNS-TLS - DNS_AGENT: "" # The namespace to treat as the administrative root namespace for Istio configuration. # When processing a leaf namespace Istio will search for declarations in that namespace first # and if none are found it will search in the root namespace. Any matching declaration found in the root namespace # is processed as if it were declared in the leaf namespace. - rootNamespace: "istio-system" + + rootNamespace: # TODO: the intent is to eventually have this enabled by default when security is used. # It is not clear if user should normally need to configure - the metadata is typically @@ -180,6 +206,8 @@ meshConfig: # No hurry to do this in 1.6, we're trying to prove the code. global: + # Used to locate istiod. + istioNamespace: istio-system # enable pod disruption budget for the control plane, which is used to # ensure Istio control plane components are gradually upgraded or recovered. defaultPodDisruptionBudget: @@ -199,15 +227,12 @@ global: # cpu: 100m # memory: 128Mi - # Used to locate istiod. - istioNamespace: istio-system - # Default hub for Istio images. # Releases are published to docker hub under 'istio' project. # Dev builds from prow are on gcr.io - hub: gcr.io/istio-testing + hub: docker.io/istio # Default tag for Istio images. - tag: latest + tag: 1.9.0 # Specify image pull policy if default behavior isn't desired. # Default behavior: latest images will be Always else IfNotPresent. @@ -353,12 +378,11 @@ global: # The customized CA address to retrieve certificates for the pods in the cluster. # CSR clients such as the Istio Agent and ingress gateways can use this to specify the CA endpoint. + # If not set explicitly, default to the Istio discovery address. caAddress: "" # External istiod controls all remote clusters: disabled by default externalIstiod: false - # Central istiod controls all remote clusters: disabled by default - centralIstiod: false # Configure the policy for validating JWT. # Currently, two options are supported: "third-party-jwt" and "first-party-jwt". @@ -483,3 +507,6 @@ global: # Deprecated, use meshConfig.trustDomain trustDomain: "" +base: + # For istioctl usage to disable istio config crds in base + enableIstioConfigCRDs: true diff --git a/charts/kubezero-istio/update.sh b/charts/kubezero-istio/update.sh index 5d6a364..00d385e 100755 --- a/charts/kubezero-istio/update.sh +++ b/charts/kubezero-istio/update.sh @@ -1,7 +1,7 @@ #!/bin/bash set -ex -export ISTIO_VERSION=1.8.2 +export ISTIO_VERSION=1.9.0 if [ ! -d istio-$ISTIO_VERSION ]; then NAME="istio-$ISTIO_VERSION" @@ -16,10 +16,10 @@ cp -r istio-${ISTIO_VERSION}/manifests/charts/base charts/ cp -r istio-${ISTIO_VERSION}/manifests/charts/istio-control/istio-discovery charts/ # Patch for istiod to control plane -patch -p3 -i istio-discovery.patch +patch -p3 -i istio-discovery.patch --no-backup-if-mismatch # remove unused old telemetry filters -rm -f charts/istio-discovery/templates/telemetryv2_1.[67].yaml +rm -f charts/istio-discovery/templates/telemetryv2_1.[678].yaml # Ingress charts rm -rf ../kubezero-istio-ingress/charts/istio-* @@ -30,8 +30,8 @@ cp -r istio-${ISTIO_VERSION}/manifests/charts/gateways/istio-ingress ../kubezero sed -i -e 's/name: istio-ingress/name: istio-private-ingress/' ../kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml # Patch for ingress for extended termination grace period -patch -i ingress-terminationgraceperiod.patch ../kubezero-istio-ingress/charts/istio-ingress/templates/deployment.yaml -patch -i ingress-terminationgraceperiod.patch ../kubezero-istio-ingress/charts/istio-private-ingress/templates/deployment.yaml +patch -i ingress-terminationgraceperiod.patch ../kubezero-istio-ingress/charts/istio-ingress/templates/deployment.yaml --no-backup-if-mismatch +patch -i ingress-terminationgraceperiod.patch ../kubezero-istio-ingress/charts/istio-private-ingress/templates/deployment.yaml --no-backup-if-mismatch # Get matching istioctl [ -x istioctl ] && [ "$(./istioctl version --remote=false)" == $ISTIO_VERSION ] || { curl -sL https://github.com/istio/istio/releases/download/${ISTIO_VERSION}/istioctl-${ISTIO_VERSION}-linux-amd64.tar.gz | tar xz; chmod +x istioctl; } diff --git a/charts/kubezero-istio/values.yaml b/charts/kubezero-istio/values.yaml index a855e1a..398dd39 100644 --- a/charts/kubezero-istio/values.yaml +++ b/charts/kubezero-istio/values.yaml @@ -1,6 +1,6 @@ global: - hub: docker.io/istio - tag: 1.8.2 + # hub: docker.io/istio + # tag: 1.9.0 logAsJson: true jwtPolicy: first-party-jwt -- 2.40.1 From 8613433699b746c23b4b09d399f25a9ca7c9c3a5 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 26 Feb 2021 00:24:12 +0100 Subject: [PATCH 19/65] Update ingress default config --- charts/kubezero-istio-ingress/values.yaml | 38 ++++++++++++++++++++--- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/charts/kubezero-istio-ingress/values.yaml b/charts/kubezero-istio-ingress/values.yaml index 4603747..639a54b 100644 --- a/charts/kubezero-istio-ingress/values.yaml +++ b/charts/kubezero-istio-ingress/values.yaml @@ -1,7 +1,7 @@ # Make sure these values match kuberzero-istio !!! global: - hub: docker.io/istio - tag: 1.8.2 + #hub: docker.io/istio + #tag: 1.9.0 logAsJson: true jwtPolicy: first-party-jwt @@ -42,17 +42,32 @@ istio-ingress: nodeSelector: node.kubernetes.io/ingress.public: "30080_30443" ports: - - name: http-status + - name: status-port port: 15021 nodePort: 30021 + protocol: http - name: http2 port: 80 targetPort: 8080 nodePort: 30080 + protocol: http2 - name: https port: 443 targetPort: 8443 nodePort: 30443 + protocol: https + ## multi-cluster - disabled on public LBs + #- name: tcp-istiod + # port: 15012 + # targetPort: 15012 + # nodePort: 30012 + # protocol: TCP + ## multi-cluster sni east-west + #- name: tls + # port: 15443 + # targetPort: 15443 + # nodePort: 30044 + # protocol: TCP dnsNames: [] # - '*.example.com' @@ -93,17 +108,32 @@ istio-private-ingress: #nodeSelector: "31080_31443_31671_31672_31224" ports: - - name: http-status + - name: status-port port: 15021 nodePort: 31021 + protocol: http - name: http2 port: 80 targetPort: 8080 nodePort: 31080 + protocol: http2 - name: https port: 443 targetPort: 8443 nodePort: 31443 + protocol: https + # multi-cluster + - name: tcp-istiod + port: 15012 + targetPort: 15012 + nodePort: 31012 + protocol: TCP + # multi-cluster sni east-west + - name: tls + port: 15443 + targetPort: 15443 + nodePort: 31044 + protocol: TCP #- name: fluentd-forward # port: 24224 # nodePort: 31224 -- 2.40.1 From f4b3bc85c76563c70a36971031c6d09355519ba4 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 26 Feb 2021 00:35:21 +0100 Subject: [PATCH 20/65] Fix gateway protocol --- charts/kubezero-istio-ingress/values.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/kubezero-istio-ingress/values.yaml b/charts/kubezero-istio-ingress/values.yaml index 639a54b..50a9aee 100644 --- a/charts/kubezero-istio-ingress/values.yaml +++ b/charts/kubezero-istio-ingress/values.yaml @@ -45,17 +45,17 @@ istio-ingress: - name: status-port port: 15021 nodePort: 30021 - protocol: http + protocol: TCP - name: http2 port: 80 targetPort: 8080 nodePort: 30080 - protocol: http2 + protocol: TCP - name: https port: 443 targetPort: 8443 nodePort: 30443 - protocol: https + protocol: TCP ## multi-cluster - disabled on public LBs #- name: tcp-istiod # port: 15012 @@ -111,17 +111,17 @@ istio-private-ingress: - name: status-port port: 15021 nodePort: 31021 - protocol: http + protocol: TCP - name: http2 port: 80 targetPort: 8080 nodePort: 31080 - protocol: http2 + protocol: TCP - name: https port: 443 targetPort: 8443 nodePort: 31443 - protocol: https + protocol: TCP # multi-cluster - name: tcp-istiod port: 15012 -- 2.40.1 From 44127fbbcb141e95660fb808c3d2ef5b8bad5e3b Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 26 Feb 2021 01:18:32 +0100 Subject: [PATCH 21/65] Minor version bump of aws-ebs-csi-driver to update livenessprobe --- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 4 ++-- .../charts/aws-ebs-csi-driver/Chart.yaml | 2 +- .../charts/aws-ebs-csi-driver/values.yaml | 2 +- charts/kubezero-aws-ebs-csi-driver/update.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index b2ea4b4..6f6e3ef 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.4.0 +version: 0.4.1 appVersion: 0.9.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -18,7 +18,7 @@ maintainers: - name: Quarky9 dependencies: - name: aws-ebs-csi-driver - version: 0.9.9 + version: 0.9.10 repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: kubezero-lib version: ">= 0.1.3" diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml index 0910aac..ec5dd3f 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml @@ -15,4 +15,4 @@ maintainers: name: aws-ebs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver -version: 0.9.9 +version: 0.9.10 diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml index fd13fad..b10aa09 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml @@ -23,7 +23,7 @@ sidecars: tag: "v3.0.3" livenessProbeImage: repository: k8s.gcr.io/sig-storage/livenessprobe - tag: "v2.1.0" + tag: "v2.2.0" resizerImage: repository: k8s.gcr.io/sig-storage/csi-resizer tag: "v1.0.0" diff --git a/charts/kubezero-aws-ebs-csi-driver/update.sh b/charts/kubezero-aws-ebs-csi-driver/update.sh index 14f4674..771beed 100755 --- a/charts/kubezero-aws-ebs-csi-driver/update.sh +++ b/charts/kubezero-aws-ebs-csi-driver/update.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION=0.9.9 +VERSION=0.9.10 rm -rf charts/aws-ebs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts -- 2.40.1 From 8a8c4fbe66b04fdeaefc0ddaf6c397e35a0d9ee9 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 26 Feb 2021 22:25:43 +0100 Subject: [PATCH 22/65] Prometheus-stack version bump --- charts/kubezero-metrics/Chart.yaml | 6 +- .../charts/kube-prometheus-stack/Chart.yaml | 17 +- .../charts/kube-prometheus-stack/README.md | 16 +- .../charts/grafana/Chart.yaml | 9 +- .../charts/grafana/README.md | 75 ++++--- .../charts/grafana/templates/_helpers.tpl | 11 + .../charts/grafana/templates/_pod.tpl | 23 ++ .../configmap-dashboard-provider.yaml | 3 + .../charts/grafana/templates/configmap.yaml | 3 + .../templates/image-renderer-deployment.yaml | 3 + .../charts/grafana/templates/ingress.yaml | 7 +- .../grafana/templates/podsecuritypolicy.yaml | 1 + .../charts/grafana/templates/pvc.yaml | 5 + .../charts/grafana/templates/role.yaml | 2 +- .../charts/grafana/templates/rolebinding.yaml | 2 +- .../charts/grafana/templates/secret.yaml | 2 +- .../grafana/templates/servicemonitor.yaml | 8 +- .../charts/grafana/templates/statefulset.yaml | 7 +- .../tests/test-podsecuritypolicy.yaml | 1 + .../charts/grafana/values.yaml | 49 ++++- .../charts/kube-state-metrics/Chart.yaml | 13 +- .../charts/kube-state-metrics/LICENSE | 202 ++++++++++++++++++ .../charts/kube-state-metrics/OWNERS | 6 + .../charts/kube-state-metrics/README.md | 141 +++++------- .../templates/clusterrolebinding.yaml | 6 +- .../templates/deployment.yaml | 12 +- .../kube-state-metrics/templates/pdb.yaml | 5 +- .../templates/{clusterrole.yaml => role.yaml} | 80 ++++--- .../templates/rolebinding.yaml | 27 +++ .../kube-state-metrics/templates/service.yaml | 6 + .../templates/servicemonitor.yaml | 9 + .../charts/kube-state-metrics/values.yaml | 22 +- .../prometheus-node-exporter/Chart.yaml | 3 +- .../templates/daemonset.yaml | 18 +- .../templates/monitor.yaml | 7 + .../templates/serviceaccount.yaml | 2 + .../prometheus-node-exporter/values.yaml | 17 +- .../crds/crd-alertmanagerconfigs.yaml | 45 ++-- .../crds/crd-alertmanagers.yaml | 2 +- .../crds/crd-podmonitors.yaml | 2 +- .../crds/crd-probes.yaml | 2 +- .../crds/crd-prometheuses.yaml | 11 +- .../crds/crd-prometheusrules.yaml | 2 +- .../crds/crd-servicemonitors.yaml | 2 +- .../crds/crd-thanosrulers.yaml | 2 +- .../templates/alertmanager/alertmanager.yaml | 22 +- .../templates/alertmanager/ingress.yaml | 11 +- .../alertmanager/ingressperreplica.yaml | 6 +- .../kube-state-metrics/serviceMonitor.yaml | 4 + .../grafana/configmaps-datasources.yaml | 5 +- .../dashboards-1.14/cluster-total.yaml | 2 +- .../grafana/dashboards-1.14/etcd.yaml | 2 +- .../grafana/dashboards-1.14/kubelet.yaml | 4 +- .../dashboards-1.14/namespace-by-pod.yaml | 2 +- .../namespace-by-workload.yaml | 2 +- .../node-cluster-rsrc-use.yaml | 22 +- .../dashboards-1.14/node-rsrc-use.yaml | 22 +- .../grafana/dashboards-1.14/nodes.yaml | 20 +- .../persistentvolumesusage.yaml | 4 +- .../grafana/dashboards-1.14/pod-total.yaml | 2 +- .../prometheus-remote-write.yaml | 20 +- .../grafana/dashboards-1.14/prometheus.yaml | 6 +- .../dashboards-1.14/workload-total.yaml | 2 +- .../templates/grafana/dashboards/etcd.yaml | 2 +- .../job-patch/clusterrole.yaml | 2 +- .../job-patch/clusterrolebinding.yaml | 2 +- .../job-patch/job-createSecret.yaml | 2 +- .../job-patch/job-patchWebhook.yaml | 2 +- .../admission-webhooks/job-patch/psp.yaml | 2 +- .../admission-webhooks/job-patch/role.yaml | 2 +- .../job-patch/rolebinding.yaml | 2 +- .../job-patch/serviceaccount.yaml | 2 +- .../mutatingWebhookConfiguration.yaml | 8 + .../validatingWebhookConfiguration.yaml | 8 + .../prometheus-operator/certmanager.yaml | 57 +++++ .../prometheus-operator/deployment.yaml | 37 ++-- .../prometheus-operator/servicemonitor.yaml | 2 +- .../templates/prometheus/ingress.yaml | 9 +- .../prometheus/ingressThanosSidecar.yaml | 13 +- .../prometheus/ingressperreplica.yaml | 6 +- .../templates/prometheus/prometheus.yaml | 19 +- .../templates/prometheus/psp.yaml | 7 + .../templates/prometheus/rules-1.14/etcd.yaml | 30 +-- .../prometheus/rules-1.14/k8s.rules.yaml | 6 +- .../kube-apiserver-availability.rules.yaml | 98 ++++----- .../rules-1.14/kube-apiserver-slos.yaml | 2 +- .../rules-1.14/kube-apiserver.rules.yaml | 7 +- .../rules-1.14/kube-scheduler.rules.yaml | 2 +- .../prometheus/rules-1.14/kubelet.rules.yaml | 2 +- .../rules-1.14/kubernetes-apps.yaml | 6 +- .../rules-1.14/kubernetes-resources.yaml | 2 +- .../rules-1.14/kubernetes-storage.yaml | 2 +- .../kubernetes-system-apiserver.yaml | 20 +- .../kubernetes-system-controller-manager.yaml | 2 +- .../rules-1.14/kubernetes-system-kubelet.yaml | 2 +- .../kubernetes-system-scheduler.yaml | 2 +- .../rules-1.14/kubernetes-system.yaml | 2 +- .../rules-1.14/node-exporter.rules.yaml | 2 +- .../prometheus/rules-1.14/node-exporter.yaml | 18 +- .../prometheus/rules-1.14/node.rules.yaml | 4 +- .../templates/prometheus/rules/etcd.yaml | 30 +-- .../templates/prometheus/service.yaml | 8 + .../prometheus/serviceThanosSIdecar.yaml | 27 +++ .../templates/prometheus/servicemonitors.yaml | 4 + .../charts/kube-prometheus-stack/values.yaml | 156 ++++++++++++-- charts/kubezero-metrics/update.sh | 4 +- 106 files changed, 1181 insertions(+), 493 deletions(-) create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/LICENSE create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/OWNERS rename charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/{clusterrole.yaml => role.yaml} (61%) create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/certmanager.yaml create mode 100644 charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSIdecar.yaml diff --git a/charts/kubezero-metrics/Chart.yaml b/charts/kubezero-metrics/Chart.yaml index 9b2dd2e..dee19cf 100644 --- a/charts/kubezero-metrics/Chart.yaml +++ b/charts/kubezero-metrics/Chart.yaml @@ -16,11 +16,11 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: kube-prometheus-stack - version: 12.8.0 + version: 13.13.0 # Switch back to upstream once all alerts are fixed eg. etcd gpcr # repository: https://prometheus-community.github.io/helm-charts - name: prometheus-adapter - version: 2.10.1 + version: 2.12.1 repository: https://prometheus-community.github.io/helm-charts condition: prometheus-adapter.enabled -kubeVersion: ">= 1.16.0" +kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml index a410ff2..a3e49b1 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/Chart.yaml @@ -6,24 +6,21 @@ annotations: url: https://github.com/prometheus-operator/kube-prometheus artifacthub.io/operator: "true" apiVersion: v2 -appVersion: 0.44.0 +appVersion: 0.45.0 dependencies: - condition: kubeStateMetrics.enabled name: kube-state-metrics - repository: https://charts.helm.sh/stable - version: 2.9.* + repository: https://kubernetes.github.io/kube-state-metrics + version: 2.13.* - condition: nodeExporter.enabled name: prometheus-node-exporter repository: https://prometheus-community.github.io/helm-charts - version: 1.12.* + version: 1.14.* - condition: grafana.enabled name: grafana repository: https://grafana.github.io/helm-charts - version: 5.8.* -description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, - and Prometheus rules combined with documentation and scripts to provide easy to - operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus - Operator. + version: 6.4.* +description: kube-prometheus-stack collects Kubernetes manifests, Grafana dashboards, and Prometheus rules combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. home: https://github.com/prometheus-operator/kube-prometheus icon: https://raw.githubusercontent.com/prometheus/prometheus.github.io/master/assets/prometheus_logo-cb55bb5c346.png keywords: @@ -47,4 +44,4 @@ sources: - https://github.com/prometheus-community/helm-charts - https://github.com/prometheus-operator/kube-prometheus type: application -version: 12.8.0 +version: 13.13.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md index 37794b1..68c1f82 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/README.md @@ -15,7 +15,6 @@ _Note: This chart was formerly named `prometheus-operator` chart, now renamed to ```console helm repo add prometheus-community https://prometheus-community.github.io/helm-charts -helm repo add stable https://charts.helm.sh/stable helm repo update ``` @@ -36,7 +35,7 @@ _See [helm install](https://helm.sh/docs/helm/helm_install/) for command documen By default this chart installs additional, dependent charts: -- [stable/kube-state-metrics](https://github.com/helm/charts/tree/master/stable/kube-state-metrics) +- [kubernetes/kube-state-metrics](https://github.com/kubernetes/kube-state-metrics/tree/master/charts/kube-state-metrics) - [prometheus-community/prometheus-node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter) - [grafana/grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) @@ -84,6 +83,15 @@ _See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documen A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. +### From 12.x to 13.x + +Version 12 upgrades prometheus-operator from 0.44.x to 0.45.x. Helm does not automatically upgrade or install new CRDs on a chart upgrade, so you have to install the CRD manually before updating: + +```console +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +``` + ### From 11.x to 12.x The chart was migrated to support only helm v3 and later. @@ -181,6 +189,8 @@ A validating and mutating webhook configuration requires the endpoint to which t It should be possible to use [jetstack/cert-manager](https://github.com/jetstack/cert-manager) if a more complete solution is required, but it has not been tested. +You can enable automatic self-signed TLS certificate provisioning via cert-manager by setting the `prometheusOperator.admissionWebhooks.certManager.enabled` value to true. + ### Limitations Because the operator can only run as a single pod, there is potential for this component failure to cause rule deployment failure. Because this risk is outweighed by the benefit of having validation, the feature is enabled by default. @@ -311,7 +321,7 @@ You can check out the tickets for this change [here](https://github.com/promethe The chart has added 3 [dependencies](#dependencies). - Node-Exporter, Kube-State-Metrics: These components are loaded as dependencies into the chart, and are relatively simple components -- Grafana: The Grafana chart is more feature-rich than this chart - it contains a sidecar that is able to load data sources and dashboards from configmaps deployed into the same cluster. For more information check out the [documentation for the chart](https://github.com/helm/charts/tree/master/stable/grafana) +- Grafana: The Grafana chart is more feature-rich than this chart - it contains a sidecar that is able to load data sources and dashboards from configmaps deployed into the same cluster. For more information check out the [documentation for the chart](https://github.com/grafana/helm-charts/blob/main/charts/grafana/README.md) #### Kubelet Service diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml index 2f6e076..bd2ea66 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/Chart.yaml @@ -1,5 +1,5 @@ -apiVersion: v1 -appVersion: 7.2.1 +apiVersion: v2 +appVersion: 7.4.2 description: The leading tool for querying and visualizing time series and metrics. home: https://grafana.net icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png @@ -13,7 +13,10 @@ maintainers: name: maorfr - email: miroslav.hadzhiev@gmail.com name: Xtigyro +- email: mail@torstenwalter.de + name: torstenwalter name: grafana sources: - https://github.com/grafana/grafana -version: 5.8.16 +type: application +version: 6.4.4 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md index 2dc1822..815f6fa 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/README.md @@ -16,7 +16,7 @@ _See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation To install the chart with the release name `my-release`: ```console -helm install --name my-release grafana/grafana +helm install my-release grafana/grafana ``` ## Uninstalling the Chart @@ -42,6 +42,10 @@ This version requires Helm >= 2.12.0. You have to add --force to your helm upgrade command as the labels of the chart have changed. +### To 6.0.0 + +This version requires Helm >= 3.1.0. + ## Configuration | Parameter | Description | Default | @@ -55,7 +59,7 @@ You have to add --force to your helm upgrade command as the labels of the chart | `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | | `priorityClassName` | Name of Priority Class to assign pods | `nil` | | `image.repository` | Image repository | `grafana/grafana` | -| `image.tag` | Image tag (`Must be >= 5.0.0`) | `7.0.3` | +| `image.tag` | Image tag (`Must be >= 5.0.0`) | `7.4.2` | | `image.sha` | Image sha (optional) | `17cbd08b9515fda889ca959e9d72ee6f3327c8f1844a3336dfd952134f38e2fe` | | `image.pullPolicy` | Image pull policy | `IfNotPresent` | | `image.pullSecrets` | Image pull secrets | `{}` | @@ -96,6 +100,8 @@ You have to add --force to your helm upgrade command as the labels of the chart | `persistence.annotations` | PersistentVolumeClaim annotations | `{}` | | `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` | | `persistence.subPath` | Mount a sub dir of the persistent volume | `nil` | +| `persistence.inMemory.enabled` | If persistence is not enabled, whether to mount the local storage in-memory to improve performance | `false` | +| `persistence.inMemory.sizeLimit` | SizeLimit for the in-memory local storage | `nil` | | `initChownData.enabled` | If false, don't reset data ownership at startup | true | | `initChownData.image.repository` | init-chown-data container image repository | `busybox` | | `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` | @@ -126,8 +132,8 @@ You have to add --force to your helm upgrade command as the labels of the chart | `podAnnotations` | Pod annotations | `{}` | | `podLabels` | Pod labels | `{}` | | `podPortName` | Name of the grafana port on the pod | `grafana` | -| `sidecar.image.repository` | Sidecar image repository | `kiwigrid/k8s-sidecar` | -| `sidecar.image.tag` | Sidecar image tag | `1.1.0` | +| `sidecar.image.repository` | Sidecar image repository | `quay.io/kiwigrid/k8s-sidecar` | +| `sidecar.image.tag` | Sidecar image tag | `1.10.6` | | `sidecar.image.sha` | Sidecar image sha (optional) | `""` | | `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | | `sidecar.resources` | Sidecar resources | `{}` | @@ -144,14 +150,16 @@ You have to add --force to your helm upgrade command as the labels of the chart | `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | | `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` | | `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` | +| `sidecar.dashboards.labelValue` | Label value that config maps with dashboards should have to be added | `nil` | | `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | | `sidecar.dashboards.folderAnnotation` | The annotation the sidecar will look for in configmaps to override the destination folder for files | `nil` | | `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | | `sidecar.dashboards.searchNamespace` | If specified, the sidecar will search for dashboard config-maps inside this namespace. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces | `nil` | | `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | | `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | +| `sidecar.datasources.labelValue` | Label value that config maps with datasources should have to be added | `nil` | | `sidecar.datasources.searchNamespace` | If specified, the sidecar will search for datasources config-maps inside this namespace. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces | `nil` | -| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana |`false` | +| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana | `false` | | `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | | `sidecar.notifiers.searchNamespace` | If specified, the sidecar will search for notifiers config-maps (or secrets) inside this namespace. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces | `nil` | | `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | @@ -173,9 +181,9 @@ You have to add --force to your helm upgrade command as the labels of the chart | `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] | | `command` | Define command to be executed by grafana container at startup | `nil` | | `testFramework.enabled` | Whether to create test-related resources | `true` | -| `testFramework.image` | `test-framework` image repository. | `bats/bats` | -| `testFramework.tag` | `test-framework` image tag. | `v1.1.0` | -| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | +| `testFramework.image` | `test-framework` image repository. | `bats/bats` | +| `testFramework.tag` | `test-framework` image tag. | `v1.1.0` | +| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | | `testFramework.securityContext` | `test-framework` securityContext | `{}` | | `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | | `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` | @@ -188,6 +196,8 @@ You have to add --force to your helm upgrade command as the labels of the chart | `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | | | `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` | | `serviceMonitor.path` | Path to scrape | `/metrics` | +| `serviceMonitor.scheme` | Scheme to use for metrics scraping | `http` | +| `serviceMonitor.tlsConfig` | TLS configuration block for the endpoint | `{}` | | `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` | | `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` | | `serviceMonitor.relabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` | @@ -198,6 +208,7 @@ You have to add --force to your helm upgrade command as the labels of the chart | `imageRenderer.image.sha` | image-renderer Image sha (optional) | `""` | | `imageRenderer.image.pullPolicy` | image-renderer ImagePullPolicy | `Always` | | `imageRenderer.env` | extra env-vars for image-renderer | `{}` | +| `imageRenderer.serviceAccountName` | image-renderer deployment serviceAccountName | `""` | | `imageRenderer.securityContext` | image-renderer deployment securityContext | `{}` | | `imageRenderer.hostAliases` | image-renderer deployment Host Aliases | `[]` | | `imageRenderer.priorityClassName` | image-renderer deployment priority class | `''` | @@ -311,35 +322,18 @@ If the parameter `sidecar.datasources.enabled` is set, an init container is depl pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, -the data sources in grafana can be imported. The secrets must be created before `helm install` so -that the datasources init container can list the secrets. +the data sources in grafana can be imported. Secrets are recommended over configmaps for this usecase because datasources usually contain private data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those. -Example datasource config adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): +Example values to add a datasource adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): ```yaml -apiVersion: v1 -kind: Secret -metadata: - name: sample-grafana-datasource - labels: - grafana_datasource: "1" -type: Opaque -stringData: - datasource.yaml: |- - # config file version - apiVersion: 1 - - # list of datasources that should be deleted from the database - deleteDatasources: - - name: Graphite - orgId: 1 - - # list of datasources to insert/update depending - # whats available in the database - datasources: +datasources: + datasources.yaml: + apiVersion: 1 + datasources: # name of the datasource. Required - name: Graphite # datasource type. Required @@ -379,7 +373,6 @@ stringData: version: 1 # allow users to edit datasources from the UI. editable: false - ``` ## Sidecar for notifiers @@ -485,6 +478,24 @@ Include in the `extraSecretMounts` configuration flag: readOnly: true ``` +### extraSecretMounts using a Container Storage Interface (CSI) provider + +This example uses a CSI driver e.g. retrieving secrets using [Azure Key Vault Provider](https://github.com/Azure/secrets-store-csi-driver-provider-azure) + +```yaml +- extraSecretMounts: + - name: secrets-store-inline + mountPath: /run/secrets + readOnly: true + csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: "my-provider" + nodePublishSecretRef: + name: akv-creds +``` + ## Image Renderer Plug-In This chart supports enabling [remote image rendering](https://github.com/grafana/grafana-image-renderer/blob/master/docs/remote_rendering_using_docker.md) diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl index 4dd8834..9ce170c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_helpers.tpl @@ -100,3 +100,14 @@ Selector labels ImageRenderer app.kubernetes.io/name: {{ include "grafana.name" . }}-image-renderer app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} + +{{/* +Return the appropriate apiVersion for rbac. +*/}} +{{- define "rbac.apiVersion" -}} +{{- if .Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }} +{{- print "rbac.authorization.k8s.io/v1" -}} +{{- else -}} +{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl index a9e471c..6b0ef5d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/_pod.tpl @@ -84,6 +84,10 @@ initContainers: value: LIST - name: LABEL value: "{{ .Values.sidecar.datasources.label }}" + {{- if .Values.sidecar.datasources.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.datasources.labelValue }} + {{- end }} - name: FOLDER value: "/etc/grafana/provisioning/datasources" - name: RESOURCE @@ -164,6 +168,10 @@ containers: value: {{ .Values.sidecar.dashboards.watchMethod }} - name: LABEL value: "{{ .Values.sidecar.dashboards.label }}" + {{- if .Values.sidecar.dashboards.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.dashboards.labelValue }} + {{- end }} - name: FOLDER value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" - name: RESOURCE @@ -203,6 +211,10 @@ containers: - {{ . }} {{- end }} {{- end}} +{{- if .Values.containerSecurityContext }} + securityContext: +{{- toYaml .Values.containerSecurityContext | nindent 6 }} +{{- end }} volumeMounts: - name: config mountPath: "/etc/grafana/grafana.ini" @@ -419,8 +431,16 @@ volumes: # nothing {{- else }} - name: storage +{{- if .Values.persistence.inMemory.enabled }} + emptyDir: + medium: Memory +{{- if .Values.persistence.inMemory.sizeLimit }} + sizeLimit: {{ .Values.persistence.inMemory.sizeLimit }} +{{- end -}} +{{- else }} emptyDir: {} {{- end -}} +{{- end -}} {{- if .Values.sidecar.dashboards.enabled }} - name: sc-dashboard-volume emptyDir: {} @@ -447,6 +467,9 @@ volumes: {{- else if .projected }} - name: {{ .name }} projected: {{- toYaml .projected | nindent 6 }} +{{- else if .csi }} + - name: {{ .name }} + csi: {{- toYaml .csi | nindent 6 }} {{- end }} {{- end }} {{- range .Values.extraVolumeMounts }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap-dashboard-provider.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap-dashboard-provider.yaml index 8bb0567..65d7385 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap-dashboard-provider.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap-dashboard-provider.yaml @@ -16,10 +16,13 @@ data: providers: - name: '{{ .Values.sidecar.dashboards.provider.name }}' orgId: {{ .Values.sidecar.dashboards.provider.orgid }} + {{- if not .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} folder: '{{ .Values.sidecar.dashboards.provider.folder }}' + {{- end}} type: {{ .Values.sidecar.dashboards.provider.type }} disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }} allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }} + updateIntervalSeconds: {{ .Values.sidecar.dashboards.provider.updateIntervalSeconds | default 30 }} options: foldersFromFilesStructure: {{ .Values.sidecar.dashboards.provider.foldersFromFilesStructure }} path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml index a9fdc3a..0d2c3e2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/configmap.yaml @@ -59,6 +59,9 @@ data: --max-time 60 \ {{- if not $value.b64content }} -H "Accept: application/json" \ + {{- if $value.token }} + -H "Authorization: token {{ $value.token }}" \ + {{- end }} -H "Content-Type: application/json;charset=UTF-8" \ {{ end }} {{- if $value.url -}}"{{ $value.url }}"{{- else -}}"https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download"{{- end -}}{{ if $value.datasource }} | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g'{{ end }}{{- if $value.b64content -}} | base64 -d {{- end -}} \ diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml index 1f60ffb..2ab9f5e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/image-renderer-deployment.yaml @@ -40,6 +40,9 @@ spec: {{- if .Values.imageRenderer.schedulerName }} schedulerName: "{{ .Values.imageRenderer.schedulerName }}" {{- end }} + {{- if .Values.imageRenderer.serviceAccountName }} + serviceAccountName: "{{ .Values.imageRenderer.serviceAccountName }}" + {{- end }} {{- if .Values.imageRenderer.securityContext }} securityContext: {{ toYaml .Values.imageRenderer.securityContext | indent 2 }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/ingress.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/ingress.yaml index 8d35662..710b82d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/ingress.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/ingress.yaml @@ -24,14 +24,17 @@ metadata: {{- end }} {{- end }} spec: + {{- if .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end -}} {{- if .Values.ingress.tls }} tls: -{{ toYaml .Values.ingress.tls | indent 4 }} +{{ tpl (toYaml .Values.ingress.tls) $ | indent 4 }} {{- end }} rules: {{- if .Values.ingress.hosts }} {{- range .Values.ingress.hosts }} - - host: {{ . }} + - host: {{ tpl . $}} http: paths: {{ if $extraPaths }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/podsecuritypolicy.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/podsecuritypolicy.yaml index 9d50471..88bf64c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/podsecuritypolicy.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/podsecuritypolicy.yaml @@ -34,6 +34,7 @@ spec: - 'configMap' - 'emptyDir' - 'projected' + - 'csi' - 'secret' - 'downwardAPI' - 'persistentVolumeClaim' diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/pvc.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/pvc.yaml index 4727d0a..8d93f5c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/pvc.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/pvc.yaml @@ -25,4 +25,9 @@ spec: {{- if .Values.persistence.storageClassName }} storageClassName: {{ .Values.persistence.storageClassName }} {{- end -}} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: +{{ toYaml . | indent 6 }} + {{- end }} {{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/role.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/role.yaml index db85355..54c3fb0 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/role.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/role.yaml @@ -1,5 +1,5 @@ {{- if and .Values.rbac.create (not .Values.rbac.useExistingRole) -}} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: {{ template "rbac.apiVersion" . }} kind: Role metadata: name: {{ template "grafana.fullname" . }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/rolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/rolebinding.yaml index 3738e58..34f1ad6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/rolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/rolebinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.rbac.create -}} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: {{ template "rbac.apiVersion" . }} kind: RoleBinding metadata: name: {{ template "grafana.fullname" . }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml index 9d2f072..4fdd817 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/secret.yaml @@ -17,6 +17,6 @@ data: {{- end }} {{- end }} {{- if not .Values.ldap.existingSecret }} - ldap-toml: {{ .Values.ldap.config | b64enc | quote }} + ldap-toml: {{ tpl .Values.ldap.config $ | b64enc | quote }} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/servicemonitor.yaml index 988956b..2328852 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/servicemonitor.yaml @@ -21,6 +21,11 @@ spec: honorLabels: true port: {{ .Values.service.portName }} path: {{ .Values.serviceMonitor.path }} + scheme: {{ .Values.serviceMonitor.scheme }} + {{- if .Values.serviceMonitor.tlsConfig }} + tlsConfig: + {{- toYaml .Values.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} {{- if .Values.serviceMonitor.relabelings }} relabelings: {{- toYaml .Values.serviceMonitor.relabelings | nindent 4 }} @@ -28,8 +33,7 @@ spec: jobLabel: "{{ .Release.Name }}" selector: matchLabels: - app: {{ template "grafana.name" . }} - release: "{{ .Release.Name }}" + {{- include "grafana.selectorLabels" . | nindent 8 }} namespaceSelector: matchNames: - {{ .Release.Namespace }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/statefulset.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/statefulset.yaml index accfa56..b2b4616 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/statefulset.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/statefulset.yaml @@ -43,5 +43,10 @@ spec: storageClassName: {{ .Values.persistence.storageClassName }} resources: requests: - storage: {{ .Values.persistence.size }} + storage: {{ .Values.persistence.size }} + {{- with .Values.persistence.selectorLabels }} + selector: + matchLabels: +{{ toYaml . | indent 10 }} + {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/tests/test-podsecuritypolicy.yaml index eb5cbbc..1acd651 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/tests/test-podsecuritypolicy.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -25,5 +25,6 @@ spec: - downwardAPI - emptyDir - projected + - csi - secret {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml index 1f22b93..c461687 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/grafana/values.yaml @@ -53,7 +53,7 @@ livenessProbe: image: repository: grafana/grafana - tag: 7.2.1 + tag: 7.4.2 sha: "" pullPolicy: IfNotPresent @@ -76,6 +76,8 @@ securityContext: runAsGroup: 472 fsGroup: 472 +containerSecurityContext: + {} extraConfigmapMounts: [] # - name: certs-configmap @@ -136,6 +138,8 @@ serviceMonitor: # namespace: monitoring (defaults to use the namespace this chart is deployed to) labels: {} interval: 1m + scheme: http + tlsConfig: {} scrapeTimeout: 30s relabelings: [] @@ -153,6 +157,9 @@ hostAliases: [] ingress: enabled: false + # For Kubernetes >= 1.18 you should specify the ingress-controller via the field ingressClassName + # See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#specifying-the-class-of-an-ingress + # ingressClassName: nginx # Values can be templated annotations: {} # kubernetes.io/ingress.class: nginx @@ -235,9 +242,21 @@ persistence: # annotations: {} finalizers: - kubernetes.io/pvc-protection + # selectorLabels: {} # subPath: "" # existingClaim: + ## If persistence is not enabled, this allows to mount the + ## local storage in-memory to improve performance + ## + inMemory: + enabled: false + ## The maximum usage on memory medium EmptyDir would be + ## the minimum value between the SizeLimit specified + ## here and the sum of memory limits of all containers in a pod + ## + # sizeLimit: 300Mi + initChownData: ## If false, data ownership will not be reset at startup ## This allows the prometheus-server to be run with an arbitrary user @@ -348,6 +367,18 @@ extraSecretMounts: [] # audience: sts.amazonaws.com # expirationSeconds: 86400 # path: token + # + # for CSI e.g. Azure Key Vault use the following + # - name: secrets-store-inline + # mountPath: /run/secrets + # readOnly: true + # csi: + # driver: secrets-store.csi.k8s.io + # readOnly: true + # volumeAttributes: + # secretProviderClass: "akv-grafana-spc" + # nodePublishSecretRef: # Only required when using service principal mode + # name: grafana-akv-creds # Only required when using service principal mode ## Additional grafana server volume mounts # Defines additional volume mounts. @@ -439,8 +470,10 @@ dashboards: {} # datasource: Prometheus # local-dashboard: # url: https://example.com/repository/test.json + # token: '' # local-dashboard-base64: # url: https://example.com/repository/test-b64.json + # token: '' # b64content: true ## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. @@ -530,8 +563,8 @@ smtp: ## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards sidecar: image: - repository: kiwigrid/k8s-sidecar - tag: 1.1.0 + repository: quay.io/kiwigrid/k8s-sidecar + tag: 1.10.6 sha: "" imagePullPolicy: IfNotPresent resources: {} @@ -549,6 +582,8 @@ sidecar: SCProvider: true # label that the configmaps with dashboards are marked with label: grafana_dashboard + # value of label that the configmaps with dashboards are set to + labelValue: null # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) folder: /tmp/dashboards # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead @@ -580,6 +615,8 @@ sidecar: enabled: false # label that the configmaps with datasources are marked with label: grafana_datasource + # value of label that the configmaps with datasources are set to + labelValue: null # If specified, the sidecar will search for datasource config-maps inside this namespace. # Otherwise the namespace in which the sidecar is running will be used. # It's also possible to specify ALL to search in all namespaces @@ -616,9 +653,12 @@ imageRenderer: # image-renderer ImagePullPolicy pullPolicy: Always # extra environment variables - env: {} + env: + HTTP_HOST: "0.0.0.0" # RENDERING_ARGS: --disable-gpu,--window-size=1280x758 # RENDERING_MODE: clustered + # image-renderer deployment serviceAccount + serviceAccountName: "" # image-renderer deployment securityContext securityContext: {} # image-renderer deployment Host Aliases @@ -630,6 +670,7 @@ imageRenderer: portName: 'http' # image-renderer service port used by both service and deployment port: 8081 + targetPort: 8081 # name of the image-renderer port on the pod podPortName: http # number of image-renderer replica sets to keep diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml index 663499a..45a1a71 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/Chart.yaml @@ -1,15 +1,18 @@ apiVersion: v1 -appVersion: 1.9.7 -deprecated: true -description: DEPRECATED - Install kube-state-metrics to generate and expose cluster-level - metrics +appVersion: 1.9.8 +description: Install kube-state-metrics to generate and expose cluster-level metrics home: https://github.com/kubernetes/kube-state-metrics/ keywords: - metric - monitoring - prometheus - kubernetes +maintainers: +- email: tariq.ibrahim@mulesoft.com + name: tariq1890 +- email: manuel@rueg.eu + name: mrueg name: kube-state-metrics sources: - https://github.com/kubernetes/kube-state-metrics/ -version: 2.9.4 +version: 2.13.0 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/LICENSE b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/LICENSE new file mode 100644 index 0000000..393b7a3 --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright The Helm Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/OWNERS b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/OWNERS new file mode 100644 index 0000000..206b4fe --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/OWNERS @@ -0,0 +1,6 @@ +approvers: +- tariq1890 +- mrueg +reviewers: +- tariq1890 +- mrueg diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md index 4e1178e..e93a3d2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/README.md @@ -1,91 +1,66 @@ -# ⚠️ Repo Archive Notice - -As of Nov 13, 2020, charts in this repo will no longer be updated. -For more information, see the Helm Charts [Deprecation and Archive Notice](https://github.com/helm/charts#%EF%B8%8F-deprecation-and-archive-notice), and [Update](https://helm.sh/blog/charts-repo-deprecation/). - # kube-state-metrics Helm Chart -* Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics). +Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics). -## DEPRECATION NOTICE +## Get Repo Info -This chart is deprecated and no longer supported. - -## Installing the Chart - -To install the chart with the release name `my-release`: - -```bash -$ helm install stable/kube-state-metrics +```console +helm repo add kube-state-metrics https://kubernetes.github.io/kube-state-metrics +helm repo update ``` +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +# Helm 3 +$ helm install [RELEASE_NAME] kube-state-metrics/kube-state-metrics [flags] + +# Helm 2 +$ helm install --name [RELEASE_NAME] kube-state-metrics/kube-state-metrics [flags] +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +# Helm 3 +$ helm uninstall [RELEASE_NAME] + +# Helm 2 +# helm delete --purge [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +# Helm 3 or 2 +$ helm upgrade [RELEASE_NAME] kube-state-metrics/kube-state-metrics [flags] +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### From stable/kube-state-metrics + +You can upgrade in-place: + +1. [get repo info](#get-repo-info) +1. [upgrade](#upgrading-chart) your existing release name using the new chart repo + ## Configuration -| Parameter | Description | Default | -|:---------------------------------------------|:--------------------------------------------------------------------------------------|:-------------------------------------------| -| `image.repository` | The image repository to pull from | `quay.io/coreos/kube-state-metrics` | -| `image.tag` | The image tag to pull from | `v1.9.7` | -| `image.pullPolicy` | Image pull policy | `IfNotPresent` | -| `imagePullSecrets` | List of container registry secrets | `[]` | -| `replicas` | Number of replicas | `1` | -| `autosharding.enabled` | Set to `true` to automatically shard data across `replicas` pods. EXPERIMENTAL | `false` | -| `service.port` | The port of the container | `8080` | -| `service.annotations` | Annotations to be added to the service | `{}` | -| `customLabels` | Custom labels to apply to service, deployment and pods | `{}` | -| `hostNetwork` | Whether or not to use the host network | `false` | -| `prometheusScrape` | Whether or not enable prom scrape | `true` | -| `rbac.create` | If true, create & use RBAC resources | `true` | -| `serviceAccount.create` | If true, create & use serviceAccount | `true` | -| `serviceAccount.name` | If not set & create is true, use template fullname | | -| `serviceAccount.imagePullSecrets` | Specify image pull secrets field | `[]` | -| `serviceAccount.annotations` | Annotations to be added to the serviceAccount | `{}` | -| `podSecurityPolicy.enabled` | If true, create & use PodSecurityPolicy resources. Note that related RBACs are created only if `rbac.enabled` is `true`. | `false` | -| `podSecurityPolicy.annotations` | Specify pod annotations in the pod security policy | `{}` | -| `podSecurityPolicy.additionalVolumes` | Specify allowed volumes in the pod security policy (`secret` is always allowed) | `[]` | -| `securityContext.enabled` | Enable security context | `true` | -| `securityContext.fsGroup` | Group ID for the filesystem | `65534` | -| `securityContext.runAsGroup` | Group ID for the container | `65534` | -| `securityContext.runAsUser` | User ID for the container | `65534` | -| `priorityClassName` | Name of Priority Class to assign pods | `nil` | -| `nodeSelector` | Node labels for pod assignment | `{}` | -| `affinity` | Affinity settings for pod assignment | `{}` | -| `tolerations` | Tolerations for pod assignment | `[]` | -| `podAnnotations` | Annotations to be added to the pod | `{}` | -| `podDisruptionBudget` | Optional PodDisruptionBudget | `{}` | -| `resources` | kube-state-metrics resource requests and limits | `{}` | -| `collectors.certificatesigningrequests` | Enable the certificatesigningrequests collector. | `true` | -| `collectors.configmaps` | Enable the configmaps collector. | `true` | -| `collectors.cronjobs` | Enable the cronjobs collector. | `true` | -| `collectors.daemonsets` | Enable the daemonsets collector. | `true` | -| `collectors.deployments` | Enable the deployments collector. | `true` | -| `collectors.endpoints` | Enable the endpoints collector. | `true` | -| `collectors.horizontalpodautoscalers` | Enable the horizontalpodautoscalers collector. | `true` | -| `collectors.ingresses` | Enable the ingresses collector. | `true` | -| `collectors.jobs` | Enable the jobs collector. | `true` | -| `collectors.limitranges` | Enable the limitranges collector. | `true` | -| `collectors.mutatingwebhookconfigurations` | Enable the mutatingwebhookconfigurations collector. | `true` | -| `collectors.namespaces` | Enable the namespaces collector. | `true` | -| `collectors.networkpolicies` | Enable the networkpolicies collector. | `true` | -| `collectors.nodes` | Enable the nodes collector. | `true` | -| `collectors.persistentvolumeclaims` | Enable the persistentvolumeclaims collector. | `true` | -| `collectors.persistentvolumes` | Enable the persistentvolumes collector. | `true` | -| `collectors.poddisruptionbudgets` | Enable the poddisruptionbudgets collector. | `true` | -| `collectors.pods` | Enable the pods collector. | `true` | -| `collectors.replicasets` | Enable the replicasets collector. | `true` | -| `collectors.replicationcontrollers` | Enable the replicationcontrollers collector. | `true` | -| `collectors.resourcequotas` | Enable the resourcequotas collector. | `true` | -| `collectors.secrets` | Enable the secrets collector. | `true` | -| `collectors.services` | Enable the services collector. | `true` | -| `collectors.statefulsets` | Enable the statefulsets collector. | `true` | -| `collectors.storageclasses` | Enable the storageclasses collector. | `true` | -| `collectors.validatingwebhookconfigurations` | Enable the validatingwebhookconfigurations collector. | `true` | -| `collectors.verticalpodautoscalers` | Enable the verticalpodautoscalers collector. | `true` | -| `collectors.volumeattachments` | Enable the volumeattachments collector. | `true` | -| `prometheus.monitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator | `false` | -| `prometheus.monitor.additionalLabels` | Additional labels that can be used so ServiceMonitor will be discovered by Prometheus | `{}` | -| `prometheus.monitor.namespace` | Namespace where servicemonitor resource should be created | `the same namespace as kube-state-metrics` | -| `prometheus.monitor.honorLabels` | Honor metric labels | `false` | -| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | -| `kubeTargetVersionOverride` | Override the k8s version of the target cluster | `""` | -| `kubeconfig.enabled` | Adds --kubeconfig arg to container at startup | `""` | -| `kubeconfig.secret` | Base64 encoded kubeconfig file | `""` | +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values kube-state-metrics/kube-state-metrics +``` + +You may also `helm show values` on this chart's [dependencies](#dependencies) for additional options. diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrolebinding.yaml index 160db8b..af158c5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrolebinding.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.create -}} +{{- if and .Values.rbac.create .Values.rbac.useClusterRole -}} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -11,7 +11,11 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole +{{- if .Values.rbac.useExistingRole }} + name: {{ .Values.rbac.useExistingRole }} +{{- else }} name: {{ template "kube-state-metrics.fullname" . }} +{{- end }} subjects: - kind: ServiceAccount name: {{ template "kube-state-metrics.fullname" . }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml index 8f491ec..5f6b644 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/deployment.yaml @@ -12,6 +12,7 @@ metadata: helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" app.kubernetes.io/instance: "{{ .Release.Name }}" app.kubernetes.io/managed-by: "{{ .Release.Service }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" {{- if .Values.customLabels }} {{ toYaml .Values.customLabels | indent 4 }} {{- end }} @@ -62,6 +63,11 @@ spec: fieldPath: metadata.namespace {{- end }} args: +{{ if .Values.extraArgs }} + {{- range .Values.extraArgs }} + - {{ . }} + {{- end }} +{{ end }} {{ if .Values.collectors.certificatesigningrequests }} - --collectors=certificatesigningrequests {{ end }} @@ -147,7 +153,7 @@ spec: - --collectors=volumeattachments {{ end }} {{ if .Values.namespace }} - - --namespace={{ .Values.namespace }} + - --namespace={{ .Values.namespace | join "," }} {{ end }} {{ if .Values.autosharding.enabled }} - --pod=$(POD_NAME) @@ -156,6 +162,10 @@ spec: {{ if .Values.kubeconfig.enabled }} - --kubeconfig=/opt/k8s/.kube/config {{ end }} +{{ if .Values.selfMonitor.telemetryHost }} + - --telemetry-host={{ .Values.selfMonitor.telemetryHost }} +{{ end }} + - --telemetry-port=8081 {{- if .Values.kubeconfig.enabled }} volumeMounts: - name: kubeconfig diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/pdb.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/pdb.yaml index 6adb50d..d3ef810 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/pdb.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/pdb.yaml @@ -9,9 +9,12 @@ metadata: helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" app.kubernetes.io/instance: "{{ .Release.Name }}" app.kubernetes.io/managed-by: "{{ .Release.Service }}" +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} spec: selector: matchLabels: app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} {{ toYaml .Values.podDisruptionBudget | indent 2 }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrole.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml similarity index 61% rename from charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrole.yaml rename to charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml index a9198b8..6259d2f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/clusterrole.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/role.yaml @@ -1,177 +1,189 @@ -{{- if .Values.rbac.create -}} +{{- if and (eq $.Values.rbac.create true) (not .Values.rbac.useExistingRole) -}} +{{- if eq .Values.rbac.useClusterRole false }} +{{- range (split "," $.Values.namespace) }} +{{- end }} +{{- end -}} +--- apiVersion: rbac.authorization.k8s.io/v1 +{{- if eq .Values.rbac.useClusterRole false }} +kind: Role +{{- else }} kind: ClusterRole +{{- end }} metadata: labels: - app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/instance: {{ .Release.Name }} - name: {{ template "kube-state-metrics.fullname" . }} + app.kubernetes.io/name: {{ template "kube-state-metrics.name" $ }} + helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version }} + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/instance: {{ $.Release.Name }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- if eq .Values.rbac.useClusterRole false }} + namespace: {{ . }} +{{- end }} rules: -{{ if .Values.collectors.certificatesigningrequests }} +{{ if $.Values.collectors.certificatesigningrequests }} - apiGroups: ["certificates.k8s.io"] resources: - certificatesigningrequests verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.configmaps }} +{{ if $.Values.collectors.configmaps }} - apiGroups: [""] resources: - configmaps verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.cronjobs }} +{{ if $.Values.collectors.cronjobs }} - apiGroups: ["batch"] resources: - cronjobs verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.daemonsets }} +{{ if $.Values.collectors.daemonsets }} - apiGroups: ["extensions", "apps"] resources: - daemonsets verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.deployments }} +{{ if $.Values.collectors.deployments }} - apiGroups: ["extensions", "apps"] resources: - deployments verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.endpoints }} +{{ if $.Values.collectors.endpoints }} - apiGroups: [""] resources: - endpoints verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.horizontalpodautoscalers }} +{{ if $.Values.collectors.horizontalpodautoscalers }} - apiGroups: ["autoscaling"] resources: - horizontalpodautoscalers verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.ingresses }} +{{ if $.Values.collectors.ingresses }} - apiGroups: ["extensions", "networking.k8s.io"] resources: - ingresses verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.jobs }} +{{ if $.Values.collectors.jobs }} - apiGroups: ["batch"] resources: - jobs verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.limitranges }} +{{ if $.Values.collectors.limitranges }} - apiGroups: [""] resources: - limitranges verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.mutatingwebhookconfigurations }} +{{ if $.Values.collectors.mutatingwebhookconfigurations }} - apiGroups: ["admissionregistration.k8s.io"] resources: - mutatingwebhookconfigurations verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.namespaces }} +{{ if $.Values.collectors.namespaces }} - apiGroups: [""] resources: - namespaces verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.networkpolicies }} +{{ if $.Values.collectors.networkpolicies }} - apiGroups: ["networking.k8s.io"] resources: - networkpolicies verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.nodes }} +{{ if $.Values.collectors.nodes }} - apiGroups: [""] resources: - nodes verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.persistentvolumeclaims }} +{{ if $.Values.collectors.persistentvolumeclaims }} - apiGroups: [""] resources: - persistentvolumeclaims verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.persistentvolumes }} +{{ if $.Values.collectors.persistentvolumes }} - apiGroups: [""] resources: - persistentvolumes verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.poddisruptionbudgets }} +{{ if $.Values.collectors.poddisruptionbudgets }} - apiGroups: ["policy"] resources: - poddisruptionbudgets verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.pods }} +{{ if $.Values.collectors.pods }} - apiGroups: [""] resources: - pods verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.replicasets }} +{{ if $.Values.collectors.replicasets }} - apiGroups: ["extensions", "apps"] resources: - replicasets verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.replicationcontrollers }} +{{ if $.Values.collectors.replicationcontrollers }} - apiGroups: [""] resources: - replicationcontrollers verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.resourcequotas }} +{{ if $.Values.collectors.resourcequotas }} - apiGroups: [""] resources: - resourcequotas verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.secrets }} +{{ if $.Values.collectors.secrets }} - apiGroups: [""] resources: - secrets verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.services }} +{{ if $.Values.collectors.services }} - apiGroups: [""] resources: - services verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.statefulsets }} +{{ if $.Values.collectors.statefulsets }} - apiGroups: ["apps"] resources: - statefulsets verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.storageclasses }} +{{ if $.Values.collectors.storageclasses }} - apiGroups: ["storage.k8s.io"] resources: - storageclasses verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.validatingwebhookconfigurations }} +{{ if $.Values.collectors.validatingwebhookconfigurations }} - apiGroups: ["admissionregistration.k8s.io"] resources: - validatingwebhookconfigurations verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.volumeattachments }} +{{ if $.Values.collectors.volumeattachments }} - apiGroups: ["storage.k8s.io"] resources: - volumeattachments verbs: ["list", "watch"] {{ end -}} -{{ if .Values.collectors.verticalpodautoscalers }} +{{ if $.Values.collectors.verticalpodautoscalers }} - apiGroups: ["autoscaling.k8s.io"] resources: - verticalpodautoscalers diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml new file mode 100644 index 0000000..89bb41b --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/rolebinding.yaml @@ -0,0 +1,27 @@ +{{- if and (eq .Values.rbac.create true) (eq .Values.rbac.useClusterRole false) -}} +{{- range (split "," $.Values.namespace) }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" $ }} + helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version }} + app.kubernetes.io/managed-by: {{ $.Release.Service }} + app.kubernetes.io/instance: {{ $.Release.Name }} + name: {{ template "kube-state-metrics.fullname" $ }} + namespace: {{ . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role +{{- if (not $.Values.rbac.useExistingRole) }} + name: {{ template "kube-state-metrics.fullname" $ }} +{{- else }} + name: {{ $.Values.rbac.useExistingRole }} +{{- end }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.fullname" $ }} + namespace: {{ template "kube-state-metrics.namespace" $ }} +{{- end -}} +{{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/service.yaml index 5dacf52..4f8e4a4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/service.yaml @@ -28,6 +28,12 @@ spec: nodePort: {{ .Values.service.nodePort }} {{- end }} targetPort: 8080 + {{ if .Values.selfMonitor.enabled }} + - name: "metrics" + protocol: TCP + port: {{ .Values.selfMonitor.telemetryPort | default 8081 }} + targetPort: 8081 + {{ end }} {{- if .Values.service.loadBalancerIP }} loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml index 54cde36..7d1cd7a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/templates/servicemonitor.yaml @@ -12,6 +12,9 @@ metadata: {{- if .Values.prometheus.monitor.additionalLabels }} {{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }} {{- end }} +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} spec: selector: matchLabels: @@ -22,4 +25,10 @@ spec: {{- if .Values.prometheus.monitor.honorLabels }} honorLabels: true {{- end }} + {{ if .Values.selfMonitor.enabled }} + - port: metrics + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} + {{ end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml index 3edd4c8..9522cfe 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/kube-state-metrics/values.yaml @@ -1,8 +1,8 @@ # Default values for kube-state-metrics. prometheusScrape: true image: - repository: quay.io/coreos/kube-state-metrics - tag: v1.9.7 + repository: k8s.gcr.io/kube-state-metrics/kube-state-metrics + tag: v1.9.8 pullPolicy: IfNotPresent imagePullSecrets: [] @@ -17,6 +17,11 @@ autosharding: replicas: 1 +# List of additional cli arguments to configure kube-state-metrics +# for example: --enable-gzip-encoding, --log-file, etc. +# all the possible args can be found here: https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md +extraArgs: [] + service: port: 8080 # Default to clusterIP for backward compatibility @@ -33,6 +38,12 @@ rbac: # If true, create & use RBAC resources create: true + # Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to it, rolename set here. + # useExistingRole: your-existing-role + + # If set to false - Run without Cluteradmin privs needed - ONLY works if namespace is also set (if useExistingRole is set this name is used as ClusterRole or Role to bind to) + useClusterRole: true + serviceAccount: # Specifies whether a ServiceAccount should be created, require rbac true create: true @@ -159,3 +170,10 @@ resources: {} ## For example: kubeTargetVersionOverride: 1.14.9 ## kubeTargetVersionOverride: "" + +# Enable self metrics configuration for service and Service Monitor +# Default values for telemetry configuration can be overriden +selfMonitor: + enabled: false + # telemetryHost: 0.0.0.0 + # telemetryPort: 8081 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml index 6594547..f35460b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/Chart.yaml @@ -10,7 +10,8 @@ maintainers: - email: gianrubio@gmail.com name: gianrubio - name: vsliouniaev +- name: bismarck name: prometheus-node-exporter sources: - https://github.com/prometheus/node_exporter/ -version: 1.12.0 +version: 1.14.2 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml index 2787dae..cd6f65f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -43,6 +43,9 @@ spec: {{- if .Values.extraArgs }} {{ toYaml .Values.extraArgs | indent 12 }} {{- end }} + {{- with .Values.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} env: - name: HOST_IP {{- if .Values.service.listenOnAllInterfaces }} @@ -55,7 +58,7 @@ spec: {{- end }} ports: - name: metrics - containerPort: {{ .Values.service.targetPort }} + containerPort: {{ .Values.service.port }} protocol: TCP livenessProbe: httpGet: @@ -100,6 +103,12 @@ spec: - name: {{ $mount.name }} mountPath: {{ $mount.mountPath }} {{- end }} + {{- if .Values.secrets }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + {{- end }} {{- end }} {{- if .Values.sidecars }} {{ toYaml .Values.sidecars | indent 8 }} @@ -157,3 +166,10 @@ spec: name: {{ $mount.name }} {{- end }} {{- end }} + {{- if .Values.secrets }} + {{- range $_, $mount := .Values.secrets }} + - name: {{ $mount.name }} + secret: + secretName: {{ $mount.name }} + {{- end }} + {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml index 4e31ba3..2f7b6ae 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/monitor.yaml @@ -15,6 +15,13 @@ spec: release: {{ .Release.Name }} endpoints: - port: metrics + scheme: {{ $.Values.prometheus.monitor.scheme }} + {{- if $.Values.prometheus.monitor.bearerTokenFile }} + bearerTokenFile: {{ $.Values.prometheus.monitor.bearerTokenFile }} + {{- end }} + {{- if $.Values.prometheus.monitor.tlsConfig }} + tlsConfig: {{ toYaml $.Values.prometheus.monitor.tlsConfig | nindent 8 }} + {{- end }} {{- if .Values.prometheus.monitor.scrapeTimeout }} scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/serviceaccount.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/serviceaccount.yaml index bd1c223..07e9f0d 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/serviceaccount.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/templates/serviceaccount.yaml @@ -10,6 +10,8 @@ metadata: chart: {{ template "prometheus-node-exporter.chart" . }} release: "{{ .Release.Name }}" heritage: "{{ .Release.Service }}" + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} imagePullSecrets: {{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} {{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml index 7edd893..4be3f9c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/charts/prometheus-node-exporter/values.yaml @@ -20,6 +20,9 @@ prometheus: enabled: false additionalLabels: {} namespace: "" + scheme: http + bearerTokenFile: + tlsConfig: {} relabelings: [] scrapeTimeout: 10s @@ -48,6 +51,7 @@ serviceAccount: # The name of the ServiceAccount to use. # If not set and create is true, a name is generated using the fullname template name: + annotations: {} imagePullSecrets: [] securityContext: @@ -56,6 +60,11 @@ securityContext: runAsNonRoot: true runAsUser: 65534 +containerSecurityContext: {} + # capabilities: + # add: + # - SYS_TIME + rbac: ## If true, create & use RBAC resources ## @@ -84,7 +93,9 @@ affinity: {} # - target-host-name # Annotations to be added to node exporter pods -podAnnotations: {} +podAnnotations: + # Fix for very slow GKE cluster upgrades + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" # Extra labels to be added to node exporter pods podLabels: {} @@ -122,7 +133,9 @@ extraHostVolumeMounts: [] configmaps: [] # - name: # mountPath: - +secrets: [] +# - name: +# mountPath: ## Override the deployment namespace ## namespaceOverride: "" diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml index fb1ad5f..a279253 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagerconfigs.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.44.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -50,6 +50,7 @@ spec: properties: name: description: Label to match. + minLength: 1 type: string regex: description: Whether to match on equality (false) or regular-expression (true). @@ -59,7 +60,6 @@ spec: type: string required: - name - - value type: object type: array targetMatch: @@ -69,6 +69,7 @@ spec: properties: name: description: Label to match. + minLength: 1 type: string regex: description: Whether to match on equality (false) or regular-expression (true). @@ -78,7 +79,6 @@ spec: type: string required: - name - - value type: object type: array type: object @@ -94,9 +94,10 @@ spec: description: EmailConfig configures notifications via Email. properties: authIdentity: + description: The identity to use for authentication. type: string authPassword: - description: SecretKeySelector selects a key of a Secret. + description: The secret's key that contains the password to use for authentication. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. @@ -111,7 +112,7 @@ spec: - key type: object authSecret: - description: SecretKeySelector selects a key of a Secret. + description: The secret's key that contains the CRAM-MD5 secret. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. @@ -126,7 +127,7 @@ spec: - key type: object authUsername: - description: SMTP authentication information. + description: The username to use for authentication. type: string from: description: The sender address. @@ -138,6 +139,7 @@ spec: properties: key: description: Key of the tuple. + minLength: 1 type: string value: description: Value of the tuple. @@ -265,6 +267,7 @@ spec: type: array name: description: Name of the receiver. Must be unique across all items from the list. + minLength: 1 type: string opsgenieConfigs: description: List of OpsGenie configurations. @@ -299,6 +302,7 @@ spec: properties: key: description: Key of the tuple. + minLength: 1 type: string value: description: Value of the tuple. @@ -469,7 +473,7 @@ spec: responders: description: List of responders responsible for notifications. items: - description: OpsGenieConfigResponder defines a responder to an incident. One of id, name or username has to be defined. + description: OpsGenieConfigResponder defines a responder to an incident. One of `id`, `name` or `username` has to be defined. properties: id: description: ID of the responder. @@ -479,10 +483,13 @@ spec: type: string type: description: Type of responder. + minLength: 1 type: string username: description: Username of the responder. type: string + required: + - type type: object type: array sendResolved: @@ -523,6 +530,7 @@ spec: properties: key: description: Key of the tuple. + minLength: 1 type: string value: description: Value of the tuple. @@ -904,7 +912,7 @@ spec: description: Notification title. type: string token: - description: Your registered application’s API token, see https://pushover.net/apps + description: The secret's key that contains the registered application’s API token, see https://pushover.net/apps. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. @@ -925,7 +933,7 @@ spec: description: A title for supplementary URL, otherwise just the URL is shown type: string userKey: - description: The recipient user’s user key. + description: The secret's key that contains the recipient user’s user key. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. @@ -959,6 +967,7 @@ spec: okText: type: string text: + minLength: 1 type: string title: type: string @@ -970,8 +979,10 @@ spec: style: type: string text: + minLength: 1 type: string type: + minLength: 1 type: string url: type: string @@ -1014,8 +1025,10 @@ spec: short: type: boolean title: + minLength: 1 type: string value: + minLength: 1 type: string required: - title @@ -1210,7 +1223,7 @@ spec: description: VictorOpsConfig configures notifications via VictorOps. See https://prometheus.io/docs/alerting/latest/configuration/#victorops_config properties: apiKey: - description: The API key to use when talking to the VictorOps API. + description: The secret's key that contains the API key to use when talking to the VictorOps API. The secret needs to be in the same namespace as the AlertmanagerConfig object and accessible by the Prometheus Operator. properties: key: description: The key of the secret to select from. Must be a valid secret key. @@ -1234,6 +1247,7 @@ spec: properties: key: description: Key of the tuple. + minLength: 1 type: string value: description: Value of the tuple. @@ -1410,8 +1424,6 @@ spec: stateMessage: description: Contains long explanation of the alerted problem. type: string - required: - - routingKey type: object type: array webhookConfigs: @@ -1569,8 +1581,9 @@ spec: type: object type: object maxAlerts: - description: Maximum number of alerts to be sent per webhook message. + description: Maximum number of alerts to be sent per webhook message. When 0, all alerts are included. format: int32 + minimum: 0 type: integer sendResolved: description: Whether or not to notify about resolved alerts. @@ -1793,7 +1806,7 @@ spec: type: object type: array route: - description: The Alertmanager route definition for alerts matching the resource’s namespace. It will be added to the generated Alertmanager configuration as a first-level route. + description: The Alertmanager route definition for alerts matching the resource’s namespace. If present, it will be added to the generated Alertmanager configuration as a first-level route. properties: continue: description: Boolean indicating whether an alert should continue matching subsequent sibling nodes. It will always be overridden to true for the first-level route by the Prometheus operator. @@ -1816,6 +1829,7 @@ spec: properties: name: description: Label to match. + minLength: 1 type: string regex: description: Whether to match on equality (false) or regular-expression (true). @@ -1825,11 +1839,10 @@ spec: type: string required: - name - - value type: object type: array receiver: - description: Name of the receiver for this route. If present, it should be listed in the `receivers` field. The field can be omitted only for nested routes otherwise it is mandatory. + description: Name of the receiver for this route. If not empty, it should be listed in the `receivers` field. type: string repeatInterval: description: How long to wait before repeating the last notification. Must match the regular expression `[0-9]+(ms|s|m|h)` (milliseconds seconds minutes hours). diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml index 86a6b98..7a4ec17 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-alertmanagers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.44.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml index 630465b..95fbafb 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-podmonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.44.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml index 41a1b6f..5ef8405 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-probes.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.44.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml index fd43ebc..6a82bc5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheuses.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.44.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml --- apiVersion: apiextensions.k8s.io/v1 @@ -2258,7 +2258,7 @@ spec: type: string type: object podMonitorNamespaceSelector: - description: Namespaces to be selected for PodMonitor discovery. If nil, only check own namespace. + description: Namespace's labels to match for PodMonitor discovery. If nil, only check own namespace. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -2849,7 +2849,7 @@ spec: description: Time duration Prometheus shall retain data for. Default is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` (milliseconds seconds minutes hours days weeks years). type: string retentionSize: - description: Maximum amount of disk space used by blocks. + description: 'Maximum amount of disk space used by blocks. Supported units: B, KB, MB, GB, TB, PB, EB. Ex: `512MB`.' type: string routePrefix: description: The route prefix Prometheus registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, but the server serves requests under a different route prefix. For example for use with `kubectl proxy`. @@ -3019,7 +3019,7 @@ spec: description: ServiceAccountName is the name of the ServiceAccount to use to run the Prometheus Pods. type: string serviceMonitorNamespaceSelector: - description: Namespaces to be selected for ServiceMonitor discovery. If nil, only check own namespace. + description: Namespace's labels to match for ServiceMonitor discovery. If nil, only check own namespace. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. The requirements are ANDed. @@ -3456,6 +3456,9 @@ spec: required: - key type: object + tracingConfigFile: + description: TracingConfig specifies the path of the tracing configuration file. When used alongside with TracingConfig, TracingConfigFile takes precedence. + type: string version: description: Version describes the version of Thanos to use. type: string diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml index 02759cd..8c0776c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-prometheusrules.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.44.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml index f5d989d..a65be71 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-servicemonitors.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.44.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml index f647e72..8fe6e81 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/crds/crd-thanosrulers.yaml @@ -1,4 +1,4 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.44.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.45.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml --- apiVersion: apiextensions.k8s.io/v1 diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml index 78d0f7c..bbdbc56 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/alertmanager.yaml @@ -19,7 +19,7 @@ spec: listenLocal: {{ .Values.alertmanager.alertmanagerSpec.listenLocal }} serviceAccountName: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} {{- if .Values.alertmanager.alertmanagerSpec.externalUrl }} - externalUrl: "{{ .Values.alertmanager.alertmanagerSpec.externalUrl }}" + externalUrl: "{{ tpl .Values.alertmanager.alertmanagerSpec.externalUrl . }}" {{- else if and .Values.alertmanager.ingress.enabled .Values.alertmanager.ingress.hosts }} externalUrl: "http://{{ tpl (index .Values.alertmanager.ingress.hosts 0) . }}{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" {{- else }} @@ -47,10 +47,14 @@ spec: {{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector }} alertmanagerConfigSelector: {{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector | indent 4}} +{{ else }} + alertmanagerConfigSelector: {} {{- end }} {{- if .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector }} alertmanagerConfigNamespaceSelector: {{ toYaml .Values.alertmanager.alertmanagerSpec.alertmanagerConfigNamespaceSelector | indent 4}} +{{ else }} + alertmanagerConfigNamespaceSelector: {} {{- end }} {{- if .Values.alertmanager.alertmanagerSpec.resources }} resources: @@ -81,9 +85,9 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} labelSelector: - matchLabels: - app: alertmanager - alertmanager: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + matchExpressions: + - {key: app, operator: In, values: [alertmanager]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-alertmanager]} {{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -91,15 +95,19 @@ spec: podAffinityTerm: topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} labelSelector: - matchLabels: - app: alertmanager - alertmanager: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + matchExpressions: + - {key: app, operator: In, values: [alertmanager]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-alertmanager]} {{- end }} {{- end }} {{- if .Values.alertmanager.alertmanagerSpec.tolerations }} tolerations: {{ toYaml .Values.alertmanager.alertmanagerSpec.tolerations | indent 4 }} {{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.alertmanager.alertmanagerSpec.topologySpreadConstraints | indent 4 }} +{{- end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | indent 4 }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml index a87ce89..50fab14 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingress.yaml @@ -1,4 +1,5 @@ {{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled }} +{{- $pathType := .Values.alertmanager.ingress.pathType | default "" }} {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} {{- $servicePort := .Values.alertmanager.service.port -}} {{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} @@ -23,7 +24,7 @@ metadata: {{- end }} {{ include "kube-prometheus-stack.labels" . | indent 4 }} spec: - {{- if or (.Capabilities.APIVersions.Has "networking.k8s.io/v1/IngressClass") (.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1/IngressClass") }} + {{- if or (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1") }} {{- if .Values.alertmanager.ingress.ingressClassName }} ingressClassName: {{ .Values.alertmanager.ingress.ingressClassName }} {{- end }} @@ -36,6 +37,9 @@ spec: paths: {{- range $p := $paths }} - path: {{ tpl $p $ }} + {{- if $pathType }} + pathType: {{ $pathType }} + {{- end }} backend: serviceName: {{ $serviceName }} servicePort: {{ $servicePort }} @@ -46,6 +50,9 @@ spec: paths: {{- range $p := $paths }} - path: {{ tpl $p $ }} + {{- if $pathType }} + pathType: {{ $pathType }} + {{- end }} backend: serviceName: {{ $serviceName }} servicePort: {{ $servicePort }} @@ -53,6 +60,6 @@ spec: {{- end -}} {{- if .Values.alertmanager.ingress.tls }} tls: -{{ toYaml .Values.alertmanager.ingress.tls | indent 4 }} +{{ tpl (toYaml .Values.alertmanager.ingress.tls | indent 4) . }} {{- end -}} {{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml index 6aef97d..3d673b2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/alertmanager/ingressperreplica.yaml @@ -1,4 +1,5 @@ {{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled .Values.alertmanager.ingressPerReplica.enabled }} +{{- $pathType := .Values.alertmanager.ingressPerReplica.pathType | default "" }} {{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} {{- $servicePort := .Values.alertmanager.service.port -}} {{- $ingressValues := .Values.alertmanager.ingressPerReplica -}} @@ -29,7 +30,7 @@ items: {{ toYaml $ingressValues.annotations | indent 8 }} {{- end }} spec: - {{- if or ($.Capabilities.APIVersions.Has "networking.k8s.io/v1/IngressClass") ($.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1/IngressClass") }} + {{- if or ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") ($.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1") }} {{- if $ingressValues.ingressClassName }} ingressClassName: {{ $ingressValues.ingressClassName }} {{- end }} @@ -40,6 +41,9 @@ items: paths: {{- range $p := $ingressValues.paths }} - path: {{ tpl $p $ }} + {{- if $pathType }} + pathType: {{ $pathType }} + {{- end }} backend: serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} servicePort: {{ $servicePort }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml index f09de5d..5b723b2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/exporters/kube-state-metrics/serviceMonitor.yaml @@ -25,6 +25,10 @@ spec: {{- end }} selector: matchLabels: +{{- if .Values.kubeStateMetrics.serviceMonitor.selectorOverride }} +{{ toYaml .Values.kubeStateMetrics.serviceMonitor.selectorOverride | indent 6 }} +{{ else }} app.kubernetes.io/name: kube-state-metrics app.kubernetes.io/instance: "{{ $.Release.Name }}" {{- end }} +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml index de904dd..db62d53 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/configmaps-datasources.yaml @@ -16,6 +16,7 @@ data: datasource.yaml: |- apiVersion: 1 datasources: +{{- $scrapeInterval := .Values.grafana.sidecar.datasources.defaultDatasourceScrapeInterval | default .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} {{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} - name: Prometheus type: prometheus @@ -23,7 +24,7 @@ data: access: proxy isDefault: true jsonData: - timeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} + timeInterval: {{ $scrapeInterval }} {{- if .Values.grafana.sidecar.datasources.createPrometheusReplicasDatasources }} {{- range until (int .Values.prometheus.prometheusSpec.replicas) }} - name: Prometheus-{{ . }} @@ -32,7 +33,7 @@ data: access: proxy isDefault: false jsonData: - timeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval | default "30s" }} + timeInterval: {{ $scrapeInterval }} {{- end }} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml index 93bf909..1d1c3e9 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -1823,7 +1823,7 @@ data: }, "datasource": "$datasource", - "hide": 2, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml index 0595cb4..66768d9 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/etcd.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'etcd' from https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/grafana.json +Generated from 'etcd' from https://raw.githubusercontent.com/etcd-io/website/master/content/docs/current/op-guide/grafana.json Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/kubelet.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/kubelet.yaml index 5f1b2a2..7238299 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/kubelet.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/kubelet.yaml @@ -191,7 +191,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "expr": "sum(kubelet_running_pods{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", @@ -275,7 +275,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "expr": "sum(kubelet_running_containers{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}instance{{`}}`}}", diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml index e814ba7..c131e68 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -1293,7 +1293,7 @@ data: }, "datasource": "$datasource", - "hide": 2, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml index a526290..097d7f5 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -1533,7 +1533,7 @@ data: }, "datasource": "$datasource", - "hide": 2, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml index 82b821a..62ab619 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -92,7 +92,7 @@ data: "timeShift": null, "title": "CPU Utilisation", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -178,7 +178,7 @@ data: "timeShift": null, "title": "CPU Saturation (load1 per CPU)", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -276,7 +276,7 @@ data: "timeShift": null, "title": "Memory Utilisation", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -362,7 +362,7 @@ data: "timeShift": null, "title": "Memory Saturation (Major Page Faults)", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -476,7 +476,7 @@ data: "timeShift": null, "title": "Net Utilisation (Bytes Receive/Transmit)", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -578,7 +578,7 @@ data: "timeShift": null, "title": "Net Saturation (Drops Receive/Transmit)", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -676,7 +676,7 @@ data: "timeShift": null, "title": "Disk IO Utilisation", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -762,7 +762,7 @@ data: "timeShift": null, "title": "Disk IO Saturation", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -860,7 +860,7 @@ data: "timeShift": null, "title": "Disk Space Utilisation", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -956,9 +956,9 @@ data: "30d" ] }, - "timezone": "UTC", + "timezone": "utc", "title": "USE Method / Cluster", - "uid": "3e97d1d02672cdd0861f4c97c64f89b2", + "uid": "", "version": 0 } {{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml index 9cdfa64..cd21961 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -92,7 +92,7 @@ data: "timeShift": null, "title": "CPU Utilisation", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -178,7 +178,7 @@ data: "timeShift": null, "title": "CPU Saturation (Load1 per CPU)", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -276,7 +276,7 @@ data: "timeShift": null, "title": "Memory Utilisation", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -362,7 +362,7 @@ data: "timeShift": null, "title": "Memory Saturation (Major Page Faults)", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -476,7 +476,7 @@ data: "timeShift": null, "title": "Net Utilisation (Bytes Receive/Transmit)", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -578,7 +578,7 @@ data: "timeShift": null, "title": "Net Saturation (Drops Receive/Transmit)", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -676,7 +676,7 @@ data: "timeShift": null, "title": "Disk IO Utilisation", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -762,7 +762,7 @@ data: "timeShift": null, "title": "Disk IO Saturation", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -860,7 +860,7 @@ data: "timeShift": null, "title": "Disk Space Utilisation", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -983,9 +983,9 @@ data: "30d" ] }, - "timezone": "UTC", + "timezone": "utc", "title": "USE Method / Node", - "uid": "fac67cfbe174d3ef53eb473d73d9212f", + "uid": "", "version": 0 } {{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml index b2935ae..2a29fc0 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/nodes.yaml @@ -107,7 +107,7 @@ data: "timeShift": null, "title": "CPU Usage", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -221,7 +221,7 @@ data: "timeShift": null, "title": "Load Average", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -348,7 +348,7 @@ data: "timeShift": null, "title": "Memory Usage", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -451,9 +451,6 @@ data: ], "thresholds": "80, 90", "title": "Memory Usage", - "tooltip": { - "shared": false - }, "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ @@ -562,7 +559,7 @@ data: "timeShift": null, "title": "Disk I/O", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -669,7 +666,7 @@ data: "timeShift": null, "title": "Disk Space Usage", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -776,7 +773,7 @@ data: "timeShift": null, "title": "Network Received", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -870,7 +867,7 @@ data: "timeShift": null, "title": "Network Transmitted", "tooltip": { - "shared": false, + "shared": true, "sort": 0, "value_type": "individual" }, @@ -993,9 +990,8 @@ data: "30d" ] }, - "timezone": "UTC", + "timezone": "browser", "title": "Nodes", - "uid": "fa49a4706d07a042595b664c87fb33ea", "version": 0 } {{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml index 4ac20ce..06fb315 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -207,7 +207,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "(\n kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "expr": "max without(instance,node) (\n(\n kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", @@ -404,7 +404,7 @@ data: "tableColumn": "", "targets": [ { - "expr": "kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "expr": "max without(instance,node) (\nkubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "", diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml index 76c2c6e..95abda4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/pod-total.yaml @@ -1025,7 +1025,7 @@ data: }, "datasource": "$datasource", - "hide": 2, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml index 950443a..b33b738 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -40,7 +40,7 @@ data: "links": [ ], - "refresh": "", + "refresh": "60s", "rows": [ { "collapse": false, @@ -92,7 +92,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}\n)\n", + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) (prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} != 0)\n)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", @@ -185,7 +185,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n)\n", + "expr": "clamp_min(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n, 0)\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", @@ -291,7 +291,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n", + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) (rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n- \n (rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", @@ -876,7 +876,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"}", + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"} or prometheus_remote_storage_samples_pending{cluster=~\"$cluster\", instance=~\"$instance\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", @@ -1181,7 +1181,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_dropped_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", @@ -1274,7 +1274,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", @@ -1367,7 +1367,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) or rate(prometheus_remote_storage_samples_retried_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", @@ -1520,7 +1520,7 @@ data: "schemaVersion": 14, "style": "dark", "tags": [ - + "prometheus-mixin" ], "templating": { "list": [ @@ -1664,7 +1664,7 @@ data: ] }, "timezone": "browser", - "title": "Prometheus Remote Write", + "title": "Prometheus / Remote Write", "version": 0 } {{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus.yaml index dfd3f3d..7095fb7 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/prometheus.yaml @@ -33,7 +33,7 @@ data: "links": [ ], - "refresh": "10s", + "refresh": "60s", "rows": [ { "collapse": false, @@ -1112,7 +1112,7 @@ data: "schemaVersion": 14, "style": "dark", "tags": [ - + "prometheus-mixin" ], "templating": { "list": [ @@ -1220,7 +1220,7 @@ data: ] }, "timezone": "utc", - "title": "Prometheus Overview", + "title": "Prometheus / Overview", "uid": "", "version": 0 } diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml index 08d8cfd..07f5353 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14/workload-total.yaml @@ -1203,7 +1203,7 @@ data: }, "datasource": "$datasource", - "hide": 2, + "hide": {{ if .Values.grafana.sidecar.dashboards.multicluster }}0{{ else }}2{{ end }}, "includeAll": false, "label": null, "multi": false, diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/etcd.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/etcd.yaml index 835a690..157843a 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/etcd.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/grafana/dashboards/etcd.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'etcd' from https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/grafana.json +Generated from 'etcd' from https://raw.githubusercontent.com/etcd-io/website/master/content/docs/current/op-guide/grafana.json Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml index 7b31ab3..249af77 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create }} +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml index 4f1f616..31fd2de 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create }} +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml index 37a19a5..f8afcb8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled }} +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: batch/v1 kind: Job metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml index b74c618..b2d8912 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled }} +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: batch/v1 kind: Job metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml index 98e002a..5834c48 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml index 3609fe9..d229f76 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create }} +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml index dcb0fbc..f4b1fbf 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create }} +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml index 5296494..2048f04 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create }} +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml index b2dacb0..b67df54 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml @@ -3,6 +3,11 @@ apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} labels: app: {{ template "kube-prometheus-stack.name" $ }}-admission {{- include "kube-prometheus-stack.labels" $ | indent 4 }} @@ -28,6 +33,9 @@ webhooks: namespace: {{ template "kube-prometheus-stack.namespace" . }} name: {{ template "kube-prometheus-stack.operator.fullname" $ }} path: /admission-prometheusrules/mutate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} admissionReviewVersions: ["v1", "v1beta1"] sideEffects: None {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml index 3d48cd8..249488e 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml @@ -3,6 +3,11 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }} + annotations: + certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-root-cert" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} + cert-manager.io/inject-ca-from: {{ printf "%s/%s-root-cert" .Release.Namespace (include "kube-prometheus-stack.fullname" .) | quote }} +{{- end }} labels: app: {{ template "kube-prometheus-stack.name" $ }}-admission {{- include "kube-prometheus-stack.labels" $ | indent 4 }} @@ -28,6 +33,9 @@ webhooks: namespace: {{ template "kube-prometheus-stack.namespace" . }} name: {{ template "kube-prometheus-stack.operator.fullname" $ }} path: /admission-prometheusrules/validate + {{- if and .Values.prometheusOperator.admissionWebhooks.caBundle (not .Values.prometheusOperator.admissionWebhooks.patch.enabled) (not .Values.prometheusOperator.admissionWebhooks.certManager.enabled) }} + caBundle: {{ .Values.prometheusOperator.admissionWebhooks.caBundle }} + {{- end }} admissionReviewVersions: ["v1", "v1beta1"] sideEffects: None {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/certmanager.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/certmanager.yaml new file mode 100644 index 0000000..090e6a5 --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/certmanager.yaml @@ -0,0 +1,57 @@ +{{- if .Values.prometheusOperator.admissionWebhooks.certManager.enabled -}} +{{- if not .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef -}} +# Create a selfsigned Issuer, in order to create a root CA certificate for +# signing webhook serving certificates +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + selfSigned: {} +--- +# Generate a CA Certificate used to sign certificates for the webhook +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert + duration: 43800h # 5y + issuerRef: + name: {{ template "kube-prometheus-stack.fullname" . }}-self-signed-issuer + commonName: "ca.webhook.kube-prometheus-stack" + isCA: true +--- +# Create an Issuer that uses the above generated CA certificate to issue certs +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + ca: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-root-cert +{{- end }} +--- +# generate a serving certificate for the apiservices to use +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +spec: + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission + duration: 8760h # 1y + issuerRef: + {{- if .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef }} + {{- toYaml .Values.prometheusOperator.admissionWebhooks.certManager.issuerRef | nindent 4 }} + {{- else }} + name: {{ template "kube-prometheus-stack.fullname" . }}-root-issuer + {{- end }} + dnsNames: + - {{ template "kube-prometheus-stack.operator.fullname" . }} + - {{ template "kube-prometheus-stack.operator.fullname" . }}.{{ template "kube-prometheus-stack.namespace" . }} + - {{ template "kube-prometheus-stack.operator.fullname" . }}.{{ template "kube-prometheus-stack.namespace" . }}.svc +{{- end -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml index c991789..15b3684 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/deployment.yaml @@ -58,24 +58,18 @@ spec: {{- end }} - --namespaces={{ $ns | join "," }} {{- end }} - {{- if (semverCompare "< v0.44.0" .Values.prometheusOperator.image.tag) -}} - - --logtostderr=true - {{- end }} - --localhost=127.0.0.1 + {{- if .Values.prometheusOperator.prometheusDefaultBaseImage }} + - --prometheus-default-base-image={{ .Values.prometheusOperator.prometheusDefaultBaseImage }} + {{- end }} + {{- if .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + - --alertmanager-default-base-image={{ .Values.prometheusOperator.alertmanagerDefaultBaseImage }} + {{- end }} {{- if .Values.prometheusOperator.prometheusConfigReloaderImage.sha }} - --prometheus-config-reloader={{ .Values.prometheusOperator.prometheusConfigReloaderImage.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloaderImage.tag }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloaderImage.sha }} {{- else }} - --prometheus-config-reloader={{ .Values.prometheusOperator.prometheusConfigReloaderImage.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloaderImage.tag }} {{- end }} - # Empty if statement to catch non-semver master tags that do not need the --config-reloader-image flag - {{- if regexMatch "master.*" .Values.prometheusOperator.image.tag -}} - {{- else if (semverCompare "< v0.43.0" .Values.prometheusOperator.image.tag) -}} - {{- if .Values.prometheusOperator.configmapReloadImage.sha }} - - --config-reloader-image={{ .Values.prometheusOperator.configmapReloadImage.repository }}:{{ .Values.prometheusOperator.configmapReloadImage.tag }}@sha256:{{ .Values.prometheusOperator.configmapReloadImage.sha }} - {{- else }} - - --config-reloader-image={{ .Values.prometheusOperator.configmapReloadImage.repository }}:{{ .Values.prometheusOperator.configmapReloadImage.tag }} - {{- end }} - {{- end }} - --config-reloader-cpu={{ .Values.prometheusOperator.configReloaderCpu }} - --config-reloader-memory={{ .Values.prometheusOperator.configReloaderMemory }} {{- if .Values.prometheusOperator.alertmanagerInstanceNamespaces }} @@ -90,14 +84,17 @@ spec: {{- if .Values.prometheusOperator.secretFieldSelector }} - --secret-field-selector={{ .Values.prometheusOperator.secretFieldSelector }} {{- end }} + {{- if .Values.prometheusOperator.clusterDomain }} + - --cluster-domain={{ .Values.prometheusOperator.clusterDomain }} + {{- end }} {{- if .Values.prometheusOperator.tls.enabled }} - --web.enable-tls=true - - --web.cert-file=cert/cert - - --web.key-file=cert/key - - --web.listen-address=:8443 + - --web.cert-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.crt{{ else }}cert{{ end }} + - --web.key-file=/cert/{{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}tls.key{{ else }}key{{ end }} + - --web.listen-address=:{{ .Values.prometheusOperator.tls.internalPort }} - --web.tls-min-version={{ .Values.prometheusOperator.tls.tlsMinVersion }} ports: - - containerPort: 8443 + - containerPort: {{ .Values.prometheusOperator.tls.internalPort }} name: https {{- else }} ports: @@ -109,19 +106,21 @@ spec: securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - {{- if .Values.prometheusOperator.tls.enabled }} +{{- if .Values.prometheusOperator.tls.enabled }} volumeMounts: - name: tls-secret mountPath: /cert readOnly: true - {{- end }} -{{- if .Values.prometheusOperator.tls.enabled }} volumes: - name: tls-secret secret: defaultMode: 420 secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission {{- end }} + {{- with .Values.prometheusOperator.dnsConfig }} + dnsConfig: +{{ toYaml . | indent 8 }} + {{- end }} {{- if .Values.prometheusOperator.securityContext }} securityContext: {{ toYaml .Values.prometheusOperator.securityContext | indent 8 }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/servicemonitor.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/servicemonitor.yaml index 7524ddf..b7bd952 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/servicemonitor.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus-operator/servicemonitor.yaml @@ -17,7 +17,7 @@ spec: ca: secret: name: {{ template "kube-prometheus-stack.fullname" . }}-admission - key: ca + key: {{ if .Values.prometheusOperator.admissionWebhooks.certManager.enabled }}ca.crt{{ else }}ca{{ end }} optional: false {{- else }} - port: http diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml index 59bd4b6..4d45873 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingress.yaml @@ -1,4 +1,5 @@ {{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled }} +{{- $pathType := .Values.prometheus.ingress.pathType | default "" }} {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} {{- $servicePort := .Values.prometheus.service.port -}} {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} @@ -23,7 +24,7 @@ metadata: {{ toYaml .Values.prometheus.ingress.labels | indent 4 }} {{- end }} spec: - {{- if or (.Capabilities.APIVersions.Has "networking.k8s.io/v1/IngressClass") (.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1/IngressClass") }} + {{- if or (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1") }} {{- if .Values.prometheus.ingress.ingressClassName }} ingressClassName: {{ .Values.prometheus.ingress.ingressClassName }} {{- end }} @@ -36,6 +37,9 @@ spec: paths: {{- range $p := $paths }} - path: {{ tpl $p $ }} + {{- if $pathType }} + pathType: {{ $pathType }} + {{- end }} backend: serviceName: {{ $serviceName }} servicePort: {{ $servicePort }} @@ -46,6 +50,9 @@ spec: paths: {{- range $p := $paths }} - path: {{ tpl $p $ }} + {{- if $pathType }} + pathType: {{ $pathType }} + {{- end }} backend: serviceName: {{ $serviceName }} servicePort: {{ $servicePort }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml index 45f9cb2..69de0f6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressThanosSidecar.yaml @@ -1,9 +1,14 @@ {{- if and .Values.prometheus.enabled .Values.prometheus.thanosIngress.enabled }} +{{- $pathType := .Values.prometheus.thanosIngress.pathType | default "" }} {{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} {{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}} {{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} {{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} +{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +apiVersion: networking.k8s.io/v1beta1 +{{ else }} apiVersion: extensions/v1beta1 +{{ end -}} kind: Ingress metadata: {{- if .Values.prometheus.thanosIngress.annotations }} @@ -18,7 +23,7 @@ metadata: {{ toYaml .Values.prometheus.thanosIngress.labels | indent 4 }} {{- end }} spec: - {{- if or (.Capabilities.APIVersions.Has "networking.k8s.io/v1/IngressClass") (.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1/IngressClass") }} + {{- if or (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1") }} {{- if .Values.prometheus.thanosIngress.ingressClassName }} ingressClassName: {{ .Values.prometheus.thanosIngress.ingressClassName }} {{- end }} @@ -31,6 +36,9 @@ spec: paths: {{- range $p := $paths }} - path: {{ tpl $p $ }} + {{- if $pathType }} + pathType: {{ $pathType }} + {{- end }} backend: serviceName: {{ $serviceName }} servicePort: {{ $thanosPort }} @@ -41,6 +49,9 @@ spec: paths: {{- range $p := $paths }} - path: {{ tpl $p $ }} + {{- if $pathType }} + pathType: {{ $pathType }} + {{- end }} backend: serviceName: {{ $serviceName }} servicePort: {{ $thanosPort }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml index c1959c6..3314377 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/ingressperreplica.yaml @@ -1,4 +1,5 @@ {{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled .Values.prometheus.ingressPerReplica.enabled }} +{{- $pathType := .Values.prometheus.ingressPerReplica.pathType | default "" }} {{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} {{- $servicePort := .Values.prometheus.servicePerReplica.port -}} {{- $ingressValues := .Values.prometheus.ingressPerReplica -}} @@ -29,7 +30,7 @@ items: {{ toYaml $ingressValues.annotations | indent 8 }} {{- end }} spec: - {{- if or ($.Capabilities.APIVersions.Has "networking.k8s.io/v1/IngressClass") ($.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1/IngressClass") }} + {{- if or ($.Capabilities.APIVersions.Has "networking.k8s.io/v1") ($.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1") }} {{- if $ingressValues.ingressClassName }} ingressClassName: {{ $ingressValues.ingressClassName }} {{- end }} @@ -40,6 +41,9 @@ items: paths: {{- range $p := $ingressValues.paths }} - path: {{ tpl $p $ }} + {{- if $pathType }} + pathType: {{ $pathType }} + {{- end }} backend: serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} servicePort: {{ $servicePort }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml index 66ac5a8..eb561e6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/prometheus.yaml @@ -40,7 +40,7 @@ spec: {{- end }} {{- if .Values.prometheus.prometheusSpec.externalLabels }} externalLabels: -{{ toYaml .Values.prometheus.prometheusSpec.externalLabels | indent 4}} +{{ tpl (toYaml .Values.prometheus.prometheusSpec.externalLabels | indent 4) . }} {{- end }} {{- if .Values.prometheus.prometheusSpec.prometheusExternalLabelNameClear }} prometheusExternalLabelName: "" @@ -65,6 +65,7 @@ spec: {{- end }} paused: {{ .Values.prometheus.prometheusSpec.paused }} replicas: {{ .Values.prometheus.prometheusSpec.replicas }} + shards: {{ .Values.prometheus.prometheusSpec.shards }} logLevel: {{ .Values.prometheus.prometheusSpec.logLevel }} logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} @@ -200,9 +201,9 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: - topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} labelSelector: - matchLabels: - app: prometheus - prometheus: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + matchExpressions: + - {key: app, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-prometheus]} {{- else if eq .Values.prometheus.prometheusSpec.podAntiAffinity "soft" }} podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: @@ -210,15 +211,19 @@ spec: podAffinityTerm: topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} labelSelector: - matchLabels: - app: prometheus - prometheus: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + matchExpressions: + - {key: app, operator: In, values: [prometheus]} + - {key: prometheus, operator: In, values: [{{ template "kube-prometheus-stack.fullname" . }}-prometheus]} {{- end }} {{- end }} {{- if .Values.prometheus.prometheusSpec.tolerations }} tolerations: {{ toYaml .Values.prometheus.prometheusSpec.tolerations | indent 4 }} {{- end }} +{{- if .Values.prometheus.prometheusSpec.topologySpreadConstraints }} + topologySpreadConstraints: +{{ toYaml .Values.prometheus.prometheusSpec.topologySpreadConstraints | indent 4 }} +{{- end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | indent 4 }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/psp.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/psp.yaml index bd2b270..08da5e1 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/psp.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/psp.yaml @@ -26,6 +26,9 @@ spec: - 'secret' - 'downwardAPI' - 'persistentVolumeClaim' +{{- if .Values.prometheus.podSecurityPolicy.volumes }} +{{ toYaml .Values.prometheus.podSecurityPolicy.volumes | indent 4 }} +{{- end }} hostNetwork: false hostIPC: false hostPID: false @@ -52,4 +55,8 @@ spec: allowedCapabilities: {{ toYaml .Values.prometheus.podSecurityPolicy.allowedCapabilities | indent 4 }} {{- end }} +{{- if .Values.prometheus.podSecurityPolicy.allowedHostPaths }} + allowedHostPaths: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedHostPaths | indent 4 }} +{{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml index 8358704..2a46523 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/etcd.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/etcd3_alert.rules.yml +Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/website/master/content/docs/v3.4.0/op-guide/etcd3_alert.rules.yml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -24,24 +24,6 @@ spec: groups: - name: etcd rules: - - alert: etcdMembersDown - annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).' - expr: |- - max by (job) ( - sum by (job) (up{job=~".*etcd.*"} == bool 0) - or - count by (job,endpoint) ( - sum by (job,endpoint,To) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[3m])) > 0.01 - ) - ) - > 0 - for: 3m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - alert: etcdInsufficientMembers annotations: message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' @@ -64,9 +46,9 @@ spec: {{- end }} - alert: etcdHighNumberOfLeaderChanges annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' - expr: increase((max by (job) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 3 - for: 5m + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": instance {{`{{`}} $labels.instance {{`}}`}} has seen {{`{{`}} $value {{`}}`}} leader changes within the last hour.' + expr: rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3 + for: 15m labels: severity: warning {{- if .Values.defaultRules.additionalRuleLabels }} @@ -98,7 +80,7 @@ spec: {{- end }} - alert: etcdHighNumberOfFailedProposals annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last hour on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 for: 15m labels: @@ -166,4 +148,4 @@ spec: {{- if .Values.defaultRules.additionalRuleLabels }} {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.yaml index 35e4ede..19511e8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -24,8 +24,6 @@ spec: groups: - name: k8s.rules rules: - - expr: sum(rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}[5m])) by (namespace) - record: namespace:container_cpu_usage_seconds_total:sum_rate - expr: |- sum by (cluster, namespace, pod, container) ( rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}[5m]) @@ -57,8 +55,6 @@ spec: max by(namespace, pod, node) (kube_pod_info{node!=""}) ) record: node_namespace_pod_container:container_memory_swap - - expr: sum(container_memory_usage_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}) by (namespace) - record: namespace:container_memory_usage_bytes:sum - expr: |- sum by (namespace) ( sum by (namespace, pod) ( diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml index 78a09e3..7b00b54 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -99,54 +99,56 @@ spec: labels: verb: write record: apiserver_request:availability30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"2.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"2.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"2.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"2.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"2.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"2.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"3.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"3.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"3.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"3.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"3.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"3.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"4.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"4.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"4.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"4.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"4.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"4.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"5.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"5.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"5.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"5.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"5.."}[30d])) - record: code_verb:apiserver_request_total:increase30d - - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"5.."}[30d])) + - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30 record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"2.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"3.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"4.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"5.."}[1h])) + record: code_verb:apiserver_request_total:increase1h - expr: sum by (code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) labels: verb: read diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml index da0de91..0f44ccc 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml index b4d1a0f..eddc1e4 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kube-apiserver.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kube-apiserver.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -343,11 +343,6 @@ spec: quantile: '0.99' verb: write record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile - - expr: |- - sum(rate(apiserver_request_duration_seconds_sum{subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod) - / - sum(rate(apiserver_request_duration_seconds_count{subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod) - record: cluster:apiserver_request_duration_seconds:mean5m - expr: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod)) labels: quantile: '0.99' diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml index 594f1bb..24df268 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubelet.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubelet.rules.yaml index f9bd1cc..8712b9f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubelet.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubelet.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubelet.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kubelet.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml index 6eb5e02..198bbb8 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -27,10 +27,10 @@ spec: rules: - alert: KubePodCrashLooping annotations: - description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is restarting {{`{{`}} printf "%.2f" $value {{`}}`}} times / 5 minutes. + description: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is restarting {{`{{`}} printf "%.2f" $value {{`}}`}} times / 10 minutes. runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodcrashlooping summary: Pod is crash looping. - expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) * 60 * 5 > 0 + expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[10m]) * 60 * 5 > 0 for: 15m labels: severity: warning diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-resources.yaml index 87933e5..898f8ee 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-resources.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml index 1fadb59..527e6e3 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml index f9acb6a..2ed298b 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-system-apiserver' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kubernetes-system-apiserver' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -48,16 +48,15 @@ spec: {{- end }} - alert: AggregatedAPIErrors annotations: - description: An aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. The number of errors have increased for it in the past five minutes. High values indicate that the availability of the service changes too often. + description: An aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. It has appeared unavailable {{`{{`}} $value | humanize {{`}}`}} times averaged over the past 10m. runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-aggregatedapierrors summary: An aggregated API has reported errors. - expr: sum by(name, namespace)(increase(aggregator_unavailable_apiservice_count[5m])) > 2 + expr: sum by(name, namespace)(increase(aggregator_unavailable_apiservice_count[10m])) > 4 labels: severity: warning {{- if .Values.defaultRules.additionalRuleLabels }} {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} {{- end }} -{{- if semverCompare ">=1.18.0-0" $kubeTargetVersion }} - alert: AggregatedAPIDown annotations: description: An aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 10m. @@ -67,7 +66,6 @@ spec: for: 5m labels: severity: warning -{{- end }} {{- if .Values.defaultRules.additionalRuleLabels }} {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} {{- end }} @@ -85,4 +83,16 @@ spec: {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} {{- end }} {{- end }} + - alert: KubeAPITerminatedRequests + annotations: + description: The apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapiterminatedrequests + summary: The apiserver has terminated {{`{{`}} $value | humanizePercentage {{`}}`}} of its incoming requests. + expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20 + for: 5m + labels: + severity: warning +{{- if .Values.defaultRules.additionalRuleLabels }} +{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml index bc9dab8..3d1ace1 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-system-controller-manager' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kubernetes-system-controller-manager' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml index cde9da4..4d536ec 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml index a2c1272..098f6fb 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-system-scheduler' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kubernetes-system-scheduler' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system.yaml index 3aca2e6..52230c6 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.rules.yaml index b6ae1bb..ddb7376 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'node-exporter.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'node-exporter.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/node-exporter-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml index 7d5ab7d..3be497c 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node-exporter.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/node-exporter-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -27,7 +27,6 @@ spec: - alert: NodeFilesystemSpaceFillingUp annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemspacefillingup summary: Filesystem is predicted to run out of space within the next 24 hours. expr: |- ( @@ -46,7 +45,6 @@ spec: - alert: NodeFilesystemSpaceFillingUp annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemspacefillingup summary: Filesystem is predicted to run out of space within the next 4 hours. expr: |- ( @@ -65,7 +63,6 @@ spec: - alert: NodeFilesystemAlmostOutOfSpace annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutofspace summary: Filesystem has less than 5% space left. expr: |- ( @@ -82,7 +79,6 @@ spec: - alert: NodeFilesystemAlmostOutOfSpace annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutofspace summary: Filesystem has less than 3% space left. expr: |- ( @@ -99,7 +95,6 @@ spec: - alert: NodeFilesystemFilesFillingUp annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemfilesfillingup summary: Filesystem is predicted to run out of inodes within the next 24 hours. expr: |- ( @@ -118,7 +113,6 @@ spec: - alert: NodeFilesystemFilesFillingUp annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemfilesfillingup summary: Filesystem is predicted to run out of inodes within the next 4 hours. expr: |- ( @@ -137,7 +131,6 @@ spec: - alert: NodeFilesystemAlmostOutOfFiles annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutoffiles summary: Filesystem has less than 5% inodes left. expr: |- ( @@ -154,7 +147,6 @@ spec: - alert: NodeFilesystemAlmostOutOfFiles annotations: description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutoffiles summary: Filesystem has less than 3% inodes left. expr: |- ( @@ -171,7 +163,6 @@ spec: - alert: NodeNetworkReceiveErrs annotations: description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.' - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodenetworkreceiveerrs summary: Network interface is reporting many receive errors. expr: rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01 for: 1h @@ -183,7 +174,6 @@ spec: - alert: NodeNetworkTransmitErrs annotations: description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.' - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodenetworktransmiterrs summary: Network interface is reporting many transmit errors. expr: rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01 for: 1h @@ -195,7 +185,6 @@ spec: - alert: NodeHighNumberConntrackEntriesUsed annotations: description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodehighnumberconntrackentriesused summary: Number of conntrack are getting close to the limit. expr: (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75 labels: @@ -206,7 +195,6 @@ spec: - alert: NodeTextFileCollectorScrapeError annotations: description: Node Exporter text file collector failed to scrape. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodetextfilecollectorscrapeerror summary: Node Exporter text file collector failed to scrape. expr: node_textfile_scrape_error{job="node-exporter"} == 1 labels: @@ -217,7 +205,6 @@ spec: - alert: NodeClockSkewDetected annotations: message: Clock on {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 300s. Ensure NTP is configured correctly on this host. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodeclockskewdetected summary: Clock skew detected. expr: |- ( @@ -240,7 +227,6 @@ spec: - alert: NodeClockNotSynchronising annotations: message: Clock on {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodeclocknotsynchronising summary: Clock not synchronising. expr: |- min_over_time(node_timex_sync_status[5m]) == 0 @@ -255,7 +241,6 @@ spec: - alert: NodeRAIDDegraded annotations: description: RAID array '{{`{{`}} $labels.device {{`}}`}}' on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-noderaiddegraded summary: RAID Array is degraded expr: node_md_disks_required - ignoring (state) (node_md_disks{state="active"}) > 0 for: 15m @@ -267,7 +252,6 @@ spec: - alert: NodeRAIDDiskFailure annotations: description: At least one device in RAID array on {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap. - runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-noderaiddiskfailure summary: Failed device in RAID array expr: node_md_disks{state="fail"} > 0 labels: diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node.rules.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node.rules.yaml index 7253b31..c841e6f 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node.rules.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules-1.14/node.rules.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -24,8 +24,6 @@ spec: groups: - name: node.rules rules: - - expr: sum(min(kube_pod_info{node!=""}) by (cluster, node)) - record: ':kube_pod_info_node_count:' - expr: |- topk by(namespace, pod) (1, max by (node, namespace, pod) ( diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/etcd.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/etcd.yaml index e9c4f6c..28cc925 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/etcd.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/rules/etcd.yaml @@ -1,5 +1,5 @@ {{- /* -Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/etcd3_alert.rules.yml +Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/website/master/content/docs/v3.4.0/op-guide/etcd3_alert.rules.yml Do not change in-place! In order to change this file first read following link: https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack */ -}} @@ -24,24 +24,6 @@ spec: groups: - name: etcd rules: - - alert: etcdMembersDown - annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).' - expr: |- - max by (job) ( - sum by (job) (up{job=~".*etcd.*"} == bool 0) - or - count by (job,endpoint) ( - sum by (job,endpoint,To) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[3m])) > 0.01 - ) - ) - > 0 - for: 3m - labels: - severity: critical -{{- if .Values.defaultRules.additionalRuleLabels }} -{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} -{{- end }} - alert: etcdInsufficientMembers annotations: message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' @@ -64,9 +46,9 @@ spec: {{- end }} - alert: etcdHighNumberOfLeaderChanges annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' - expr: increase((max by (job) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 3 - for: 5m + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": instance {{`{{`}} $labels.instance {{`}}`}} has seen {{`{{`}} $value {{`}}`}} leader changes within the last hour.' + expr: rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3 + for: 15m labels: severity: warning {{- if .Values.defaultRules.additionalRuleLabels }} @@ -126,7 +108,7 @@ spec: {{- end }} - alert: etcdHighNumberOfFailedProposals annotations: - message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last hour on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 for: 15m labels: @@ -194,4 +176,4 @@ spec: {{- if .Values.defaultRules.additionalRuleLabels }} {{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/service.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/service.yaml index cb831c7..8676b81 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/service.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/service.yaml @@ -39,6 +39,14 @@ spec: {{- end }} port: {{ .Values.prometheus.service.port }} targetPort: {{ .Values.prometheus.service.targetPort }} + {{- if .Values.prometheus.thanosIngress.enabled }} + - name: grpc + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.thanosIngress.nodePort }} + {{- end }} + port: {{ .Values.prometheus.thanosIngress.servicePort }} + targetPort: {{ .Values.prometheus.thanosIngress.servicePort }} + {{- end }} {{- if .Values.prometheus.service.additionalPorts }} {{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSIdecar.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSIdecar.yaml new file mode 100644 index 0000000..6ae1b14 --- /dev/null +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/serviceThanosSIdecar.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosService.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-discovery + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-thanos-discovery +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosService.labels }} +{{ toYaml .Values.prometheus.thanosService.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.thanosService.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosService.annotations | indent 4 }} +{{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: {{ .Values.prometheus.thanosService.portName }} + port: {{ .Values.prometheus.thanosService.port }} + targetPort: {{ .Values.prometheus.thanosService.targetPort }} + selector: + app: prometheus + prometheus: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/servicemonitors.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/servicemonitors.yaml index 959df21..a78d1cd 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/servicemonitors.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/templates/prometheus/servicemonitors.yaml @@ -30,5 +30,9 @@ items: targetLabels: {{ toYaml .targetLabels | indent 8 }} {{- end }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} {{- end }} {{- end }} diff --git a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml index 3aa810f..66114e2 100644 --- a/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml +++ b/charts/kubezero-metrics/charts/kube-prometheus-stack/values.yaml @@ -158,6 +158,8 @@ alertmanager: receiver: 'null' receivers: - name: 'null' + templates: + - '/etc/alertmanager/config/*.tmpl' ## Pass the Alertmanager configuration directives through Helm's templating ## engine. If the Alertmanager configuration contains Alertmanager templates, @@ -170,6 +172,10 @@ alertmanager: tplConfig: false ## Alertmanager template files to format alerts + ## By default, templateFiles are placed in /etc/alertmanager/config/ and if + ## they have a .tmpl file suffix will be loaded. See config.templates above + ## to change, add other suffixes. If adding other suffixes, be sure to update + ## config.templates above to include those suffixes. ## ref: https://prometheus.io/docs/alerting/notifications/ ## https://prometheus.io/docs/alerting/notification_examples/ ## @@ -214,6 +220,10 @@ alertmanager: paths: [] # - / + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + ## TLS configuration for Alertmanager Ingress ## Secret must be manually created in the namespace ## @@ -254,6 +264,10 @@ alertmanager: paths: [] # - / + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + ## Secret name containing the TLS certificate for alertmanager per replica ingress ## Secret must be manually created in the namespace tlsSecretName: "" @@ -520,6 +534,17 @@ alertmanager: # value: "value" # effect: "NoSchedule" + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: alertmanager + ## SecurityContext holds pod-level security attributes and common container settings. ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ @@ -628,6 +653,9 @@ grafana: enabled: true defaultDatasourceEnabled: true + # If not defined, will use prometheus.prometheusSpec.scrapeInterval or its default + # defaultDatasourceScrapeInterval: 15s + ## Annotations for Grafana datasource configmaps ## annotations: {} @@ -1145,6 +1173,9 @@ kubeStateMetrics: ## Scrape interval. If not set, the Prometheus default scrape interval is used. ## interval: "" + ## Override serviceMonitor selector + ## + selectorOverride: {} ## metric relabel configs to apply to samples before ingestion. ## @@ -1218,25 +1249,31 @@ prometheus-node-exporter: ## jobLabel: node-exporter extraArgs: - - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/) - - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$ + - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ ## Manages Prometheus and Alertmanager components ## prometheusOperator: enabled: true - # Prometheus-Operator v0.39.0 and later support TLS natively. + ## Prometheus-Operator v0.39.0 and later support TLS natively. + ## tls: enabled: true # Value must match version names from https://golang.org/pkg/crypto/tls/#pkg-constants tlsMinVersion: VersionTLS13 + # The default webhook port is 10250 in order to work out-of-the-box in GKE private clusters and avoid adding firewall rules. + internalPort: 10250 ## Admission webhook support for PrometheusRules resources added in Prometheus Operator 0.30 can be enabled to prevent incorrectly formatted ## rules from making their way into prometheus and potentially preventing the container from starting admissionWebhooks: failurePolicy: Fail enabled: true + ## A PEM encoded CA bundle which will be used to validate the webhook's server certificate. + ## If unspecified, system trust roots on the apiserver are used. + caBundle: "" ## If enabled, generate a self-signed certificate, then patch the webhook configurations with the generated data. ## On chart upgrades (or if the secret exists) the cert will not be re-generated. You can use this to provide your own ## certs ahead of time if you wish. @@ -1256,6 +1293,12 @@ prometheusOperator: nodeSelector: {} affinity: {} tolerations: [] + # Use certmanager to generate webhook certs + certManager: + enabled: false + # issuerRef: + # name: "issuer" + # kind: "ClusterIssuer" ## Namespaces to scope the interaction of the Prometheus Operator and the apiserver (allow list). ## This is mutually exclusive with denyNamespaces. Setting this to an empty object will disable the configuration @@ -1275,6 +1318,12 @@ prometheusOperator: prometheusInstanceNamespaces: [] thanosRulerInstanceNamespaces: [] + ## The clusterDomain value will be added to the cluster.peer option of the alertmanager. + ## Without this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated:9094 (default value) + ## With this specified option cluster.peer will have value alertmanager-monitoring-alertmanager-0.alertmanager-operated.namespace.svc.cluster-domain:9094 + ## + # clusterDomain: "cluster.local" + ## Service account for Alertmanager to use. ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ ## @@ -1411,7 +1460,16 @@ prometheusOperator: # values: # - e2e-az1 # - e2e-az2 - + dnsConfig: {} + # nameservers: + # - 1.2.3.4 + # searches: + # - ns1.svc.cluster-domain.example + # - my.dns.search.suffix + # options: + # - name: ndots + # value: "2" + # - name: edns0 securityContext: fsGroup: 65534 runAsGroup: 65534 @@ -1422,22 +1480,23 @@ prometheusOperator: ## image: repository: quay.io/prometheus-operator/prometheus-operator - tag: v0.44.0 + tag: v0.45.0 sha: "" pullPolicy: IfNotPresent - ## Configmap-reload image to use for reloading configmaps + ## Prometheus image to use for prometheuses managed by the operator ## - configmapReloadImage: - repository: docker.io/jimmidyson/configmap-reload - tag: v0.4.0 - sha: "" + # prometheusDefaultBaseImage: quay.io/prometheus/prometheus + + ## Alertmanager image to use for alertmanagers managed by the operator + ## + # alertmanagerDefaultBaseImage: quay.io/prometheus/alertmanager ## Prometheus-config-reloader image to use for config and rule reloading ## prometheusConfigReloaderImage: repository: quay.io/prometheus-operator/prometheus-config-reloader - tag: v0.44.0 + tag: v0.45.0 sha: "" ## Set the prometheus config reloader side-car CPU limit @@ -1446,7 +1505,7 @@ prometheusOperator: ## Set the prometheus config reloader side-car memory limit ## - configReloaderMemory: 25Mi + configReloaderMemory: 50Mi ## Set a Field Selector to filter watched secrets ## @@ -1469,6 +1528,19 @@ prometheus: create: true name: "" + # Service for thanos service discovery on sidecar + # Enable this can make Thanos Query can use + # `--store=dnssrv+_grpc._tcp.${kube-prometheus-stack.fullname}-thanos-discovery.${namespace}.svc.cluster.local` to discovery + # Thanos sidecar on prometheus nodes + # (Please remember to change ${kube-prometheus-stack.fullname} and ${namespace}. Not just copy and paste!) + thanosService: + enabled: false + annotations: {} + labels: {} + portName: grpc + port: 10901 + targetPort: "grpc" + ## Configuration for Prometheus service ## service: @@ -1538,7 +1610,7 @@ prometheus: minAvailable: 1 maxUnavailable: "" - # Ingress exposes thanos sidecar outside the clsuter + # Ingress exposes thanos sidecar outside the cluster thanosIngress: enabled: false @@ -1549,6 +1621,12 @@ prometheus: annotations: {} labels: {} servicePort: 10901 + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30901 + ## Hosts must be provided if Ingress is enabled. ## hosts: [] @@ -1559,7 +1637,11 @@ prometheus: paths: [] # - / - ## TLS configuration for Alertmanager Ingress + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + + ## TLS configuration for Thanos Ingress ## Secret must be manually created in the namespace ## tls: [] @@ -1589,6 +1671,10 @@ prometheus: paths: [] # - / + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + ## TLS configuration for Prometheus Ingress ## Secret must be manually created in the namespace ## @@ -1624,6 +1710,10 @@ prometheus: paths: [] # - / + ## For Kubernetes >= 1.18 you should specify the pathType (determines how Ingress paths should be matched) + ## See https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#better-path-matching-with-path-types + # pathType: ImplementationSpecific + ## Secret name containing the TLS certificate for Prometheus per replica ingress ## Secret must be manually created in the namespace tlsSecretName: "" @@ -1641,6 +1731,8 @@ prometheus: ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ podSecurityPolicy: allowedCapabilities: [] + allowedHostPaths: [] + volumes: [] serviceMonitor: ## Scrape interval. If not set, the Prometheus default scrape interval is used. @@ -1714,7 +1806,7 @@ prometheus: ## image: repository: quay.io/prometheus/prometheus - tag: v2.22.1 + tag: v2.24.0 sha: "" ## Tolerations for use with node taints @@ -1726,6 +1818,17 @@ prometheus: # value: "value" # effect: "NoSchedule" + ## If specified, the pod's topology spread constraints. + ## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ + ## + topologySpreadConstraints: [] + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: DoNotSchedule + # labelSelector: + # matchLabels: + # app: prometheus + ## Alertmanagers to which alerts will be sent ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#alertmanagerendpoints ## @@ -1834,9 +1937,12 @@ prometheus: # prometheus: somelabel ## Namespaces to be selected for ServiceMonitor discovery. - ## See https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#namespaceselector for usage ## serviceMonitorNamespaceSelector: {} + ## Example which selects ServiceMonitors in namespaces with label "prometheus" set to "somelabel" + # serviceMonitorNamespaceSelector: + # matchLabels: + # prometheus: somelabel ## If true, a nil or {} value for prometheus.prometheusSpec.podMonitorSelector will cause the ## prometheus resource to be created with selectors based on values in the helm deployment, @@ -1894,10 +2000,20 @@ prometheus: ## paused: false - ## Number of Prometheus replicas desired + ## Number of replicas of each shard to deploy for a Prometheus deployment. + ## Number of replicas multiplied by shards is the total number of Pods created. ## replicas: 1 + ## EXPERIMENTAL: Number of shards to distribute targets onto. + ## Number of replicas multiplied by shards is the total number of Pods created. + ## Note that scaling down shards will not reshard data onto remaining instances, it must be manually moved. + ## Increasing shards will not reshard data either but it will continue to be available from the same instances. + ## To query globally use Thanos sidecar and Thanos querier or remote write data to a central location. + ## Sharding is done on the content of the `__address__` target meta-label. + ## + shards: 1 + ## Log level for Prometheus be configured in ## logLevel: info @@ -2160,7 +2276,11 @@ prometheus: ## labels to transfer from the kubernetes service to the target ## - # targetLabels: "" + # targetLabels: [] + + ## labels to transfer from the kubernetes pods to the target + ## + # podTargetLabels: [] ## Label selector for services to which this ServiceMonitor applies ## diff --git a/charts/kubezero-metrics/update.sh b/charts/kubezero-metrics/update.sh index c757053..4be6f0f 100755 --- a/charts/kubezero-metrics/update.sh +++ b/charts/kubezero-metrics/update.sh @@ -1,8 +1,8 @@ #!/bin/bash -VERSION=12.8.0 +VERSION=13.13.0 rm -rf charts/kube-prometheus-stack curl -L -s -o - https://github.com/prometheus-community/helm-charts/releases/download/kube-prometheus-stack-${VERSION}/kube-prometheus-stack-${VERSION}.tgz | tar xfz - -C charts -patch -p3 -i remove_etcd_grpc_alerts.patch +patch -p3 -i remove_etcd_grpc_alerts.patch --no-backup-if-mismatch -- 2.40.1 From ef75ae73f67f06c3641a7597b71c2330ac971003 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 2 Mar 2021 10:33:12 +0100 Subject: [PATCH 23/65] Istio version bump due to security release --- charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml | 2 +- charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml | 2 +- .../charts/istio-private-ingress/Chart.yaml | 2 +- .../charts/istio-private-ingress/values.yaml | 2 +- charts/kubezero-istio/charts/base/Chart.yaml | 2 +- charts/kubezero-istio/charts/istio-discovery/Chart.yaml | 2 +- charts/kubezero-istio/charts/istio-discovery/values.yaml | 2 +- charts/kubezero-istio/update.sh | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml b/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml index 78bd0c4..d402a1f 100644 --- a/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-ingress/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: istio-ingress -version: 1.9.0 +version: 1.9.1 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio gateways keywords: diff --git a/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml b/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml index cc01ef8..249b3a8 100644 --- a/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-ingress/values.yaml @@ -171,7 +171,7 @@ global: hub: docker.io/istio # Default tag for Istio images. - tag: 1.9.0 + tag: 1.9.1 # Specify image pull policy if default behavior isn't desired. # Default behavior: latest images will be Always else IfNotPresent. diff --git a/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml b/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml index fac8f49..3786335 100644 --- a/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-private-ingress/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: istio-private-ingress -version: 1.9.0 +version: 1.9.1 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio gateways keywords: diff --git a/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml b/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml index cc01ef8..249b3a8 100644 --- a/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml +++ b/charts/kubezero-istio-ingress/charts/istio-private-ingress/values.yaml @@ -171,7 +171,7 @@ global: hub: docker.io/istio # Default tag for Istio images. - tag: 1.9.0 + tag: 1.9.1 # Specify image pull policy if default behavior isn't desired. # Default behavior: latest images will be Always else IfNotPresent. diff --git a/charts/kubezero-istio/charts/base/Chart.yaml b/charts/kubezero-istio/charts/base/Chart.yaml index 8cf22dc..e6142de 100644 --- a/charts/kubezero-istio/charts/base/Chart.yaml +++ b/charts/kubezero-istio/charts/base/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: base -version: 1.9.0 +version: 1.9.1 tillerVersion: ">=2.7.2" description: Helm chart for deploying Istio cluster resources and CRDs keywords: diff --git a/charts/kubezero-istio/charts/istio-discovery/Chart.yaml b/charts/kubezero-istio/charts/istio-discovery/Chart.yaml index 60e1a49..a43db08 100644 --- a/charts/kubezero-istio/charts/istio-discovery/Chart.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: istio-discovery -version: 1.9.0 +version: 1.9.1 tillerVersion: ">=2.7.2" description: Helm chart for istio control plane keywords: diff --git a/charts/kubezero-istio/charts/istio-discovery/values.yaml b/charts/kubezero-istio/charts/istio-discovery/values.yaml index 5dab58d..6491504 100644 --- a/charts/kubezero-istio/charts/istio-discovery/values.yaml +++ b/charts/kubezero-istio/charts/istio-discovery/values.yaml @@ -232,7 +232,7 @@ global: # Dev builds from prow are on gcr.io hub: docker.io/istio # Default tag for Istio images. - tag: 1.9.0 + tag: 1.9.1 # Specify image pull policy if default behavior isn't desired. # Default behavior: latest images will be Always else IfNotPresent. diff --git a/charts/kubezero-istio/update.sh b/charts/kubezero-istio/update.sh index 00d385e..db27e79 100755 --- a/charts/kubezero-istio/update.sh +++ b/charts/kubezero-istio/update.sh @@ -1,7 +1,7 @@ #!/bin/bash set -ex -export ISTIO_VERSION=1.9.0 +export ISTIO_VERSION=1.9.1 if [ ! -d istio-$ISTIO_VERSION ]; then NAME="istio-$ISTIO_VERSION" -- 2.40.1 From ef254a149bf7e8ad2f952306e3f82875a54d80a7 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 2 Mar 2021 11:22:34 +0100 Subject: [PATCH 24/65] aws-ebs-csi-driver version bump and resource limits --- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 4 +- .../charts/aws-ebs-csi-driver/Chart.yaml | 2 +- .../aws-ebs-csi-driver/templates/NOTES.txt | 2 +- .../clusterrolebinding-attacher.yaml | 2 +- .../clusterrolebinding-provisioner.yaml | 2 +- .../templates/clusterrolebinding-resizer.yaml | 2 +- ...lusterrolebinding-snapshot-controller.yaml | 2 +- .../clusterrolebinding-snapshotter.yaml | 2 +- .../templates/controller.yaml | 48 +++- .../templates/controller.yaml.orig | 231 ++++++++++++++++++ .../aws-ebs-csi-driver/templates/node.yaml | 17 +- .../templates/node.yaml.orig | 174 +++++++++++++ ...le-snapshot-controller-leaderelection.yaml | 1 - ...ng-snapshot-controller-leaderelection.yaml | 3 +- .../serviceaccount-csi-controller.yaml | 1 - .../templates/serviceaccount-csi-node.yaml | 1 - .../serviceaccount-snapshot-controller.yaml | 1 - .../templates/statefulset.yaml | 10 +- .../templates/statefulset.yaml.orig | 60 +++++ .../charts/aws-ebs-csi-driver/values.yaml | 18 ++ .../loglevel_leader.patch | 9 - charts/kubezero-aws-ebs-csi-driver/update.sh | 2 +- .../kubezero-aws-ebs-csi-driver/values.yaml | 8 + 23 files changed, 574 insertions(+), 28 deletions(-) create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml.orig create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml.orig create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml.orig diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index 6f6e3ef..ddfe3fe 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.4.1 +version: 0.4.2 appVersion: 0.9.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -18,7 +18,7 @@ maintainers: - name: Quarky9 dependencies: - name: aws-ebs-csi-driver - version: 0.9.10 + version: 0.9.12 repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: kubezero-lib version: ">= 0.1.3" diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml index ec5dd3f..d9476e4 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml @@ -15,4 +15,4 @@ maintainers: name: aws-ebs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver -version: 0.9.10 +version: 0.9.12 diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt index 34db916..3717647 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt @@ -1,3 +1,3 @@ To verify that aws-ebs-csi-driver has started, run: - kubectl get pod -n kube-system -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" + kubectl get pod -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml index 92a8b40..c75cb9b 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml @@ -8,7 +8,7 @@ metadata: subjects: - kind: ServiceAccount name: {{ .Values.serviceAccount.controller.name }} - namespace: kube-system + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: ebs-external-attacher-role diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml index e2478b9..4a9174b 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml @@ -8,7 +8,7 @@ metadata: subjects: - kind: ServiceAccount name: {{ .Values.serviceAccount.controller.name }} - namespace: kube-system + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: ebs-external-provisioner-role diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml index 997dc28..6fe42d1 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml @@ -9,7 +9,7 @@ metadata: subjects: - kind: ServiceAccount name: {{ .Values.serviceAccount.controller.name }} - namespace: kube-system + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: ebs-external-resizer-role diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml index cb46730..b74484f 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml @@ -9,7 +9,7 @@ metadata: subjects: - kind: ServiceAccount name: {{ .Values.serviceAccount.snapshot.name }} - namespace: kube-system + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: ebs-snapshot-controller-role diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml index f55c38e..cbc1169 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml @@ -9,7 +9,7 @@ metadata: subjects: - kind: ServiceAccount name: {{ .Values.serviceAccount.controller.name }} - namespace: kube-system + namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole name: ebs-external-snapshotter-role diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml index 7feff6f..af5e0f1 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -3,7 +3,6 @@ kind: Deployment apiVersion: apps/v1 metadata: name: ebs-csi-controller - namespace: kube-system labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} spec: @@ -38,6 +37,13 @@ spec: {{- with .Values.tolerations }} {{ toYaml . | indent 8 }} {{- end }} +{{- if .Values.topologySpreadConstraints }} +{{- $tscLabelSelector := dict "labelSelector" ( dict "matchLabels" ( dict "app" "ebs-csi-controller" ) ) }} + topologySpreadConstraints: + {{- range .Values.topologySpreadConstraints }} + - {{ mergeOverwrite . $tscLabelSelector | toJson }} + {{- end }} +{{- end }} containers: - name: ebs-plugin image: {{ .Values.image.repository }}:{{ .Values.image.tag }} @@ -76,6 +82,14 @@ spec: - name: AWS_REGION value: {{ .Values.region }} {{- end }} +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -110,6 +124,14 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -125,6 +147,14 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -140,6 +170,14 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ @@ -157,6 +195,14 @@ spec: env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml.orig b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml.orig new file mode 100644 index 0000000..2e46432 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml.orig @@ -0,0 +1,231 @@ +# Controller Service +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ebs-csi-controller + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: ebs-csi-controller + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: ebs-csi-controller + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.podAnnotations }} + annotations: {{ toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} + spec: + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + serviceAccountName: {{ .Values.serviceAccount.controller.name }} + priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + tolerations: + {{- if .Values.tolerateAllTaints }} + - operator: Exists + {{- end }} + {{- with .Values.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} +{{- if .Values.topologySpreadConstraints }} +{{- $tscLabelSelector := dict "labelSelector" ( dict "matchLabels" ( dict "app" "ebs-csi-controller" ) ) }} + topologySpreadConstraints: + {{- range .Values.topologySpreadConstraints }} + - {{ mergeOverwrite . $tscLabelSelector | toJson }} + {{- end }} +{{- end }} + containers: + - name: ebs-plugin + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + {{- if ne .Release.Name "kustomize" }} + - controller + {{- else }} + # - {all,controller,node} # specify the driver mode + {{- end }} + - --endpoint=$(CSI_ENDPOINT) + {{- if .Values.extraVolumeTags }} + {{- include "aws-ebs-csi-driver.extra-volume-tags" . | nindent 12 }} + {{- end }} + {{- if .Values.k8sTagClusterId }} + - --k8s-tag-cluster-id={{ .Values.k8sTagClusterId }} + {{- end }} + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-secret + key: key_id + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-secret + key: access_key + optional: true + {{- if .Values.region }} + - name: AWS_REGION + value: {{ .Values.region }} + {{- end }} +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: csi-provisioner + image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --v=5 + {{- if .Values.enableVolumeScheduling }} + - --feature-gates=Topology=true + {{- end}} + {{- if .Values.extraCreateMetadata }} + - --extra-create-metadata + {{- end}} + - --leader-election=true + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: csi-attacher + image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --v=5 + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.enableVolumeSnapshot }} + - name: csi-snapshotter + image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.enableVolumeResizing }} + - name: csi-resizer + image: {{ printf "%s:%s" .Values.sidecars.resizerImage.repository .Values.sidecars.resizerImage.tag }} + imagePullPolicy: Always + args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: socket-dir + emptyDir: {} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml index 6fd1110..5de7469 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml @@ -3,7 +3,6 @@ kind: DaemonSet apiVersion: apps/v1 metadata: name: ebs-csi-node - namespace: kube-system labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} spec: @@ -60,6 +59,14 @@ spec: env: - name: CSI_ENDPOINT value: unix:/csi/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet @@ -104,6 +111,14 @@ spec: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} volumeMounts: - name: plugin-dir mountPath: /csi diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml.orig b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml.orig new file mode 100644 index 0000000..bf9c6e0 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml.orig @@ -0,0 +1,174 @@ +# Node Service +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ebs-csi-node + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.node.podAnnotations }} + annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- end }} + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.node.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + hostNetwork: true + serviceAccountName: {{ .Values.serviceAccount.node.name }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-cluster-critical" }} + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- end }} + {{- with .Values.node.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: ebs-plugin + securityContext: + privileged: true + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + args: + - node + - --endpoint=$(CSI_ENDPOINT) + {{- if .Values.volumeAttachLimit }} + - --volume-attach-limit={{ .Values.volumeAttachLimit }} + {{- end }} + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: node-driver-registrar + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml index 947d241..4d09e4c 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml @@ -4,7 +4,6 @@ kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: name: ebs-snapshot-controller-leaderelection - namespace: kube-system labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} rules: diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml index 0670c70..e8248bd 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml @@ -4,13 +4,12 @@ kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: ebs-snapshot-controller-leaderelection - namespace: kube-system labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} subjects: - kind: ServiceAccount name: {{ .Values.serviceAccount.snapshot.name }} - namespace: kube-system + namespace: {{ .Release.Namespace }} roleRef: kind: Role name: ebs-snapshot-controller-leaderelection diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml index 8ec4c4e..0490c32 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml @@ -3,7 +3,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ .Values.serviceAccount.controller.name }} - namespace: kube-system labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} {{- with .Values.serviceAccount.controller.annotations }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml index afe0218..2e93f72 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml @@ -3,7 +3,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ .Values.serviceAccount.node.name }} - namespace: kube-system labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} {{- with .Values.serviceAccount.node.annotations }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml index 3b5ef2b..19d27cb 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml @@ -5,7 +5,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ .Values.serviceAccount.snapshot.name }} - namespace: kube-system labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} {{- with .Values.serviceAccount.snapshot.annotations }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml index aeb8351..ffde0ba 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml @@ -4,7 +4,6 @@ kind: StatefulSet apiVersion: apps/v1 metadata: name: ebs-snapshot-controller - namespace: kube-system labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} spec: @@ -40,6 +39,15 @@ spec: containers: - name: snapshot-controller image: {{ printf "%s:%s" .Values.snapshotController.repository .Values.snapshotController.tag }} + env: +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} args: - --v={{ .Values.logLevel }} - --leader-election=false diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml.orig b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml.orig new file mode 100644 index 0000000..7c594c3 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml.orig @@ -0,0 +1,60 @@ +{{- if .Values.enableVolumeSnapshot }} +#Snapshot controller +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: ebs-snapshot-controller + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + serviceName: ebs-snapshot-controller + replicas: 1 + selector: + matchLabels: + app: ebs-snapshot-controller + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: ebs-snapshot-controller + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + spec: + serviceAccountName: {{ .Values.serviceAccount.snapshot.name }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + tolerations: + {{- if .Values.tolerateAllTaints }} + - operator: Exists + {{- end }} + {{- with .Values.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: snapshot-controller + image: {{ printf "%s:%s" .Values.snapshotController.repository .Values.snapshotController.tag }} + env: +{{- if .Values.proxy.http_proxy }} + - name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} + - name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end }} + args: + - --v=5 + - --leader-election=false + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml index b10aa09..624d0e9 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml @@ -35,6 +35,10 @@ snapshotController: repository: k8s.gcr.io/sig-storage/snapshot-controller tag: "v3.0.3" +proxy: {} +# http_proxy: +# no_proxy: + imagePullSecrets: [] nameOverride: "" fullnameOverride: "" @@ -72,6 +76,20 @@ tolerateAllTaints: true tolerations: [] affinity: {} +# TSCs without the label selector stanza +# +# Example: +# +# topologySpreadConstraints: +# - maxSkew: 1 +# topologyKey: topology.kubernetes.io/zone +# whenUnsatisfiable: ScheduleAnyway +# - maxSkew: 1 +# topologyKey: kubernetes.io/hostname +# whenUnsatisfiable: ScheduleAnyway + +topologySpreadConstraints: [] + # Extra volume tags to attach to each dynamically provisioned volume. # --- # extraVolumeTags: diff --git a/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch b/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch index 39563e9..938d304 100644 --- a/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch +++ b/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch @@ -89,15 +89,6 @@ diff -rtub aws-ebs-csi-driver/templates/statefulset.yaml /tmp/aws-ebs-csi-driver - --leader-election=false {{- if .Values.imagePullSecrets }} imagePullSecrets: -diff -rtub aws-ebs-csi-driver/templates/storageclass.yaml /tmp/aws-ebs-csi-driver/templates/storageclass.yaml ---- aws-ebs-csi-driver/templates/storageclass.yaml 2021-02-23 18:54:24.000000000 +0100 -+++ /tmp/aws-ebs-csi-driver/templates/storageclass.yaml 2021-02-24 17:43:46.866722719 +0100 -@@ -1,4 +1,5 @@ - {{- range .Values.storageClasses }} -+--- - kind: StorageClass - apiVersion: storage.k8s.io/v1 - metadata: diff -rtub aws-ebs-csi-driver/values.yaml /tmp/aws-ebs-csi-driver/values.yaml --- aws-ebs-csi-driver/values.yaml 2021-02-23 18:54:24.000000000 +0100 +++ /tmp/aws-ebs-csi-driver/values.yaml 2021-02-24 18:41:15.513545244 +0100 diff --git a/charts/kubezero-aws-ebs-csi-driver/update.sh b/charts/kubezero-aws-ebs-csi-driver/update.sh index 771beed..1cb381a 100755 --- a/charts/kubezero-aws-ebs-csi-driver/update.sh +++ b/charts/kubezero-aws-ebs-csi-driver/update.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION=0.9.10 +VERSION=0.9.12 rm -rf charts/aws-ebs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts diff --git a/charts/kubezero-aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/values.yaml index 77bbef0..1f0051c 100644 --- a/charts/kubezero-aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/values.yaml @@ -21,6 +21,14 @@ aws-ebs-csi-driver: extraVolumeTags: {} # Name: KubeZero-Cluster + resources: + requests: + cpu: 5m + memory: 24Mi + limits: + cpu: 20m + memory: 40Mi + storageClasses: - name: ebs-sc-gp2-xfs volumeBindingMode: WaitForFirstConsumer -- 2.40.1 From b9af73a9ad93ace67e9ad85fbe33d943d9257f25 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 2 Mar 2021 11:28:13 +0100 Subject: [PATCH 25/65] Version bump charts --- charts/kubezero-istio-ingress/Chart.yaml | 6 +++--- charts/kubezero-istio/Chart.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/kubezero-istio-ingress/Chart.yaml b/charts/kubezero-istio-ingress/Chart.yaml index 76d0a00..f5aa064 100644 --- a/charts/kubezero-istio-ingress/Chart.yaml +++ b/charts/kubezero-istio-ingress/Chart.yaml @@ -3,7 +3,7 @@ name: kubezero-istio-ingress description: KubeZero Umbrella Chart for Istio based Ingress type: application version: 0.5.0 -appVersion: 1.9.0 +appVersion: 1.9.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -16,9 +16,9 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: istio-ingress - version: 1.9.0 + version: 1.9.1 condition: istio-ingress.enabled - name: istio-private-ingress - version: 1.9.0 + version: 1.9.1 condition: istio-private-ingress.enabled kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-istio/Chart.yaml b/charts/kubezero-istio/Chart.yaml index 708ab9e..bae339a 100644 --- a/charts/kubezero-istio/Chart.yaml +++ b/charts/kubezero-istio/Chart.yaml @@ -3,7 +3,7 @@ name: kubezero-istio description: KubeZero Umbrella Chart for Istio type: application version: 0.5.0 -appVersion: 1.9.0 +appVersion: 1.9.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -16,7 +16,7 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: base - version: 1.9.0 + version: 1.9.1 - name: istio-discovery - version: 1.9.0 + version: 1.9.1 kubeVersion: ">= 1.18.0" -- 2.40.1 From 2f5af18b984c1eea9bcbe65891831782dfdcbadf Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 2 Mar 2021 11:32:00 +0100 Subject: [PATCH 26/65] Slightly increase cpu limits for aws-ebs --- charts/kubezero-aws-ebs-csi-driver/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/values.yaml index 1f0051c..7ca7b72 100644 --- a/charts/kubezero-aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/values.yaml @@ -26,7 +26,7 @@ aws-ebs-csi-driver: cpu: 5m memory: 24Mi limits: - cpu: 20m + cpu: 50m memory: 40Mi storageClasses: -- 2.40.1 From 1f6c8c21ba92ada556bc8398df759921554d488b Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 2 Mar 2021 11:37:02 +0100 Subject: [PATCH 27/65] remove patch left overs --- .../templates/controller.yaml.orig | 231 ------------------ .../templates/node.yaml.orig | 174 ------------- .../templates/statefulset.yaml.orig | 60 ----- charts/kubezero-aws-ebs-csi-driver/update.sh | 2 +- 4 files changed, 1 insertion(+), 466 deletions(-) delete mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml.orig delete mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml.orig delete mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml.orig diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml.orig b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml.orig deleted file mode 100644 index 2e46432..0000000 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml.orig +++ /dev/null @@ -1,231 +0,0 @@ -# Controller Service -kind: Deployment -apiVersion: apps/v1 -metadata: - name: ebs-csi-controller - labels: - {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - app: ebs-csi-controller - {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app: ebs-csi-controller - {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} - {{- if .Values.podAnnotations }} - annotations: {{ toYaml .Values.podAnnotations | nindent 8 }} - {{- end }} - spec: - nodeSelector: - kubernetes.io/os: linux - {{- with .Values.nodeSelector }} -{{ toYaml . | indent 8 }} - {{- end }} - serviceAccountName: {{ .Values.serviceAccount.controller.name }} - priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} - {{- with .Values.affinity }} - affinity: {{ toYaml . | nindent 8 }} - {{- end }} - tolerations: - {{- if .Values.tolerateAllTaints }} - - operator: Exists - {{- end }} - {{- with .Values.tolerations }} -{{ toYaml . | indent 8 }} - {{- end }} -{{- if .Values.topologySpreadConstraints }} -{{- $tscLabelSelector := dict "labelSelector" ( dict "matchLabels" ( dict "app" "ebs-csi-controller" ) ) }} - topologySpreadConstraints: - {{- range .Values.topologySpreadConstraints }} - - {{ mergeOverwrite . $tscLabelSelector | toJson }} - {{- end }} -{{- end }} - containers: - - name: ebs-plugin - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: - {{- if ne .Release.Name "kustomize" }} - - controller - {{- else }} - # - {all,controller,node} # specify the driver mode - {{- end }} - - --endpoint=$(CSI_ENDPOINT) - {{- if .Values.extraVolumeTags }} - {{- include "aws-ebs-csi-driver.extra-volume-tags" . | nindent 12 }} - {{- end }} - {{- if .Values.k8sTagClusterId }} - - --k8s-tag-cluster-id={{ .Values.k8sTagClusterId }} - {{- end }} - - --logtostderr - - --v=5 - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: aws-secret - key: key_id - optional: true - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: aws-secret - key: access_key - optional: true - {{- if .Values.region }} - - name: AWS_REGION - value: {{ .Values.region }} - {{- end }} -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - ports: - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 10 - failureThreshold: 5 - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - - name: csi-provisioner - image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} - args: - - --csi-address=$(ADDRESS) - - --v=5 - {{- if .Values.enableVolumeScheduling }} - - --feature-gates=Topology=true - {{- end}} - {{- if .Values.extraCreateMetadata }} - - --extra-create-metadata - {{- end}} - - --leader-election=true - - --default-fstype=ext4 - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - - name: csi-attacher - image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} - args: - - --csi-address=$(ADDRESS) - - --v=5 - - --leader-election=true - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.enableVolumeSnapshot }} - - name: csi-snapshotter - image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} - args: - - --csi-address=$(ADDRESS) - - --leader-election=true - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- end }} - {{- if .Values.enableVolumeResizing }} - - name: csi-resizer - image: {{ printf "%s:%s" .Values.sidecars.resizerImage.repository .Values.sidecars.resizerImage.tag }} - imagePullPolicy: Always - args: - - --csi-address=$(ADDRESS) - - --v=5 - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} - volumeMounts: - - name: socket-dir - mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- end }} - - name: liveness-probe - image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} - args: - - --csi-address=/csi/csi.sock - volumeMounts: - - name: socket-dir - mountPath: /csi - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- if .Values.imagePullSecrets }} - imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} - volumes: - - name: socket-dir - emptyDir: {} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml.orig b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml.orig deleted file mode 100644 index bf9c6e0..0000000 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml.orig +++ /dev/null @@ -1,174 +0,0 @@ -# Node Service -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: ebs-csi-node - labels: - {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app: ebs-csi-node - {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} - {{- if .Values.node.podAnnotations }} - annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} - {{- end }} - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: eks.amazonaws.com/compute-type - operator: NotIn - values: - - fargate - nodeSelector: - kubernetes.io/os: linux - {{- with .Values.node.nodeSelector }} -{{ toYaml . | indent 8 }} - {{- end }} - hostNetwork: true - serviceAccountName: {{ .Values.serviceAccount.node.name }} - priorityClassName: {{ .Values.node.priorityClassName | default "system-cluster-critical" }} - tolerations: - {{- if .Values.node.tolerateAllTaints }} - - operator: Exists - {{- end }} - {{- with .Values.node.tolerations }} -{{ toYaml . | indent 8 }} - {{- end }} - containers: - - name: ebs-plugin - securityContext: - privileged: true - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - args: - - node - - --endpoint=$(CSI_ENDPOINT) - {{- if .Values.volumeAttachLimit }} - - --volume-attach-limit={{ .Values.volumeAttachLimit }} - {{- end }} - - --logtostderr - - --v=5 - env: - - name: CSI_ENDPOINT - value: unix:/csi/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} - volumeMounts: - - name: kubelet-dir - mountPath: /var/lib/kubelet - mountPropagation: "Bidirectional" - - name: plugin-dir - mountPath: /csi - - name: device-dir - mountPath: /dev - ports: - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 10 - failureThreshold: 5 - {{- if .Values.node.resources }} - {{- with .Values.node.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- else }} - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- end }} - - name: node-driver-registrar - image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v=5 - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"] - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - {{- if .Values.node.resources }} - {{- with .Values.node.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- else }} - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- end }} - - name: liveness-probe - image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} - args: - - --csi-address=/csi/csi.sock - volumeMounts: - - name: plugin-dir - mountPath: /csi - {{- if .Values.node.resources }} - {{- with .Values.node.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- else }} - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- end }} - {{- if .Values.imagePullSecrets }} - imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} - volumes: - - name: kubelet-dir - hostPath: - path: /var/lib/kubelet - type: Directory - - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ - type: DirectoryOrCreate - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - - name: device-dir - hostPath: - path: /dev - type: Directory diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml.orig b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml.orig deleted file mode 100644 index 7c594c3..0000000 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml.orig +++ /dev/null @@ -1,60 +0,0 @@ -{{- if .Values.enableVolumeSnapshot }} -#Snapshot controller -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: ebs-snapshot-controller - labels: - {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} -spec: - serviceName: ebs-snapshot-controller - replicas: 1 - selector: - matchLabels: - app: ebs-snapshot-controller - {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app: ebs-snapshot-controller - {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} - spec: - serviceAccountName: {{ .Values.serviceAccount.snapshot.name }} - nodeSelector: - kubernetes.io/os: linux - {{- with .Values.nodeSelector }} -{{ toYaml . | indent 8 }} - {{- end }} - priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} - {{- with .Values.affinity }} - affinity: {{ toYaml . | nindent 8 }} - {{- end }} - tolerations: - {{- if .Values.tolerateAllTaints }} - - operator: Exists - {{- end }} - {{- with .Values.tolerations }} -{{ toYaml . | indent 8 }} - {{- end }} - containers: - - name: snapshot-controller - image: {{ printf "%s:%s" .Values.snapshotController.repository .Values.snapshotController.tag }} - env: -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} - args: - - --v=5 - - --leader-election=false - {{- if .Values.imagePullSecrets }} - imagePullSecrets: - {{- range .Values.imagePullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/update.sh b/charts/kubezero-aws-ebs-csi-driver/update.sh index 1cb381a..a2ccd69 100755 --- a/charts/kubezero-aws-ebs-csi-driver/update.sh +++ b/charts/kubezero-aws-ebs-csi-driver/update.sh @@ -5,4 +5,4 @@ VERSION=0.9.12 rm -rf charts/aws-ebs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts -patch -d charts -i ../loglevel_leader.patch -p0 +patch -d charts -i ../loglevel_leader.patch -p0 --no-backup-if-mismatch -- 2.40.1 From 1dd10cc1a8c3847aec1c885fff6642796619b3c7 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 3 Mar 2021 10:59:12 +0100 Subject: [PATCH 28/65] aws-ebs-csi-driver version bump introducing readiness probes --- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 2 +- .../charts/aws-ebs-csi-driver/Chart.yaml | 2 +- .../charts/aws-ebs-csi-driver/templates/controller.yaml | 8 ++++++++ charts/kubezero-aws-ebs-csi-driver/update.sh | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index ddfe3fe..30ac280 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -18,7 +18,7 @@ maintainers: - name: Quarky9 dependencies: - name: aws-ebs-csi-driver - version: 0.9.12 + version: 0.9.13 repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: kubezero-lib version: ">= 0.1.3" diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml index d9476e4..5804f5c 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml @@ -15,4 +15,4 @@ maintainers: name: aws-ebs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver -version: 0.9.12 +version: 0.9.13 diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml index af5e0f1..f84b1bc 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -105,6 +105,14 @@ spec: timeoutSeconds: 3 periodSeconds: 10 failureThreshold: 5 + readinessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 {{- with .Values.resources }} resources: {{ toYaml . | nindent 12 }} {{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/update.sh b/charts/kubezero-aws-ebs-csi-driver/update.sh index a2ccd69..afa8703 100755 --- a/charts/kubezero-aws-ebs-csi-driver/update.sh +++ b/charts/kubezero-aws-ebs-csi-driver/update.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION=0.9.12 +VERSION=0.9.13 rm -rf charts/aws-ebs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts -- 2.40.1 From ff3510ec4bccb41010cb454b36c6d4e51e2a5f8e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 5 Mar 2021 10:22:54 +0100 Subject: [PATCH 29/65] remove default cpu limmits for kiam --- charts/kubezero-kiam/Chart.yaml | 2 +- charts/kubezero-kiam/values.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index 5b69919..2a323db 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-kiam description: KubeZero Umbrella Chart for Kiam type: application -version: 0.3.0 +version: 0.3.1 appVersion: "4.0" home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-kiam/values.yaml b/charts/kubezero-kiam/values.yaml index 00e2530..cb25cc6 100644 --- a/charts/kubezero-kiam/values.yaml +++ b/charts/kubezero-kiam/values.yaml @@ -22,10 +22,10 @@ kiam: resources: requests: memory: "50Mi" - cpu: "100m" + cpu: "50m" limits: memory: "50Mi" - cpu: "300m" + # cpu: "300m" tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule @@ -61,7 +61,7 @@ kiam: cpu: "50m" limits: memory: "20Mi" - cpu: "50m" + # cpu: "50m" tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule -- 2.40.1 From add271c4473563ca565243c6b3e9c3539932b4c2 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 5 Mar 2021 10:32:42 +0100 Subject: [PATCH 30/65] aws-ebs-csi-driver version bump, remove cpu limts --- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 4 +- .../charts/aws-ebs-csi-driver/Chart.yaml | 2 +- .../templates/controller.yaml | 6 +++ .../aws-ebs-csi-driver/templates/node.yaml | 10 ++-- .../loglevel_leader.patch | 50 +++++++++---------- charts/kubezero-aws-ebs-csi-driver/update.sh | 2 +- .../kubezero-aws-ebs-csi-driver/values.yaml | 4 +- 7 files changed, 43 insertions(+), 35 deletions(-) diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index 30ac280..23599cf 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.4.2 +version: 0.4.3 appVersion: 0.9.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -18,7 +18,7 @@ maintainers: - name: Quarky9 dependencies: - name: aws-ebs-csi-driver - version: 0.9.13 + version: 0.9.14 repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: kubezero-lib version: ">= 0.1.3" diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml index 5804f5c..9a5a2b9 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml @@ -15,4 +15,4 @@ maintainers: name: aws-ebs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver -version: 0.9.13 +version: 0.9.14 diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml index f84b1bc..ccd7489 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -33,6 +33,12 @@ spec: tolerations: {{- if .Values.tolerateAllTaints }} - operator: Exists + {{- else }} + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 {{- end }} {{- with .Values.tolerations }} {{ toYaml . | indent 8 }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml index 5de7469..5caeb33 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml @@ -39,6 +39,12 @@ spec: tolerations: {{- if .Values.node.tolerateAllTaints }} - operator: Exists + {{- else }} + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 {{- end }} {{- with .Values.node.tolerations }} {{ toYaml . | indent 8 }} @@ -102,10 +108,6 @@ spec: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - --v={{ .Values.logLevel }} - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"] env: - name: ADDRESS value: /csi/csi.sock diff --git a/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch b/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch index 938d304..349eea0 100644 --- a/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch +++ b/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch @@ -1,7 +1,7 @@ -diff -rtub aws-ebs-csi-driver/templates/controller.yaml /tmp/aws-ebs-csi-driver/templates/controller.yaml ---- aws-ebs-csi-driver/templates/controller.yaml 2021-02-23 18:54:24.000000000 +0100 -+++ /tmp/aws-ebs-csi-driver/templates/controller.yaml 2021-02-24 18:40:00.753541864 +0100 -@@ -56,7 +56,7 @@ +diff -rtubN aws-ebs-csi-driver.orig/templates/controller.yaml aws-ebs-csi-driver/templates/controller.yaml +--- aws-ebs-csi-driver.orig/templates/controller.yaml 2021-03-05 03:10:41.000000000 +0100 ++++ aws-ebs-csi-driver/templates/controller.yaml 2021-03-05 10:29:31.878615411 +0100 +@@ -68,7 +68,7 @@ - --k8s-tag-cluster-id={{ .Values.k8sTagClusterId }} {{- end }} - --logtostderr @@ -10,7 +10,7 @@ diff -rtub aws-ebs-csi-driver/templates/controller.yaml /tmp/aws-ebs-csi-driver/ env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock -@@ -98,14 +98,14 @@ +@@ -126,14 +126,14 @@ image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} args: - --csi-address=$(ADDRESS) @@ -27,7 +27,7 @@ diff -rtub aws-ebs-csi-driver/templates/controller.yaml /tmp/aws-ebs-csi-driver/ - --default-fstype=ext4 env: - name: ADDRESS -@@ -120,8 +120,8 @@ +@@ -156,8 +156,8 @@ image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} args: - --csi-address=$(ADDRESS) @@ -38,7 +38,7 @@ diff -rtub aws-ebs-csi-driver/templates/controller.yaml /tmp/aws-ebs-csi-driver/ env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock -@@ -136,7 +136,7 @@ +@@ -180,7 +180,7 @@ image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} args: - --csi-address=$(ADDRESS) @@ -47,7 +47,7 @@ diff -rtub aws-ebs-csi-driver/templates/controller.yaml /tmp/aws-ebs-csi-driver/ env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock -@@ -153,7 +153,7 @@ +@@ -205,7 +205,7 @@ imagePullPolicy: Always args: - --csi-address=$(ADDRESS) @@ -56,10 +56,10 @@ diff -rtub aws-ebs-csi-driver/templates/controller.yaml /tmp/aws-ebs-csi-driver/ env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock -diff -rtub aws-ebs-csi-driver/templates/node.yaml /tmp/aws-ebs-csi-driver/templates/node.yaml ---- aws-ebs-csi-driver/templates/node.yaml 2021-02-23 18:54:24.000000000 +0100 -+++ /tmp/aws-ebs-csi-driver/templates/node.yaml 2021-02-24 18:41:44.630213228 +0100 -@@ -56,7 +56,7 @@ +diff -rtubN aws-ebs-csi-driver.orig/templates/node.yaml aws-ebs-csi-driver/templates/node.yaml +--- aws-ebs-csi-driver.orig/templates/node.yaml 2021-03-05 03:10:41.000000000 +0100 ++++ aws-ebs-csi-driver/templates/node.yaml 2021-03-05 10:30:07.391950366 +0100 +@@ -61,7 +61,7 @@ - --volume-attach-limit={{ .Values.volumeAttachLimit }} {{- end }} - --logtostderr @@ -68,30 +68,30 @@ diff -rtub aws-ebs-csi-driver/templates/node.yaml /tmp/aws-ebs-csi-driver/templa env: - name: CSI_ENDPOINT value: unix:/csi/csi.sock -@@ -94,7 +94,7 @@ +@@ -107,7 +107,7 @@ args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v=5 + - --v={{ .Values.logLevel }} - lifecycle: - preStop: - exec: -diff -rtub aws-ebs-csi-driver/templates/statefulset.yaml /tmp/aws-ebs-csi-driver/templates/statefulset.yaml ---- aws-ebs-csi-driver/templates/statefulset.yaml 2021-02-23 18:54:24.000000000 +0100 -+++ /tmp/aws-ebs-csi-driver/templates/statefulset.yaml 2021-02-24 18:42:07.223547582 +0100 -@@ -41,7 +41,7 @@ - - name: snapshot-controller - image: {{ printf "%s:%s" .Values.snapshotController.repository .Values.snapshotController.tag }} + env: + - name: ADDRESS + value: /csi/csi.sock +diff -rtubN aws-ebs-csi-driver.orig/templates/statefulset.yaml aws-ebs-csi-driver/templates/statefulset.yaml +--- aws-ebs-csi-driver.orig/templates/statefulset.yaml 2021-03-05 03:10:41.000000000 +0100 ++++ aws-ebs-csi-driver/templates/statefulset.yaml 2021-03-05 10:29:31.881948744 +0100 +@@ -49,7 +49,7 @@ + value: {{ .Values.proxy.no_proxy | quote }} + {{- end }} args: - - --v=5 + - --v={{ .Values.logLevel }} - --leader-election=false {{- if .Values.imagePullSecrets }} imagePullSecrets: -diff -rtub aws-ebs-csi-driver/values.yaml /tmp/aws-ebs-csi-driver/values.yaml ---- aws-ebs-csi-driver/values.yaml 2021-02-23 18:54:24.000000000 +0100 -+++ /tmp/aws-ebs-csi-driver/values.yaml 2021-02-24 18:41:15.513545244 +0100 +diff -rtubN aws-ebs-csi-driver.orig/values.yaml aws-ebs-csi-driver/values.yaml +--- aws-ebs-csi-driver.orig/values.yaml 2021-03-05 03:10:41.000000000 +0100 ++++ aws-ebs-csi-driver/values.yaml 2021-03-05 10:29:31.881948744 +0100 @@ -9,6 +9,8 @@ tag: "v0.9.0" pullPolicy: IfNotPresent diff --git a/charts/kubezero-aws-ebs-csi-driver/update.sh b/charts/kubezero-aws-ebs-csi-driver/update.sh index afa8703..e748fa3 100755 --- a/charts/kubezero-aws-ebs-csi-driver/update.sh +++ b/charts/kubezero-aws-ebs-csi-driver/update.sh @@ -1,6 +1,6 @@ #!/bin/bash -VERSION=0.9.13 +VERSION=0.9.14 rm -rf charts/aws-ebs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts diff --git a/charts/kubezero-aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/values.yaml index 7ca7b72..b7cee72 100644 --- a/charts/kubezero-aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/values.yaml @@ -23,10 +23,10 @@ aws-ebs-csi-driver: resources: requests: - cpu: 5m + cpu: 50m memory: 24Mi limits: - cpu: 50m + # cpu: 50m memory: 40Mi storageClasses: -- 2.40.1 From b4ccbe6955b5ac07418a64b946c51454fd3fccae Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 5 Mar 2021 13:58:54 +0100 Subject: [PATCH 31/65] More request tuning for aws-ebs-csi --- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 2 +- charts/kubezero-aws-ebs-csi-driver/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index 23599cf..0f0e48a 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.4.3 +version: 0.4.4 appVersion: 0.9.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/values.yaml index b7cee72..8519805 100644 --- a/charts/kubezero-aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/values.yaml @@ -23,7 +23,7 @@ aws-ebs-csi-driver: resources: requests: - cpu: 50m + cpu: 10m memory: 24Mi limits: # cpu: 50m -- 2.40.1 From f6abd9b8947300bbe4335c48c8179b1559da63a8 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 5 Mar 2021 14:00:00 +0100 Subject: [PATCH 32/65] remove cpu limit for aws-iam-auth, enable cpufs kubelet feature flag --- charts/kubeadm/Chart.yaml | 2 +- charts/kubeadm/templates/KubeletConfiguration.yaml | 1 + charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/charts/kubeadm/Chart.yaml b/charts/kubeadm/Chart.yaml index c73bea0..ba8a347 100644 --- a/charts/kubeadm/Chart.yaml +++ b/charts/kubeadm/Chart.yaml @@ -10,4 +10,4 @@ keywords: - kubeadm maintainers: - name: Quarky9 -kubeVersion: ">= 1.16.0" +kubeVersion: ">= 1.18.0" diff --git a/charts/kubeadm/templates/KubeletConfiguration.yaml b/charts/kubeadm/templates/KubeletConfiguration.yaml index 7002a7f..442e4bf 100644 --- a/charts/kubeadm/templates/KubeletConfiguration.yaml +++ b/charts/kubeadm/templates/KubeletConfiguration.yaml @@ -20,6 +20,7 @@ tlsCipherSuites: [TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES featureGates: CSIMigrationAWS: true CSIMigrationAWSComplete: true + CustomCPUCFSQuotaPeriod: true {{- end }} kubeReserved: cpu: 50m diff --git a/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml b/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml index fe40f3c..7224642 100644 --- a/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml +++ b/charts/kubeadm/templates/aws-iam-authenticator/deployment.yaml @@ -137,7 +137,7 @@ spec: cpu: 10m limits: memory: 20Mi - cpu: 100m + #cpu: 100m volumeMounts: - name: config -- 2.40.1 From 5d57cf548065f3a56be9d9c9364ef31c8b79e224 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 5 Mar 2021 16:53:02 +0100 Subject: [PATCH 33/65] Version upgrade ES/Kibana and Fluentbit, various tunings --- charts/kubezero-logging/Chart.yaml | 8 +- .../charts/fluent-bit/Chart.yaml | 35 +- .../fluent-bit/conf/custom_parsers.conf | 6 - .../charts/fluent-bit/conf/fluent-bit.conf | 76 - .../charts/fluent-bit/conf/functions.lua | 28 - .../fluent-bit/dashboards/fluent-bit.json | 1305 +++++++++++++++++ .../charts/fluent-bit/templates/_pod.tpl | 29 +- .../templates/configmap-dashboards.yaml | 20 + .../templates/configmap-luascripts.yaml | 12 + .../fluent-bit/templates/configmap.yaml | 9 +- .../fluent-bit/templates/daemonset.yaml | 1 + .../fluent-bit/templates/deployment.yaml | 1 + .../fluent-bit/templates/networkpolicy.yaml | 22 + .../fluent-bit/templates/prometheusrule.yaml | 20 + .../templates/tests/test-connection.yaml | 9 +- .../charts/fluent-bit/values.yaml | 126 +- charts/kubezero-logging/fluent-bit.patch | 37 + .../templates/eck/elasticsearch.yaml | 26 +- .../templates/eck/kibana.yaml | 2 +- charts/kubezero-logging/update.sh | 8 + charts/kubezero-logging/values.yaml | 131 +- 21 files changed, 1734 insertions(+), 177 deletions(-) delete mode 100644 charts/kubezero-logging/charts/fluent-bit/conf/custom_parsers.conf delete mode 100644 charts/kubezero-logging/charts/fluent-bit/conf/fluent-bit.conf delete mode 100644 charts/kubezero-logging/charts/fluent-bit/conf/functions.lua create mode 100644 charts/kubezero-logging/charts/fluent-bit/dashboards/fluent-bit.json create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/configmap-dashboards.yaml create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/configmap-luascripts.yaml create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/networkpolicy.yaml create mode 100644 charts/kubezero-logging/charts/fluent-bit/templates/prometheusrule.yaml create mode 100644 charts/kubezero-logging/fluent-bit.patch create mode 100755 charts/kubezero-logging/update.sh diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index a78f85d..f030c77 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application -version: 0.5.3 -appVersion: 1.3.1 +version: 0.6.0 +appVersion: 1.4.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -19,13 +19,13 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: eck-operator - version: 1.3.1 + version: 1.4.0 repository: https://helm.elastic.co condition: eck-operator.enabled - name: fluentd version: 2.5.3 condition: fluentd.enabled - name: fluent-bit - version: 0.7.14 + version: 0.12.3 condition: fluent-bit.enabled kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml index f29b870..f83e759 100644 --- a/charts/kubezero-logging/charts/fluent-bit/Chart.yaml +++ b/charts/kubezero-logging/charts/fluent-bit/Chart.yaml @@ -1,20 +1,23 @@ +annotations: + artifacthub.io/changes: | + - add custom annotations on dashboards configmap apiVersion: v2 -name: fluent-bit +appVersion: 1.7.1 description: Fast and lightweight log processor and forwarder or Linux, OSX and BSD family operating systems. -keywords: - - logging - - fluent-bit - - fluentd -version: 0.7.14 -appVersion: 1.6.8 -icon: https://fluentbit.io/assets/img/logo1-default.png home: https://fluentbit.io/ -sources: - - https://github.com/fluent/fluent-bit/ +icon: https://fluentbit.io/assets/img/logo1-default.png +keywords: +- logging +- fluent-bit +- fluentd maintainers: - - name: edsiper - email: eduardo@treasure-data.com - - name: naseemkullah - email: naseem@transit.app - - name: Towmeykaw - email: towmeykaw@gmail.com +- email: eduardo@treasure-data.com + name: edsiper +- email: naseem@transit.app + name: naseemkullah +- email: towmeykaw@gmail.com + name: Towmeykaw +name: fluent-bit +sources: +- https://github.com/fluent/fluent-bit/ +version: 0.12.3 diff --git a/charts/kubezero-logging/charts/fluent-bit/conf/custom_parsers.conf b/charts/kubezero-logging/charts/fluent-bit/conf/custom_parsers.conf deleted file mode 100644 index a2055db..0000000 --- a/charts/kubezero-logging/charts/fluent-bit/conf/custom_parsers.conf +++ /dev/null @@ -1,6 +0,0 @@ -[PARSER] - Name cri-log - Format regex - Regex ^(?