From 5da613de64fd4f752d945beb7b2464feb18b041f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 15 May 2020 16:07:01 +0100 Subject: [PATCH 01/31] First kiam implementation incl. docs --- charts/kubezero-kiam/Chart.yaml | 2 +- charts/kubezero-kiam/README.md | 69 + charts/kubezero-kiam/README.md.gotmpl | 40 + .../kubezero-kiam/kiam-grafana-dashboard.json | 2933 +++++++++++++++++ charts/kubezero-kiam/kiam_architecure.png | Bin 0 -> 43992 bytes .../kubezero-kiam/templates/certificates.yaml | 28 + charts/kubezero-kiam/templates/namespace.yaml | 6 + charts/kubezero-kiam/values.yaml | 15 +- 8 files changed, 3085 insertions(+), 8 deletions(-) create mode 100644 charts/kubezero-kiam/README.md create mode 100644 charts/kubezero-kiam/README.md.gotmpl create mode 100644 charts/kubezero-kiam/kiam-grafana-dashboard.json create mode 100644 charts/kubezero-kiam/kiam_architecure.png create mode 100644 charts/kubezero-kiam/templates/certificates.yaml create mode 100644 charts/kubezero-kiam/templates/namespace.yaml diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index f341d4a..d2fc880 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-kiam description: KubeZero Umbrella Chart for Kiam type: application -version: 0.1.0 +version: 0.1.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: diff --git a/charts/kubezero-kiam/README.md b/charts/kubezero-kiam/README.md new file mode 100644 index 0000000..9eee72a --- /dev/null +++ b/charts/kubezero-kiam/README.md @@ -0,0 +1,69 @@ +kubezero-kiam +============= +KubeZero Umbrella Chart for Kiam + +Current chart version is `0.1.1` + +Source code can be found [here](https://kubezero.com) + +## Chart Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://uswitch.github.io/kiam-helm-charts/charts/ | kiam | 5.7.0 | + +## KubeZero default configuration +We run agents on the controllers as well, so we force eg. ebs csi controllers and others to assume roles etc. +This means we need to run kiam containers on the controllers using `hostnetwork: true`. +Therefore we also change the default port from 443 to 6444 to not collide with the potential api-server port on the controllers. +Make sure any firewall rules between controllers and workers are adjusted accordingly. + +## Kiam Certificates +The required certificates for Kiam server and agents are provided by a local cert-manager, which is configured to have a cluster local self-signing CA as part of the KubeZero platform. +[Kiam TLS Config](https://github.com/uswitch/kiam/blob/master/docs/TLS.md#cert-manager) +[KubeZero cert-manager](../kubezero-cert-manager/README.md) + +## Metadata restrictions +Required for the *csi ebs plugin* and most likely various others assuming basic AWS information. + +- `/latest/meta-data/instance-id` +- `/latest/dynamic/instance-identity/document` + +## Chart Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| kiam.agent.host.iptables | bool | `true` | | +| kiam.agent.log.level | string | `"warn"` | | +| kiam.agent.prometheus.servicemonitor.enabled | bool | `false` | | +| kiam.agent.sslCertHostPath | string | `"/etc/ssl/certs"` | | +| kiam.agent.tlsSecret | string | `"kiam-agent-tls"` | | +| kiam.agent.tolerations[0].effect | string | `"NoSchedule"` | | +| kiam.agent.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| kiam.agent.whiteListRouteRegexp | string | `"^/latest/(meta-data/instance-id|dynamic)"` | | +| kiam.server.assumeRoleArn | string | `"arn:aws:iam::123456789012:role/kiam-server-role"` | kiam server IAM role to assume, required as we run the agents next to the servers normally | +| kiam.server.deployment.enabled | bool | `true` | | +| kiam.server.deployment.replicas | int | `1` | | +| kiam.server.log.level | string | `"warn"` | | +| kiam.server.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| kiam.server.prometheus.servicemonitor.enabled | bool | `false` | | +| kiam.server.service.port | int | `6444` | | +| kiam.server.service.targetPort | int | `6444` | | +| kiam.server.sslCertHostPath | string | `"/etc/ssl/certs"` | | +| kiam.server.tlsSecret | string | `"kiam-server-tls"` | | +| kiam.server.tolerations[0].effect | string | `"NoSchedule"` | | +| kiam.server.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| kiam.server.useHostNetwork | bool | `true` | | + +## Debugging +- Verify iptables rules on hosts to be set by the kiam agent: + `iptables -L -t nat -n --line-numbers` + `iptables -t nat -D PREROUTING ` + +## Resources +- https://github.com/uswitch/kiam +- https://www.bluematador.com/blog/iam-access-in-kubernetes-kube2iam-vs-kiam + +--- +![Architecture](kiam_architecure.png) +Image Credits: Blue Matador, Inc. diff --git a/charts/kubezero-kiam/README.md.gotmpl b/charts/kubezero-kiam/README.md.gotmpl new file mode 100644 index 0000000..037f292 --- /dev/null +++ b/charts/kubezero-kiam/README.md.gotmpl @@ -0,0 +1,40 @@ +{{ template "chart.header" . }} +{{ template "chart.description" . }} + +{{ template "chart.versionLine" . }} + +{{ template "chart.sourceLinkLine" . }} + +{{ template "chart.requirementsSection" . }} + +## KubeZero default configuration +We run agents on the controllers as well, so we force eg. ebs csi controllers and others to assume roles etc. +This means we need to run kiam containers on the controllers using `hostnetwork: true`. +Therefore we also change the default port from 443 to 6444 to not collide with the potential api-server port on the controllers. +Make sure any firewall rules between controllers and workers are adjusted accordingly. + +## Kiam Certificates +The required certificates for Kiam server and agents are provided by a local cert-manager, which is configured to have a cluster local self-signing CA as part of the KubeZero platform. +[Kiam TLS Config](https://github.com/uswitch/kiam/blob/master/docs/TLS.md#cert-manager) +[KubeZero cert-manager](../kubezero-cert-manager/README.md) + +## Metadata restrictions +Required for the *csi ebs plugin* and most likely various others assuming basic AWS information. + +- `/latest/meta-data/instance-id` +- `/latest/dynamic/instance-identity/document` + +{{ template "chart.valuesSection" . }} + +## Debugging +- Verify iptables rules on hosts to be set by the kiam agent: + `iptables -L -t nat -n --line-numbers` + `iptables -t nat -D PREROUTING ` + +## Resources +- https://github.com/uswitch/kiam +- https://www.bluematador.com/blog/iam-access-in-kubernetes-kube2iam-vs-kiam + +--- +![Architecture](kiam_architecure.png) +Image Credits: Blue Matador, Inc. diff --git a/charts/kubezero-kiam/kiam-grafana-dashboard.json b/charts/kubezero-kiam/kiam-grafana-dashboard.json new file mode 100644 index 0000000..61dde81 --- /dev/null +++ b/charts/kubezero-kiam/kiam-grafana-dashboard.json @@ -0,0 +1,2933 @@ +{ + "__inputs": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Kiam cache, response & health metrics", + "editable": true, + "gnetId": 3831, + "graphTooltip": 0, + "id": null, + "iteration": 1533922855243, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 17, + "panels": [], + "title": "Stats", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (pod, handler) (delta(kiam_metadata_success_total[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{handler}} - {{pod}} ", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of successful responses from a handler", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 37, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code, pod) (delta(kiam_metadata_responses_total[$interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{code}} {{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Responses from mocked out metadata handlers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (pod) (delta(kiam_metadata_empty_role_total[$interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of empty roles returned", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (handler, pod) (delta(kiam_metadata_find_role_errors_total[$interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{handler}} {{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of errors finding the role for a pod", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 39, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (handler, pod) (delta(kiam_metadata_find_role_errors_total[$interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{handler}} {{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of errors finding the role for a pod", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 41, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (pod) (delta(kiam_sts_issuing_errors_total[$interval]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of errors issuing credentials", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 45, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (pod) (kiam_k8s_dropped_pods_total)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of dropped pods because of full buffer", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Number of assume role calls currently executing", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 60, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dimensions": {}, + "expr": "sum by (pod) (kiam_sts_assumerole_current)", + "format": "time_series", + "highResolution": false, + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "metricName": "", + "namespace": "", + "period": "", + "refId": "A", + "region": "default", + "statistics": [ + "Average" + ], + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of assume role calls currently executing", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (delta(kiam_sts_cache_hit_total[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of cache hits to the metadata cache", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 40, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (delta(kiam_sts_cache_miss_total[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of cache misses to the metadata cache", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "linear", + "colorScheme": "interpolateReds", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "$datasource", + "description": "Bucketed histogram of handler timings. Tagged by handler", + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 24 + }, + "heatmap": {}, + "highlightCards": true, + "id": 9, + "legend": { + "show": false + }, + "links": [], + "targets": [ + { + "expr": "sum(rate(kiam_metadata_handler_latency_seconds_bucket{handler=\"credentials\"}[$interval])) by (le)", + "format": "heatmap", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A", + "target": "" + } + ], + "title": "Credentials responsetime", + "tooltip": { + "show": true, + "showHistogram": false + }, + "transparent": false, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": 0 + }, + "color": { + "cardColor": "#bf1b00", + "colorScale": "sqrt", + "colorScheme": "interpolateReds", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "$datasource", + "description": "Bucketed histogram of handler timings. Tagged by handler", + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 24 + }, + "heatmap": {}, + "highlightCards": true, + "id": 10, + "legend": { + "show": false + }, + "links": [], + "targets": [ + { + "expr": "sum(rate(kiam_metadata_handler_latency_seconds_bucket{handler=\"roleName\"}[$interval])) by (le)", + "format": "heatmap", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A", + "target": "" + } + ], + "title": "roleHandler responsetime", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "ms", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "upper", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "linear", + "colorScheme": "interpolateReds", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "$datasource", + "description": "Bucketed histogram of assumeRole timings", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 29 + }, + "heatmap": {}, + "highlightCards": true, + "id": 42, + "legend": { + "show": false + }, + "links": [], + "targets": [ + { + "expr": "sum(rate(kiam_sts_assumerole_timing_seconds_bucket[$interval])) by (le)", + "format": "heatmap", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A", + "target": "" + } + ], + "title": "assumeRole timings", + "tooltip": { + "show": true, + "showHistogram": false + }, + "transparent": false, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 15, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 25 + }, + "id": 27, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(kube_daemonset_status_number_available{namespace=\"$namespace\",daemonset=~\".*kiam-server\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Available Replicas", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 10, + "x": 2, + "y": 25 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",pod_name=~\".*kiam-server.*\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "CPU usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Out", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"$namespace\",pod_name=~\".*kiam-server-.*\"}[$interval])) by (pod_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "In {{pod_name}}", + "refId": "B" + }, + { + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"$namespace\",pod_name=~\".*kiam-server-.*\"}[$interval])) by (pod_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Out {{pod_name}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network traffic", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 28 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(kube_daemonset_status_desired_number_scheduled{namespace=\"$namespace\",daemonset=~\".*kiam-server\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Desired Replicas", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "decbytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 10, + "x": 2, + "y": 28 + }, + "id": 31, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{namespace=\"$namespace\",pod_name=~\".*kiam-server-.*\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Memory usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + } + ], + "title": "Server", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 19, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 26 + }, + "id": 28, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(kube_daemonset_status_number_available{namespace=\"$namespace\",daemonset=~\".*kiam-agent\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Available Replicas", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 10, + "x": 2, + "y": 26 + }, + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\",pod_name=~\".*kiam-agent.*\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "CPU usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Out/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"$namespace\",pod_name=~\".*kiam-agent-.*\"}[$interval])) by (pod_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "In {{pod_name}}", + "refId": "B" + }, + { + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"$namespace\",pod_name=~\".*kiam-agent-.*\"}[$interval])) by (pod_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Out {{pod_name}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network traffic", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 29 + }, + "id": 22, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(kube_daemonset_status_desired_number_scheduled{namespace=\"$namespace\",daemonset=~\".*kiam-agent\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Desired Replicas", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "decbytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 10, + "x": 2, + "y": 29 + }, + "id": 32, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{namespace=\"$namespace\",pod_name=~\".*kiam-agent-.*\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Memory usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + } + ], + "title": "Agent", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 47, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total number of RPCs completed on the server, regardless of success or failure.", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 49, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (increase(grpc_server_handled_total{namespace=\"$namespace\",pod=~\".*kiam-.*\"}[$interval])) by (grpc_code, grpc_method, pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{grpc_method}} {{grpc_code}} {{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "grpc server handled total", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total number of RPCs started on the server.", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 52, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Rate.*/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (increase(grpc_server_started_total{namespace=\"$namespace\",pod=~\".*kiam-server-.*\"}[$interval])) by (grpc_method, pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{grpc_method}} {{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "grpc server started total", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total number of RPC stream messages received on the server.", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 50, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (increase(grpc_server_msg_received_total{namespace=\"$namespace\",pod=~\".*kiam-.*\"}[$interval])) by (grpc_method, pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{grpc_method}} {{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "grpc server msg received total", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total number of gRPC stream messages sent by the server.", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 51, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (increase(grpc_server_msg_sent_total{namespace=\"$namespace\",pod=~\".*kiam-.*\"}[$interval])) by (grpc_method, pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{grpc_method}} {{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "grpc server msg sent total", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Grpc server stats", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 54, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total number of RPCs completed by the client, regardless of success or failure.", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 55, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (increase(grpc_client_handled_total{namespace=\"$namespace\",pod=~\".*kiam-.*\"}[$interval])) by (grpc_code, grpc_method, pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{grpc_method}} {{grpc_code}} {{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "grpc client handled total", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total number of RPCs started on the client.", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 58, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Rate.*/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (increase(grpc_client_started_total{namespace=\"$namespace\",pod=~\".*kiam-.*\"}[$interval])) by (grpc_method, pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{grpc_method}} {{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "grpc client started total", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total number of RPC stream messages received on the client.", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 56, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (increase(grpc_client_msg_received_total{namespace=\"$namespace\",pod=~\".*kiam-.*\"}[$interval])) by (grpc_method, pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{grpc_method}} {{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "grpc client msg received total", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Total number of gRPC stream messages sent by the client.", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 57, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (increase(grpc_client_msg_sent_total{namespace=\"$namespace\",pod=~\".*kiam-.*\"}[$interval])) by (grpc_method, pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{grpc_method}} {{pod}}", + "refId": "A", + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "grpc client msg sent total", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Grpc client stats", + "type": "row" + } + ], + "refresh": "30s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "Kubernetes", + "Kiam" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "staging", + "value": "staging" + }, + "hide": 0, + "label": "Source", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(kiam_metadata_responses_total, namespace)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "tags": [], + "text": "5m", + "value": "5m" + }, + "hide": 0, + "includeAll": false, + "label": "Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "3m", + "value": "3m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,3m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kiam", + "uid": "000000070", + "version": 32 +} diff --git a/charts/kubezero-kiam/kiam_architecure.png b/charts/kubezero-kiam/kiam_architecure.png new file mode 100644 index 0000000000000000000000000000000000000000..0ef4bde70b5273e7918275ebfbbd3194d0416f90 GIT binary patch literal 43992 zcmce-byQqUv?tm)1OfyJB)BvXoZu3i5L|-0Yp_NeI!H)x?M8xo2p-&>KpJ;(Yi7N>)_b$w`KM29t9^Fus`~Y=b0Rg>6(2u({s;g7JXU%muMGfT!2tlw zcw7t=WfLu&0sufS*HqO}czAgD{YeN#%+dr-PEC1dN*Wp(uC1*V6&0yia=wQ^C@CrX z`uc8A*L2g}@lS4U?(_5WKi?o1NgA^t-e13dg@lC2%ge)Hu*u0uS65dJ4UPKx`unqk z-(jNJdW?v&u)E#G%RHOkQIfyoUn7@CuDjE+w1Bs(@T;cii*&=wukP1_UpK>*>+0%` zj*ccKCf2p(X1#beHaC%bTX%Q2ov|JgN-w_Zv7UYqq#(eY93L<02+?%lUhb|LU2bW! z;^3sfYqNXtGgNZjpYKl<uR$7YpR~OfV#gU)Ae>icj?M~h<^`)n$PyHA`UE2`; z?VX!a60B@N~;j@IYbCV#Yy<}?PHulVo{4GqoIbj?FMeA;d0gY1YaK?XNV5!^3Q#3 zA0bLaMAXtz+t^!NT%9tPnNr#cgJ=p4WQGPudLE`*zB5p+bdieFezGTR?E`T?s`A`rq6C zcbA3FA!WTK<)f|>ISKJF5n>|8#Ys6b*(&-ID7?0qm`zlgTN4x_yPN(3XV+wRqI8~C znZA-bN^|cagm|73d>wB8cTRO357A))PQrh_<i|Y-{L&OHIO0!I zn~~7fYSV~eiZUFHo{<*Fj5PHpgUA zquI>%ly*{njt=56=RsvhgGVgKfTi-4y8Umw&E&EM^zWTPJbU&Z7nyoys~M}|&%&-U z{7#=0B|Yd>g5H;QC##nhsA{{u80xe~PHFh?gZm=#3-KiFCe6G~+I)|qJ{pVX`Q4OUB4$MA4vB~NXEid!UTH12YTO7 zG$*9L03i@-il;y${7FefHPcIY|9TWBj%D(dHH*}o6N6+4)uo1;GxB1ORHwRuV38P0 zq?)_ciXl>Ux)#nwofe_#rW0Hk>7z#FIKA+PVJ+7#0I%uROoO2J7Q-f9A4Zol)cLY` zdi;sz3}Th>ku=xV;ggrqD!;;R)uF9~9 znq)51WMW?f=S(YBDNA+@&r(o;}@cP`)RWE>@3z$n#Vwykp2cJdij(3iE ztkVduf=1Q#Ns{(cIX=;a#F~*#vri5R$6qcA6IftFDN3biKH>5zoT@$2v{9T9uLZui z0{94O?b8P`;6y_|N%SDt-zqCUx+Z=t{JPgF@jo|2w1!(@YB?#=f{ zM#9@9MoSqc`Fwh?OLsIoN07qf(D~Ig)gy+ld4jRLb%g_(t2Oz|dav0Ge_+(YpKY7x z`a^_J4`h;`?-XVmCtnouG^bi~O+JStlA0{`go}wm;0@#KSshUBW6@(4H*YxArLVCD zZMwOj3173~WXNoKCZp4${pT};ZJ4yT;gUXXDj=^ zy)@uSPoN*A$UI3PF(PYm%h$j%-@dG?X#Uwl_Wm{HB6<0>9XQ0MsY zft{rom%v$V)r6C4nF4nC#wBh}W+xC)Sb$fdvP1a1^3PHto>(Gg)C~qIUcz?qV=Pjf zsf`j~dG;#U9wsxI^Ab+p{s+mA=?ppQyEI5#_$GfZ3#<{{$Yo+soh3)AG_-<6;mdqS&=5mk-}=vu;TARsuT1~ZiIr9)Y4((C=Au~_N^7fN9*fqyFrS(2d4iga z7zQR%rL&2z+Bz+O9TSfHG`FVaI<0%Qv%US`z}2@FfzF6pn&?h2;-$p0!MB0! zLph{pReJSzv3~Fp@~|r@1FAT>9MLOK?4ZVzvjNq>&4rn=Ms;e(wr2m|3wVLoRK2`X zw(mtc5ECjwSLU(s95dsJ%2!-pBt5+ATl_!dY{7Yl$F7t_BwKG;GH%`M^wpP*Dk|6R zZU?e(%4Vnn8Tos8GyXV=bg0&RgFjeh-wry}x1^d^nQz$d^Apr2(na5X53kw+p^Rsq zww*<0iQBm_5@3kjyBAhakbDcJVa~~85LqTZCbikMVt}cnK7Hv$JZwwC=<&paU_)AD zyjq0<@8c%)#j~2aP+1wNsN=Ly(z+_&Zj-gtn}-yW9@{%2p0Ts+Mk7-1A1})_L4aoE zj@qgCw)zZ?Cs!vwh2VaOmZY}vh2)FEBH7hSlmJ+za+s0ldN#T5ls$_7NE`V&jfy;;& z;9m#H7S2Grs&L}u5dU9VAb{_Fxe3r7KDDH;j=8jOdRtGK6vsr!CJGoy=4e(P&9~(n+bfN~tvUh5s4Dp_Jx3zU1# z20r!4!k7>()de*bBBb}UkehHlzpl%0P5A?IMZ-DydaGI%3CzQ8(!^W+)XLqas$N7qlT=v>(c zCF?~Dnh-QY@g&uiM|sY>>-Y|HvOE<7iPG{4zY)VKMLq6W8=(~Fn2QMZ7gD$^Z zXu0R_sls%`np(@G7o5?8?ci6(gI6OuJL-$38MLmL0Jf{~FQqDb?$*~$PXW0IIpaN( z7Lzoy4t#(nGQ=>pXo$j^(hN-+u6XDMe6UHs_~huz3Xq0l5Evp$9Or*tb=Ua=QSBSb z4n#)ykH3$W1d%{j=@5=5G-u)|MkUcz1ZFbiEP|;cbkwNMX@DbN` zKGtMNzFyp9VT(?Ax-Ny(QE3Bs)~u<2Bw0n6!dREe-i3$_;Gc&W+hB2R+UikRsu8@)3Gr7FJ|!ITcRRJqS+ zueGm!9?^5Mp}-oF7LsOV2ThRj*e5^0q~Z|AcRs}spHTkxt@^y|n>O*Z9zOm=&c+y@ zy`B3B2X^9qJBZe&*4Gtj=homSl$ohoDd86wGZrWRJ$5PPohBtw7b3nn^e2%jEiae( z&$E6I^JvMjsyd#flDu@9o+-Q~tOk)FSW)jXAYoC;gRLRpNdHqe3!7yKyyQ&7m3(JK zFGGCsqXllbIp8Oj{lfJOW0Hl5%Hr3LvGI)^mg5UMWSPER`V)P%(F^s3zE`jN+N|!o z5riQG!~wA-Y0Ch)4}|ZzJKS6DDo6<4+lIYt58a8NrLJSe5+1o?v)^46`9ZEJGmpW+ z@<7MDs%qA*K$ai9Q8ohcVjtQr`zcXc8nCjnA0YO8Xp67Zp%(6PynU;;e!eDr^<9i& zGuo5mtwTlnvTUu&4Kjz-X~^~Rm;DU+K)5r^mBn6o>ErH3geQ$4U4CtzJVFHM=XF(ltx@rw*a@Tt1z*2ge(&z!5I9AiCU0E2Xp(Kz+rAHPZqXr7%u~-sA4SO z?(X;6vnUBNaxw|7w6;`NhCC)2;=s_)5DZPV3G6|b{%#E%0}2$c-feNB7V?gM{?)1P z8LS>J&VN_B88k8DrI?{ar5bquVoYgN>Ypx~>Us69$S)HmpCQ5V-##9fa>OX)U#km(gY%-s{a%zc^l=`k8A;AOQI?eeFQ#_|B*r~$Bsr{mc2#OZl<>SgB z^7sNhg_CC7Ym0bgw1NpUr$ZV?6YW5twp2q75=={}@{HS;o)<^^*6)CBEDuKH&!Uf< zURmB6V)dm`R2F~yv8t$F#?hVPiA+Co5U=HU^{ec&P{^#))L;udKz`7#$qM-_F3=Km zyiPXWxSG7aPL1QY+p&_ZFDp)Om{PAE@o3RkP<8}S5T+8$@b4T;6TXZ;Yj#txn5}*- zm$IxGP>4ftFwa0NNJe%u%la8)+>wmN=E2+H!HF1c!< z___J!1N}j=H;muDGE=;t^sIDDdv-?iHJYT#Aa{XiDh{p#Ye_H_39;c7B^enn$X$R(pR7RjwhBA z-_lm6nnt(Wd_{nI(Ig@$k@gKrz~htymeYK4f4=wd1$gS_{-bvBUO|?%h+IZQOwVqd z1a>@zV6Cb&@AL$fB2TY>>$4)yg=L8Q5aGS3!AceOX{F-P;A5ThZQW-@QfvI>;Pid@ zH3-jIIoj_1`MsUBbwv@(SWAy zf3~cjJ&VWGX1K!1vTVdHnGFd3mQaC?|3diVr+0~rv8Gr;g=(ZQxxxj;Zd2EFc8Vzb z*A_@?%Md1sp&N?=OTrVzzTzkD^U%)Q=RZq06m7?#Id1*>Pjl8uR@S3oM^6CW9~C#s(%vS>Ve@$VkJ{ z`XZn>Xh0ldCS_X80aK(W8bEnfh7MSz(eg+B?{>cWtfFv4*MG7b1BfFH56~~Oo7`ME z@o{e{4&OLY1q4GcIp2;@m&>*h;nh^NL^OvEc<`#dZb$HUqXFMC zRaA~$^uoviF0*AWaC;t105QWH5?_oEpk-Xey(0TyBu{Ikx|v9ulj!P-9kw-iMw z0BqDFsgQ*`b14)~CZQ@$1hVkA`llxR=Ez~zl{gX<2<5{C(mB4+c0N?$e#`N4A{`|} z27#-Lz?hW;iGUj3jLCK=D4M^nV&=F$+aEPwa)W|O5g$0fc7h(*M$M1)FN|3KR?OCw zvz>t1GsCHEI?kU^MyO5TiW!~4vx})FAey%*mA?6;Pie}U=}xr_j3qDRl}Am{{-c;! zSPziaVENFi;#r?o$0jMc@b*dvkh_wU^nla+>ubV_S(ev{5#ZM9cvZI$w}+SD0a7=E zrTchBMxSms+MPGkWWMjm{2R>E;53I)KMdNpPspsTCsaIt1sCm?S{GQ02NIPVwTjmO zk*;TMYD9N$FnG>e4K0%Bn)L3X%4FrwnV!Dh_01R`iZ}Th2p4Byybi%z_xsuSb{xqR zUlz4R&UkJ5Ja{X4{TndIAj@Z~@VY}0rS<;lx*t&F9o)rxFgcoHO=r#9EFz-A4PC?x z@7@W{(<#yD`3cCHuRtJT_JYt9Nu_(ft$=%_kdhhk^ch;5KD0OW7C98{A##6U;(q#Z zik#7?kLlM(LNnx0s#S-Zi5P{6Scw4$MWuiK5v31{K2^S5_)dl#B3DtDkB)hhId0Ad z_Gm2qE7OlbT}2k2Q)xrTfwG_S5-`B=Juhezd}jhGvVuoClsrNCD(VL>X!X#!|MHi@ zN95iHrEWvOZ~P!XyMT>i9Zx1a+);Y)Zk23Pyl}mgq_!|KFKe!m{#ZQ)mw>enpkGL^$l=>Ndnv_vUs=W`_`MP;nBd3(|rxy{m;@ zo`zJy+IAmu9+>4H=%wQlb(PhO#ts8p>;3IV!Bj{&B10rhe@J=Nx`%Sf80xAF-%91% zi5I9qKp&D`b1_zgmjHPrH1VHQOeLn6FlAgUR>;1DG>?aKT-CaNu$6&}tC}w6hiNGJ zs%Nn`pYO^$5wYF;^eW;HcIsnzy84*~68HkVo+rseT~0h9M&kmve*Z4sqZ2yExN?x2 z-sa$+E2;^1Gl>s>LXjjf-5L*^rQF!U*>+Pt8jr+k6Z)(S=i)u*&5G+yJ=t|r;pY9B zPEDOeS~261YVyt@RiNS6CDW2oI1RoZM+5B6C|%bi#vD?uo_lhGZ)sBv2(0*W3{k%G z7>!XakXFo2q_pXa2Rj#rs({s61=%aaUXcAl;(Yq*W}8R&Enrn<-?jPEY*lL8YoVeG zLWfuBJSvqy=W&%l4Q?vNsx(pDeZrW%bPdhEU?AXUbgk_&Ubk zEW`20({#@NG!t^LfnWs^T>^Qsxx_V&^7gwB_&bEPryx6ck>cWx!8YQxypf9nwt=d4 zlHcWtt>qskjr@9)#qeuYN~_;(Zw`_HR+c(mvw~oJ*i*3G{r>aAz`)>%Z7K&_{Za*> zTLKRhXSZ&Hma_9LC|0J>$3ZF;(#5%HxmDS_HCXrJ&L#HHGQBOqu8Uc|PF>Ly*-wA| zyd7bLp`9unWDJTg;RIZH*Dh@-RJt^W=trnXsEBJpyc~F^;js0ikz9IFgxTTl)&)*QRA-q;Ck`js!uX8+uKlrQdnQvXqZAlPvtayq^@3+RuSxT=p(l)f$e2wCBG|3nGegWluwfo zGvWz<)VNQ9yPmyv#>nw0vkXH!G0%}bH@Xb2hEjCklyh2#_{2lQVTg_K!L`n^3ONq> zb0Tphw2`m09_RM6PljIq&>WB`$!)0Tk9x|>8_9t)gthM7Ayjy)@9dNOLlYC%m7@!j zwm?PZPr4S0O}D}jvxwcCi=3oe*83f(ub?7_{DWAEga`!Q=;pT4(pgbM)acs2f}B~4 zhya?$zpsRwudTjw?X3K8STV9an^Lu0m~d-?+uzmD z0GYhdintW+e)UhIexVkzw3IP&$LibDP5uwbd!Y)Ex1sTNEJw-$9=S^@?i)j*M22b6 zTv`Z8{DoG!&q5X^q^la+7QT+0L5%G5QX29eeD z?<)Zoj$kOpZJ;t?LKYOm1_d7$UWiJg9RB~qkzeyBWsHBoP;fD6nba6ArOx;d=X*}JjgaPj!w5y==qy1DeaJ*aa5|6kns(U z&IQEk-g#)IXE$$5Cmnw?@$K2H1%mD+j+<~%@8qLp<-7g)p;owUIx-kwkOIcIw^zwxR^qz8maNn8~|z|zYr zW`~R}Y(Ni6g*WfJ%6+*h`J>AcRI&BM{FMdL^*Eo!XFfr2yn`1k%}8X!I$bI?- zp0fDB%@-SrlY_VJhZC|&GHzp*_1n33A z9OA;9RbfIONg2s@p^OEjI1rFYAf{I{aH(ZL6ntghG}0`Fw+G|fFv&hS(O7wQF8lKi zIdX+t5umoJshik~Fup7&+uS#o_-=kb_H54zIrUwfB=KYMW6eZoD^~8us+fJvZ%M|P z=~+k6cyQ3lwqv)H=Whhi>uexhNUd^&+D`{Lw`DA^?|>v3pAg!wq#5C&=B&e`7u4>xVOU4v zX;o*1yzxm~{_|sX0rBQBhGpG3*E^LsV8Dj?PnOK!-r?twq3$?@A?pqarJu*ul3p`m z3Y4m2s7qDkM$})}KV|s$k|7hlYc)E!e;WR?=aL_;IUeJp#Nq8;7 zmnEKY?!!bt$i`F1p1^xd_p+f?U>+sLfk*YcsgtQIJ?%yQmcK$F4ofN+&Pr#duVJ=+ z5T!9TF*6iBSJ8^AoNaB8Mine!t1{>5A&b!75p#= z%Vp6|!VtVl(vh>SkGV>W7~oAK1jHZrx-x(AZC2F-;d5oVMG2~5=z&u$S`jW1)!goSdErvXWrfH z=SHUYl7h{>JsFO*qKP*fX4)1B3TI^0QK)Bk)r3VC%F`kW<_DtSTMN3|VcnBC-vg)& z<5F=Qb@jv{PgD75-bp{%^&Y5r0=Tw@4PhE-*CEAVZmu>Yj2R}SI-_dYjQMEGTCGqf ztB+n;oJYUl9Mo0vf1<3`r_$h|1b7tr-E14~KT|x$cAx;`<+)B_V)Y3yw?K!2+<;?u!4`l|f%-;{3 z9a5Ox>cYY$HQ#6@*>5YU1!}wzHojsMD0*hwt6;z)7LUJ2_Zk0Gcr#czDqlkIArk(; zl|6`9aB@=6%BB$%JsVKF-`*M;x)Gk5$uT#R?3geQdP}CDcAwlX-e5_I*~7N{ZQOiK znQ}sQ=+WzUUC#}<7_<-7M?co(~?kXp3|I*{X>m#R5uJ*5<|0DHX7kNb^E779E|F?L<(+w;=j-=vWeU#1}*@E68EeHkI)Zu88DC8yv#Cd~yn!O}ydK8{3dJ6^sRsb1=Y38SbGiK0Tpit}a%6{1X3h+r6n zo~RK07ffQuu+dW0u%NhTm-lRGiIbcCota=92ox+O^lQO5Xl7EK zR&d5A3Dz5)C2r(Gjh)Xcks19gggJqmd71AL{`McjAVFl-kx!YTS> z;*MQ4j=+??XVK!1qw=@8Z{B{!G!oZ;oYWc_S-(0vAlGv>iOiW<{`^XmM;dUDplU^4 zjkw7L6`NSsnapKN;hykXc?*&>UKJVg#5j7Ya2t}j3c2`w9UECw&rvAplNC-qGSR3L zYnLC4Xk0_zh>D??d|Ueb>Mg7}m-#`_9G5YuL-He&B-N{Rj2XM0Ms7! z2W+q! z(0IuwUpQy6Yke{Y71v4JZ8_AC*jl`&54(Q%fIh&QEiW&evEeP)UYfp9@Q$P=lslU7 zhh&6el~NTjo)!-N(DxLmd64~k*FQl{9aJ$#Hi~DLU z7oAQ`9H}dQ>F=fkx>9Pyw5at3YVb`lj`I&hPg>4Sy{laUsW!E3uVmWNXPwQ4*1GpZ zER|E4jM}%XdcF%e!_5-~Fl(aTEyjb5P#DJh0E)s7Rjn5vVdSy$5xOzaK}6a}ft}I1p2fSo*ekCbd%J_oM)fWj*N*D4T|X0b)n#Uy z@9-TxsWf_8xRjp-Qpl`NRmf{H#tVSqV(q#Wn*>T+vwO!39kaQ>qW3uTV_qjBmPh*X;YT$z1>VX{=L5!&s&BCjnj@ zjqw(KD{w;h18cg05w7lP8TmjMA2n^PX#a4aV3Vik=zN0g{1C0IHe6pPbYql%Nhh(f zZ0D$yyZ85TRhp{>)0Z0wbX@FV3Guj_34)tUDM05?+OQ^m@@kR*vKq~N$ zPX1rZ)fk`p}(WBvZ95SJz}e^|m$!z_wk$iS6vh1r1Dq(2)umjFEoj-z>RuVD=a zuxfYNN+n9s%2!vHAqrZ?TYHc;2&M{K{QN#dnH#AOtm-FrAf%V_$%~t^u$e06c*sBZ z@l9PloVuB7>oi;IRF}UuIsdG@k43aPX6jar{wAQYW8Ya{UjAN+y*c8?FCEa6#F-KV zBV*ZHxPrW1F*G`&TpdX9#4t;_Bj4%NIxh!F&&YlSrM_$Y6mfg0gCu{k&X^CaI${TV z1DiioN0Ad*?_6M7@c5OGYwUS#eUr#OIx~p#em3^!P}vO4q@21K zB_3_6wh&uzaybWpTll|q!HggJ2KH0=`R9B)-QKj|~-TcKA2tY;pZOR&C#0)J$CW#dKhdCq0T(Zm6Ly32(9{&NbuVbX+v$?Vm}u<78*Tf)xXVR*rA&9HQ%x-0)K`xjS4R>sX19!!2nbg<&!(rW1SpTSk-` z^rvd@w`r|0*6P|lir;)MLb1;7B{bm*dPUGG2QM)fi5#UhsJKXWCmWd1``|h1PhWr? z1WVkW7QiSRPD!Z?O4KbuaHMBdhIn(bKcl1u-Nmg=PcgUnj;WU*soXQ=&|FxR=#VBwDEJP&>}LL$^mvx`Qa4txg^ zAID%9P?<;gT3~C}x!puUFzkl1M;`bGr)2 zL`Q0VSZ0`H^fB^u%lTkG6)jz$7Y$it!f^$~f*z5M{)kQU^{yd}uzr`m0xx&S6>1)( zZjL@l$Wro2B*WIJRk}yBcbH9YOGAj1Z3d@b{@_cuqr=*ogvPSW>BWV5Y2XPS|8mI* z0%RNlNzvSt36~LS2?c&w=t9R82f1&lEk=BhbEU`2H!1}2 z)o6dRX*%;Rw_WzL+6vsz!_d)}5n>*H#g0uRZF-l11480I=$e{Acg80_NC%%!nDt0UwstTzsK=`HMTWtqwM-kDJ}>&i_c@HLy~wXrN~1^TWfFb)YDf&7(Xl{ z2E`+Cq<7kDAvlS4u{bx6PgpLIkAgnSb)QMjo0(x`(EP-p^)q{TYVs%hH|V269$>U} z(adbBeDB2~)wcLimFHepZp-zytQd1@EV@_XpJA-du#sW9q)1HLTi9&u_bTJ zmPU4WgAG+CLnNufhsf9>s+Fjy1kGoI3!f>8moZcOY`^sm&m@p3Uwt)_PFmc)iwyjl zgu7S&@JmT!ED^f1+sS?fEA|!QxW_0P$tu>UF|tRR76;fKUj{tZth9yJCvL#p?twVzW?@ zpF=JnaPj^)!p{7S>Xsa^)_>TRRq4pPt{hEwopWD&+C_W;oC6q%<NN9PZ#Xr6o9i2oWj1-n)<6{36D%1`YZ;$arI_ zoXwCs4&@OI)JHbR2?Hz&!|iW75O-dV#ZtWT8cUO-cx$O`^^ZAMAX}6(sZ1PH`MTZn zXz4x#k2*4FtEiO;`si2}OCcCq;t;zI=U6459mCd+sz1_;lINvy6hrk7$Zsa``IK(?4vs`24N#_Y7Uu>gv|A6a$xwVw@ z@2EaQkKz*T-~z960;V4eW>f*^Jj-4=T}Y2D5}OvY{?0>Wxr~nkCeZ5{OkHA1lF^4D z4F1nov@u!=eun{!U!@%f0)bWipxZn;13F*ViD%V- z96XsQYwiqfaXmJy;Zf~|CXyKNy@T$RkyEe#W$GeW@n8l~qH&-Cf6MUi=nO+2+_NFL zpFg2ZQ=*0mMCl~Yu7DvYAS7=T4`GoRA;}+ijAMTpD9<^X`RYqCyxmH=72Fd0`>6RueS}Cnd;u~= zW!}ng5iBgX|E*vpH9CYY{Q2m#Sm_o0sKc}!<(*gTE6?i!ng3;+2iWn(C=-&p>}T-i zOzr845K6-vJ%xy$J?6fR$~_-c*A-!OP5f?tX1-Y~$xv3laC{gv6<<0HSS94x&PHS} zxtZn$H`lRE{C9eza-80nx!~0$RALRfRyC-79vH*~V*^!__u{Z+?%d9m9)(5|LXk12;u?P zn?XYdlCMHcDyPiswUGkP-d9!5?x&7RfDYLdA;gJxQo{d?S+Dl-`rcX+ z{wH2|%CXgIz%5fpd`N5DPW6Adr>A^B1p^MPI3SrW~UAx zKjdW`PQ{=WYl&DMIh0JMr+g|>_1PjQaA z`*&#krs4J6t7-(t?>;Eb$)i;-P%u*pgk)k;*aF_nsX`0q5ZT+s)#too&M$K7f_fUJ zNI>=7$F=bP6AAUz!F2R1lWGL)bIQfQKT;^tWH;fc0TTZ)ZBO?JIMSF1)6iDNW`F2M+uJ%Kq-))z`1n>YECdEP|BhB z6V5-XC~rBNHx+4qzat{z6jkZCfU1l$nxz0A_$vJGQSXYO-~mFiiBlyo9D1z#o1E4F zv?{K06gpIc@MRPADLp>vf}X7!g6K?AfSrnP<2xccPSN8m;AOY1Au<59u~wZtQG~lV zRVNg;M_nGVO+$T$xIGM|IGby~D#xveLr^E#HN=TK*!(2~Q7&=KE`tFN+Mj_uIX92_ zgt?s$kowIB-ki+^2Fmw1jkwJSp&`}UL@PsNPo~Su5BsN#2BauX!aU>ssCw4*dXHekjH{*rEW;IX1&e&0z)4NO4`VqSI z)k76+%RE)(VN5*`lii{($WVx4kG1wod2rjal={1x2IE=6p~Z^#!L}33f+CMk*6I>K-Mw9mp@6Y8g=zcT$xvBE39d4SIa`>cbZ|kHBOr;O!3LjKvWU1w5jKm%>JQR2An&l>zM$4^4wcEi!jrr zg?0U7&*CD(W??X+;f?N`+r>QMQK%s{Wf-Pb2nM!bmmxdlX9rSM^;dhKOgU{Cdgm`g z%?vE1H z;!e|Q!fbKnS|`%fTz3nkSi295DFL6w>8o}rko5Ms!=!aJ*qKsgLIp4Liz(~MTBU&E zKFi#&Ut#MdV%CyzkJw;7LxLQ3;|e0dHJKYhJjLh7;0*D5D-miZZPlM=X6fDlw>|_7 zG*Iy#OO34}tc67l)wr4M*_^(5k25nBcKEUf(0pyk!_@v#ozW)qT(~)en}rVbLCRxm zr58PfA-@B2-1J}UqRhwL+JWM!(z`}5(oBhQdt>Oc9g#-MANy#GYoC0PaAAiE)ucPE z5UwCNr0;fH(oMcGG+IboSo%GYevp9>9t`EmyUwCfwpaOU$o*LQ>jH1u(rH@h~({v#8H3BjY4zeC7c zziFPn+>R?N42B7~f>>-~81W$X$m?r}&yN@dNlog6x$2DDKWQe`UtdVSTwbmGh==6* z#=NrUrcdVN0dSkPquE|OxPVebp<86O!kxj_MetX`=SLj!TE^Xdk3SW4nmVS;s;M8& zyi3!HqLR5Xu6yjC$;7YqqW#E=+BYeA)l7^AL2QMfTb3^z7q9~?TZNoCT#E__fhlb$k5c3a;8DbQ*!ZY$S zZqfhR{qXqC8fIKuX(>SI!&4^vK_$^f6*B1& zE~Mifwr#?CWE8EHvh+``s|AK^o!G0?4J?UCBjv9}Iuqiyx`?(W4Y?h>rHw74g@OY!1XTnhv!?k=G?lmLaG zAJ=u?@Ao{%``7zlj&nzTGqamLcXno$F&Fo%SV30Dnl+L2W4bf^Da#p-MH%4?-zeg5 z-{m_22dL|4|gyDWR=J)OYlZ zgas998JI2eM#h}%Fj=N|{1>5n(tUmYH}Z4;XnLW8&|?z*sm6pl%HLfu@1%Pp3wfBD zf{5{=T3aA(xZ#3 zw3eHqk$`fl#0Rc1sboTg#Ye5we6Ri_r*t~(Y>r+h8%$utrQT$(e$8wDlNYvNOGL7( z2D$%+E8e(ZaKmT+R3t z;cu+NuZL+7HO};)D`coKw8%saGz#E%x1l)Xu@`c+K@WDtR;FdqTfG+yar&V9C0yO6 zyw*T4r|0dvzP%?|srIl2HHa{le~04G{#DB!#?{-+e*C5+e;or(=-$H!uxk$6oort~ z*Bhp~ty>{c_->RD+a@Hc%t<1Gn_Y9JP2|gY|g`G;rGP;4EW7=;aghYYS>k%S#r^@fyB5UL8+P zBC%0jB}*QVw95zcLEWedrCs^)x;wC`3WyQowACr6YANTPJtBz z&tr;H#|94r-)f+TT4J^kiCn>YKq#z6_dd>(d#1ekV0fQh2?qMKpv-aa;^%hW>;~9d z41mhQ=j^WlEu-OPIVcaTE9ZR`OFF#CA<2u&Kc%tZKWoG@QloPjp3AXpKH&e#=zOEk z82qr}3Ca^^AB0~HenA=ZDIp_4i^JE*G}=$J&?e=EqBgrumik}_!@DKmlmbXAndE#- zPh52lZ&kToE17L-feqi`f!m)|pV9h=)C%-Qvh<_|zUz@!nJY;Bt@PdPOYsJh7Ie9_ zb0W!T{3b*QBp;KpuVG;MxVB+?zFnku+7TJsqm-bb9F`ljd}imWz_6K&X7bTB6AL9- z_(eNCV-`dE2l+3tVr#|gr!g%|j<{wOpnN_>9SgQ<6lM(~zkRX=5yVUNg+=3M>r?Dc z0Nwej@=Yr>%FTO(1nCZKZ}yh#(>~5IVbTIxaEKW)ZRAB+Si`FDnTMB=->xU#lZI0g z-wy^7z9{dx0Zf|Ui`oqxXyJV&m*2g1+cH7_R>WE%ZS-IaF5)L}T;m~DY^o)J^y@mQ zxxFm1o48O_^5VfHxY&4hfD5e;`jKFkI8u(%;6u=qAL$o`U-JPrVKDa+4eY@^1(QRD zmejV=Yw_$uR~qWIwxtwU82jR;W`ZK_YQFl+k1uH@{O)H>dbX!NgWSbj1QuELR^@Tl z3kUoPI6{29hXd+GmfUO>`1NB*Gf95V@9%#Uy#xSTg4+iFy&|<7n0!#PA_e6@MW+hR zy(<-{R;eMiS6YjA;LFw zR8c56B-BI)GHAFyXi$RR{DR@DGI2lTUBBh6I*lS7c@-m#@p_rUDde4vY_jCQ?!U-L zz63UBG6M4eaF*w01$)UW9_TJa_gm*{w*sYR^PBQJ_LU?faduf~|2799_-?rD6Q`B2 z6TmKKgr9}8l}B2%;-}onDBepypw`wXpxZU!C7%O|moXGk_8XG6H`md#s33#7I`DrC zL}WdTeC3aaRn7hggOFn#zps_%k^?C$5$~1CpY}$b$waYEz=rq%YN#!3lI6+Y(ZP44 z+^tO4&0Z6*MJ%XAq=^hjK*)=ei4?rT8E2^sSxA5GhdDL(X6)X}SZ&1x@Sk$eZ&sjV z{E*1$q!S1;ygVcWw_hu96W#)HMAUZ*BgXMPaof_dYgeZi-UPEYwABIUe{KOf4nRb`X@*E%$YT<0{%_H4L^8MI zIx0eQTGP`7J!T*&oBqo!J(N2~HuAGd`>b7SM?3xten5g26`TCVkOukmxiR?cO0*~n zkaOF`+Ar-(lT@iB2a599U}{Y#j{C&s*QZiD=O1tXA=kvBg|RymRHs zLT;mo{dFdh?~dZ^$M>diA#f$;FA{~Wj`N({9;FJv*?9mw2glnnIeTM<^K%;PO(|C1 z^u4Y!xdKpLz!&5R`j+^F51v@5(3=A-v)W*Dh{a8G#jK^=gTR&K>HOIZT30yGN^bkp zbe`^R*Lx?MpQgl9xh6~`IUbFdxA{t2XClyqb2uGKk4AW%;X1^B-h|}BhE?%+k!19W zV{#WuKDUFeb=RyNKt2m%2NxqqHp!oj(ACqeWRM&4;L`*>UxnX8U<}@Wj{pe_ z=B@ts&{rzjK>&d=4ab3O+(i2`W3Y`JHFbpGW=1r^0i2$;+d)+g*!!|*+A2a}{W&XZ z$?|SBfr`Q+S1YfyV+RN1OU}aH`ky~}%^-FtYBJI659D)4!J(@_Nb{ckV2T-&=ZujXMLL-SO1G%5!?SZ$e9zY+@A0D^Qgp8+s{yF-@c&nul z4}17*MGam&y|%dwf>7SnjBl$P%r7xDCi(xoU6Ie-Q88mcKyJ$cPZ6CBOP=)M1%J+8 zex!Ym-;2VWlr62n#r?dS#Oc0(WqyBw153bW%F zaz)ZGN8(jg6C8id#rt&;nVsjA@KRN(SC0G~P<#`l_BUb+0Yn7hr7^Zj$zvDb0bDg( zDpfWk$t^39AmwylGfD64Jb1I_&O(t)gYnqMWA zqx}m&Ce_QFn!bdxL7NKv_>ja)ln9u}MzB1Oz@*|Rs}vfsH0%gWAoCoFt$G^D!hX^S!8h*L%pOwF}v(@H|wT(&sL2e zWuQ$Y&TG#^Jl&Y^F%0_GV%=04L85J#Arz9@H5>J_S_;%t3$!1mCMH3H*5)qj6&#q?9YNMvtu>J@ZYa^4CX0^}0R_BKXk<+XP zR_Q+1^;ZDie;9wQUfZw`NzvB)$^HmH0k+AC;4L&emQb>_q+8Ti!0+4c^~%+smRVkN_3DUpbz_55I+3bP*-PS$D~gN-}| zb+dpKdQ1xo*hf{BKr_FBXPrbG%lA&9hLbV@nGNp}943Joj&a>|KA{7<9^vlw*pE-o zb8qN;F@mCy#Mp$27+Vj(s0OBgT}#^(sy7J5cAlSyfrvZrnr0g)LTyEg4a4pv#6(Nq9lg0rQfI0;<}zCX`Z5d{Xo+^>#nfA^*cVe zX{Bp;e^R#jsJUnMHPq9EgY%3T?S+{3P<0T)(-Xw_Ocm83*&W$V!MRO(1v!6$$)&*BL3zDHV3a1{KXc6kARLAvrk=P^5dq-rpS%2 ziizs4$mlj1P&ohEo7Uc65o}jRU>oY9q_Fg9ychC3e^?p#d8I(OpQ!3gX2?k2@nK*c zVkxICx5f!tajp<$f!!Y>{t z7CtKzrZ!Jj8HBNmk}PrOngJYxC|*VqK#N{i%^LJg57ydCG9Uk~=m56rQW-G%@r6(N z<|p_Qw4pir56~KxHOq^n(0QB-&#`CfdV=GEjw1#}6NR1CQ@wQZjQbuO0}-cl$F)Ke z)hjScfe~3fP~B3VPEcjUzDM9jca^DGClesgEn4Zi@xiM{V?A_1lZ1rdy5+(`GE~?x z_K^{4WjW!-iqn7v7L9duq*?)hEy$PjDM$6iIKA*Yr-y+67i7enIZk3)KmN2J=em;@ zMphfJ-IYGum!D8emI{;bQ*S33Re6=`z_`|+&Og8Bl^2}19yn^63yavVbA0X zubw`2W)L#RCEp~&-}U_D;nz_l^av^yF)C_1bMQjX<6E)sf+!J)cf_hKi`lAK*oT$J zLO7lxkg5g#E8j@CvD2BYCKxN8{{r!9kJLrAfd~&vw_!)LjkCS~&M?5p$$X}Oy3okD zR~#PwQvJ>LGFnevEHjF(9s`tN_-B3$6RZ9}S%a+t_Gs`oMWdn$kK7W};p#q3vkeWY z;;)KP3+2_TbC)w)NkB%Q8e^kHiluJvnI|f*<)D9@b++$GES}#bjd+B4^9Vp^a$b+8 zwNo6o1QULD{aE9g6JUtEIKXe&Nmi%~k95!>l#${?%7`@9C*L|($ypV&=ah$Qb54ts zQLQG{hFgJv%?y)L>|<=kf*-w@KS|t@R6sWVQ65rac(&r3H6}=|L4`1hi9uYY6G2k1 zNq2LAr*@Lyv%^oZVyj9x(~WV6gryU=v@yJ07djv~W>vE~QA^3k?RPjV#EC1|DTV7B z`0bq8fbZPskcn|o2%52@>ukQq{8W)8Cm|-fb zD&`+&ngNTOH~3B?Z*m_B`&K@H6jJhbnMR>{NeQ*f%`yTk8+DJi$Y|*CU47CRKLb`& zl81wP>3Adpxfe)LcqDt66Kow$_gLycqC)aBG@ep=$yxSIxBfl}rU$nyCQIZ&T6v+A zQvC$sg3A39`~9VPKOn;W9ps+-_732(k9@RW@DX{J4a%@<$eBAXyK}6j(6!&;p{E;8 zT$X8t(#%858WPR9m1PA)r2MPfDgySG3IK`5^ zEBW>A$wPc2Rr~#%`vbbxuF#$J#Ih{j*R7=x#!Y&>b_K|b6}#;|lzt*6OdJ{%v^S$S z0}_)&s_GU9y)up;kwU!evmc9R6~ms4Eum5Kr3w0Wz25D)r5YEZF-rl1zG_22y{)@( zptpa0b^0ptA`FKTJ?5-^5ij=`SPQ)nHN#3!^a6^l3N5e2e(9{yrTXV6$CkTqJ&&B9 zus&aF%nuj%buIQ?1o3|kZ6Y;q&Ad@dRXL>|&$>ry-pNY;(kcgwSl~v8I{xgEAz9mo z)(l+{*=Ir_5G7~?FjG4qHO%L9f;>)o^CNE)AG6A5rkrxp!gEF@YJ3o zJuZ)eYkf~_s3lauof16>AoNOl4;gYoj7?fV9G+{@@b**!u>o$z;|hWty9tY#)+ysg zMi?Ugmti_ED6FbZa0$u!PYW9gUu#$`lrHuCK~`pm#y9TLQpp57w!2bcIpFm&9L<}< zDnX@-`d=c$DCIX<86DxDWv$*tW?m_NO{$Hi%(z!A*?$Y%2=E=$qR_VfD zhULKNuensV$uc^k4rPEVupi0o5-clNqEY zl+WjV(u*rQD{6|2dvtpJnr!;RITpG?t1Cqy@cIzE8JI~*p0O%-dBtrxee%pyY6fu` z@T}l79@;@N`}5Yy%p63M^j+^i=>Aa_1w~chco~e&ydn$4eIxboI>W zSxm-1zt%@-pcG(+E+KNR>rJ2q1EiNrBlwZ1UZN$zv;IlniB+yqQH*RX{(joX&opWs2t`5phx!v^Oi3hpbtb{U`!?Rn^gy zkJBfTK=R>rO5@2Jj4a`a4mJp?hxL26zuSvPtPUeIDtVv7BP~6rm2D!M)Cq}`=4$Ys z?P}isf$r2g>`@;;0MRS2QnZ= z_EE3K@)Nx8divb0{11T%{xG2k2M2z&o8o+epU#bESF(T0P5s5((lhWssE;f4 zOjcktf^usig7g?yZT>$L+<2IUv!_)l3K6Dqg7V)Cv#@qe#(oK~o)a#khnFSnv2 zc2cOagtuNwakQRqhEOI7W!~f0SJ3lX@2=NO{i{3N2_Cwj1)CiB51Y}D!>p^!M(16h zvx?{*_E>Mv9BeO9hp3#B+3|32U%d+YJMEVz*_B}*cwILQ=pD!n_ zg%=@U)OCzhqcO7Jeb!ZL2uahaA8M81q)b&{g=oywpBJ_LS^mWpyl_2x2xapR9wgP^-MOnyePsmx>jeT7V*I zZ70N*{k3yi z2KPyJMq|lF2I_E+a+EN%U|X+VzT^@LRpw#a(cQzKX ztae5mz^y+DOB}IK`~8Yh3Fz`>jiVKjaVkmR16UUPLJpDpLVME_xbq(d2)aPY^HE&psAFDGR~w@#eu%&-(Qvj zAAx$1hJ>*MgIU*~6Md?Ed2R@sOD1vPnaeFVm)P|ke}<5v5Z^zS-s z0$GCR<~5)kFncGWPnLW)?P_u*6+P|moV;z6# zSCNZx8JIUcUiAwoMqLQ~8*6#>rtOyh(a50Y+0uVcdK9_z7ZUT6X9Srg%}cP2KbBHL z`n-})eT3`(Lj*(MI-3P)(*WvVQ%OFQy`h{rpPQFqB)R!e9PRp0203b(WsC;`tIk>wgg)_L?}4)_GIbm!1GQ*0alU!tGjN{{Luya_$l_mEEa1-F03 zSY~J~62}++kgso*4@8YldV9>C$|$EnPiRQr%PWFgOZ7hZGyV%UBDA#7RB$*`$d0Z0 zcH)Z@Uu&;1|Fx1u{fVBRtWS%_bJ1>IhJJhgyH?76PYo(VCOE9TBpdW|xo2%?q&YZl8S9`FO zSB+!+JZ!B$(}*ze_6Dl4qIUEjSd~fH$6gCJI7x%(6Iu05vWyNYfoKjO%$9pN+tuCf z-=&zJke^3GE6nFr%%FeakQURM4R47`Dk=&xsXNO}B(2@i>Va0(3tOt5=7QB|qq zSY|P{E9>sMCrfEc8x=v*rYO+@N@Z@}zMZ)-ucxqsre-y&>dwG<2ERkwC76_{^*omz zpkvqt0g?+@H&;^~(G&N`cFZe$jeiB|g_DE_Yd;=w`WVn|t;QNB2CuY}MR8WKEk#;Z zJNS@W%)_Yiv4Azh77zLuT$r7ryrLrSr>9gVu=4{}&)50VFR-kxsV1nz@4lWoPLuR$ zIDt&2Q?QK_M?PzwIs0?!+tMh`OBBkJ6*wb)lQsbPsV`JaT8vGWvf@mwR3N`^vY;Q3 zGGZeMq!_G*C|lf`&?>h4d{qwmls zn#B#ypU?MdLAtU};&Nq|FieRJED6EhSVw}Kk@ykzJY-CpaiUD&;v7#2?2w4a${%%I zg_d@z^I8T&aPwGwNEXhWPAL7R|^C7r47Abs{Iu?o&5;8@O!QMyFW&kpGpBX%G7 zgj7+`=VFO68kf~9z~_69MuWVZlM8GdTMR@ zEeqG^sjMH;S@om$LvSbuV(kob2h(|5Wm=(H(j=!LdwrfpXR*oxIZ+Z={IF@FZ6pu8 zSdJ*eqdO@jzDrI6;C?VM3eBGK#=j?KP@OklXVR=VDf{{tW6dQNA(o&`uh+E z?GV#+8}ArWFzmxnE{m-h4PH{S1af0(@L9L)p9BnYnwoxV-QFXYVAUY4q=TBb$YP>w zP*W;_BQ&>kO}mdjAq{ghLM%Rjo^RK1PaAHNPTZ)=G}!8?@R`a(ATIF@{QI383okKIFYMTqoy&;bmLFS^xrQ_piDnK?E=RLj)HalR6AnLOkMN z^MYUe$vtF+-cfc3D1w8$_Z^}7E^ZpsOfPbChzg4fs>XFOPq8_v@MV%^wd*p*YbB)q zrl(Qk&obB!{q=|macc8O?IQ{WCfOTfJhCY`7p&2Mg&*hK%JM|SnMw@}H{+)l)A+#~ zmmPp7YM-D^xgQv%fQh(BR5gG%2JBPn>(V;x)j*{cW7bg7mFEFvR!2K9l%%mQdEn^5x70(0!n<5ZJMysl#9D1w768@xB2ONtn#uxRZ2S8elDjPp;{|YE0l_&g`S>X}w zwyQzlH6cNkjIaW~umB5|=OhQ#@m!9kEd-^*XECz+QIo(NP?(cT*+cRITmOkUn3s;h zLNO~;p7BF!MIF^39iH7tu9$07;#Tn+y&mFTfe$04zMt?X*2LC7#FEJ^YN?p)PcBhF zU3^WzsLny15j~EHk=S9miDQazw z@_W&2H;1t&XO$*B9EX)t%gg~;iwq2!p-S>FsUkn09!kinQgM=I58Dh%cTxu_0iJLQ z@)uhP5j@7rgS85?VWeOITAv{jH3d-5so+s@S(ru+QA-a+;EtmNWya@pG%wQ3QW;7j zugI)}Sc#j?2kh&_d@OuUVg1kgZz>|@{XL?7B= zXl!m>e-rh7db^V@U%e)&BXTmb74wgos$iNLZ}f|Hz>)7+acW%u?e>^5f4%j99mm1=MczYAPsF~|v@_}pQ$*wOL$WDvH?5Pa>xz^1 z6a&BbV3Ja->en4bIap$cudEm^snM0a%CwIcw@}UBl7L)Xb=BH@te`~?QUkSr|Co7s z#0w_j93+&cO7>t4EJQz&_?)e;PUIwz1M!#W$gBEt_4o@i4;!s+__SGymb3Xn|2HOZ zP!j8VW^o-DYSZP-CqL=8CC?X^1@%Kp*j%(88k654jmG)fk=jw6xGbUz+ zgob8d9NPrIQN-A+LkcHV(dz_TV&W%8 zeUkn2ocI~Ih+kL(v$EK(lGzFp*I(yjX83oas}RL^6@11Nefcp#mCTt@pg?mh9*B&H zggJzrX*3nFO1e|YGkzY)EWW(`Y}B%(sd(+jtvYi1Ldz_#9jNzEd-h#K!nLx%p4QN`^=G=;H8MMYDq&`!z1YBdxi0S2YwJuSfU-!b~?2f%&%hM>FgKZ zD9KZx@5E&V$eSwTIRm%=Tx5^JeDFqBA9ms_k_m_&q(SJ2?z!jM7Nz$SHLFvTHY>aO zXKy<1wmf`^r7p+&wS3)D>Al2NFHf(KbNS)Kb!y-Cm=N9^LbMI34CQSN%9d-lHux(_ zmhIHGltyYZ8+HRTyniiJ4x+4_b~Mpy(evG2Wr8p=f1r|^qM;~gzmY`Olo8L5gT>O? zc-pr$tL6TzX<|voKNNtn9WpC%E+rklhN(1LkV=$kaOWb~%H`buY3yzXU*}w{=*17S zOOLCL1LN(@405ja#!Ke(HCvq(oQ(fwSEW7Yt7PAyalGjte_{MyKC#x;ak|w>+K{1I z97zrfP%B{yKyqV57`*Q|Vanx7D0(b+Bjr4L>i&u(zFN@mqyX-|CYxe}inX=3`$=3L zSlu&ucvB1J94Foz(aczV@m{{uc?eQ?WWYPf(?pqk0@xnY*=movz;aYvvc&PiH*%!n zUVL}dRo#B8^fQ9BR$%vc?s5f~kwu`fbNB2zA(?E^V{N{|m&0&MFB;0#AM4cDF!}-h zC!N2?H_7~}Hp%XVS=wbt)nnhucHuk8WHW$a<7WR5t20Cz4BVxv_lCgTDHGxge;t6{m8Ix`dFv+ zZcnnqs&n&bP%UW7NBI8XLcVaAitQ>}$;PYXsogvg!Fsi2P`;5?IG%)G)&+HhTjPTB!kuCAKheS~u!;HImI<6VS0$P9$j)g!4sOy3ru|*|= zQB1M@eh8z)55d8bJe|I70)a8T`fN>ra1}_O{$P8XE;<2*c~-F}84;BcX0kPoG@yv5 zvDA(&sb3(AYjL-+VDBeYN`g2Ikz`If^ylVLR0Lj9?h$sfm^G|6ykj`ODI>-oaLblj zR(3;~LAssuRt4w{xdy?Rr;=>r#0}mA=?l9Ro^bP) zf}&xa>`B5T`2AkR_EV%qbyN#5=MUBOhJ@`e&9q7M&!*lbG5Cb_!Gs7~k^N#`*=iBL zWeoLUOSgIGns4|`Y)s?&S`%s855#r1!lR zHFL64FrC#=2rM)^bCSHgmu|r*W~4$b=`Jo^3OpM@1M5)jWS>n94G0pDiP)@LSTFTB`IsCvc<&- z&p8E{-@toGwfs$&Uo1ZOP71r3OETxeMc$bwxLT9}k;_lsqy`c>C8)gnrLWBW@=epe zRT4HP%ZIWlr+CZJWmEeDcbM4$ zy>)W9;;|fz+URA3qD8Ab&>CQ=7c>dyJ#2tkS3MVfO<<$Z*W<8M<2TxJcGup z9^(Rp$n8L~1NQWoXx`tTE0;&UfEDgs>KXZ}of6+@AuU!}6eifCfJZCal{gECl?u~b zZHO_a+K0ps)_C}Wz*%EP^sVm_cXc8p9Us*%Tx0kqc*oGdEsNY=ovNc3dg6X*&pq5n z+34-E8Em|2*Rw)i+k1OQ+l<@y?$c3)%+rk`XirE8Jc<_7-XjgJd^lp-^+<*h*zcOU zMFYpj zNu)%&+D>g_D8)zYc#}RiFTH#-P+n5UQlI2Pm!}J|2jw?c3F%bszs~avh)p3eaOKSN zbf)hdVY=7Sm%FaI>lyWZU|H!)1f|y~cFFW@T#q$9xU%2-?|4414=qOjk^lfTiqSMw zR_66b%t(G0C>uRSFJ(o`;}Z>>lR^>QK3dXy#&J)8%}3y%T5F4ud-XS z?s|XO^%3N41gZN1EbH4&N&rx8iMt*wZD->B(|V=qT&ih-qWPzDCI3asJ_$5DFuv7q zso0mF%IlkqcZo`0*ST^`auxp3x5XMURY|?Yy7lhkmUk-nRN|bDXWX&9FvTWbPk$Jh zHNI6(TTWO+Qch}&d2q$UyY_OMFG21|KeCeD;aFE)<{k&PTGROIX&9bpn};Y$&3C(_@RZ<+%Sq7Ps!&AyI03 zDAG(-9xvs&Ati0^g`#ke{NVedLu|pva58wo2?Ta0q_3tzqju0DC-dmc|3GNB$IyT6 zb`58;B&zITH8gLnjr%Yyg^lWLW|))zGKYVNds7KyQr&wMIVLyo_C|tU`qF~-ume$2 zZ(CcBe0z(sEb;6)0?0Kju*C*c2z^hKUewmbzhSu`KJd8u)rkvl_^9Jq3Cq~jwy3cywY>> zV^3k`F|0fIf3KW({9h|i$2pww$Fz9TmG8xo+!#XM^rheBowcJs<*d&+b`2UMACiw~ z|3y)<=&JmoraLO{4Cr=gN-}aNwr4K10FB5$&r&w&C4>9}$Uf(oJlRym>{+|!!h z=KeQp_ABE&VPrvQMahw5^I)hmRc`8<0|}#eNxppUPreL+sM1I}_7J{5Yfq^zfFQeP z>s#j)=Mv};8aX=(pgJ7;R|MUtGTAsWY-Zl?=*En+0b*(Kf(gvqLg23m4^W+z8n+Ha1!*r{Ee4MZ$djS@+8$hGzDjU(00fVjXLEzd=kO`2Cy@+QHm z9`}-P4E|DD)d(gig^E*+(G(z0EY}|L++?qazPJu~E;>~GH!_D2iNdrp^h`(SU@a-Y z`N}~dPRtr@`;-lRYMd10UfW_R70>lk;6+Y-8aKLN74U!J_Z5~j1=@Cx%X_XIk5DU< z7W$jtXKsgn_l5K8C^POoEh0Wyq)65DsXaJVO15zKGNDswP=YkPfWy(07ImGa35v4) zxK{)!v&2L18Gm|7OFu>NF6a>H9eA%w7M`v+7b9QIR9&Bt75M-*N&rsBD|CJR)phBz zd%*vD?{>Wq_UKJ7c6{W}(3Ka`z0l4m2OpS|r`9U>C4%ZyRoX22W9Qvh1-Zl@lyGM` zT?}~gQ1jib3R4pWwS~f>DwBjuG_M4~1b`<SWN}oeLA0Y z`)C8c_iSFzJboreRXEKMnD!BX>;p4yEsNXTj)un`r!!s~YnrQ0sKIaYERW)#B^m#a zDkWwYyo&Eq>M@m)rRgRQjYo?UMQbScV5I@=W&N#6?}{qBs_^sMKzrA(mzFyBRzoBI z%dh)czy{5!QT=a@B{X}tw}LPIdt>>5N?abf;V1&T)+D9#7pJDl&*HNV3jqA|;E>%{ z(3Qio$wI%6ZCNmslwXg6SuEO!2>D!@0i=iJ>BBMN=D4ne0+cPNLYESsDV>ig6&43v z9>YTnT369{9c(3RMzE`&%%4U_kuId}a0zAn)yS%%cS`B^$ zA0X=aEe-qG=Y{N$rpLB9b!$)3BovL0WGNR;8r!j(&-(Uqbci%Mq zEjlZIzjSciUWT5Z&N$sRPJdqAr6mp!QzaOD#fch*O~}MFj?-|~j{9B$#N`URlW!j6 zUD_Pes7KQf!x$!aFZ#&F0hA?sPZ-1v{ne|J?Oo!R<8yGtr1I6P-c`tf5ne^EP^t6n z?rN>Q_)XQ3_VxUK@q#+9y=uW+C`mnnBuJt+T^B=j!-Napj4@KtSE_59msuR zUgJ(fcBB~nGf#cZupjk?yJg=rS9F>t*FnFu{=Dd*nTWJGo(9xLB857C0(9)0m!DX6 z;WT>(KqzGucv#nrR(j{f-!l}=-bV}V(#_})j}-b^PCPOpzKqZ?wQ~3HGL+-N%SpZk zru5M6i=%l=cY3cvO@!*5+*D6LWpnRK7fpTwLks~t8{pA*#M=Eos5MZv^FzOAAB)Vw zKkv!3AN@|c@+ta=dXY=dtN2p#S4ZnL!NjqC&;unAC6wJD-B-7Wp@NO<>o+M(w1Dpg zmMRa@swRKXzd0m-|JkPsf>W>EpxtyxYz#^i#AhPEvbNu}`K8f#fd2P52rO21k7#EB zZK8hgX~a_O{NO9QP4*%hCHB}Sb>AbIoDNspRp~DjdrzNn#)zX(W1?SK@(d`khwU4Y z$=i>0bZ=!o$4j2J;c7Ms>wFoO*!V}XgT*%9yEs`!Z=D|s`K!rUKw!*9|Tnx*gKzFt=@L5o)C)D?&TSsPc(RQ2*fEWoI{&=4U!i6-pR=(R0h1v@( zaq|yvcX_-gSvO{`aY){m&dU3w3o)>J=VoHUd^K(rVGy-Jv)J&22j$Nq$!d-`doA?c zr05^Zk%C9+Si|);o`L8%Kxcd+4Xbg_@w#5xHSH|^7p?9%2^en0v)%A&TF z#}U|j(LuG1fIhoU-#B=h{JOZ39f;68-1;ab<%dIhx4?+jkmjGw)MH4@kM{!Rcvab5 zT{t?{DnyF;n|09^?y8`NpO=tx)pbQOwldRI6*ji9w9mM;*Ex`^slEMgtYyDVoPX;_ zwEaC$qKxyGvza*j%460mih$YS4`~;A&ahHq*qu~wHm*Q-295UsdXu1lM@!6jzVZ*U#cJ_M&uf3n*D zAQ;U4q2Y2sOFAlUp^?wZW=78jD=jd!N?r2&U6UvNE-M zNbf+OkD7P)vv*b7^;$}n4VIVsr>dftIvDn+OXOuQY7V~66nYm+${7f+Y+L8Zqsz^hWIi=Wv*rD^accPn)@!JNoBq``O+nzR`eOhyu+^Q&!I;vpUfw)}d14sbO+a4Eg=8TMRWefe5?0^M%i$@T-)PI6Yr`th^{D>g-jjSBdHK#3=J@w5Ek zVq58hZzse~wQ=Knpt$9%+OX?*zbO;+3)#iOm2np423eI#x~21YI&DZ`Q&`EnMe}I0 zuU(@G?A>uNzckw~il0XE*A*?B1b+M5eZ3N`K#P+<{Y?F~B@%Z3((dbX?lYzuH>}ml z`FmX>9j{U`?)3vnW8)Nom|m;FTY}%Y*n=PJ+LuQZg(;kKjaA|5`r#am-b;a?y!%YH zn{eYZWS()Z7M6m;NSHQUxcRf?#5w^K(orz;;!HZ?yQs*_Upr5FL8q6p{BH>OrhVNL z+vLAq(($?DbZwZ9PY^P<#(y`$`W_)N= z#ONn3wYZL-8-f`TF3B|U*ZFIJLmn$Xin7^NQJms>r-t9lnr&`L+ErQ^LTy4f76CEa zIn*aL%ZCboPhX`q=|W~gjCq~sWq9HOhURGPC+Sj_&J#|SgcuNi>q{PH7W-h-?0XEA z4%^8&@kXK-LIM4MD02s+?yab@CQy9CZ?uj-f7_#C$o(ee{PE;kh`MFZu0?OBN0!<> zv2`MAE&9h5TaK<6{iOx`wEym!Ml}947T?E5B&b+t=;ypq2DS$mh(CKm<7KxL9Q#d< z4d&s>3~iboKD1@bRX3xq@6Q$A2=|eG{0g^AG6%fX$f=chlmeO%do*#J-R-=>ffo@( z*$AR-&_t36O(d*2Asp}tbFn#(C2b)grsolQ2aHY;*=|^} zwja3!v;Wj&i!iBbCRd z4+DF6|6g@q6%|*|t$BbXNN@>(;O?%$J-EBOYjF2Ka2j_>a0woQ25;Qm0*!kI*Xi8* zzubA5rPA^1FmB;* z-cyL+S=Y9~?tkf6f%0)t-CLF*dxGw^<9<9o6}hk!E&4@E0?*$9f>r3pH5Q-=*Zpye zb!Qh`-qwa_PfwHuFD6d3Snox5_uFZYwUd>=i}n$k7x4O0gB+Tq>8%iI**7G<&seG) zi5NYnyG3z~p=9sgwX6&$Fn2^PM&nl>OTJ}(JZ7qD#C`iasGy@MXdnl4y#CXPiv7cy z;V!+n8%Or43*Q6{s8t}_&(0dku;PCB;s_;n<3eaMcxoxe^vVJA-po24K9zIcy~$n9 z^jGgDw@xr>_r7!pdqLHc04bM~XrD3fcdfG);NcrswQx1WDFkK`jfsNeIE}!N+n_t= zr;%KR)Ablf`wjt+v%OU=SQ{n$!o0TU=^{T+%$`{B8A?32ec18(X`xM=F=ga9&8j&G z%sQzDaIGWf6RNDtL)a-VvJ8Kn>h$zIL?mpBJXSa=({#C%%l8;MAC}BMG**L4SsGR{ zmuOb~jVl_4&_41#+muGNyfboi@VKbw(M2&IAA@L$@emaQ-NUX{#oZ@pAkd|J~WN&#%#} zpCHJC63=fTCI_n!z1MsnC_Z9 zi`LbV=vCS)GOoK$L;knAMwpw{$SqA`1=SZ=sz z({hyH?bd1$%DrFO2!8${r0RO~h_KL^oqLd;eZcV`sphNA`#I`lX_cL6Fuftd@ig%% zdr9)9^{Z5x70-b(wX|ECBbJr|OD;XZbSBub#*G0k8H##tDL}>_?qKcU{WU2$EpyA= z)~#2cXQqz(@c?&f6@qmpcWa;ttZq8tLUPXS_^6@RGz-^GtHQn~Ne;sP%8ZccLknp3 z@LH>*G%O^X|AmA;EDU4uO1pxvJ*xnnb)x+oOsjAW&tmLsWhzy6euC;R2)7Q_=wEuV zQd7nde^B1`lcUti>!UH-o3_47rCAs%qniGRR=7vgA~s~ld--;V|GrJncqq=0_5;^QQ{1rDss{9qT9}`TE7P zyG-ECyYQn@Wn)CP&StaJE=`m*h0e3X0EEM*Ry$|6t3FH5IF`0Z+6r1w<=(7_>rP(# ztZm5e8Yb0fW_BMn1tRUN99d7S=V^c7Ab_5a%ZKZwV#tEo+;CzW8wtB;tpbemjS}|W zF~qOWi=uk#O@)7lrlBytYL#kSh<9P&!PDwSvk~kYR5454xbe4n&qlfn8V0n#^-d|R-&FY7#>TU>TdN7@)_Nh9;T{P#n3Xt`cASmdX# zasG%AEG3bDp>tc|9ZoR;i!eM&DgzVCbU9e_N!|^1KM~z8t2YBQSp4N2XQh7kPq21FEZ@hNJ;B~d0d3wb{fj^mC zp8}hDNPSSf*Ud*e1Tms+#FC%;Up4gCKKX>w&AoPPo&Nrlc5@5L6h56h{)!hjO{r)T zMVwAdeBm7BfK^TS+c&xFbY!G=G5zn~C4_}M2?0UX*4B^YBUR3(Lt;^1M27!R-<)*> zYU*A58M`Wx4MbGe%!;PH^GlOG-Bv)-UENYhN!+|&BMIs{5;Iyl@qCLgq1lW%qEyW! zlJg-}PAN-d@1PS?RmdlKA!=yky$InZk%e3I=k4aB8w#eC*}+y49lK2f2!npOWi~~Q zy(kGo8Q2r7duBI7M_Z(PF_VgqL28fL6?x07@2y{?72@C`Mt%A4(im{fE>(KdhNR*L zFRxq%uBkkJPVcB3+d>Y0UBs8xY^gJ90vBS67PG1lx z*-r_)DFV#Nbx_!O;@h{F>e|;~_Ov`5G}1OGSuSMyZ*d9{E zmej_A4Q9)dn;+cxjy!%<$n#m!jQCD*h}dJfj*Tuj&?!lY$%B|=^D%p@wK>y%0<0!a zUu4xw0XRE@6*9>gooLS_p63Ip7@MX^v>)?2jyq-gDcmF~T;`tXh4pDWyt2ku5iSYi z%S#zh0|Xs7JaDROa=HMfQ`-k{Rj&U9?aX=aqB-CBhz_#O`dQxFbH%SHNQuzw$R_Eu!{G#$ za;`)a*e9=~6mIt$Jry%@n)(A(Q0JY~Bi{U_B}BU`7T=P%rfuO-@s#I$`2&;opjZf| zNFwgUujr8#<{#rvK>*j z;>&2|7uLny>ToiyjATq6K0$Brjz~7D@wCI@GCaEo(?j36|2V4X9l>+RYpQH~l4?KJ z_a}NjM|0L4cn!oX1N7Q9aB;HY#U5vI<7>!@rLe9%yXW|J4w*!YOAYqc^>f>B(u9wy zGfvJ!`Ggsrvl0q8v_8Sit6aAKDWpf1Nq`}gaMn$=2?(3?CW#wVY}Wje_UA&#K9{T- z!}?f=U3f2$H3;j*AI~>=q<7N!XMaKPepA(x!OLWN9#e0t)WDNj%1%^#i&HF$uOv-LMAds zU7wSAUoFu>Fv&zuTOrIox~qy!QhtI0=7ZUa0}LkC)UBn%y=v*Ua@SAax>Bc%pC1oO z%I``)te=%VttbZts%~Hz=NTF0<=;g7JqypzhY@?_J@<2Nekmn2e)jFGJ|S(|I3rV( zetC|I3%tMDDcwsL8xvkT4^%$c+sii!3{ZO^@b~fc^)(g5SfoAi`u#Twf57x-*OB}! z=@;7bz19umgj<=D8w~l+CU47rBX^O!(R<3i{X=%UeR>T0E#NoTD?;+-9co!2U-cNH~N93Cbpd1w`p_7s6wcvJgVPYNnJJ7Q71A`UyF~7SI&Gb(^p5$RWuVA zE6XxDu<0GO7R6D6Lm^@$7iFBz(3Q9wu>9!dsA&!Q)$F7yB)F~!{SCEQIAvjD=&idu z9CgbDq57>x8hQx~puFZPGEvUb5ia}X)JjgChND)FoDoUo^;s>J&tBGzx8=6%3pzU35KqdV*yi=9RGJHmn zTAaQ0?t2YT#A!ulL{1_JUYy~rM?Yajjua&pdnj)u&r2Hl3BlrmevKUT>kqv=+-*TH z+A2I*L%!$xYt&E@#uSi8)@ihPykcsYL1=jRWs-*7=qp-TVg35MA0Id`toJD|!kQm6 zWre|ge8lO)>#453#WMO_i2>9%u1os?%N^RH+{v!TtyFG zIDR?C$`G3n1a?x!1F`B3=VV@{L)7`aX71e#_@XS+GGRu=4@tgtMX`9uzo}yi1eNVXu5EyK>!3hvGJNacy*uelstN$1w!`HCt zO}zPl>zih}uoiDwRR%Qnf15s(70d-`e^iL#`3W(ZgP9>Q4N!Lbve{lo79p#ib);m# zyk-9Hs+?T&zMi;yyrvtULEf|-H5WUsEg(?rf9+AG@>Ud`k6XRtw>uB*1f3BF)%RE* z%2Yz9r956Z{sDzHxCSpczB25KSW9K7J{;WAj**YC^2*1mx8#{PWUha~-_Jw2G~QGE z9iST|y5i!%Ja(noMF__H=S<$@E&uy{O-iynn-O*Q3|+rLZf!w&j-wUR;Jl(E@gew| zVYe7QjZw+WSY07oHrk31@y=fTU_awe1f!$B*+ZTWt`2Z*=9BT>g()i8eE>6R)Y=WD%^;CH*(nC!( zdX>X9I7q?%40ttDI7{K#o>wQUKOuN7Eme>)o1KuZ5~s^_sZ#fe!UBTu5PL97q^||# zE6fPRj?e&l_Y~_oLZrfmOWsNO{q`@I$k6c{lu0&V>PmzV?2ba4Jv-l|y<^p7!FwIt zUT3Wgjb`YXf(G~FDj0o2?jb7RSM63jF~Mzct4~i|6trp!A_F&=p0D70glaf^ly{pb zR47?Fe!RF3=dohRi2O=5J`i|{yWdcmsRPb#;@9UQU$Yxpucx`2MvH4w9u`H3h(fqD z*Yn$$!wZu_JCAW6{h`G2G$3zCeVuE8RRR{rz{6>;T3{bf7W6h|W6b>JN`iEd20^1Y z+of8NLDR%9`fK<>#b6>*JsdjY@Ad6PkOy!XVadTbD26T!oJYLUeu%Y4U<;?B#$wNz4 zebdQ}^{(cfzNz)TBOK0t(?B?^Uu8a7M|+oB<9)wAsVlE-tmMX0A@xp?voRGc_R^mrsbGcYLfda17g-HwQXvA z(k+|JI(_!~px&;&3`&w!Y9;=Ws&$yH{a23^x-Q<3j|CR#omLZLt@rrK8u@UYaXKg` z5p~mtHuTuWJb8vaD#p3Y3L8@y|7V;=at@9#)H21EyD{B9IO?Vi^DGVG|0L$h%G|g; zn=MsIGux#^Hi(q4jhz30J!>$Cm7MUN-oBS$--T zD4+Zgyqgf9%>SU3Jn{qIhcPAUfC@|u?2Od2%uqkgKS44*mPJ3(w6-A0?A-CulT{}dr$vrz5sm&t{xf#y5ir`tBRTI(Jwx2_ux$yN7UHwE2UUI8J zF?i*DXMj8_cf1v*=QpAJ#`YAJZj7>&H;*{oPsQqz=S5i+&spX4yQidD9FAseNW-)U zj$!9MV!~TG1(wLV`l*G=D;TtrQ^oV`LSF1D@D$&}eO6Kg!9y8ZDMBzSVq)LG^BTvI zH4BrK#^;BoV{e{nQQff#h+?LHOZ3NDI7-3gc_cv`@?n?1lnw-)0ZCo*=IR<`Cl!HwLIF{-(U0#8TD``(21KoOh(l}F9KO7`Nm_$R2hFldHYtr(_ zxrkTJXkSoJfB5WMV~Ptnm_>B{{KYX+D+#O-rNeb--o~Z0adQl2K#uInt-`*Nd>W)u z_MeubsB18mpwO=xGv^#Cb(1|+0NX7r96AqQ`Qrk zznwx#nyL7&;D(I_Y&-u(p4g)^XcFKh8+UtW+J1?Q*Tg zkl!!AL18q!V1M>^SHE14vhh9cmwwrcGjQ8I!gHVHV83uPx^D&K`VYpxbVt8rOgsPK ze2RN$Sw3C7@bE9ed13#)(PMr8;f89XZu{s~f%e&4t}9~VlJ``$bKCj))TYVxeH8q@ z3#oKyov7~*CW4n2-s3$-s@6((iO>Ue>WRXXa!R+l_a%uuN%syV16|ga0-S5bN^akU zv&%2e2%g3jZpuIg@2_M0!|A8eeT9*0E%GS(U zb80B>lFjo!~sC(*mZnP^{ggLlrbe{G-&sV;wcefQ7_WMccRK`X}z!^ z^_N}A04ZmdF>Pz_?w|c!%>CRzQTZNEQ`ewj-4@smuJ1Kj(8j^pkpj6d7odW&yBML*aCX-7qGOX_nXP{?h^b$4|tS z8$NssC8zMR@T9(`v7ml=C2SzDiV2QJs3sN<1f#D7DW3PGgfM2rI>&LE{}_v*Ap{y5(06$4-M#G= zYb0yhz@g?a`Fzq~^1Vv0E;oUcN1>r3%$m_-P9aXjq%q=qx@;#Uj-_PLm46vNx&sMM z6$DG0!EUOR=ymrgU1Pt8J7o@MK)(?sC${FBDM6$mWQPn}KbZ-w!-`D8p!R*9f}<7| z;2O@Qv0LrJAP3bFj_UQHR0|2byo~XqN6n=ia;vl+k%n$g6gpfxT7$#c`I4L=xB7Mh zeg|b*=1Fzo_?Ekh^tY|n%|JT|C6W(UTe^`y5gr!I3Jet>5oP)8#(&L%hN*q~bep|y z|9cJwW2|zRJV=%Vna(A9t_mpr*0Sx`?Fo+Fn60U|!Jh7e0vRY5jAjq)CeB}WnM{1^ zQ7%28#>xLZ`(;y`JZdj~BYSUVl)h}_qwFO`S4n;x#xu)k*r!~ZmVgK0WHkRIH5}2I z7*bRa_(H-48LoJx?$)o^)PeAyVYB19eO$5b*D2G(q<_nUQQ{nN(=9$8h7;yDRRv_g zfhFbjbM&SCCR>K?H)*0IwYuST>5=#xe|&a4>;BKx6ONUSIvSO^jLtUN*}kaW8vICT z?VRt~q`ch1MGyQDzr4k~lZ;iqY_AaGe(f*U;-Js5Lq8MwH0kx`Deb?9;s1PZ1TB># zU<1JWS0+#TBEQiJ^x?21mCxRclz)Jvf08oQx1#1OYtc&{bdz%m1%GSpB>lve^JB)yuytxP=sNCoKSJ zQ797bWF3;>fLIL^JCdenw52YcSr~0DXCJfC-~UT&(4ki%rSN1E&BlRmgGFxwTk(Mq zr2)KP_Z6c6Tp-D_V;K)fiZnudEm|VOO7dE?EZ6ll+3IABJzS#Q@sbYEm*vZY2h`R4 zgB1T0%|2-Y;e0tk@egBWL64mQ^T$h1OaL~KQ~Frj$lazbJXs&9pI22S9j=tAcP)N! zpA~~_4nO%)9E8_7VRKnGfCQ>dkrLqp^p0)Xp&2%=#g%!uUSU z?KNOUmf$`!vElp%Kr%SEj5e10i~zhkuAUaUS_fbRQT#%iCP1)gs;`0_5KNYet%?qO z){>Lt2S#yoLzqahq-6fwh0C+RH@+FyrMWDo>aawxO7V zDa^pR+V{>Pt`$;7pZ*Xw#NdVUD?AU~gXEYYH^4gy^k!?Z<~2}TH>E<9F9x@X=eE`( z`4$-M6+>`)ti%KaDQ4Zit6nVO05qj-IxtPK7;d(EJm4qeChVlA_kf8SY!@COU?LZ> zK9z#d9gBy3g#wJ~6&B9D0e(_K$ot+Mk(Mb)UBv|$He!*~+QThcAzv_GC*eDYG7{q$~XdR=NovO^64@CL08 z@xCzqGW00#Y#L~edAJ&MwcM>Hf{l?n!|Vv+-}@p$T?0w~ih&{_E6)S16iy>_fRe9b z0po9S<Z)gYV5tAa&qD^t zGI$5JlXjb>Jz3@)yrT0gKH5baR!!jbugKatUCRMLp&%!TI&pBqrJ?p(nC^g=+8QPo z9dF>Zq@nKU`{l5#tADx96wPw;G8WzU>qVa3;^_@Ggm)T53ezQ8vT^ z0K88}V^~HCCAFjLvY-noV29-=5T(SUE$mK}{u3$xkX0H? zsKM3`ArU|Ub4vuqI7E;b39}8I&dajrpyf`5^3AB{ zATx0D@-rtUSl-)@V}BwKiw#e!YN00+t+Q!8c?2v#VWJ9=oA)5d&IM9iY&uukx zrUj~|pZmsrup@InLzMu4@yh$9uVx$>#|b>4C%&AqFoW2M{IDl1umtu3NKW z0*?rOrY_Mqz_UQLw7GhX{>nQHp|?QU%u^YCE8TJqakc!ZlDj_ScL$wUEqP24_r+ub z5KPzv{jp9*;V?lminmqyLh2nut}okMy5u?^aW!Gg(5$(7XmM6Hs_*=c>^9&Fk;#a4NxCqEeh`Tjnmk{Jv2sg7)MI|l@ z6K!%PTE6D$pZ!zIHc+kxSuYJ{)8~#%g*XxM@tE7Dj)8<{CVub-gM(FUftfgif1aPY zYY}A=u+`sQH?~o8P)~o1cTxzHLFlod_ClZl6WVm=escQfnE*Kpkx{=a{wvRQh=}k2tcX6>#uZfjkJ3Xzg!E4Y+;#DU{twCj p{$=_ujkrnZ{{lxEA6EbX literal 0 HcmV?d00001 diff --git a/charts/kubezero-kiam/templates/certificates.yaml b/charts/kubezero-kiam/templates/certificates.yaml new file mode 100644 index 0000000..ca9bc01 --- /dev/null +++ b/charts/kubezero-kiam/templates/certificates.yaml @@ -0,0 +1,28 @@ +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: kiam-agent +spec: + secretName: kiam-agent-tls + issuerRef: + name: kubezero-local-ca-issuer + usages: + - "any" + dnsNames: + - "kiam-agent" +--- +apiVersion: cert-manager.io/v1alpha2 +kind: Certificate +metadata: + name: kiam-server +spec: + secretName: kiam-server-tls + issuerRef: + name: kubezero-local-ca-issuer + usages: + - "any" + dnsNames: + - "localhost" + - "kiam-server" + ipAddresses: + - "127.0.0.1" diff --git a/charts/kubezero-kiam/templates/namespace.yaml b/charts/kubezero-kiam/templates/namespace.yaml new file mode 100644 index 0000000..f0690ca --- /dev/null +++ b/charts/kubezero-kiam/templates/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kube-system + annotations: + iam.amazonaws.com/permitted: ".*" diff --git a/charts/kubezero-kiam/values.yaml b/charts/kubezero-kiam/values.yaml index f397fe7..3eea878 100644 --- a/charts/kubezero-kiam/values.yaml +++ b/charts/kubezero-kiam/values.yaml @@ -1,6 +1,7 @@ kiam: server: - # assumeRoleArn: + # kiam.server.assumeRoleArn -- kiam server IAM role to assume, required as we run the agents next to the servers normally + assumeRoleArn: arn:aws:iam::123456789012:role/kiam-server-role useHostNetwork: true sslCertHostPath: /etc/ssl/certs tlsSecret: kiam-server-tls @@ -9,7 +10,7 @@ kiam: targetPort: 6444 deployment: enabled: true - replicas: 2 + replicas: 1 tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule @@ -18,9 +19,9 @@ kiam: prometheus: servicemonitor: enabled: false - # log: - # level: warn - + log: + level: warn + agent: host: iptables: true @@ -33,8 +34,8 @@ kiam: prometheus: servicemonitor: enabled: false - # log: - # level: warn + log: + level: warn # extraEnv: # - name: GRPC_GO_LOG_SEVERITY_LEVEL # value: "info" -- 2.40.1 From bfcf6a36a2007891445b1f6ff86118f61f8bf955 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 15 May 2020 16:35:33 +0100 Subject: [PATCH 02/31] Fix secret names to match cert-manager --- charts/kubezero-kiam/Chart.yaml | 2 +- charts/kubezero-kiam/values.yaml | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index d2fc880..d2ee70e 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-kiam description: KubeZero Umbrella Chart for Kiam type: application -version: 0.1.1 +version: 0.1.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: diff --git a/charts/kubezero-kiam/values.yaml b/charts/kubezero-kiam/values.yaml index 3eea878..66c2255 100644 --- a/charts/kubezero-kiam/values.yaml +++ b/charts/kubezero-kiam/values.yaml @@ -5,6 +5,10 @@ kiam: useHostNetwork: true sslCertHostPath: /etc/ssl/certs tlsSecret: kiam-server-tls + tlsCerts: + certFileName: tls.crt + keyFileName: tls.key + caFileName: ca.crt service: port: 6444 targetPort: 6444 @@ -28,6 +32,10 @@ kiam: whiteListRouteRegexp: '^/latest/(meta-data/instance-id|dynamic)' sslCertHostPath: /etc/ssl/certs tlsSecret: kiam-agent-tls + tlsCerts: + certFileName: tls.crt + keyFileName: tls.key + caFileName: ca.crt tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule -- 2.40.1 From ed91d467fe113f9996e12d8f3f3d7c6ff491979f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 15 May 2020 16:48:32 +0100 Subject: [PATCH 03/31] Remove kube-system NS, move to annoted via sync hook later on --- charts/kubezero-kiam/Chart.yaml | 2 +- charts/kubezero-kiam/templates/namespace.yaml | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 charts/kubezero-kiam/templates/namespace.yaml diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index d2ee70e..f11c886 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-kiam description: KubeZero Umbrella Chart for Kiam type: application -version: 0.1.2 +version: 0.1.3 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: diff --git a/charts/kubezero-kiam/templates/namespace.yaml b/charts/kubezero-kiam/templates/namespace.yaml deleted file mode 100644 index f0690ca..0000000 --- a/charts/kubezero-kiam/templates/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: kube-system - annotations: - iam.amazonaws.com/permitted: ".*" -- 2.40.1 From 09499b2bbbba407eab0e7f8e8b1489041b0a1d2f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 15 May 2020 17:23:25 +0100 Subject: [PATCH 04/31] Enable rollingUpdate support kiam, turn off iptables support --- charts/kubezero-kiam/Chart.yaml | 2 +- charts/kubezero-kiam/values.yaml | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index f11c886..4381f27 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-kiam description: KubeZero Umbrella Chart for Kiam type: application -version: 0.1.3 +version: 0.2.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: diff --git a/charts/kubezero-kiam/values.yaml b/charts/kubezero-kiam/values.yaml index 66c2255..1f91f82 100644 --- a/charts/kubezero-kiam/values.yaml +++ b/charts/kubezero-kiam/values.yaml @@ -1,5 +1,7 @@ kiam: server: + image: + tag: "v3.6-rc1" # kiam.server.assumeRoleArn -- kiam server IAM role to assume, required as we run the agents next to the servers normally assumeRoleArn: arn:aws:iam::123456789012:role/kiam-server-role useHostNetwork: true @@ -15,6 +17,7 @@ kiam: deployment: enabled: true replicas: 1 + updateStrategy: RollingUpdate tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule @@ -27,8 +30,14 @@ kiam: level: warn agent: + image: + tag: "v3.6-rc1" + gatewayTimeoutCreation: "5s" + updateStrategy: RollingUpdate + # IP tables set on each node at boot, see CloudBender host: - iptables: true + iptables: false + interface: "cali+" whiteListRouteRegexp: '^/latest/(meta-data/instance-id|dynamic)' sslCertHostPath: /etc/ssl/certs tlsSecret: kiam-agent-tls -- 2.40.1 From a4a21eee2df8ec0932b2946c12a479fcb8a9bb13 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 18 May 2020 14:56:37 +0100 Subject: [PATCH 05/31] Update docs for kiam chart --- charts/kubezero-kiam/README.md | 20 ++++++++++++++------ charts/kubezero-kiam/README.md.gotmpl | 4 ---- charts/kubezero-kiam/kiam_architecure.png | Bin 43992 -> 0 bytes 3 files changed, 14 insertions(+), 10 deletions(-) delete mode 100644 charts/kubezero-kiam/kiam_architecure.png diff --git a/charts/kubezero-kiam/README.md b/charts/kubezero-kiam/README.md index 9eee72a..55254ed 100644 --- a/charts/kubezero-kiam/README.md +++ b/charts/kubezero-kiam/README.md @@ -2,7 +2,7 @@ kubezero-kiam ============= KubeZero Umbrella Chart for Kiam -Current chart version is `0.1.1` +Current chart version is `0.2.0` Source code can be found [here](https://kubezero.com) @@ -33,26 +33,38 @@ Required for the *csi ebs plugin* and most likely various others assuming basic | Key | Type | Default | Description | |-----|------|---------|-------------| -| kiam.agent.host.iptables | bool | `true` | | +| kiam.agent.gatewayTimeoutCreation | string | `"5s"` | | +| kiam.agent.host.interface | string | `"cali+"` | | +| kiam.agent.host.iptables | bool | `false` | | +| kiam.agent.image.tag | string | `"v3.6-rc1"` | | | kiam.agent.log.level | string | `"warn"` | | | kiam.agent.prometheus.servicemonitor.enabled | bool | `false` | | | kiam.agent.sslCertHostPath | string | `"/etc/ssl/certs"` | | +| kiam.agent.tlsCerts.caFileName | string | `"ca.crt"` | | +| kiam.agent.tlsCerts.certFileName | string | `"tls.crt"` | | +| kiam.agent.tlsCerts.keyFileName | string | `"tls.key"` | | | kiam.agent.tlsSecret | string | `"kiam-agent-tls"` | | | kiam.agent.tolerations[0].effect | string | `"NoSchedule"` | | | kiam.agent.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| kiam.agent.updateStrategy | string | `"RollingUpdate"` | | | kiam.agent.whiteListRouteRegexp | string | `"^/latest/(meta-data/instance-id|dynamic)"` | | | kiam.server.assumeRoleArn | string | `"arn:aws:iam::123456789012:role/kiam-server-role"` | kiam server IAM role to assume, required as we run the agents next to the servers normally | | kiam.server.deployment.enabled | bool | `true` | | | kiam.server.deployment.replicas | int | `1` | | +| kiam.server.image.tag | string | `"v3.6-rc1"` | | | kiam.server.log.level | string | `"warn"` | | | kiam.server.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | kiam.server.prometheus.servicemonitor.enabled | bool | `false` | | | kiam.server.service.port | int | `6444` | | | kiam.server.service.targetPort | int | `6444` | | | kiam.server.sslCertHostPath | string | `"/etc/ssl/certs"` | | +| kiam.server.tlsCerts.caFileName | string | `"ca.crt"` | | +| kiam.server.tlsCerts.certFileName | string | `"tls.crt"` | | +| kiam.server.tlsCerts.keyFileName | string | `"tls.key"` | | | kiam.server.tlsSecret | string | `"kiam-server-tls"` | | | kiam.server.tolerations[0].effect | string | `"NoSchedule"` | | | kiam.server.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| kiam.server.updateStrategy | string | `"RollingUpdate"` | | | kiam.server.useHostNetwork | bool | `true` | | ## Debugging @@ -63,7 +75,3 @@ Required for the *csi ebs plugin* and most likely various others assuming basic ## Resources - https://github.com/uswitch/kiam - https://www.bluematador.com/blog/iam-access-in-kubernetes-kube2iam-vs-kiam - ---- -![Architecture](kiam_architecure.png) -Image Credits: Blue Matador, Inc. diff --git a/charts/kubezero-kiam/README.md.gotmpl b/charts/kubezero-kiam/README.md.gotmpl index 037f292..9a441b2 100644 --- a/charts/kubezero-kiam/README.md.gotmpl +++ b/charts/kubezero-kiam/README.md.gotmpl @@ -34,7 +34,3 @@ Required for the *csi ebs plugin* and most likely various others assuming basic ## Resources - https://github.com/uswitch/kiam - https://www.bluematador.com/blog/iam-access-in-kubernetes-kube2iam-vs-kiam - ---- -![Architecture](kiam_architecure.png) -Image Credits: Blue Matador, Inc. diff --git a/charts/kubezero-kiam/kiam_architecure.png b/charts/kubezero-kiam/kiam_architecure.png deleted file mode 100644 index 0ef4bde70b5273e7918275ebfbbd3194d0416f90..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 43992 zcmce-byQqUv?tm)1OfyJB)BvXoZu3i5L|-0Yp_NeI!H)x?M8xo2p-&>KpJ;(Yi7N>)_b$w`KM29t9^Fus`~Y=b0Rg>6(2u({s;g7JXU%muMGfT!2tlw zcw7t=WfLu&0sufS*HqO}czAgD{YeN#%+dr-PEC1dN*Wp(uC1*V6&0yia=wQ^C@CrX z`uc8A*L2g}@lS4U?(_5WKi?o1NgA^t-e13dg@lC2%ge)Hu*u0uS65dJ4UPKx`unqk z-(jNJdW?v&u)E#G%RHOkQIfyoUn7@CuDjE+w1Bs(@T;cii*&=wukP1_UpK>*>+0%` zj*ccKCf2p(X1#beHaC%bTX%Q2ov|JgN-w_Zv7UYqq#(eY93L<02+?%lUhb|LU2bW! z;^3sfYqNXtGgNZjpYKl<uR$7YpR~OfV#gU)Ae>icj?M~h<^`)n$PyHA`UE2`; z?VX!a60B@N~;j@IYbCV#Yy<}?PHulVo{4GqoIbj?FMeA;d0gY1YaK?XNV5!^3Q#3 zA0bLaMAXtz+t^!NT%9tPnNr#cgJ=p4WQGPudLE`*zB5p+bdieFezGTR?E`T?s`A`rq6C zcbA3FA!WTK<)f|>ISKJF5n>|8#Ys6b*(&-ID7?0qm`zlgTN4x_yPN(3XV+wRqI8~C znZA-bN^|cagm|73d>wB8cTRO357A))PQrh_<i|Y-{L&OHIO0!I zn~~7fYSV~eiZUFHo{<*Fj5PHpgUA zquI>%ly*{njt=56=RsvhgGVgKfTi-4y8Umw&E&EM^zWTPJbU&Z7nyoys~M}|&%&-U z{7#=0B|Yd>g5H;QC##nhsA{{u80xe~PHFh?gZm=#3-KiFCe6G~+I)|qJ{pVX`Q4OUB4$MA4vB~NXEid!UTH12YTO7 zG$*9L03i@-il;y${7FefHPcIY|9TWBj%D(dHH*}o6N6+4)uo1;GxB1ORHwRuV38P0 zq?)_ciXl>Ux)#nwofe_#rW0Hk>7z#FIKA+PVJ+7#0I%uROoO2J7Q-f9A4Zol)cLY` zdi;sz3}Th>ku=xV;ggrqD!;;R)uF9~9 znq)51WMW?f=S(YBDNA+@&r(o;}@cP`)RWE>@3z$n#Vwykp2cJdij(3iE ztkVduf=1Q#Ns{(cIX=;a#F~*#vri5R$6qcA6IftFDN3biKH>5zoT@$2v{9T9uLZui z0{94O?b8P`;6y_|N%SDt-zqCUx+Z=t{JPgF@jo|2w1!(@YB?#=f{ zM#9@9MoSqc`Fwh?OLsIoN07qf(D~Ig)gy+ld4jRLb%g_(t2Oz|dav0Ge_+(YpKY7x z`a^_J4`h;`?-XVmCtnouG^bi~O+JStlA0{`go}wm;0@#KSshUBW6@(4H*YxArLVCD zZMwOj3173~WXNoKCZp4${pT};ZJ4yT;gUXXDj=^ zy)@uSPoN*A$UI3PF(PYm%h$j%-@dG?X#Uwl_Wm{HB6<0>9XQ0MsY zft{rom%v$V)r6C4nF4nC#wBh}W+xC)Sb$fdvP1a1^3PHto>(Gg)C~qIUcz?qV=Pjf zsf`j~dG;#U9wsxI^Ab+p{s+mA=?ppQyEI5#_$GfZ3#<{{$Yo+soh3)AG_-<6;mdqS&=5mk-}=vu;TARsuT1~ZiIr9)Y4((C=Au~_N^7fN9*fqyFrS(2d4iga z7zQR%rL&2z+Bz+O9TSfHG`FVaI<0%Qv%US`z}2@FfzF6pn&?h2;-$p0!MB0! zLph{pReJSzv3~Fp@~|r@1FAT>9MLOK?4ZVzvjNq>&4rn=Ms;e(wr2m|3wVLoRK2`X zw(mtc5ECjwSLU(s95dsJ%2!-pBt5+ATl_!dY{7Yl$F7t_BwKG;GH%`M^wpP*Dk|6R zZU?e(%4Vnn8Tos8GyXV=bg0&RgFjeh-wry}x1^d^nQz$d^Apr2(na5X53kw+p^Rsq zww*<0iQBm_5@3kjyBAhakbDcJVa~~85LqTZCbikMVt}cnK7Hv$JZwwC=<&paU_)AD zyjq0<@8c%)#j~2aP+1wNsN=Ly(z+_&Zj-gtn}-yW9@{%2p0Ts+Mk7-1A1})_L4aoE zj@qgCw)zZ?Cs!vwh2VaOmZY}vh2)FEBH7hSlmJ+za+s0ldN#T5ls$_7NE`V&jfy;;& z;9m#H7S2Grs&L}u5dU9VAb{_Fxe3r7KDDH;j=8jOdRtGK6vsr!CJGoy=4e(P&9~(n+bfN~tvUh5s4Dp_Jx3zU1# z20r!4!k7>()de*bBBb}UkehHlzpl%0P5A?IMZ-DydaGI%3CzQ8(!^W+)XLqas$N7qlT=v>(c zCF?~Dnh-QY@g&uiM|sY>>-Y|HvOE<7iPG{4zY)VKMLq6W8=(~Fn2QMZ7gD$^Z zXu0R_sls%`np(@G7o5?8?ci6(gI6OuJL-$38MLmL0Jf{~FQqDb?$*~$PXW0IIpaN( z7Lzoy4t#(nGQ=>pXo$j^(hN-+u6XDMe6UHs_~huz3Xq0l5Evp$9Or*tb=Ua=QSBSb z4n#)ykH3$W1d%{j=@5=5G-u)|MkUcz1ZFbiEP|;cbkwNMX@DbN` zKGtMNzFyp9VT(?Ax-Ny(QE3Bs)~u<2Bw0n6!dREe-i3$_;Gc&W+hB2R+UikRsu8@)3Gr7FJ|!ITcRRJqS+ zueGm!9?^5Mp}-oF7LsOV2ThRj*e5^0q~Z|AcRs}spHTkxt@^y|n>O*Z9zOm=&c+y@ zy`B3B2X^9qJBZe&*4Gtj=homSl$ohoDd86wGZrWRJ$5PPohBtw7b3nn^e2%jEiae( z&$E6I^JvMjsyd#flDu@9o+-Q~tOk)FSW)jXAYoC;gRLRpNdHqe3!7yKyyQ&7m3(JK zFGGCsqXllbIp8Oj{lfJOW0Hl5%Hr3LvGI)^mg5UMWSPER`V)P%(F^s3zE`jN+N|!o z5riQG!~wA-Y0Ch)4}|ZzJKS6DDo6<4+lIYt58a8NrLJSe5+1o?v)^46`9ZEJGmpW+ z@<7MDs%qA*K$ai9Q8ohcVjtQr`zcXc8nCjnA0YO8Xp67Zp%(6PynU;;e!eDr^<9i& zGuo5mtwTlnvTUu&4Kjz-X~^~Rm;DU+K)5r^mBn6o>ErH3geQ$4U4CtzJVFHM=XF(ltx@rw*a@Tt1z*2ge(&z!5I9AiCU0E2Xp(Kz+rAHPZqXr7%u~-sA4SO z?(X;6vnUBNaxw|7w6;`NhCC)2;=s_)5DZPV3G6|b{%#E%0}2$c-feNB7V?gM{?)1P z8LS>J&VN_B88k8DrI?{ar5bquVoYgN>Ypx~>Us69$S)HmpCQ5V-##9fa>OX)U#km(gY%-s{a%zc^l=`k8A;AOQI?eeFQ#_|B*r~$Bsr{mc2#OZl<>SgB z^7sNhg_CC7Ym0bgw1NpUr$ZV?6YW5twp2q75=={}@{HS;o)<^^*6)CBEDuKH&!Uf< zURmB6V)dm`R2F~yv8t$F#?hVPiA+Co5U=HU^{ec&P{^#))L;udKz`7#$qM-_F3=Km zyiPXWxSG7aPL1QY+p&_ZFDp)Om{PAE@o3RkP<8}S5T+8$@b4T;6TXZ;Yj#txn5}*- zm$IxGP>4ftFwa0NNJe%u%la8)+>wmN=E2+H!HF1c!< z___J!1N}j=H;muDGE=;t^sIDDdv-?iHJYT#Aa{XiDh{p#Ye_H_39;c7B^enn$X$R(pR7RjwhBA z-_lm6nnt(Wd_{nI(Ig@$k@gKrz~htymeYK4f4=wd1$gS_{-bvBUO|?%h+IZQOwVqd z1a>@zV6Cb&@AL$fB2TY>>$4)yg=L8Q5aGS3!AceOX{F-P;A5ThZQW-@QfvI>;Pid@ zH3-jIIoj_1`MsUBbwv@(SWAy zf3~cjJ&VWGX1K!1vTVdHnGFd3mQaC?|3diVr+0~rv8Gr;g=(ZQxxxj;Zd2EFc8Vzb z*A_@?%Md1sp&N?=OTrVzzTzkD^U%)Q=RZq06m7?#Id1*>Pjl8uR@S3oM^6CW9~C#s(%vS>Ve@$VkJ{ z`XZn>Xh0ldCS_X80aK(W8bEnfh7MSz(eg+B?{>cWtfFv4*MG7b1BfFH56~~Oo7`ME z@o{e{4&OLY1q4GcIp2;@m&>*h;nh^NL^OvEc<`#dZb$HUqXFMC zRaA~$^uoviF0*AWaC;t105QWH5?_oEpk-Xey(0TyBu{Ikx|v9ulj!P-9kw-iMw z0BqDFsgQ*`b14)~CZQ@$1hVkA`llxR=Ez~zl{gX<2<5{C(mB4+c0N?$e#`N4A{`|} z27#-Lz?hW;iGUj3jLCK=D4M^nV&=F$+aEPwa)W|O5g$0fc7h(*M$M1)FN|3KR?OCw zvz>t1GsCHEI?kU^MyO5TiW!~4vx})FAey%*mA?6;Pie}U=}xr_j3qDRl}Am{{-c;! zSPziaVENFi;#r?o$0jMc@b*dvkh_wU^nla+>ubV_S(ev{5#ZM9cvZI$w}+SD0a7=E zrTchBMxSms+MPGkWWMjm{2R>E;53I)KMdNpPspsTCsaIt1sCm?S{GQ02NIPVwTjmO zk*;TMYD9N$FnG>e4K0%Bn)L3X%4FrwnV!Dh_01R`iZ}Th2p4Byybi%z_xsuSb{xqR zUlz4R&UkJ5Ja{X4{TndIAj@Z~@VY}0rS<;lx*t&F9o)rxFgcoHO=r#9EFz-A4PC?x z@7@W{(<#yD`3cCHuRtJT_JYt9Nu_(ft$=%_kdhhk^ch;5KD0OW7C98{A##6U;(q#Z zik#7?kLlM(LNnx0s#S-Zi5P{6Scw4$MWuiK5v31{K2^S5_)dl#B3DtDkB)hhId0Ad z_Gm2qE7OlbT}2k2Q)xrTfwG_S5-`B=Juhezd}jhGvVuoClsrNCD(VL>X!X#!|MHi@ zN95iHrEWvOZ~P!XyMT>i9Zx1a+);Y)Zk23Pyl}mgq_!|KFKe!m{#ZQ)mw>enpkGL^$l=>Ndnv_vUs=W`_`MP;nBd3(|rxy{m;@ zo`zJy+IAmu9+>4H=%wQlb(PhO#ts8p>;3IV!Bj{&B10rhe@J=Nx`%Sf80xAF-%91% zi5I9qKp&D`b1_zgmjHPrH1VHQOeLn6FlAgUR>;1DG>?aKT-CaNu$6&}tC}w6hiNGJ zs%Nn`pYO^$5wYF;^eW;HcIsnzy84*~68HkVo+rseT~0h9M&kmve*Z4sqZ2yExN?x2 z-sa$+E2;^1Gl>s>LXjjf-5L*^rQF!U*>+Pt8jr+k6Z)(S=i)u*&5G+yJ=t|r;pY9B zPEDOeS~261YVyt@RiNS6CDW2oI1RoZM+5B6C|%bi#vD?uo_lhGZ)sBv2(0*W3{k%G z7>!XakXFo2q_pXa2Rj#rs({s61=%aaUXcAl;(Yq*W}8R&Enrn<-?jPEY*lL8YoVeG zLWfuBJSvqy=W&%l4Q?vNsx(pDeZrW%bPdhEU?AXUbgk_&Ubk zEW`20({#@NG!t^LfnWs^T>^Qsxx_V&^7gwB_&bEPryx6ck>cWx!8YQxypf9nwt=d4 zlHcWtt>qskjr@9)#qeuYN~_;(Zw`_HR+c(mvw~oJ*i*3G{r>aAz`)>%Z7K&_{Za*> zTLKRhXSZ&Hma_9LC|0J>$3ZF;(#5%HxmDS_HCXrJ&L#HHGQBOqu8Uc|PF>Ly*-wA| zyd7bLp`9unWDJTg;RIZH*Dh@-RJt^W=trnXsEBJpyc~F^;js0ikz9IFgxTTl)&)*QRA-q;Ck`js!uX8+uKlrQdnQvXqZAlPvtayq^@3+RuSxT=p(l)f$e2wCBG|3nGegWluwfo zGvWz<)VNQ9yPmyv#>nw0vkXH!G0%}bH@Xb2hEjCklyh2#_{2lQVTg_K!L`n^3ONq> zb0Tphw2`m09_RM6PljIq&>WB`$!)0Tk9x|>8_9t)gthM7Ayjy)@9dNOLlYC%m7@!j zwm?PZPr4S0O}D}jvxwcCi=3oe*83f(ub?7_{DWAEga`!Q=;pT4(pgbM)acs2f}B~4 zhya?$zpsRwudTjw?X3K8STV9an^Lu0m~d-?+uzmD z0GYhdintW+e)UhIexVkzw3IP&$LibDP5uwbd!Y)Ex1sTNEJw-$9=S^@?i)j*M22b6 zTv`Z8{DoG!&q5X^q^la+7QT+0L5%G5QX29eeD z?<)Zoj$kOpZJ;t?LKYOm1_d7$UWiJg9RB~qkzeyBWsHBoP;fD6nba6ArOx;d=X*}JjgaPj!w5y==qy1DeaJ*aa5|6kns(U z&IQEk-g#)IXE$$5Cmnw?@$K2H1%mD+j+<~%@8qLp<-7g)p;owUIx-kwkOIcIw^zwxR^qz8maNn8~|z|zYr zW`~R}Y(Ni6g*WfJ%6+*h`J>AcRI&BM{FMdL^*Eo!XFfr2yn`1k%}8X!I$bI?- zp0fDB%@-SrlY_VJhZC|&GHzp*_1n33A z9OA;9RbfIONg2s@p^OEjI1rFYAf{I{aH(ZL6ntghG}0`Fw+G|fFv&hS(O7wQF8lKi zIdX+t5umoJshik~Fup7&+uS#o_-=kb_H54zIrUwfB=KYMW6eZoD^~8us+fJvZ%M|P z=~+k6cyQ3lwqv)H=Whhi>uexhNUd^&+D`{Lw`DA^?|>v3pAg!wq#5C&=B&e`7u4>xVOU4v zX;o*1yzxm~{_|sX0rBQBhGpG3*E^LsV8Dj?PnOK!-r?twq3$?@A?pqarJu*ul3p`m z3Y4m2s7qDkM$})}KV|s$k|7hlYc)E!e;WR?=aL_;IUeJp#Nq8;7 zmnEKY?!!bt$i`F1p1^xd_p+f?U>+sLfk*YcsgtQIJ?%yQmcK$F4ofN+&Pr#duVJ=+ z5T!9TF*6iBSJ8^AoNaB8Mine!t1{>5A&b!75p#= z%Vp6|!VtVl(vh>SkGV>W7~oAK1jHZrx-x(AZC2F-;d5oVMG2~5=z&u$S`jW1)!goSdErvXWrfH z=SHUYl7h{>JsFO*qKP*fX4)1B3TI^0QK)Bk)r3VC%F`kW<_DtSTMN3|VcnBC-vg)& z<5F=Qb@jv{PgD75-bp{%^&Y5r0=Tw@4PhE-*CEAVZmu>Yj2R}SI-_dYjQMEGTCGqf ztB+n;oJYUl9Mo0vf1<3`r_$h|1b7tr-E14~KT|x$cAx;`<+)B_V)Y3yw?K!2+<;?u!4`l|f%-;{3 z9a5Ox>cYY$HQ#6@*>5YU1!}wzHojsMD0*hwt6;z)7LUJ2_Zk0Gcr#czDqlkIArk(; zl|6`9aB@=6%BB$%JsVKF-`*M;x)Gk5$uT#R?3geQdP}CDcAwlX-e5_I*~7N{ZQOiK znQ}sQ=+WzUUC#}<7_<-7M?co(~?kXp3|I*{X>m#R5uJ*5<|0DHX7kNb^E779E|F?L<(+w;=j-=vWeU#1}*@E68EeHkI)Zu88DC8yv#Cd~yn!O}ydK8{3dJ6^sRsb1=Y38SbGiK0Tpit}a%6{1X3h+r6n zo~RK07ffQuu+dW0u%NhTm-lRGiIbcCota=92ox+O^lQO5Xl7EK zR&d5A3Dz5)C2r(Gjh)Xcks19gggJqmd71AL{`McjAVFl-kx!YTS> z;*MQ4j=+??XVK!1qw=@8Z{B{!G!oZ;oYWc_S-(0vAlGv>iOiW<{`^XmM;dUDplU^4 zjkw7L6`NSsnapKN;hykXc?*&>UKJVg#5j7Ya2t}j3c2`w9UECw&rvAplNC-qGSR3L zYnLC4Xk0_zh>D??d|Ueb>Mg7}m-#`_9G5YuL-He&B-N{Rj2XM0Ms7! z2W+q! z(0IuwUpQy6Yke{Y71v4JZ8_AC*jl`&54(Q%fIh&QEiW&evEeP)UYfp9@Q$P=lslU7 zhh&6el~NTjo)!-N(DxLmd64~k*FQl{9aJ$#Hi~DLU z7oAQ`9H}dQ>F=fkx>9Pyw5at3YVb`lj`I&hPg>4Sy{laUsW!E3uVmWNXPwQ4*1GpZ zER|E4jM}%XdcF%e!_5-~Fl(aTEyjb5P#DJh0E)s7Rjn5vVdSy$5xOzaK}6a}ft}I1p2fSo*ekCbd%J_oM)fWj*N*D4T|X0b)n#Uy z@9-TxsWf_8xRjp-Qpl`NRmf{H#tVSqV(q#Wn*>T+vwO!39kaQ>qW3uTV_qjBmPh*X;YT$z1>VX{=L5!&s&BCjnj@ zjqw(KD{w;h18cg05w7lP8TmjMA2n^PX#a4aV3Vik=zN0g{1C0IHe6pPbYql%Nhh(f zZ0D$yyZ85TRhp{>)0Z0wbX@FV3Guj_34)tUDM05?+OQ^m@@kR*vKq~N$ zPX1rZ)fk`p}(WBvZ95SJz}e^|m$!z_wk$iS6vh1r1Dq(2)umjFEoj-z>RuVD=a zuxfYNN+n9s%2!vHAqrZ?TYHc;2&M{K{QN#dnH#AOtm-FrAf%V_$%~t^u$e06c*sBZ z@l9PloVuB7>oi;IRF}UuIsdG@k43aPX6jar{wAQYW8Ya{UjAN+y*c8?FCEa6#F-KV zBV*ZHxPrW1F*G`&TpdX9#4t;_Bj4%NIxh!F&&YlSrM_$Y6mfg0gCu{k&X^CaI${TV z1DiioN0Ad*?_6M7@c5OGYwUS#eUr#OIx~p#em3^!P}vO4q@21K zB_3_6wh&uzaybWpTll|q!HggJ2KH0=`R9B)-QKj|~-TcKA2tY;pZOR&C#0)J$CW#dKhdCq0T(Zm6Ly32(9{&NbuVbX+v$?Vm}u<78*Tf)xXVR*rA&9HQ%x-0)K`xjS4R>sX19!!2nbg<&!(rW1SpTSk-` z^rvd@w`r|0*6P|lir;)MLb1;7B{bm*dPUGG2QM)fi5#UhsJKXWCmWd1``|h1PhWr? z1WVkW7QiSRPD!Z?O4KbuaHMBdhIn(bKcl1u-Nmg=PcgUnj;WU*soXQ=&|FxR=#VBwDEJP&>}LL$^mvx`Qa4txg^ zAID%9P?<;gT3~C}x!puUFzkl1M;`bGr)2 zL`Q0VSZ0`H^fB^u%lTkG6)jz$7Y$it!f^$~f*z5M{)kQU^{yd}uzr`m0xx&S6>1)( zZjL@l$Wro2B*WIJRk}yBcbH9YOGAj1Z3d@b{@_cuqr=*ogvPSW>BWV5Y2XPS|8mI* z0%RNlNzvSt36~LS2?c&w=t9R82f1&lEk=BhbEU`2H!1}2 z)o6dRX*%;Rw_WzL+6vsz!_d)}5n>*H#g0uRZF-l11480I=$e{Acg80_NC%%!nDt0UwstTzsK=`HMTWtqwM-kDJ}>&i_c@HLy~wXrN~1^TWfFb)YDf&7(Xl{ z2E`+Cq<7kDAvlS4u{bx6PgpLIkAgnSb)QMjo0(x`(EP-p^)q{TYVs%hH|V269$>U} z(adbBeDB2~)wcLimFHepZp-zytQd1@EV@_XpJA-du#sW9q)1HLTi9&u_bTJ zmPU4WgAG+CLnNufhsf9>s+Fjy1kGoI3!f>8moZcOY`^sm&m@p3Uwt)_PFmc)iwyjl zgu7S&@JmT!ED^f1+sS?fEA|!QxW_0P$tu>UF|tRR76;fKUj{tZth9yJCvL#p?twVzW?@ zpF=JnaPj^)!p{7S>Xsa^)_>TRRq4pPt{hEwopWD&+C_W;oC6q%<NN9PZ#Xr6o9i2oWj1-n)<6{36D%1`YZ;$arI_ zoXwCs4&@OI)JHbR2?Hz&!|iW75O-dV#ZtWT8cUO-cx$O`^^ZAMAX}6(sZ1PH`MTZn zXz4x#k2*4FtEiO;`si2}OCcCq;t;zI=U6459mCd+sz1_;lINvy6hrk7$Zsa``IK(?4vs`24N#_Y7Uu>gv|A6a$xwVw@ z@2EaQkKz*T-~z960;V4eW>f*^Jj-4=T}Y2D5}OvY{?0>Wxr~nkCeZ5{OkHA1lF^4D z4F1nov@u!=eun{!U!@%f0)bWipxZn;13F*ViD%V- z96XsQYwiqfaXmJy;Zf~|CXyKNy@T$RkyEe#W$GeW@n8l~qH&-Cf6MUi=nO+2+_NFL zpFg2ZQ=*0mMCl~Yu7DvYAS7=T4`GoRA;}+ijAMTpD9<^X`RYqCyxmH=72Fd0`>6RueS}Cnd;u~= zW!}ng5iBgX|E*vpH9CYY{Q2m#Sm_o0sKc}!<(*gTE6?i!ng3;+2iWn(C=-&p>}T-i zOzr845K6-vJ%xy$J?6fR$~_-c*A-!OP5f?tX1-Y~$xv3laC{gv6<<0HSS94x&PHS} zxtZn$H`lRE{C9eza-80nx!~0$RALRfRyC-79vH*~V*^!__u{Z+?%d9m9)(5|LXk12;u?P zn?XYdlCMHcDyPiswUGkP-d9!5?x&7RfDYLdA;gJxQo{d?S+Dl-`rcX+ z{wH2|%CXgIz%5fpd`N5DPW6Adr>A^B1p^MPI3SrW~UAx zKjdW`PQ{=WYl&DMIh0JMr+g|>_1PjQaA z`*&#krs4J6t7-(t?>;Eb$)i;-P%u*pgk)k;*aF_nsX`0q5ZT+s)#too&M$K7f_fUJ zNI>=7$F=bP6AAUz!F2R1lWGL)bIQfQKT;^tWH;fc0TTZ)ZBO?JIMSF1)6iDNW`F2M+uJ%Kq-))z`1n>YECdEP|BhB z6V5-XC~rBNHx+4qzat{z6jkZCfU1l$nxz0A_$vJGQSXYO-~mFiiBlyo9D1z#o1E4F zv?{K06gpIc@MRPADLp>vf}X7!g6K?AfSrnP<2xccPSN8m;AOY1Au<59u~wZtQG~lV zRVNg;M_nGVO+$T$xIGM|IGby~D#xveLr^E#HN=TK*!(2~Q7&=KE`tFN+Mj_uIX92_ zgt?s$kowIB-ki+^2Fmw1jkwJSp&`}UL@PsNPo~Su5BsN#2BauX!aU>ssCw4*dXHekjH{*rEW;IX1&e&0z)4NO4`VqSI z)k76+%RE)(VN5*`lii{($WVx4kG1wod2rjal={1x2IE=6p~Z^#!L}33f+CMk*6I>K-Mw9mp@6Y8g=zcT$xvBE39d4SIa`>cbZ|kHBOr;O!3LjKvWU1w5jKm%>JQR2An&l>zM$4^4wcEi!jrr zg?0U7&*CD(W??X+;f?N`+r>QMQK%s{Wf-Pb2nM!bmmxdlX9rSM^;dhKOgU{Cdgm`g z%?vE1H z;!e|Q!fbKnS|`%fTz3nkSi295DFL6w>8o}rko5Ms!=!aJ*qKsgLIp4Liz(~MTBU&E zKFi#&Ut#MdV%CyzkJw;7LxLQ3;|e0dHJKYhJjLh7;0*D5D-miZZPlM=X6fDlw>|_7 zG*Iy#OO34}tc67l)wr4M*_^(5k25nBcKEUf(0pyk!_@v#ozW)qT(~)en}rVbLCRxm zr58PfA-@B2-1J}UqRhwL+JWM!(z`}5(oBhQdt>Oc9g#-MANy#GYoC0PaAAiE)ucPE z5UwCNr0;fH(oMcGG+IboSo%GYevp9>9t`EmyUwCfwpaOU$o*LQ>jH1u(rH@h~({v#8H3BjY4zeC7c zziFPn+>R?N42B7~f>>-~81W$X$m?r}&yN@dNlog6x$2DDKWQe`UtdVSTwbmGh==6* z#=NrUrcdVN0dSkPquE|OxPVebp<86O!kxj_MetX`=SLj!TE^Xdk3SW4nmVS;s;M8& zyi3!HqLR5Xu6yjC$;7YqqW#E=+BYeA)l7^AL2QMfTb3^z7q9~?TZNoCT#E__fhlb$k5c3a;8DbQ*!ZY$S zZqfhR{qXqC8fIKuX(>SI!&4^vK_$^f6*B1& zE~Mifwr#?CWE8EHvh+``s|AK^o!G0?4J?UCBjv9}Iuqiyx`?(W4Y?h>rHw74g@OY!1XTnhv!?k=G?lmLaG zAJ=u?@Ao{%``7zlj&nzTGqamLcXno$F&Fo%SV30Dnl+L2W4bf^Da#p-MH%4?-zeg5 z-{m_22dL|4|gyDWR=J)OYlZ zgas998JI2eM#h}%Fj=N|{1>5n(tUmYH}Z4;XnLW8&|?z*sm6pl%HLfu@1%Pp3wfBD zf{5{=T3aA(xZ#3 zw3eHqk$`fl#0Rc1sboTg#Ye5we6Ri_r*t~(Y>r+h8%$utrQT$(e$8wDlNYvNOGL7( z2D$%+E8e(ZaKmT+R3t z;cu+NuZL+7HO};)D`coKw8%saGz#E%x1l)Xu@`c+K@WDtR;FdqTfG+yar&V9C0yO6 zyw*T4r|0dvzP%?|srIl2HHa{le~04G{#DB!#?{-+e*C5+e;or(=-$H!uxk$6oort~ z*Bhp~ty>{c_->RD+a@Hc%t<1Gn_Y9JP2|gY|g`G;rGP;4EW7=;aghYYS>k%S#r^@fyB5UL8+P zBC%0jB}*QVw95zcLEWedrCs^)x;wC`3WyQowACr6YANTPJtBz z&tr;H#|94r-)f+TT4J^kiCn>YKq#z6_dd>(d#1ekV0fQh2?qMKpv-aa;^%hW>;~9d z41mhQ=j^WlEu-OPIVcaTE9ZR`OFF#CA<2u&Kc%tZKWoG@QloPjp3AXpKH&e#=zOEk z82qr}3Ca^^AB0~HenA=ZDIp_4i^JE*G}=$J&?e=EqBgrumik}_!@DKmlmbXAndE#- zPh52lZ&kToE17L-feqi`f!m)|pV9h=)C%-Qvh<_|zUz@!nJY;Bt@PdPOYsJh7Ie9_ zb0W!T{3b*QBp;KpuVG;MxVB+?zFnku+7TJsqm-bb9F`ljd}imWz_6K&X7bTB6AL9- z_(eNCV-`dE2l+3tVr#|gr!g%|j<{wOpnN_>9SgQ<6lM(~zkRX=5yVUNg+=3M>r?Dc z0Nwej@=Yr>%FTO(1nCZKZ}yh#(>~5IVbTIxaEKW)ZRAB+Si`FDnTMB=->xU#lZI0g z-wy^7z9{dx0Zf|Ui`oqxXyJV&m*2g1+cH7_R>WE%ZS-IaF5)L}T;m~DY^o)J^y@mQ zxxFm1o48O_^5VfHxY&4hfD5e;`jKFkI8u(%;6u=qAL$o`U-JPrVKDa+4eY@^1(QRD zmejV=Yw_$uR~qWIwxtwU82jR;W`ZK_YQFl+k1uH@{O)H>dbX!NgWSbj1QuELR^@Tl z3kUoPI6{29hXd+GmfUO>`1NB*Gf95V@9%#Uy#xSTg4+iFy&|<7n0!#PA_e6@MW+hR zy(<-{R;eMiS6YjA;LFw zR8c56B-BI)GHAFyXi$RR{DR@DGI2lTUBBh6I*lS7c@-m#@p_rUDde4vY_jCQ?!U-L zz63UBG6M4eaF*w01$)UW9_TJa_gm*{w*sYR^PBQJ_LU?faduf~|2799_-?rD6Q`B2 z6TmKKgr9}8l}B2%;-}onDBepypw`wXpxZU!C7%O|moXGk_8XG6H`md#s33#7I`DrC zL}WdTeC3aaRn7hggOFn#zps_%k^?C$5$~1CpY}$b$waYEz=rq%YN#!3lI6+Y(ZP44 z+^tO4&0Z6*MJ%XAq=^hjK*)=ei4?rT8E2^sSxA5GhdDL(X6)X}SZ&1x@Sk$eZ&sjV z{E*1$q!S1;ygVcWw_hu96W#)HMAUZ*BgXMPaof_dYgeZi-UPEYwABIUe{KOf4nRb`X@*E%$YT<0{%_H4L^8MI zIx0eQTGP`7J!T*&oBqo!J(N2~HuAGd`>b7SM?3xten5g26`TCVkOukmxiR?cO0*~n zkaOF`+Ar-(lT@iB2a599U}{Y#j{C&s*QZiD=O1tXA=kvBg|RymRHs zLT;mo{dFdh?~dZ^$M>diA#f$;FA{~Wj`N({9;FJv*?9mw2glnnIeTM<^K%;PO(|C1 z^u4Y!xdKpLz!&5R`j+^F51v@5(3=A-v)W*Dh{a8G#jK^=gTR&K>HOIZT30yGN^bkp zbe`^R*Lx?MpQgl9xh6~`IUbFdxA{t2XClyqb2uGKk4AW%;X1^B-h|}BhE?%+k!19W zV{#WuKDUFeb=RyNKt2m%2NxqqHp!oj(ACqeWRM&4;L`*>UxnX8U<}@Wj{pe_ z=B@ts&{rzjK>&d=4ab3O+(i2`W3Y`JHFbpGW=1r^0i2$;+d)+g*!!|*+A2a}{W&XZ z$?|SBfr`Q+S1YfyV+RN1OU}aH`ky~}%^-FtYBJI659D)4!J(@_Nb{ckV2T-&=ZujXMLL-SO1G%5!?SZ$e9zY+@A0D^Qgp8+s{yF-@c&nul z4}17*MGam&y|%dwf>7SnjBl$P%r7xDCi(xoU6Ie-Q88mcKyJ$cPZ6CBOP=)M1%J+8 zex!Ym-;2VWlr62n#r?dS#Oc0(WqyBw153bW%F zaz)ZGN8(jg6C8id#rt&;nVsjA@KRN(SC0G~P<#`l_BUb+0Yn7hr7^Zj$zvDb0bDg( zDpfWk$t^39AmwylGfD64Jb1I_&O(t)gYnqMWA zqx}m&Ce_QFn!bdxL7NKv_>ja)ln9u}MzB1Oz@*|Rs}vfsH0%gWAoCoFt$G^D!hX^S!8h*L%pOwF}v(@H|wT(&sL2e zWuQ$Y&TG#^Jl&Y^F%0_GV%=04L85J#Arz9@H5>J_S_;%t3$!1mCMH3H*5)qj6&#q?9YNMvtu>J@ZYa^4CX0^}0R_BKXk<+XP zR_Q+1^;ZDie;9wQUfZw`NzvB)$^HmH0k+AC;4L&emQb>_q+8Ti!0+4c^~%+smRVkN_3DUpbz_55I+3bP*-PS$D~gN-}| zb+dpKdQ1xo*hf{BKr_FBXPrbG%lA&9hLbV@nGNp}943Joj&a>|KA{7<9^vlw*pE-o zb8qN;F@mCy#Mp$27+Vj(s0OBgT}#^(sy7J5cAlSyfrvZrnr0g)LTyEg4a4pv#6(Nq9lg0rQfI0;<}zCX`Z5d{Xo+^>#nfA^*cVe zX{Bp;e^R#jsJUnMHPq9EgY%3T?S+{3P<0T)(-Xw_Ocm83*&W$V!MRO(1v!6$$)&*BL3zDHV3a1{KXc6kARLAvrk=P^5dq-rpS%2 ziizs4$mlj1P&ohEo7Uc65o}jRU>oY9q_Fg9ychC3e^?p#d8I(OpQ!3gX2?k2@nK*c zVkxICx5f!tajp<$f!!Y>{t z7CtKzrZ!Jj8HBNmk}PrOngJYxC|*VqK#N{i%^LJg57ydCG9Uk~=m56rQW-G%@r6(N z<|p_Qw4pir56~KxHOq^n(0QB-&#`CfdV=GEjw1#}6NR1CQ@wQZjQbuO0}-cl$F)Ke z)hjScfe~3fP~B3VPEcjUzDM9jca^DGClesgEn4Zi@xiM{V?A_1lZ1rdy5+(`GE~?x z_K^{4WjW!-iqn7v7L9duq*?)hEy$PjDM$6iIKA*Yr-y+67i7enIZk3)KmN2J=em;@ zMphfJ-IYGum!D8emI{;bQ*S33Re6=`z_`|+&Og8Bl^2}19yn^63yavVbA0X zubw`2W)L#RCEp~&-}U_D;nz_l^av^yF)C_1bMQjX<6E)sf+!J)cf_hKi`lAK*oT$J zLO7lxkg5g#E8j@CvD2BYCKxN8{{r!9kJLrAfd~&vw_!)LjkCS~&M?5p$$X}Oy3okD zR~#PwQvJ>LGFnevEHjF(9s`tN_-B3$6RZ9}S%a+t_Gs`oMWdn$kK7W};p#q3vkeWY z;;)KP3+2_TbC)w)NkB%Q8e^kHiluJvnI|f*<)D9@b++$GES}#bjd+B4^9Vp^a$b+8 zwNo6o1QULD{aE9g6JUtEIKXe&Nmi%~k95!>l#${?%7`@9C*L|($ypV&=ah$Qb54ts zQLQG{hFgJv%?y)L>|<=kf*-w@KS|t@R6sWVQ65rac(&r3H6}=|L4`1hi9uYY6G2k1 zNq2LAr*@Lyv%^oZVyj9x(~WV6gryU=v@yJ07djv~W>vE~QA^3k?RPjV#EC1|DTV7B z`0bq8fbZPskcn|o2%52@>ukQq{8W)8Cm|-fb zD&`+&ngNTOH~3B?Z*m_B`&K@H6jJhbnMR>{NeQ*f%`yTk8+DJi$Y|*CU47CRKLb`& zl81wP>3Adpxfe)LcqDt66Kow$_gLycqC)aBG@ep=$yxSIxBfl}rU$nyCQIZ&T6v+A zQvC$sg3A39`~9VPKOn;W9ps+-_732(k9@RW@DX{J4a%@<$eBAXyK}6j(6!&;p{E;8 zT$X8t(#%858WPR9m1PA)r2MPfDgySG3IK`5^ zEBW>A$wPc2Rr~#%`vbbxuF#$J#Ih{j*R7=x#!Y&>b_K|b6}#;|lzt*6OdJ{%v^S$S z0}_)&s_GU9y)up;kwU!evmc9R6~ms4Eum5Kr3w0Wz25D)r5YEZF-rl1zG_22y{)@( zptpa0b^0ptA`FKTJ?5-^5ij=`SPQ)nHN#3!^a6^l3N5e2e(9{yrTXV6$CkTqJ&&B9 zus&aF%nuj%buIQ?1o3|kZ6Y;q&Ad@dRXL>|&$>ry-pNY;(kcgwSl~v8I{xgEAz9mo z)(l+{*=Ir_5G7~?FjG4qHO%L9f;>)o^CNE)AG6A5rkrxp!gEF@YJ3o zJuZ)eYkf~_s3lauof16>AoNOl4;gYoj7?fV9G+{@@b**!u>o$z;|hWty9tY#)+ysg zMi?Ugmti_ED6FbZa0$u!PYW9gUu#$`lrHuCK~`pm#y9TLQpp57w!2bcIpFm&9L<}< zDnX@-`d=c$DCIX<86DxDWv$*tW?m_NO{$Hi%(z!A*?$Y%2=E=$qR_VfD zhULKNuensV$uc^k4rPEVupi0o5-clNqEY zl+WjV(u*rQD{6|2dvtpJnr!;RITpG?t1Cqy@cIzE8JI~*p0O%-dBtrxee%pyY6fu` z@T}l79@;@N`}5Yy%p63M^j+^i=>Aa_1w~chco~e&ydn$4eIxboI>W zSxm-1zt%@-pcG(+E+KNR>rJ2q1EiNrBlwZ1UZN$zv;IlniB+yqQH*RX{(joX&opWs2t`5phx!v^Oi3hpbtb{U`!?Rn^gy zkJBfTK=R>rO5@2Jj4a`a4mJp?hxL26zuSvPtPUeIDtVv7BP~6rm2D!M)Cq}`=4$Ys z?P}isf$r2g>`@;;0MRS2QnZ= z_EE3K@)Nx8divb0{11T%{xG2k2M2z&o8o+epU#bESF(T0P5s5((lhWssE;f4 zOjcktf^usig7g?yZT>$L+<2IUv!_)l3K6Dqg7V)Cv#@qe#(oK~o)a#khnFSnv2 zc2cOagtuNwakQRqhEOI7W!~f0SJ3lX@2=NO{i{3N2_Cwj1)CiB51Y}D!>p^!M(16h zvx?{*_E>Mv9BeO9hp3#B+3|32U%d+YJMEVz*_B}*cwILQ=pD!n_ zg%=@U)OCzhqcO7Jeb!ZL2uahaA8M81q)b&{g=oywpBJ_LS^mWpyl_2x2xapR9wgP^-MOnyePsmx>jeT7V*I zZ70N*{k3yi z2KPyJMq|lF2I_E+a+EN%U|X+VzT^@LRpw#a(cQzKX ztae5mz^y+DOB}IK`~8Yh3Fz`>jiVKjaVkmR16UUPLJpDpLVME_xbq(d2)aPY^HE&psAFDGR~w@#eu%&-(Qvj zAAx$1hJ>*MgIU*~6Md?Ed2R@sOD1vPnaeFVm)P|ke}<5v5Z^zS-s z0$GCR<~5)kFncGWPnLW)?P_u*6+P|moV;z6# zSCNZx8JIUcUiAwoMqLQ~8*6#>rtOyh(a50Y+0uVcdK9_z7ZUT6X9Srg%}cP2KbBHL z`n-})eT3`(Lj*(MI-3P)(*WvVQ%OFQy`h{rpPQFqB)R!e9PRp0203b(WsC;`tIk>wgg)_L?}4)_GIbm!1GQ*0alU!tGjN{{Luya_$l_mEEa1-F03 zSY~J~62}++kgso*4@8YldV9>C$|$EnPiRQr%PWFgOZ7hZGyV%UBDA#7RB$*`$d0Z0 zcH)Z@Uu&;1|Fx1u{fVBRtWS%_bJ1>IhJJhgyH?76PYo(VCOE9TBpdW|xo2%?q&YZl8S9`FO zSB+!+JZ!B$(}*ze_6Dl4qIUEjSd~fH$6gCJI7x%(6Iu05vWyNYfoKjO%$9pN+tuCf z-=&zJke^3GE6nFr%%FeakQURM4R47`Dk=&xsXNO}B(2@i>Va0(3tOt5=7QB|qq zSY|P{E9>sMCrfEc8x=v*rYO+@N@Z@}zMZ)-ucxqsre-y&>dwG<2ERkwC76_{^*omz zpkvqt0g?+@H&;^~(G&N`cFZe$jeiB|g_DE_Yd;=w`WVn|t;QNB2CuY}MR8WKEk#;Z zJNS@W%)_Yiv4Azh77zLuT$r7ryrLrSr>9gVu=4{}&)50VFR-kxsV1nz@4lWoPLuR$ zIDt&2Q?QK_M?PzwIs0?!+tMh`OBBkJ6*wb)lQsbPsV`JaT8vGWvf@mwR3N`^vY;Q3 zGGZeMq!_G*C|lf`&?>h4d{qwmls zn#B#ypU?MdLAtU};&Nq|FieRJED6EhSVw}Kk@ykzJY-CpaiUD&;v7#2?2w4a${%%I zg_d@z^I8T&aPwGwNEXhWPAL7R|^C7r47Abs{Iu?o&5;8@O!QMyFW&kpGpBX%G7 zgj7+`=VFO68kf~9z~_69MuWVZlM8GdTMR@ zEeqG^sjMH;S@om$LvSbuV(kob2h(|5Wm=(H(j=!LdwrfpXR*oxIZ+Z={IF@FZ6pu8 zSdJ*eqdO@jzDrI6;C?VM3eBGK#=j?KP@OklXVR=VDf{{tW6dQNA(o&`uh+E z?GV#+8}ArWFzmxnE{m-h4PH{S1af0(@L9L)p9BnYnwoxV-QFXYVAUY4q=TBb$YP>w zP*W;_BQ&>kO}mdjAq{ghLM%Rjo^RK1PaAHNPTZ)=G}!8?@R`a(ATIF@{QI383okKIFYMTqoy&;bmLFS^xrQ_piDnK?E=RLj)HalR6AnLOkMN z^MYUe$vtF+-cfc3D1w8$_Z^}7E^ZpsOfPbChzg4fs>XFOPq8_v@MV%^wd*p*YbB)q zrl(Qk&obB!{q=|macc8O?IQ{WCfOTfJhCY`7p&2Mg&*hK%JM|SnMw@}H{+)l)A+#~ zmmPp7YM-D^xgQv%fQh(BR5gG%2JBPn>(V;x)j*{cW7bg7mFEFvR!2K9l%%mQdEn^5x70(0!n<5ZJMysl#9D1w768@xB2ONtn#uxRZ2S8elDjPp;{|YE0l_&g`S>X}w zwyQzlH6cNkjIaW~umB5|=OhQ#@m!9kEd-^*XECz+QIo(NP?(cT*+cRITmOkUn3s;h zLNO~;p7BF!MIF^39iH7tu9$07;#Tn+y&mFTfe$04zMt?X*2LC7#FEJ^YN?p)PcBhF zU3^WzsLny15j~EHk=S9miDQazw z@_W&2H;1t&XO$*B9EX)t%gg~;iwq2!p-S>FsUkn09!kinQgM=I58Dh%cTxu_0iJLQ z@)uhP5j@7rgS85?VWeOITAv{jH3d-5so+s@S(ru+QA-a+;EtmNWya@pG%wQ3QW;7j zugI)}Sc#j?2kh&_d@OuUVg1kgZz>|@{XL?7B= zXl!m>e-rh7db^V@U%e)&BXTmb74wgos$iNLZ}f|Hz>)7+acW%u?e>^5f4%j99mm1=MczYAPsF~|v@_}pQ$*wOL$WDvH?5Pa>xz^1 z6a&BbV3Ja->en4bIap$cudEm^snM0a%CwIcw@}UBl7L)Xb=BH@te`~?QUkSr|Co7s z#0w_j93+&cO7>t4EJQz&_?)e;PUIwz1M!#W$gBEt_4o@i4;!s+__SGymb3Xn|2HOZ zP!j8VW^o-DYSZP-CqL=8CC?X^1@%Kp*j%(88k654jmG)fk=jw6xGbUz+ zgob8d9NPrIQN-A+LkcHV(dz_TV&W%8 zeUkn2ocI~Ih+kL(v$EK(lGzFp*I(yjX83oas}RL^6@11Nefcp#mCTt@pg?mh9*B&H zggJzrX*3nFO1e|YGkzY)EWW(`Y}B%(sd(+jtvYi1Ldz_#9jNzEd-h#K!nLx%p4QN`^=G=;H8MMYDq&`!z1YBdxi0S2YwJuSfU-!b~?2f%&%hM>FgKZ zD9KZx@5E&V$eSwTIRm%=Tx5^JeDFqBA9ms_k_m_&q(SJ2?z!jM7Nz$SHLFvTHY>aO zXKy<1wmf`^r7p+&wS3)D>Al2NFHf(KbNS)Kb!y-Cm=N9^LbMI34CQSN%9d-lHux(_ zmhIHGltyYZ8+HRTyniiJ4x+4_b~Mpy(evG2Wr8p=f1r|^qM;~gzmY`Olo8L5gT>O? zc-pr$tL6TzX<|voKNNtn9WpC%E+rklhN(1LkV=$kaOWb~%H`buY3yzXU*}w{=*17S zOOLCL1LN(@405ja#!Ke(HCvq(oQ(fwSEW7Yt7PAyalGjte_{MyKC#x;ak|w>+K{1I z97zrfP%B{yKyqV57`*Q|Vanx7D0(b+Bjr4L>i&u(zFN@mqyX-|CYxe}inX=3`$=3L zSlu&ucvB1J94Foz(aczV@m{{uc?eQ?WWYPf(?pqk0@xnY*=movz;aYvvc&PiH*%!n zUVL}dRo#B8^fQ9BR$%vc?s5f~kwu`fbNB2zA(?E^V{N{|m&0&MFB;0#AM4cDF!}-h zC!N2?H_7~}Hp%XVS=wbt)nnhucHuk8WHW$a<7WR5t20Cz4BVxv_lCgTDHGxge;t6{m8Ix`dFv+ zZcnnqs&n&bP%UW7NBI8XLcVaAitQ>}$;PYXsogvg!Fsi2P`;5?IG%)G)&+HhTjPTB!kuCAKheS~u!;HImI<6VS0$P9$j)g!4sOy3ru|*|= zQB1M@eh8z)55d8bJe|I70)a8T`fN>ra1}_O{$P8XE;<2*c~-F}84;BcX0kPoG@yv5 zvDA(&sb3(AYjL-+VDBeYN`g2Ikz`If^ylVLR0Lj9?h$sfm^G|6ykj`ODI>-oaLblj zR(3;~LAssuRt4w{xdy?Rr;=>r#0}mA=?l9Ro^bP) zf}&xa>`B5T`2AkR_EV%qbyN#5=MUBOhJ@`e&9q7M&!*lbG5Cb_!Gs7~k^N#`*=iBL zWeoLUOSgIGns4|`Y)s?&S`%s855#r1!lR zHFL64FrC#=2rM)^bCSHgmu|r*W~4$b=`Jo^3OpM@1M5)jWS>n94G0pDiP)@LSTFTB`IsCvc<&- z&p8E{-@toGwfs$&Uo1ZOP71r3OETxeMc$bwxLT9}k;_lsqy`c>C8)gnrLWBW@=epe zRT4HP%ZIWlr+CZJWmEeDcbM4$ zy>)W9;;|fz+URA3qD8Ab&>CQ=7c>dyJ#2tkS3MVfO<<$Z*W<8M<2TxJcGup z9^(Rp$n8L~1NQWoXx`tTE0;&UfEDgs>KXZ}of6+@AuU!}6eifCfJZCal{gECl?u~b zZHO_a+K0ps)_C}Wz*%EP^sVm_cXc8p9Us*%Tx0kqc*oGdEsNY=ovNc3dg6X*&pq5n z+34-E8Em|2*Rw)i+k1OQ+l<@y?$c3)%+rk`XirE8Jc<_7-XjgJd^lp-^+<*h*zcOU zMFYpj zNu)%&+D>g_D8)zYc#}RiFTH#-P+n5UQlI2Pm!}J|2jw?c3F%bszs~avh)p3eaOKSN zbf)hdVY=7Sm%FaI>lyWZU|H!)1f|y~cFFW@T#q$9xU%2-?|4414=qOjk^lfTiqSMw zR_66b%t(G0C>uRSFJ(o`;}Z>>lR^>QK3dXy#&J)8%}3y%T5F4ud-XS z?s|XO^%3N41gZN1EbH4&N&rx8iMt*wZD->B(|V=qT&ih-qWPzDCI3asJ_$5DFuv7q zso0mF%IlkqcZo`0*ST^`auxp3x5XMURY|?Yy7lhkmUk-nRN|bDXWX&9FvTWbPk$Jh zHNI6(TTWO+Qch}&d2q$UyY_OMFG21|KeCeD;aFE)<{k&PTGROIX&9bpn};Y$&3C(_@RZ<+%Sq7Ps!&AyI03 zDAG(-9xvs&Ati0^g`#ke{NVedLu|pva58wo2?Ta0q_3tzqju0DC-dmc|3GNB$IyT6 zb`58;B&zITH8gLnjr%Yyg^lWLW|))zGKYVNds7KyQr&wMIVLyo_C|tU`qF~-ume$2 zZ(CcBe0z(sEb;6)0?0Kju*C*c2z^hKUewmbzhSu`KJd8u)rkvl_^9Jq3Cq~jwy3cywY>> zV^3k`F|0fIf3KW({9h|i$2pww$Fz9TmG8xo+!#XM^rheBowcJs<*d&+b`2UMACiw~ z|3y)<=&JmoraLO{4Cr=gN-}aNwr4K10FB5$&r&w&C4>9}$Uf(oJlRym>{+|!!h z=KeQp_ABE&VPrvQMahw5^I)hmRc`8<0|}#eNxppUPreL+sM1I}_7J{5Yfq^zfFQeP z>s#j)=Mv};8aX=(pgJ7;R|MUtGTAsWY-Zl?=*En+0b*(Kf(gvqLg23m4^W+z8n+Ha1!*r{Ee4MZ$djS@+8$hGzDjU(00fVjXLEzd=kO`2Cy@+QHm z9`}-P4E|DD)d(gig^E*+(G(z0EY}|L++?qazPJu~E;>~GH!_D2iNdrp^h`(SU@a-Y z`N}~dPRtr@`;-lRYMd10UfW_R70>lk;6+Y-8aKLN74U!J_Z5~j1=@Cx%X_XIk5DU< z7W$jtXKsgn_l5K8C^POoEh0Wyq)65DsXaJVO15zKGNDswP=YkPfWy(07ImGa35v4) zxK{)!v&2L18Gm|7OFu>NF6a>H9eA%w7M`v+7b9QIR9&Bt75M-*N&rsBD|CJR)phBz zd%*vD?{>Wq_UKJ7c6{W}(3Ka`z0l4m2OpS|r`9U>C4%ZyRoX22W9Qvh1-Zl@lyGM` zT?}~gQ1jib3R4pWwS~f>DwBjuG_M4~1b`<SWN}oeLA0Y z`)C8c_iSFzJboreRXEKMnD!BX>;p4yEsNXTj)un`r!!s~YnrQ0sKIaYERW)#B^m#a zDkWwYyo&Eq>M@m)rRgRQjYo?UMQbScV5I@=W&N#6?}{qBs_^sMKzrA(mzFyBRzoBI z%dh)czy{5!QT=a@B{X}tw}LPIdt>>5N?abf;V1&T)+D9#7pJDl&*HNV3jqA|;E>%{ z(3Qio$wI%6ZCNmslwXg6SuEO!2>D!@0i=iJ>BBMN=D4ne0+cPNLYESsDV>ig6&43v z9>YTnT369{9c(3RMzE`&%%4U_kuId}a0zAn)yS%%cS`B^$ zA0X=aEe-qG=Y{N$rpLB9b!$)3BovL0WGNR;8r!j(&-(Uqbci%Mq zEjlZIzjSciUWT5Z&N$sRPJdqAr6mp!QzaOD#fch*O~}MFj?-|~j{9B$#N`URlW!j6 zUD_Pes7KQf!x$!aFZ#&F0hA?sPZ-1v{ne|J?Oo!R<8yGtr1I6P-c`tf5ne^EP^t6n z?rN>Q_)XQ3_VxUK@q#+9y=uW+C`mnnBuJt+T^B=j!-Napj4@KtSE_59msuR zUgJ(fcBB~nGf#cZupjk?yJg=rS9F>t*FnFu{=Dd*nTWJGo(9xLB857C0(9)0m!DX6 z;WT>(KqzGucv#nrR(j{f-!l}=-bV}V(#_})j}-b^PCPOpzKqZ?wQ~3HGL+-N%SpZk zru5M6i=%l=cY3cvO@!*5+*D6LWpnRK7fpTwLks~t8{pA*#M=Eos5MZv^FzOAAB)Vw zKkv!3AN@|c@+ta=dXY=dtN2p#S4ZnL!NjqC&;unAC6wJD-B-7Wp@NO<>o+M(w1Dpg zmMRa@swRKXzd0m-|JkPsf>W>EpxtyxYz#^i#AhPEvbNu}`K8f#fd2P52rO21k7#EB zZK8hgX~a_O{NO9QP4*%hCHB}Sb>AbIoDNspRp~DjdrzNn#)zX(W1?SK@(d`khwU4Y z$=i>0bZ=!o$4j2J;c7Ms>wFoO*!V}XgT*%9yEs`!Z=D|s`K!rUKw!*9|Tnx*gKzFt=@L5o)C)D?&TSsPc(RQ2*fEWoI{&=4U!i6-pR=(R0h1v@( zaq|yvcX_-gSvO{`aY){m&dU3w3o)>J=VoHUd^K(rVGy-Jv)J&22j$Nq$!d-`doA?c zr05^Zk%C9+Si|);o`L8%Kxcd+4Xbg_@w#5xHSH|^7p?9%2^en0v)%A&TF z#}U|j(LuG1fIhoU-#B=h{JOZ39f;68-1;ab<%dIhx4?+jkmjGw)MH4@kM{!Rcvab5 zT{t?{DnyF;n|09^?y8`NpO=tx)pbQOwldRI6*ji9w9mM;*Ex`^slEMgtYyDVoPX;_ zwEaC$qKxyGvza*j%460mih$YS4`~;A&ahHq*qu~wHm*Q-295UsdXu1lM@!6jzVZ*U#cJ_M&uf3n*D zAQ;U4q2Y2sOFAlUp^?wZW=78jD=jd!N?r2&U6UvNE-M zNbf+OkD7P)vv*b7^;$}n4VIVsr>dftIvDn+OXOuQY7V~66nYm+${7f+Y+L8Zqsz^hWIi=Wv*rD^accPn)@!JNoBq``O+nzR`eOhyu+^Q&!I;vpUfw)}d14sbO+a4Eg=8TMRWefe5?0^M%i$@T-)PI6Yr`th^{D>g-jjSBdHK#3=J@w5Ek zVq58hZzse~wQ=Knpt$9%+OX?*zbO;+3)#iOm2np423eI#x~21YI&DZ`Q&`EnMe}I0 zuU(@G?A>uNzckw~il0XE*A*?B1b+M5eZ3N`K#P+<{Y?F~B@%Z3((dbX?lYzuH>}ml z`FmX>9j{U`?)3vnW8)Nom|m;FTY}%Y*n=PJ+LuQZg(;kKjaA|5`r#am-b;a?y!%YH zn{eYZWS()Z7M6m;NSHQUxcRf?#5w^K(orz;;!HZ?yQs*_Upr5FL8q6p{BH>OrhVNL z+vLAq(($?DbZwZ9PY^P<#(y`$`W_)N= z#ONn3wYZL-8-f`TF3B|U*ZFIJLmn$Xin7^NQJms>r-t9lnr&`L+ErQ^LTy4f76CEa zIn*aL%ZCboPhX`q=|W~gjCq~sWq9HOhURGPC+Sj_&J#|SgcuNi>q{PH7W-h-?0XEA z4%^8&@kXK-LIM4MD02s+?yab@CQy9CZ?uj-f7_#C$o(ee{PE;kh`MFZu0?OBN0!<> zv2`MAE&9h5TaK<6{iOx`wEym!Ml}947T?E5B&b+t=;ypq2DS$mh(CKm<7KxL9Q#d< z4d&s>3~iboKD1@bRX3xq@6Q$A2=|eG{0g^AG6%fX$f=chlmeO%do*#J-R-=>ffo@( z*$AR-&_t36O(d*2Asp}tbFn#(C2b)grsolQ2aHY;*=|^} zwja3!v;Wj&i!iBbCRd z4+DF6|6g@q6%|*|t$BbXNN@>(;O?%$J-EBOYjF2Ka2j_>a0woQ25;Qm0*!kI*Xi8* zzubA5rPA^1FmB;* z-cyL+S=Y9~?tkf6f%0)t-CLF*dxGw^<9<9o6}hk!E&4@E0?*$9f>r3pH5Q-=*Zpye zb!Qh`-qwa_PfwHuFD6d3Snox5_uFZYwUd>=i}n$k7x4O0gB+Tq>8%iI**7G<&seG) zi5NYnyG3z~p=9sgwX6&$Fn2^PM&nl>OTJ}(JZ7qD#C`iasGy@MXdnl4y#CXPiv7cy z;V!+n8%Or43*Q6{s8t}_&(0dku;PCB;s_;n<3eaMcxoxe^vVJA-po24K9zIcy~$n9 z^jGgDw@xr>_r7!pdqLHc04bM~XrD3fcdfG);NcrswQx1WDFkK`jfsNeIE}!N+n_t= zr;%KR)Ablf`wjt+v%OU=SQ{n$!o0TU=^{T+%$`{B8A?32ec18(X`xM=F=ga9&8j&G z%sQzDaIGWf6RNDtL)a-VvJ8Kn>h$zIL?mpBJXSa=({#C%%l8;MAC}BMG**L4SsGR{ zmuOb~jVl_4&_41#+muGNyfboi@VKbw(M2&IAA@L$@emaQ-NUX{#oZ@pAkd|J~WN&#%#} zpCHJC63=fTCI_n!z1MsnC_Z9 zi`LbV=vCS)GOoK$L;knAMwpw{$SqA`1=SZ=sz z({hyH?bd1$%DrFO2!8${r0RO~h_KL^oqLd;eZcV`sphNA`#I`lX_cL6Fuftd@ig%% zdr9)9^{Z5x70-b(wX|ECBbJr|OD;XZbSBub#*G0k8H##tDL}>_?qKcU{WU2$EpyA= z)~#2cXQqz(@c?&f6@qmpcWa;ttZq8tLUPXS_^6@RGz-^GtHQn~Ne;sP%8ZccLknp3 z@LH>*G%O^X|AmA;EDU4uO1pxvJ*xnnb)x+oOsjAW&tmLsWhzy6euC;R2)7Q_=wEuV zQd7nde^B1`lcUti>!UH-o3_47rCAs%qniGRR=7vgA~s~ld--;V|GrJncqq=0_5;^QQ{1rDss{9qT9}`TE7P zyG-ECyYQn@Wn)CP&StaJE=`m*h0e3X0EEM*Ry$|6t3FH5IF`0Z+6r1w<=(7_>rP(# ztZm5e8Yb0fW_BMn1tRUN99d7S=V^c7Ab_5a%ZKZwV#tEo+;CzW8wtB;tpbemjS}|W zF~qOWi=uk#O@)7lrlBytYL#kSh<9P&!PDwSvk~kYR5454xbe4n&qlfn8V0n#^-d|R-&FY7#>TU>TdN7@)_Nh9;T{P#n3Xt`cASmdX# zasG%AEG3bDp>tc|9ZoR;i!eM&DgzVCbU9e_N!|^1KM~z8t2YBQSp4N2XQh7kPq21FEZ@hNJ;B~d0d3wb{fj^mC zp8}hDNPSSf*Ud*e1Tms+#FC%;Up4gCKKX>w&AoPPo&Nrlc5@5L6h56h{)!hjO{r)T zMVwAdeBm7BfK^TS+c&xFbY!G=G5zn~C4_}M2?0UX*4B^YBUR3(Lt;^1M27!R-<)*> zYU*A58M`Wx4MbGe%!;PH^GlOG-Bv)-UENYhN!+|&BMIs{5;Iyl@qCLgq1lW%qEyW! zlJg-}PAN-d@1PS?RmdlKA!=yky$InZk%e3I=k4aB8w#eC*}+y49lK2f2!npOWi~~Q zy(kGo8Q2r7duBI7M_Z(PF_VgqL28fL6?x07@2y{?72@C`Mt%A4(im{fE>(KdhNR*L zFRxq%uBkkJPVcB3+d>Y0UBs8xY^gJ90vBS67PG1lx z*-r_)DFV#Nbx_!O;@h{F>e|;~_Ov`5G}1OGSuSMyZ*d9{E zmej_A4Q9)dn;+cxjy!%<$n#m!jQCD*h}dJfj*Tuj&?!lY$%B|=^D%p@wK>y%0<0!a zUu4xw0XRE@6*9>gooLS_p63Ip7@MX^v>)?2jyq-gDcmF~T;`tXh4pDWyt2ku5iSYi z%S#zh0|Xs7JaDROa=HMfQ`-k{Rj&U9?aX=aqB-CBhz_#O`dQxFbH%SHNQuzw$R_Eu!{G#$ za;`)a*e9=~6mIt$Jry%@n)(A(Q0JY~Bi{U_B}BU`7T=P%rfuO-@s#I$`2&;opjZf| zNFwgUujr8#<{#rvK>*j z;>&2|7uLny>ToiyjATq6K0$Brjz~7D@wCI@GCaEo(?j36|2V4X9l>+RYpQH~l4?KJ z_a}NjM|0L4cn!oX1N7Q9aB;HY#U5vI<7>!@rLe9%yXW|J4w*!YOAYqc^>f>B(u9wy zGfvJ!`Ggsrvl0q8v_8Sit6aAKDWpf1Nq`}gaMn$=2?(3?CW#wVY}Wje_UA&#K9{T- z!}?f=U3f2$H3;j*AI~>=q<7N!XMaKPepA(x!OLWN9#e0t)WDNj%1%^#i&HF$uOv-LMAds zU7wSAUoFu>Fv&zuTOrIox~qy!QhtI0=7ZUa0}LkC)UBn%y=v*Ua@SAax>Bc%pC1oO z%I``)te=%VttbZts%~Hz=NTF0<=;g7JqypzhY@?_J@<2Nekmn2e)jFGJ|S(|I3rV( zetC|I3%tMDDcwsL8xvkT4^%$c+sii!3{ZO^@b~fc^)(g5SfoAi`u#Twf57x-*OB}! z=@;7bz19umgj<=D8w~l+CU47rBX^O!(R<3i{X=%UeR>T0E#NoTD?;+-9co!2U-cNH~N93Cbpd1w`p_7s6wcvJgVPYNnJJ7Q71A`UyF~7SI&Gb(^p5$RWuVA zE6XxDu<0GO7R6D6Lm^@$7iFBz(3Q9wu>9!dsA&!Q)$F7yB)F~!{SCEQIAvjD=&idu z9CgbDq57>x8hQx~puFZPGEvUb5ia}X)JjgChND)FoDoUo^;s>J&tBGzx8=6%3pzU35KqdV*yi=9RGJHmn zTAaQ0?t2YT#A!ulL{1_JUYy~rM?Yajjua&pdnj)u&r2Hl3BlrmevKUT>kqv=+-*TH z+A2I*L%!$xYt&E@#uSi8)@ihPykcsYL1=jRWs-*7=qp-TVg35MA0Id`toJD|!kQm6 zWre|ge8lO)>#453#WMO_i2>9%u1os?%N^RH+{v!TtyFG zIDR?C$`G3n1a?x!1F`B3=VV@{L)7`aX71e#_@XS+GGRu=4@tgtMX`9uzo}yi1eNVXu5EyK>!3hvGJNacy*uelstN$1w!`HCt zO}zPl>zih}uoiDwRR%Qnf15s(70d-`e^iL#`3W(ZgP9>Q4N!Lbve{lo79p#ib);m# zyk-9Hs+?T&zMi;yyrvtULEf|-H5WUsEg(?rf9+AG@>Ud`k6XRtw>uB*1f3BF)%RE* z%2Yz9r956Z{sDzHxCSpczB25KSW9K7J{;WAj**YC^2*1mx8#{PWUha~-_Jw2G~QGE z9iST|y5i!%Ja(noMF__H=S<$@E&uy{O-iynn-O*Q3|+rLZf!w&j-wUR;Jl(E@gew| zVYe7QjZw+WSY07oHrk31@y=fTU_awe1f!$B*+ZTWt`2Z*=9BT>g()i8eE>6R)Y=WD%^;CH*(nC!( zdX>X9I7q?%40ttDI7{K#o>wQUKOuN7Eme>)o1KuZ5~s^_sZ#fe!UBTu5PL97q^||# zE6fPRj?e&l_Y~_oLZrfmOWsNO{q`@I$k6c{lu0&V>PmzV?2ba4Jv-l|y<^p7!FwIt zUT3Wgjb`YXf(G~FDj0o2?jb7RSM63jF~Mzct4~i|6trp!A_F&=p0D70glaf^ly{pb zR47?Fe!RF3=dohRi2O=5J`i|{yWdcmsRPb#;@9UQU$Yxpucx`2MvH4w9u`H3h(fqD z*Yn$$!wZu_JCAW6{h`G2G$3zCeVuE8RRR{rz{6>;T3{bf7W6h|W6b>JN`iEd20^1Y z+of8NLDR%9`fK<>#b6>*JsdjY@Ad6PkOy!XVadTbD26T!oJYLUeu%Y4U<;?B#$wNz4 zebdQ}^{(cfzNz)TBOK0t(?B?^Uu8a7M|+oB<9)wAsVlE-tmMX0A@xp?voRGc_R^mrsbGcYLfda17g-HwQXvA z(k+|JI(_!~px&;&3`&w!Y9;=Ws&$yH{a23^x-Q<3j|CR#omLZLt@rrK8u@UYaXKg` z5p~mtHuTuWJb8vaD#p3Y3L8@y|7V;=at@9#)H21EyD{B9IO?Vi^DGVG|0L$h%G|g; zn=MsIGux#^Hi(q4jhz30J!>$Cm7MUN-oBS$--T zD4+Zgyqgf9%>SU3Jn{qIhcPAUfC@|u?2Od2%uqkgKS44*mPJ3(w6-A0?A-CulT{}dr$vrz5sm&t{xf#y5ir`tBRTI(Jwx2_ux$yN7UHwE2UUI8J zF?i*DXMj8_cf1v*=QpAJ#`YAJZj7>&H;*{oPsQqz=S5i+&spX4yQidD9FAseNW-)U zj$!9MV!~TG1(wLV`l*G=D;TtrQ^oV`LSF1D@D$&}eO6Kg!9y8ZDMBzSVq)LG^BTvI zH4BrK#^;BoV{e{nQQff#h+?LHOZ3NDI7-3gc_cv`@?n?1lnw-)0ZCo*=IR<`Cl!HwLIF{-(U0#8TD``(21KoOh(l}F9KO7`Nm_$R2hFldHYtr(_ zxrkTJXkSoJfB5WMV~Ptnm_>B{{KYX+D+#O-rNeb--o~Z0adQl2K#uInt-`*Nd>W)u z_MeubsB18mpwO=xGv^#Cb(1|+0NX7r96AqQ`Qrk zznwx#nyL7&;D(I_Y&-u(p4g)^XcFKh8+UtW+J1?Q*Tg zkl!!AL18q!V1M>^SHE14vhh9cmwwrcGjQ8I!gHVHV83uPx^D&K`VYpxbVt8rOgsPK ze2RN$Sw3C7@bE9ed13#)(PMr8;f89XZu{s~f%e&4t}9~VlJ``$bKCj))TYVxeH8q@ z3#oKyov7~*CW4n2-s3$-s@6((iO>Ue>WRXXa!R+l_a%uuN%syV16|ga0-S5bN^akU zv&%2e2%g3jZpuIg@2_M0!|A8eeT9*0E%GS(U zb80B>lFjo!~sC(*mZnP^{ggLlrbe{G-&sV;wcefQ7_WMccRK`X}z!^ z^_N}A04ZmdF>Pz_?w|c!%>CRzQTZNEQ`ewj-4@smuJ1Kj(8j^pkpj6d7odW&yBML*aCX-7qGOX_nXP{?h^b$4|tS z8$NssC8zMR@T9(`v7ml=C2SzDiV2QJs3sN<1f#D7DW3PGgfM2rI>&LE{}_v*Ap{y5(06$4-M#G= zYb0yhz@g?a`Fzq~^1Vv0E;oUcN1>r3%$m_-P9aXjq%q=qx@;#Uj-_PLm46vNx&sMM z6$DG0!EUOR=ymrgU1Pt8J7o@MK)(?sC${FBDM6$mWQPn}KbZ-w!-`D8p!R*9f}<7| z;2O@Qv0LrJAP3bFj_UQHR0|2byo~XqN6n=ia;vl+k%n$g6gpfxT7$#c`I4L=xB7Mh zeg|b*=1Fzo_?Ekh^tY|n%|JT|C6W(UTe^`y5gr!I3Jet>5oP)8#(&L%hN*q~bep|y z|9cJwW2|zRJV=%Vna(A9t_mpr*0Sx`?Fo+Fn60U|!Jh7e0vRY5jAjq)CeB}WnM{1^ zQ7%28#>xLZ`(;y`JZdj~BYSUVl)h}_qwFO`S4n;x#xu)k*r!~ZmVgK0WHkRIH5}2I z7*bRa_(H-48LoJx?$)o^)PeAyVYB19eO$5b*D2G(q<_nUQQ{nN(=9$8h7;yDRRv_g zfhFbjbM&SCCR>K?H)*0IwYuST>5=#xe|&a4>;BKx6ONUSIvSO^jLtUN*}kaW8vICT z?VRt~q`ch1MGyQDzr4k~lZ;iqY_AaGe(f*U;-Js5Lq8MwH0kx`Deb?9;s1PZ1TB># zU<1JWS0+#TBEQiJ^x?21mCxRclz)Jvf08oQx1#1OYtc&{bdz%m1%GSpB>lve^JB)yuytxP=sNCoKSJ zQ797bWF3;>fLIL^JCdenw52YcSr~0DXCJfC-~UT&(4ki%rSN1E&BlRmgGFxwTk(Mq zr2)KP_Z6c6Tp-D_V;K)fiZnudEm|VOO7dE?EZ6ll+3IABJzS#Q@sbYEm*vZY2h`R4 zgB1T0%|2-Y;e0tk@egBWL64mQ^T$h1OaL~KQ~Frj$lazbJXs&9pI22S9j=tAcP)N! zpA~~_4nO%)9E8_7VRKnGfCQ>dkrLqp^p0)Xp&2%=#g%!uUSU z?KNOUmf$`!vElp%Kr%SEj5e10i~zhkuAUaUS_fbRQT#%iCP1)gs;`0_5KNYet%?qO z){>Lt2S#yoLzqahq-6fwh0C+RH@+FyrMWDo>aawxO7V zDa^pR+V{>Pt`$;7pZ*Xw#NdVUD?AU~gXEYYH^4gy^k!?Z<~2}TH>E<9F9x@X=eE`( z`4$-M6+>`)ti%KaDQ4Zit6nVO05qj-IxtPK7;d(EJm4qeChVlA_kf8SY!@COU?LZ> zK9z#d9gBy3g#wJ~6&B9D0e(_K$ot+Mk(Mb)UBv|$He!*~+QThcAzv_GC*eDYG7{q$~XdR=NovO^64@CL08 z@xCzqGW00#Y#L~edAJ&MwcM>Hf{l?n!|Vv+-}@p$T?0w~ih&{_E6)S16iy>_fRe9b z0po9S<Z)gYV5tAa&qD^t zGI$5JlXjb>Jz3@)yrT0gKH5baR!!jbugKatUCRMLp&%!TI&pBqrJ?p(nC^g=+8QPo z9dF>Zq@nKU`{l5#tADx96wPw;G8WzU>qVa3;^_@Ggm)T53ezQ8vT^ z0K88}V^~HCCAFjLvY-noV29-=5T(SUE$mK}{u3$xkX0H? zsKM3`ArU|Ub4vuqI7E;b39}8I&dajrpyf`5^3AB{ zATx0D@-rtUSl-)@V}BwKiw#e!YN00+t+Q!8c?2v#VWJ9=oA)5d&IM9iY&uukx zrUj~|pZmsrup@InLzMu4@yh$9uVx$>#|b>4C%&AqFoW2M{IDl1umtu3NKW z0*?rOrY_Mqz_UQLw7GhX{>nQHp|?QU%u^YCE8TJqakc!ZlDj_ScL$wUEqP24_r+ub z5KPzv{jp9*;V?lminmqyLh2nut}okMy5u?^aW!Gg(5$(7XmM6Hs_*=c>^9&Fk;#a4NxCqEeh`Tjnmk{Jv2sg7)MI|l@ z6K!%PTE6D$pZ!zIHc+kxSuYJ{)8~#%g*XxM@tE7Dj)8<{CVub-gM(FUftfgif1aPY zYY}A=u+`sQH?~o8P)~o1cTxzHLFlod_ClZl6WVm=escQfnE*Kpkx{=a{wvRQh=}k2tcX6>#uZfjkJ3Xzg!E4Y+;#DU{twCj p{$=_ujkrnZ{{lxEA6EbX -- 2.40.1 From 718440d155629476ecb5160b8d1456c9922a1e5a Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 18 May 2020 16:24:57 +0100 Subject: [PATCH 06/31] First trial of sync hook to annotate system ns --- charts/kubezero-kiam/Chart.yaml | 2 +- .../kubezero-kiam/templates/postsync-ns.yaml | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 charts/kubezero-kiam/templates/postsync-ns.yaml diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index 4381f27..5313592 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-kiam description: KubeZero Umbrella Chart for Kiam type: application -version: 0.2.0 +version: 0.2.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: diff --git a/charts/kubezero-kiam/templates/postsync-ns.yaml b/charts/kubezero-kiam/templates/postsync-ns.yaml new file mode 100644 index 0000000..a8dbdcb --- /dev/null +++ b/charts/kubezero-kiam/templates/postsync-ns.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: kiam-kube-system-ns-annotation + namespace: kube-system + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: HookSucceeded + labels: + app.kubernetes.io/name: {{ .name }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/part-of: kubezero +spec: + template: + spec: + serviceAccountName: default + containers: + - name: kubectl + image: "bitnami/kubectl:latest" + imagePullPolicy: "IfNotPresent" + command: + - /bin/sh + - -c + - kubectl annotate --overwrite namespace kube-system 'iam.amazonaws.com/permitted=.*' + restartPolicy: Never -- 2.40.1 From 3f9515a1600124ef317109acb924d1a26fd4ac11 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 18 May 2020 18:12:02 +0100 Subject: [PATCH 07/31] Add ServiceAccount and roles to allow namespace annotation --- charts/kubezero-kiam/Chart.yaml | 2 +- .../kubezero-kiam/templates/postsync-ns.yaml | 48 ++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index 5313592..2dceee6 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-kiam description: KubeZero Umbrella Chart for Kiam type: application -version: 0.2.1 +version: 0.2.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: diff --git a/charts/kubezero-kiam/templates/postsync-ns.yaml b/charts/kubezero-kiam/templates/postsync-ns.yaml index a8dbdcb..441c070 100644 --- a/charts/kubezero-kiam/templates/postsync-ns.yaml +++ b/charts/kubezero-kiam/templates/postsync-ns.yaml @@ -1,3 +1,49 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: {{ .name }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/part-of: kubezero + name: kiam-namespace-annotate +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: {{ .name }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/part-of: kubezero + name: kiam-namespace-annotate +rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: {{ .name }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/part-of: kubezero + name: kiam-namespace-annotate +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kiam-namespace-annotate +subjects: + - kind: ServiceAccount + name: kiam-namespace-annotate + namespace: kube-system +--- apiVersion: batch/v1 kind: Job metadata: @@ -14,7 +60,7 @@ metadata: spec: template: spec: - serviceAccountName: default + serviceAccountName: kiam-namespace-annotate containers: - name: kubectl image: "bitnami/kubectl:latest" -- 2.40.1 From 49c183a71892e447f59ff063f3199df5778b911d Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 18 May 2020 18:25:41 +0100 Subject: [PATCH 08/31] Looks like we need to allow read as well --- charts/kubezero-kiam/templates/postsync-ns.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/kubezero-kiam/templates/postsync-ns.yaml b/charts/kubezero-kiam/templates/postsync-ns.yaml index 441c070..590279f 100644 --- a/charts/kubezero-kiam/templates/postsync-ns.yaml +++ b/charts/kubezero-kiam/templates/postsync-ns.yaml @@ -23,6 +23,8 @@ rules: resources: - namespaces verbs: + - get + - list - update - patch --- -- 2.40.1 From ad5007266abda4f51e3d0e2ca7ca10927e01cb86 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 19 May 2020 11:14:11 +0100 Subject: [PATCH 09/31] Add draft for central helm library --- charts/kubezero-lib/.helmignore | 23 ++++++++++++++++++++++ charts/kubezero-lib/Chart.yaml | 12 +++++++++++ charts/kubezero-lib/README.md | 11 +++++++++++ charts/kubezero-lib/templates/_helpers.tpl | 9 +++++++++ 4 files changed, 55 insertions(+) create mode 100644 charts/kubezero-lib/.helmignore create mode 100644 charts/kubezero-lib/Chart.yaml create mode 100644 charts/kubezero-lib/README.md create mode 100644 charts/kubezero-lib/templates/_helpers.tpl diff --git a/charts/kubezero-lib/.helmignore b/charts/kubezero-lib/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/kubezero-lib/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubezero-lib/Chart.yaml b/charts/kubezero-lib/Chart.yaml new file mode 100644 index 0000000..78f7791 --- /dev/null +++ b/charts/kubezero-lib/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: kubezero-lib +description: KubeZero helm library - common helm functions and blocks +type: library +version: 0.1.0 +home: https://kubezero.com +icon: https://cdn.zero-downtime.net/assets/logo_small.png +keywords: + - kubezero +maintainers: + - name: Quarky9 +kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-lib/README.md b/charts/kubezero-lib/README.md new file mode 100644 index 0000000..479c259 --- /dev/null +++ b/charts/kubezero-lib/README.md @@ -0,0 +1,11 @@ +kubezero-lib +============ +KubeZero helm library - common helm functions and blocks + +Current chart version is `0.1.0` + +Source code can be found [here](https://kubezero.com) + + + + diff --git a/charts/kubezero-lib/templates/_helpers.tpl b/charts/kubezero-lib/templates/_helpers.tpl new file mode 100644 index 0000000..498dd1b --- /dev/null +++ b/charts/kubezero-lib/templates/_helpers.tpl @@ -0,0 +1,9 @@ +{{- /* +Common set of labels +*/ -}} +{{- define "kubezero-library.labels" -}} +app.kubernetes.io/name: {{ .name }} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/part-of: kubezero +{{- end -}} -- 2.40.1 From 77b7762154d1e03fdd2e04c6e89ba933edd75d9f Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 19 May 2020 11:26:27 +0100 Subject: [PATCH 10/31] Fix lib namespace --- charts/kubezero-lib/Chart.yaml | 2 +- charts/kubezero-lib/templates/_helpers.tpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-lib/Chart.yaml b/charts/kubezero-lib/Chart.yaml index 78f7791..4b385a8 100644 --- a/charts/kubezero-lib/Chart.yaml +++ b/charts/kubezero-lib/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-lib description: KubeZero helm library - common helm functions and blocks type: library -version: 0.1.0 +version: 0.1.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: diff --git a/charts/kubezero-lib/templates/_helpers.tpl b/charts/kubezero-lib/templates/_helpers.tpl index 498dd1b..67bfa26 100644 --- a/charts/kubezero-lib/templates/_helpers.tpl +++ b/charts/kubezero-lib/templates/_helpers.tpl @@ -1,7 +1,7 @@ {{- /* Common set of labels */ -}} -{{- define "kubezero-library.labels" -}} +{{- define "kubezero-lib.labels" -}} app.kubernetes.io/name: {{ .name }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} app.kubernetes.io/managed-by: {{ .Release.Service }} -- 2.40.1 From 40766ca7daed433ab72a62ef3453a8e610a4d146 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 19 May 2020 11:28:38 +0100 Subject: [PATCH 11/31] Switch kiam to use helm lib --- charts/kubezero-kiam/Chart.yaml | 5 ++++- .../kubezero-kiam/templates/postsync-ns.yaml | 22 +++++-------------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index 2dceee6..c0c7d24 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-kiam description: KubeZero Umbrella Chart for Kiam type: application -version: 0.2.2 +version: 0.2.3 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: @@ -11,6 +11,9 @@ keywords: maintainers: - name: Quarky9 dependencies: + - name: kubezero-lib + version: ">= 0.1.1" + repository: https://zero-down-time.github.io/kubezero/ - name: kiam version: 5.7.0 repository: https://uswitch.github.io/kiam-helm-charts/charts/ diff --git a/charts/kubezero-kiam/templates/postsync-ns.yaml b/charts/kubezero-kiam/templates/postsync-ns.yaml index 590279f..8170b9e 100644 --- a/charts/kubezero-kiam/templates/postsync-ns.yaml +++ b/charts/kubezero-kiam/templates/postsync-ns.yaml @@ -2,20 +2,14 @@ apiVersion: v1 kind: ServiceAccount metadata: labels: - app.kubernetes.io/name: {{ .name }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/part-of: kubezero +{{ include "kubezero-lib.labels" . | indent 4 }} name: kiam-namespace-annotate --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: labels: - app.kubernetes.io/name: {{ .name }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/part-of: kubezero +{{ include "kubezero-lib.labels" . | indent 4 }} name: kiam-namespace-annotate rules: - apiGroups: @@ -32,10 +26,7 @@ apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: labels: - app.kubernetes.io/name: {{ .name }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/part-of: kubezero +{{ include "kubezero-lib.labels" . | indent 4 }} name: kiam-namespace-annotate roleRef: apiGroup: rbac.authorization.k8s.io @@ -52,13 +43,10 @@ metadata: name: kiam-kube-system-ns-annotation namespace: kube-system annotations: - argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook: Sync argocd.argoproj.io/hook-delete-policy: HookSucceeded labels: - app.kubernetes.io/name: {{ .name }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/part-of: kubezero +{{ include "kubezero-lib.labels" . | indent 4 }} spec: template: spec: -- 2.40.1 From 851d9d3d6041bace3b5b07c6d22e445101ba1bee Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 19 May 2020 11:48:15 +0100 Subject: [PATCH 12/31] Add labels to certs --- charts/kubezero-kiam/Chart.yaml | 2 +- charts/kubezero-kiam/templates/certificates.yaml | 4 ++++ .../templates/{postsync-ns.yaml => sync-ns.yaml} | 0 3 files changed, 5 insertions(+), 1 deletion(-) rename charts/kubezero-kiam/templates/{postsync-ns.yaml => sync-ns.yaml} (100%) diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index c0c7d24..1fccd1c 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-kiam description: KubeZero Umbrella Chart for Kiam type: application -version: 0.2.3 +version: 0.2.4 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: diff --git a/charts/kubezero-kiam/templates/certificates.yaml b/charts/kubezero-kiam/templates/certificates.yaml index ca9bc01..c2a9775 100644 --- a/charts/kubezero-kiam/templates/certificates.yaml +++ b/charts/kubezero-kiam/templates/certificates.yaml @@ -2,6 +2,8 @@ apiVersion: cert-manager.io/v1alpha2 kind: Certificate metadata: name: kiam-agent + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: secretName: kiam-agent-tls issuerRef: @@ -15,6 +17,8 @@ apiVersion: cert-manager.io/v1alpha2 kind: Certificate metadata: name: kiam-server + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: secretName: kiam-server-tls issuerRef: diff --git a/charts/kubezero-kiam/templates/postsync-ns.yaml b/charts/kubezero-kiam/templates/sync-ns.yaml similarity index 100% rename from charts/kubezero-kiam/templates/postsync-ns.yaml rename to charts/kubezero-kiam/templates/sync-ns.yaml -- 2.40.1 From 29faa8b003e404dc4139517102858837df224171 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 19 May 2020 11:57:24 +0100 Subject: [PATCH 13/31] Add labels via central lib to cert-manager --- charts/kubezero-cert-manager/Chart.yaml | 5 ++++- charts/kubezero-cert-manager/templates/cluster-ca.yaml | 8 ++++++++ .../kubezero-cert-manager/templates/cluster-issuer.yaml | 2 ++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/charts/kubezero-cert-manager/Chart.yaml b/charts/kubezero-cert-manager/Chart.yaml index 09e2842..6fe01b3 100644 --- a/charts/kubezero-cert-manager/Chart.yaml +++ b/charts/kubezero-cert-manager/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-cert-manager description: KubeZero Umbrella Chart for cert-manager type: application -version: 0.3.2 +version: 0.3.3 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: @@ -11,6 +11,9 @@ keywords: maintainers: - name: Quarky9 dependencies: + - name: kubezero-lib + version: ">= 0.1.1" + repository: https://zero-down-time.github.io/kubezero/ - name: cert-manager version: 0.15.0 repository: https://charts.jetstack.io diff --git a/charts/kubezero-cert-manager/templates/cluster-ca.yaml b/charts/kubezero-cert-manager/templates/cluster-ca.yaml index 7ac9665..f5f70b0 100644 --- a/charts/kubezero-cert-manager/templates/cluster-ca.yaml +++ b/charts/kubezero-cert-manager/templates/cluster-ca.yaml @@ -7,6 +7,8 @@ kind: Issuer metadata: name: kubezero-selfsigning-issuer namespace: kube-system + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: selfSigned: {} --- @@ -15,6 +17,8 @@ kind: Certificate metadata: name: kubezero-local-ca namespace: kube-system + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: secretName: kubezero-ca-tls commonName: "kubezero-local-ca" @@ -31,6 +35,8 @@ kind: Secret metadata: name: kubezero-ca-tls namespace: kube-system + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} data: tls.crt: {{ .Values.localCA.ca.crt | b64enc }} tls.key: {{ .Values.localCA.ca.key | b64enc }} @@ -42,6 +48,8 @@ kind: Issuer metadata: name: kubezero-local-ca-issuer namespace: kube-system + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: ca: secretName: kubezero-ca-tls diff --git a/charts/kubezero-cert-manager/templates/cluster-issuer.yaml b/charts/kubezero-cert-manager/templates/cluster-issuer.yaml index c84a034..4861733 100644 --- a/charts/kubezero-cert-manager/templates/cluster-issuer.yaml +++ b/charts/kubezero-cert-manager/templates/cluster-issuer.yaml @@ -3,6 +3,8 @@ apiVersion: cert-manager.io/v1alpha2 kind: ClusterIssuer metadata: name: {{ .Values.clusterIssuer.name }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} spec: acme: server: {{ .Values.clusterIssuer.server }} -- 2.40.1 From ed49c9dc93b894c358c328b92f839ef8294fdc56 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 19 May 2020 12:07:58 +0100 Subject: [PATCH 14/31] Switch root app to use common labels --- charts/kubezero-app/Chart.yaml | 6 +++++- .../templates/{_apps_common.yaml => _app.yaml} | 7 ++----- charts/kubezero-app/templates/calico.yaml | 2 +- charts/kubezero-app/templates/cert-manager.yaml | 4 +++- charts/kubezero-app/templates/kiam.yaml | 2 +- .../kubezero-app/templates/local-volume-provisioner.yaml | 2 +- 6 files changed, 13 insertions(+), 10 deletions(-) rename charts/kubezero-app/templates/{_apps_common.yaml => _app.yaml} (79%) diff --git a/charts/kubezero-app/Chart.yaml b/charts/kubezero-app/Chart.yaml index 3110b30..f14be38 100644 --- a/charts/kubezero-app/Chart.yaml +++ b/charts/kubezero-app/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-app description: KubeZero ArgoCD Application - Root chart of the KubeZero type: application -version: 0.2.0 +version: 0.2.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: @@ -11,4 +11,8 @@ keywords: - gitops maintainers: - name: Quarky9 +dependencies: + - name: kubezero-lib + version: ">= 0.1.1" + repository: https://zero-down-time.github.io/kubezero/ kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-app/templates/_apps_common.yaml b/charts/kubezero-app/templates/_app.yaml similarity index 79% rename from charts/kubezero-app/templates/_apps_common.yaml rename to charts/kubezero-app/templates/_app.yaml index 2b44ab7..1734694 100644 --- a/charts/kubezero-app/templates/_apps_common.yaml +++ b/charts/kubezero-app/templates/_app.yaml @@ -1,14 +1,11 @@ -{{- define "kubezero.app" }} +{{- define "kubezero-app.app" }} apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: {{ .name | lower }} namespace: argocd labels: - app.kubernetes.io/name: {{ .name }} - helm.sh/chart: {{ .root.Chart.Name }}-{{ .root.Chart.Version | replace "+" "_" }} - app.kubernetes.io/managed-by: {{ .root.Release.Service }} - app.kubernetes.io/part-of: kubezero +{{ include "kubezero-lib.labels" .root | indent 4 }} {{- if not .retain }} finalizers: - resources-finalizer.argocd.argoproj.io diff --git a/charts/kubezero-app/templates/calico.yaml b/charts/kubezero-app/templates/calico.yaml index 8b0ee58..33aa844 100644 --- a/charts/kubezero-app/templates/calico.yaml +++ b/charts/kubezero-app/templates/calico.yaml @@ -1,3 +1,3 @@ {{- if .Values.calico.enabled }} -{{ template "kubezero.app" dict "root" . "name" "calico" "type" "kustomize" "retain" true }} +{{ template "kubezero-app.app" dict "root" . "name" "calico" "type" "kustomize" "retain" true }} {{- end }} diff --git a/charts/kubezero-app/templates/cert-manager.yaml b/charts/kubezero-app/templates/cert-manager.yaml index 5774f5b..b744db8 100644 --- a/charts/kubezero-app/templates/cert-manager.yaml +++ b/charts/kubezero-app/templates/cert-manager.yaml @@ -1,10 +1,12 @@ {{- if index .Values "cert-manager" "enabled" }} -{{ template "kubezero.app" dict "root" . "name" "cert-manager" "type" "helm" "namespace" "cert-manager" }} +{{ template "kubezero-app.app" dict "root" . "name" "cert-manager" "type" "helm" "namespace" "cert-manager" }} --- apiVersion: v1 kind: Namespace metadata: name: cert-manager +{{- if index .Values "kiam" "enabled" }} annotations: iam.amazonaws.com/permitted: ".*CertManagerRole.*" {{- end }} +{{- end }} diff --git a/charts/kubezero-app/templates/kiam.yaml b/charts/kubezero-app/templates/kiam.yaml index 298f6fb..78c34af 100644 --- a/charts/kubezero-app/templates/kiam.yaml +++ b/charts/kubezero-app/templates/kiam.yaml @@ -1,3 +1,3 @@ {{- if index .Values "kiam" "enabled" }} -{{ template "kubezero.app" dict "root" . "name" "kiam" "type" "helm" }} +{{ template "kubezero-app.app" dict "root" . "name" "kiam" "type" "helm" }} {{- end }} diff --git a/charts/kubezero-app/templates/local-volume-provisioner.yaml b/charts/kubezero-app/templates/local-volume-provisioner.yaml index 490efbb..e52c920 100644 --- a/charts/kubezero-app/templates/local-volume-provisioner.yaml +++ b/charts/kubezero-app/templates/local-volume-provisioner.yaml @@ -1,3 +1,3 @@ {{- if index .Values "local-volume-provisioner" "enabled" }} -{{ template "kubezero.app" dict "root" . "name" "local-volume-provisioner" "type" "kustomize" }} +{{ template "kubezero-app.app" dict "root" . "name" "local-volume-provisioner" "type" "kustomize" }} {{- end }} -- 2.40.1 From 23535a242ecc235071e7505199a81f6c4258ebbe Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 19 May 2020 12:15:40 +0100 Subject: [PATCH 15/31] Remove name label --- charts/kubezero-lib/Chart.yaml | 2 +- charts/kubezero-lib/templates/_helpers.tpl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/kubezero-lib/Chart.yaml b/charts/kubezero-lib/Chart.yaml index 4b385a8..6ed57f6 100644 --- a/charts/kubezero-lib/Chart.yaml +++ b/charts/kubezero-lib/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-lib description: KubeZero helm library - common helm functions and blocks type: library -version: 0.1.1 +version: 0.1.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: diff --git a/charts/kubezero-lib/templates/_helpers.tpl b/charts/kubezero-lib/templates/_helpers.tpl index 67bfa26..e211b10 100644 --- a/charts/kubezero-lib/templates/_helpers.tpl +++ b/charts/kubezero-lib/templates/_helpers.tpl @@ -2,7 +2,6 @@ Common set of labels */ -}} {{- define "kubezero-lib.labels" -}} -app.kubernetes.io/name: {{ .name }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/part-of: kubezero -- 2.40.1 From c25e4b022d402f34fc2816550eef67f750a8ec18 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 19 May 2020 12:16:22 +0100 Subject: [PATCH 16/31] Use common label for main chart --- charts/kubezero/Chart.yaml | 6 +++++- charts/kubezero/templates/app.yaml | 5 +---- charts/kubezero/templates/istio-service.yaml | 7 +------ charts/kubezero/templates/project.yaml | 5 +---- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index e7979b2..3194bf9 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 description: KubeZero Helm chart to install Zero Down Time Kuberenetes platform name: kubezero -version: 0.2.5 +version: 0.2.6 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: @@ -11,6 +11,10 @@ keywords: maintainers: - name: Quarky9 dependencies: +dependencies: + - name: kubezero-lib + version: ">= 0.1.1" + repository: https://zero-down-time.github.io/kubezero/ - name: argo-cd version: 2.3.2 repository: https://argoproj.github.io/argo-helm diff --git a/charts/kubezero/templates/app.yaml b/charts/kubezero/templates/app.yaml index 14e94b7..24b4a83 100644 --- a/charts/kubezero/templates/app.yaml +++ b/charts/kubezero/templates/app.yaml @@ -4,10 +4,7 @@ metadata: name: kubezero namespace: argocd labels: - app.kubernetes.io/name: kubezero - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} +{{ include "kubezero-lib.labels" . | indent 4 }} spec: project: kubezero source: diff --git a/charts/kubezero/templates/istio-service.yaml b/charts/kubezero/templates/istio-service.yaml index e13e477..195494c 100644 --- a/charts/kubezero/templates/istio-service.yaml +++ b/charts/kubezero/templates/istio-service.yaml @@ -4,12 +4,7 @@ kind: VirtualService metadata: name: argocd-server labels: - app.kubernetes.io/name: {{ .Chart.Name }}-argocd-virtualservice - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/part-of: argocd - app.kubernetes.io/component: server +{{ include "kubezero-lib.labels" . | indent 4 }} spec: gateways: - {{ .Values.istio.gateway }} diff --git a/charts/kubezero/templates/project.yaml b/charts/kubezero/templates/project.yaml index 5ab30ca..81b0fcc 100644 --- a/charts/kubezero/templates/project.yaml +++ b/charts/kubezero/templates/project.yaml @@ -4,10 +4,7 @@ metadata: name: kubezero namespace: argocd labels: - app.kubernetes.io/name: {{ .Chart.Name }}-project - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} +{{ include "kubezero-lib.labels" . | indent 4 }} spec: description: KubeZero - ZeroDownTime Kubernetes Platform -- 2.40.1 From cdd75cb5656e99a4d2a28d18c9ed64abb36f0c44 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 19 May 2020 13:44:01 +0100 Subject: [PATCH 17/31] Update README.md --- charts/kubezero-app/README.md | 23 +++++++++------- charts/kubezero-cert-manager/README.md | 36 ++++++++++++++++++++++++++ charts/kubezero-kiam/README.md | 3 ++- charts/kubezero/README.md | 9 +++++-- 4 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 charts/kubezero-cert-manager/README.md diff --git a/charts/kubezero-app/README.md b/charts/kubezero-app/README.md index dcf5372..b2b9bf6 100644 --- a/charts/kubezero-app/README.md +++ b/charts/kubezero-app/README.md @@ -1,21 +1,26 @@ -kubezeroApp -=========== +kubezero-app +============ KubeZero ArgoCD Application - Root chart of the KubeZero -Current chart version is `0.1.4` +Current chart version is `0.2.1` Source code can be found [here](https://kubezero.com) +## Chart Requirements +| Repository | Name | Version | +|------------|------|---------| +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | ## Chart Values | Key | Type | Default | Description | |-----|------|---------|-------------| | calico.enabled | bool | `false` | | -| certManager.enabled | bool | `false` | | -| defaultDestination.server | string | `"https://kubernetes.default.svc"` | | -| defaultSource.pathPrefix | string | `""` | optional path prefix within repoURL to support eg. remote subtrees | -| defaultSource.repoURL | string | `"https://github.com/zero-down-time/kubezero"` | default repository for argocd applications | -| defaultSource.targetRevision | string | `"HEAD"` | default tracking of repoURL | -| localVolumeProvisioner.enabled | bool | `false` | | +| cert-manager.enabled | bool | `false` | | +| global.defaultDestination.server | string | `"https://kubernetes.default.svc"` | | +| global.defaultSource.pathPrefix | string | `""` | | +| global.defaultSource.repoURL | string | `"https://github.com/zero-down-time/kubezero"` | | +| global.defaultSource.targetRevision | string | `"HEAD"` | | +| kiam.enabled | bool | `false` | | +| local-volume-provisioner.enabled | bool | `false` | | diff --git a/charts/kubezero-cert-manager/README.md b/charts/kubezero-cert-manager/README.md new file mode 100644 index 0000000..756f630 --- /dev/null +++ b/charts/kubezero-cert-manager/README.md @@ -0,0 +1,36 @@ +kubezero-cert-manager +===================== +KubeZero Umbrella Chart for cert-manager + +Current chart version is `0.3.3` + +Source code can be found [here](https://kubezero.com) + +## Chart Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.jetstack.io | cert-manager | 0.15.0 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | + +## Chart Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| cert-manager.cainjector.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| cert-manager.cainjector.tolerations[0].effect | string | `"NoSchedule"` | | +| cert-manager.cainjector.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| cert-manager.extraArgs[0] | string | `"--dns01-recursive-nameservers-only"` | | +| cert-manager.ingressShim.defaultIssuerKind | string | `"ClusterIssuer"` | | +| cert-manager.ingressShim.defaultIssuerName | string | `"letsencrypt-dns-prod"` | | +| cert-manager.installCRDs | bool | `true` | | +| cert-manager.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| cert-manager.prometheus.servicemonitor.enabled | bool | `false` | | +| cert-manager.tolerations[0].effect | string | `"NoSchedule"` | | +| cert-manager.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| cert-manager.webhook.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| cert-manager.webhook.tolerations[0].effect | string | `"NoSchedule"` | | +| cert-manager.webhook.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| clusterIssuer | object | `{}` | | +| localCA.enabled | bool | `true` | | +| localCA.selfsigning | bool | `true` | | diff --git a/charts/kubezero-kiam/README.md b/charts/kubezero-kiam/README.md index 55254ed..b897c6b 100644 --- a/charts/kubezero-kiam/README.md +++ b/charts/kubezero-kiam/README.md @@ -2,7 +2,7 @@ kubezero-kiam ============= KubeZero Umbrella Chart for Kiam -Current chart version is `0.2.0` +Current chart version is `0.2.4` Source code can be found [here](https://kubezero.com) @@ -11,6 +11,7 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| | https://uswitch.github.io/kiam-helm-charts/charts/ | kiam | 5.7.0 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | ## KubeZero default configuration We run agents on the controllers as well, so we force eg. ebs csi controllers and others to assume roles etc. diff --git a/charts/kubezero/README.md b/charts/kubezero/README.md index 2f0a6dd..0de00b9 100644 --- a/charts/kubezero/README.md +++ b/charts/kubezero/README.md @@ -2,7 +2,7 @@ kubezero ======== KubeZero Helm chart to install Zero Down Time Kuberenetes platform -Current chart version is `0.2.0` +Current chart version is `0.2.6` Source code can be found [here](https://kubezero.com) @@ -10,7 +10,8 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| -| https://argoproj.github.io/argo-helm | argo-cd | 2.2.13 | +| https://argoproj.github.io/argo-helm | argo-cd | 2.3.2 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | ## Chart Values @@ -33,6 +34,10 @@ Source code can be found [here](https://kubezero.com) | argo-cd.server.service.servicePortHttpsName | string | `"grpc"` | | | argo-cd.server.tolerations[0].effect | string | `"NoSchedule"` | | | argo-cd.server.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| global.defaultDestination.server | string | `"https://kubernetes.default.svc"` | | +| global.defaultSource.pathPrefix | string | `""` | | +| global.defaultSource.repoURL | string | `"https://github.com/zero-down-time/kubezero"` | | +| global.defaultSource.targetRevision | string | `"HEAD"` | | | istio.enabled | bool | `false` | Deploy Istio VirtualService to expose ArgoCD | | istio.gateway | string | `"ingressgateway.istio-system.svc.cluster.local"` | Name of the Istio gateway to add the VirtualService to | | kubezero | object | `{}` | Kubezero configuration, values.yaml please see kubezeroApp | -- 2.40.1 From 80d4c172abfe7f5c095c44de6c8c42304e5e32e7 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 19 May 2020 15:13:41 +0100 Subject: [PATCH 18/31] First version of aws-ebs-csi-driver umbrella chart, updated docs --- charts/kubezero-aws-ebs/.helmignore | 22 ++ charts/kubezero-aws-ebs/Chart.yaml | 22 ++ charts/kubezero-aws-ebs/README.md | 27 ++ .../charts/aws-ebs-csi-driver/.helmignore | 22 ++ .../charts/aws-ebs-csi-driver/Chart.yaml | 16 ++ .../aws-ebs-csi-driver/templates/NOTES.txt | 3 + .../aws-ebs-csi-driver/templates/_helpers.tpl | 58 ++++ .../templates/csidriver.yaml | 7 + .../templates/daemonset.yaml | 108 ++++++++ .../templates/deployment.yaml | 151 +++++++++++ .../aws-ebs-csi-driver/templates/rbac.yaml | 251 ++++++++++++++++++ .../templates/serviceaccount.yaml | 18 ++ .../templates/statefulset.yaml | 26 ++ .../charts/aws-ebs-csi-driver/values.yaml | 86 ++++++ .../templates/snapshot-class.yaml | 10 + .../templates/storage-class.yaml | 41 +++ charts/kubezero-aws-ebs/update.sh | 10 + charts/kubezero-aws-ebs/values.yaml | 21 ++ charts/kubezero-cert-manager/README.md | 1 + charts/kubezero-cert-manager/values.yaml | 5 +- charts/kubezero-kiam/README.md | 2 +- charts/kubezero-kiam/values.yaml | 4 +- 22 files changed, 906 insertions(+), 5 deletions(-) create mode 100644 charts/kubezero-aws-ebs/.helmignore create mode 100644 charts/kubezero-aws-ebs/Chart.yaml create mode 100644 charts/kubezero-aws-ebs/README.md create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/.helmignore create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/Chart.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/NOTES.txt create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/_helpers.tpl create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/csidriver.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/daemonset.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/deployment.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/rbac.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/statefulset.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/values.yaml create mode 100644 charts/kubezero-aws-ebs/templates/snapshot-class.yaml create mode 100644 charts/kubezero-aws-ebs/templates/storage-class.yaml create mode 100755 charts/kubezero-aws-ebs/update.sh create mode 100644 charts/kubezero-aws-ebs/values.yaml diff --git a/charts/kubezero-aws-ebs/.helmignore b/charts/kubezero-aws-ebs/.helmignore new file mode 100644 index 0000000..50af031 --- /dev/null +++ b/charts/kubezero-aws-ebs/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubezero-aws-ebs/Chart.yaml b/charts/kubezero-aws-ebs/Chart.yaml new file mode 100644 index 0000000..2349320 --- /dev/null +++ b/charts/kubezero-aws-ebs/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +name: kubezero-aws-ebs +description: KubeZero Umbrella Chart for aws-ebs-csi-driver +type: application +version: 0.1.0 +home: https://kubezero.com +icon: https://cdn.zero-downtime.net/assets/logo_small.png +sources: + - https://github.com/kubernetes-sigs/aws-ebs-csi-driver + - https://github.com/Zero-Down-Time/kubezero +keywords: + - kubezero + - aws + - ebs + - csi +maintainers: + - name: Quarky9 +dependencies: + - name: kubezero-lib + version: ">= 0.1.1" + repository: https://zero-down-time.github.io/kubezero/ +kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-aws-ebs/README.md b/charts/kubezero-aws-ebs/README.md new file mode 100644 index 0000000..1e2f2a3 --- /dev/null +++ b/charts/kubezero-aws-ebs/README.md @@ -0,0 +1,27 @@ +kubezero-aws-ebs +================ +KubeZero Umbrella Chart for aws-ebs-csi-driver + +Current chart version is `0.1.0` + +Source code can be found [here](https://kubezero.com) + +## Chart Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | + +## Chart Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| aws-ebs-csi-driver.enableVolumeResizing | bool | `false` | | +| aws-ebs-csi-driver.enableVolumeScheduling | bool | `true` | | +| aws-ebs-csi-driver.enableVolumeSnapshot | bool | `false` | | +| aws-ebs-csi-driver.extraVolumeTags | object | `{}` | Optional tags to be added to each EBS volume | +| aws-ebs-csi-driver.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| aws-ebs-csi-driver.podAnnotations | object | `{}` | iam.amazonaws.com/role: to assume | +| aws-ebs-csi-driver.replicaCount | int | `1` | | +| aws-ebs-csi-driver.tolerations[0].effect | string | `"NoSchedule"` | | +| aws-ebs-csi-driver.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/.helmignore b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/.helmignore new file mode 100644 index 0000000..50af031 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/Chart.yaml new file mode 100644 index 0000000..df6d0fc --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +appVersion: "0.5.0" +name: aws-ebs-csi-driver +description: A Helm chart for AWS EBS CSI Driver +version: 0.3.0 +kubeVersion: ">=1.13.0-0" +home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver +sources: + - https://github.com/kubernetes-sigs/aws-ebs-csi-driver +keywords: + - aws + - ebs + - csi +maintainers: + - name: leakingtapan + email: chengpan@amazon.com diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/NOTES.txt b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/NOTES.txt new file mode 100644 index 0000000..34db916 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/NOTES.txt @@ -0,0 +1,3 @@ +To verify that aws-ebs-csi-driver has started, run: + + kubectl get pod -n kube-system -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/_helpers.tpl b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/_helpers.tpl new file mode 100644 index 0000000..7fa1330 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/_helpers.tpl @@ -0,0 +1,58 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "aws-ebs-csi-driver.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "aws-ebs-csi-driver.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "aws-ebs-csi-driver.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "aws-ebs-csi-driver.labels" -}} +app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} +helm.sh/chart: {{ include "aws-ebs-csi-driver.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Convert the `--extra-volume-tags` command line arg from a map. +*/}} +{{- define "aws-ebs-csi-driver.extra-volume-tags" -}} +{{- $result := dict "pairs" (list) -}} +{{- range $key, $value := .Values.extraVolumeTags -}} +{{- $noop := printf "%s=%s" $key $value | append $result.pairs | set $result "pairs" -}} +{{- end -}} +{{- if gt (len $result.pairs) 0 -}} +- --extra-volume-tags={{- join "," $result.pairs -}} +{{- end -}} +{{- end -}} diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/csidriver.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/csidriver.yaml new file mode 100644 index 0000000..6e427fd --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/csidriver.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: ebs.csi.aws.com +spec: + attachRequired: true + podInfoOnMount: false diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/daemonset.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/daemonset.yaml new file mode 100644 index 0000000..1e6e817 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/daemonset.yaml @@ -0,0 +1,108 @@ +# Node Service +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ebs-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.node.podAnnotations }} + annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- end }} + spec: + nodeSelector: + beta.kubernetes.io/os: linux + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - operator: Exists + {{- with .Values.node.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: ebs-plugin + securityContext: + privileged: true + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + args: + - node + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + - name: node-driver-registrar + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/deployment.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/deployment.yaml new file mode 100644 index 0000000..3316e96 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/deployment.yaml @@ -0,0 +1,151 @@ +# Controller Service +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ebs-csi-controller + namespace: kube-system +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.podAnnotations }} + annotations: {{ toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} + spec: + nodeSelector: + beta.kubernetes.io/os: linux + {{- with .Values.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + serviceAccountName: ebs-csi-controller-sa + priorityClassName: system-cluster-critical + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + tolerations: + - operator: Exists + {{- with .Values.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: ebs-plugin + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - controller + - --endpoint=$(CSI_ENDPOINT) + {{ include "aws-ebs-csi-driver.extra-volume-tags" . }} + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-secret + key: key_id + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-secret + key: access_key + optional: true + {{- if .Values.region }} + - name: AWS_REGION + value: {{ .Values.region }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: csi-provisioner + image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --v=5 + {{- if .Values.enableVolumeScheduling }} + - --feature-gates=Topology=true + {{- end}} + - --enable-leader-election + - --leader-election-type=leases + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --v=5 + - --leader-election=true + - --leader-election-type=leases + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- if .Values.enableVolumeSnapshot }} + - name: csi-snapshotter + image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- end }} + {{- if .Values.enableVolumeResizing }} + - name: csi-resizer + image: {{ printf "%s:%s" .Values.sidecars.resizerImage.repository .Values.sidecars.resizerImage.tag }} + imagePullPolicy: Always + args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + volumes: + - name: socket-dir + emptyDir: {} diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/rbac.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/rbac.yaml new file mode 100644 index 0000000..464c648 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/rbac.yaml @@ -0,0 +1,251 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-attacher-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-snapshotter-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-snapshot-controller-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshot-controller-binding +subjects: + - kind: ServiceAccount + name: ebs-snapshot-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-snapshot-controller-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-snapshot-controller-leaderelection + namespace: kube-system +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-leaderelection + namespace: kube-system +subjects: + - kind: ServiceAccount + name: ebs-snapshot-controller + namespace: kube-system +roleRef: + kind: Role + name: snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io + +{{- end }} + +{{- if .Values.enableVolumeResizing }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-resizer-role +rules: + # The following rule should be uncommented for plugins that require secrets + # for provisioning. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-resizer-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-resizer-role + apiGroup: rbac.authorization.k8s.io +{{- end}} diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml new file mode 100644 index 0000000..95396d6 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ebs-csi-controller-sa + namespace: kube-system + {{- with .Values.serviceAccount.controller.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ebs-snapshot-controller + namespace: kube-system + {{- with .Values.serviceAccount.snapshot.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/statefulset.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/statefulset.yaml new file mode 100644 index 0000000..01f36b7 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/statefulset.yaml @@ -0,0 +1,26 @@ +{{- if .Values.enableVolumeSnapshot }} +#Snapshot controller +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: ebs-snapshot-controller + namespace: kube-system +spec: + serviceName: ebs-snapshot-controller + replicas: 1 + selector: + matchLabels: + app: ebs-snapshot-controller + template: + metadata: + labels: + app: ebs-snapshot-controller + spec: + serviceAccount: ebs-snapshot-controller + containers: + - name: snapshot-controller + image: quay.io/k8scsi/snapshot-controller:v2.0.1 + args: + - --v=5 + - --leader-election=false +{{- end }} diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/values.yaml new file mode 100644 index 0000000..b899721 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/values.yaml @@ -0,0 +1,86 @@ +# Default values for aws-ebs-csi-driver. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 2 + +image: + repository: amazon/aws-ebs-csi-driver + tag: "v0.5.0" + pullPolicy: IfNotPresent + +sidecars: + provisionerImage: + repository: quay.io/k8scsi/csi-provisioner + tag: "v1.5.0" + attacherImage: + repository: quay.io/k8scsi/csi-attacher + tag: "v1.2.0" + snapshotterImage: + repository: quay.io/k8scsi/csi-snapshotter + tag: "v2.0.1" + livenessProbeImage: + repository: quay.io/k8scsi/livenessprobe + tag: "v1.1.0" + resizerImage: + repository: quay.io/k8scsi/csi-resizer + tag: "v0.3.0" + nodeDriverRegistrarImage: + repository: quay.io/k8scsi/csi-node-driver-registrar + tag: "v1.1.0" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +podAnnotations: {} + +# True if enable volume scheduling for dynamic volume provisioning +enableVolumeScheduling: false + +# True if enable volume resizing +enableVolumeResizing: false + +# True if enable volume snapshot +enableVolumeSnapshot: false + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# Extra volume tags to attach to each dynamically provisioned volume. +# --- +# extraVolumeTags: +# key1: value1 +# key2: value2 +extraVolumeTags: {} + +# AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata +# service. +# --- +# region: us-east-1 +region: "" + +node: + podAnnotations: {} + tolerations: [] + +serviceAccount: + controller: + annotations: {} + snapshot: + annotations: {} diff --git a/charts/kubezero-aws-ebs/templates/snapshot-class.yaml b/charts/kubezero-aws-ebs/templates/snapshot-class.yaml new file mode 100644 index 0000000..1b4a831 --- /dev/null +++ b/charts/kubezero-aws-ebs/templates/snapshot-class.yaml @@ -0,0 +1,10 @@ +{{- if index .Values "aws-ebs-csi-driver" "enableVolumeSnapshot" }} +apiVersion: snapshot.storage.k8s.io/v1beta1 +kind: VolumeSnapshotClass +metadata: + name: csi-aws-vsc + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +driver: ebs.csi.aws.com +deletionPolicy: Delete +{{- end }} diff --git a/charts/kubezero-aws-ebs/templates/storage-class.yaml b/charts/kubezero-aws-ebs/templates/storage-class.yaml new file mode 100644 index 0000000..cd714bb --- /dev/null +++ b/charts/kubezero-aws-ebs/templates/storage-class.yaml @@ -0,0 +1,41 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: ebs-sc-gp2-xfs + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer +parameters: + csi.storage.k8s.io/fstype: xfs + type: gp2 + encrypted: "true" +{{- if index .Values "aws-ebs-csi-driver" "enableVolumeResizing" }} +allowVolumeExpansion: true +{{- end }} + +{{- range .Values.storageClassZones }} +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: ebs-sc-gp2-xfs-{{ . }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer +parameters: + csi.storage.k8s.io/fstype: xfs + type: gp2 + encrypted: "true" +{{- if index .Values "aws-ebs-csi-driver" "enableVolumeResizing" }} +allowVolumeExpansion: true +{{- end }} +allowedTopologies: +- matchLabelExpressions: + - key: failure-domain.beta.kubernetes.io/zone + values: + - {{ . }} +{{- end }} diff --git a/charts/kubezero-aws-ebs/update.sh b/charts/kubezero-aws-ebs/update.sh new file mode 100755 index 0000000..c7b5df7 --- /dev/null +++ b/charts/kubezero-aws-ebs/update.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -ex + +REPO="kubernetes-sigs/aws-ebs-csi-driver" +LATEST_RELEASE=$(curl -sL -s https://api.github.com/repos/${REPO}/releases | grep '"tag_name":' | cut -d'"' -f4 | grep -v -E "(alpha|beta|rc)" | sort -t"." -k 1,1 -k 2,2 -k 3,3 -k 4,4 | tail -n 1) + +URL="https://github.com/${REPO}/releases/download/${LATEST_RELEASE}/helm-chart.tgz" + +rm -rf charts/aws-ebs-csi-driver +curl -sL "$URL" | tar xz -C charts diff --git a/charts/kubezero-aws-ebs/values.yaml b/charts/kubezero-aws-ebs/values.yaml new file mode 100644 index 0000000..cac6c1a --- /dev/null +++ b/charts/kubezero-aws-ebs/values.yaml @@ -0,0 +1,21 @@ +aws-ebs-csi-driver: + replicaCount: 1 + + enableVolumeScheduling: true + enableVolumeResizing: false + enableVolumeSnapshot: false + + nodeSelector: + node-role.kubernetes.io/master: "" + + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + + # aws-ebs-csi-driver.podAnnotations -- iam.amazonaws.com/role: to assume + podAnnotations: {} + # iam.amazonaws.com/role: '' + + # aws-ebs-csi-driver.extraVolumeTags -- Optional tags to be added to each EBS volume + extraVolumeTags: {} + # Name: KubeZero-Cluster diff --git a/charts/kubezero-cert-manager/README.md b/charts/kubezero-cert-manager/README.md index 756f630..8027bbf 100644 --- a/charts/kubezero-cert-manager/README.md +++ b/charts/kubezero-cert-manager/README.md @@ -25,6 +25,7 @@ Source code can be found [here](https://kubezero.com) | cert-manager.ingressShim.defaultIssuerName | string | `"letsencrypt-dns-prod"` | | | cert-manager.installCRDs | bool | `true` | | | cert-manager.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| cert-manager.podAnnotations."iam.amazonaws.com/role" | string | `""` | IAM role ARN the cert-manager might use via kiam eg."arn:aws:iam::123456789012:role/certManagerRoleArn" | | cert-manager.prometheus.servicemonitor.enabled | bool | `false` | | | cert-manager.tolerations[0].effect | string | `"NoSchedule"` | | | cert-manager.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | diff --git a/charts/kubezero-cert-manager/values.yaml b/charts/kubezero-cert-manager/values.yaml index e23fcfb..e8b6eec 100644 --- a/charts/kubezero-cert-manager/values.yaml +++ b/charts/kubezero-cert-manager/values.yaml @@ -45,5 +45,6 @@ cert-manager: prometheus: servicemonitor: enabled: false - #podAnnotations: - # iam.amazonaws.com/role: "INSERT_CLOUDFORMATION_OUTPUT_CertManagerRoleArn" + # cert-manager.podAnnotations."iam.amazonaws.com/role" -- IAM role ARN the cert-manager might use via kiam eg."arn:aws:iam::123456789012:role/certManagerRoleArn" + podAnnotations: + iam.amazonaws.com/role: "" diff --git a/charts/kubezero-kiam/README.md b/charts/kubezero-kiam/README.md index b897c6b..079edc3 100644 --- a/charts/kubezero-kiam/README.md +++ b/charts/kubezero-kiam/README.md @@ -49,7 +49,7 @@ Required for the *csi ebs plugin* and most likely various others assuming basic | kiam.agent.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | kiam.agent.updateStrategy | string | `"RollingUpdate"` | | | kiam.agent.whiteListRouteRegexp | string | `"^/latest/(meta-data/instance-id|dynamic)"` | | -| kiam.server.assumeRoleArn | string | `"arn:aws:iam::123456789012:role/kiam-server-role"` | kiam server IAM role to assume, required as we run the agents next to the servers normally | +| kiam.server.assumeRoleArn | string | `""` | kiam server IAM role to assume, required as we run the agents next to the servers normally, eg. arn:aws:iam::123456789012:role/kiam-server-role | | kiam.server.deployment.enabled | bool | `true` | | | kiam.server.deployment.replicas | int | `1` | | | kiam.server.image.tag | string | `"v3.6-rc1"` | | diff --git a/charts/kubezero-kiam/values.yaml b/charts/kubezero-kiam/values.yaml index 1f91f82..2affcd8 100644 --- a/charts/kubezero-kiam/values.yaml +++ b/charts/kubezero-kiam/values.yaml @@ -2,8 +2,8 @@ kiam: server: image: tag: "v3.6-rc1" - # kiam.server.assumeRoleArn -- kiam server IAM role to assume, required as we run the agents next to the servers normally - assumeRoleArn: arn:aws:iam::123456789012:role/kiam-server-role + # kiam.server.assumeRoleArn -- kiam server IAM role to assume, required as we run the agents next to the servers normally, eg. arn:aws:iam::123456789012:role/kiam-server-role + assumeRoleArn: '' useHostNetwork: true sslCertHostPath: /etc/ssl/certs tlsSecret: kiam-server-tls -- 2.40.1 From 12f73c02111848659df0f71fae7680d617d40de7 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 19 May 2020 16:25:29 +0100 Subject: [PATCH 19/31] Add aws-ebs-csi-driver into root app, rename module properly --- charts/kubezero-app/Chart.yaml | 2 +- charts/kubezero-app/templates/aws-ebs-csi-driver.yaml | 3 +++ charts/kubezero-app/values-all.yaml | 3 +++ charts/kubezero-app/values.yaml | 3 +++ .../.helmignore | 0 .../Chart.yaml | 2 +- .../README.md | 0 .../charts/aws-ebs-csi-driver/.helmignore | 0 .../charts/aws-ebs-csi-driver/Chart.yaml | 0 .../charts/aws-ebs-csi-driver/templates/NOTES.txt | 0 .../charts/aws-ebs-csi-driver/templates/_helpers.tpl | 0 .../charts/aws-ebs-csi-driver/templates/csidriver.yaml | 0 .../charts/aws-ebs-csi-driver/templates/daemonset.yaml | 0 .../charts/aws-ebs-csi-driver/templates/deployment.yaml | 0 .../charts/aws-ebs-csi-driver/templates/rbac.yaml | 0 .../charts/aws-ebs-csi-driver/templates/serviceaccount.yaml | 0 .../charts/aws-ebs-csi-driver/templates/statefulset.yaml | 0 .../charts/aws-ebs-csi-driver/values.yaml | 0 .../templates/snapshot-class.yaml | 0 .../templates/storage-class.yaml | 0 .../update.sh | 0 .../values.yaml | 0 22 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 charts/kubezero-app/templates/aws-ebs-csi-driver.yaml rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/.helmignore (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/Chart.yaml (93%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/README.md (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/charts/aws-ebs-csi-driver/.helmignore (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/charts/aws-ebs-csi-driver/Chart.yaml (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/charts/aws-ebs-csi-driver/templates/NOTES.txt (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/charts/aws-ebs-csi-driver/templates/_helpers.tpl (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/charts/aws-ebs-csi-driver/templates/csidriver.yaml (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/charts/aws-ebs-csi-driver/templates/daemonset.yaml (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/charts/aws-ebs-csi-driver/templates/deployment.yaml (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/charts/aws-ebs-csi-driver/templates/rbac.yaml (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/charts/aws-ebs-csi-driver/templates/statefulset.yaml (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/charts/aws-ebs-csi-driver/values.yaml (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/templates/snapshot-class.yaml (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/templates/storage-class.yaml (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/update.sh (100%) rename charts/{kubezero-aws-ebs => kubezero-aws-ebs-csi-driver}/values.yaml (100%) diff --git a/charts/kubezero-app/Chart.yaml b/charts/kubezero-app/Chart.yaml index f14be38..adab879 100644 --- a/charts/kubezero-app/Chart.yaml +++ b/charts/kubezero-app/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-app description: KubeZero ArgoCD Application - Root chart of the KubeZero type: application -version: 0.2.1 +version: 0.2.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/logo_small.png keywords: diff --git a/charts/kubezero-app/templates/aws-ebs-csi-driver.yaml b/charts/kubezero-app/templates/aws-ebs-csi-driver.yaml new file mode 100644 index 0000000..cf4e108 --- /dev/null +++ b/charts/kubezero-app/templates/aws-ebs-csi-driver.yaml @@ -0,0 +1,3 @@ +{{- if index .Values "aws-ebs-csi-driver" "enabled" }} +{{ template "kubezero-app.app" dict "root" . "name" "aws-ebs-csi-driver" "type" "helm" }} +{{- end }} diff --git a/charts/kubezero-app/values-all.yaml b/charts/kubezero-app/values-all.yaml index acfbcfb..d939434 100644 --- a/charts/kubezero-app/values-all.yaml +++ b/charts/kubezero-app/values-all.yaml @@ -24,3 +24,6 @@ cert-manager: kiam: enabled: true + +aws-ebs-csi-driver: + enabled: true diff --git a/charts/kubezero-app/values.yaml b/charts/kubezero-app/values.yaml index 0b88a44..c888333 100644 --- a/charts/kubezero-app/values.yaml +++ b/charts/kubezero-app/values.yaml @@ -24,3 +24,6 @@ cert-manager: kiam: enabled: false + +aws-ebs-csi-driver: + enabled: false diff --git a/charts/kubezero-aws-ebs/.helmignore b/charts/kubezero-aws-ebs-csi-driver/.helmignore similarity index 100% rename from charts/kubezero-aws-ebs/.helmignore rename to charts/kubezero-aws-ebs-csi-driver/.helmignore diff --git a/charts/kubezero-aws-ebs/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml similarity index 93% rename from charts/kubezero-aws-ebs/Chart.yaml rename to charts/kubezero-aws-ebs-csi-driver/Chart.yaml index 2349320..111306d 100644 --- a/charts/kubezero-aws-ebs/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: kubezero-aws-ebs +name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application version: 0.1.0 diff --git a/charts/kubezero-aws-ebs/README.md b/charts/kubezero-aws-ebs-csi-driver/README.md similarity index 100% rename from charts/kubezero-aws-ebs/README.md rename to charts/kubezero-aws-ebs-csi-driver/README.md diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/.helmignore b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/.helmignore similarity index 100% rename from charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/.helmignore rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/.helmignore diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml similarity index 100% rename from charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/Chart.yaml rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/NOTES.txt b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt similarity index 100% rename from charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/NOTES.txt rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/_helpers.tpl b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl similarity index 100% rename from charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/_helpers.tpl rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/csidriver.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml similarity index 100% rename from charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/csidriver.yaml rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/daemonset.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/daemonset.yaml similarity index 100% rename from charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/daemonset.yaml rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/daemonset.yaml diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/deployment.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml similarity index 100% rename from charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/deployment.yaml rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/deployment.yaml diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/rbac.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rbac.yaml similarity index 100% rename from charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/rbac.yaml rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rbac.yaml diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml similarity index 100% rename from charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/statefulset.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml similarity index 100% rename from charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/statefulset.yaml rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml similarity index 100% rename from charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/values.yaml rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml diff --git a/charts/kubezero-aws-ebs/templates/snapshot-class.yaml b/charts/kubezero-aws-ebs-csi-driver/templates/snapshot-class.yaml similarity index 100% rename from charts/kubezero-aws-ebs/templates/snapshot-class.yaml rename to charts/kubezero-aws-ebs-csi-driver/templates/snapshot-class.yaml diff --git a/charts/kubezero-aws-ebs/templates/storage-class.yaml b/charts/kubezero-aws-ebs-csi-driver/templates/storage-class.yaml similarity index 100% rename from charts/kubezero-aws-ebs/templates/storage-class.yaml rename to charts/kubezero-aws-ebs-csi-driver/templates/storage-class.yaml diff --git a/charts/kubezero-aws-ebs/update.sh b/charts/kubezero-aws-ebs-csi-driver/update.sh similarity index 100% rename from charts/kubezero-aws-ebs/update.sh rename to charts/kubezero-aws-ebs-csi-driver/update.sh diff --git a/charts/kubezero-aws-ebs/values.yaml b/charts/kubezero-aws-ebs-csi-driver/values.yaml similarity index 100% rename from charts/kubezero-aws-ebs/values.yaml rename to charts/kubezero-aws-ebs-csi-driver/values.yaml -- 2.40.1 From 92cacb700baada35ed550cc7b89f70f800265e64 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 19 May 2020 16:54:07 +0100 Subject: [PATCH 20/31] Update README --- charts/kubezero-app/README.md | 3 ++- charts/kubezero-aws-ebs-csi-driver/README.md | 15 +++++++++++-- .../README.md.gotmpl | 21 +++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 charts/kubezero-aws-ebs-csi-driver/README.md.gotmpl diff --git a/charts/kubezero-app/README.md b/charts/kubezero-app/README.md index b2b9bf6..479f387 100644 --- a/charts/kubezero-app/README.md +++ b/charts/kubezero-app/README.md @@ -2,7 +2,7 @@ kubezero-app ============ KubeZero ArgoCD Application - Root chart of the KubeZero -Current chart version is `0.2.1` +Current chart version is `0.2.2` Source code can be found [here](https://kubezero.com) @@ -16,6 +16,7 @@ Source code can be found [here](https://kubezero.com) | Key | Type | Default | Description | |-----|------|---------|-------------| +| aws-ebs-csi-driver.enabled | bool | `false` | | | calico.enabled | bool | `false` | | | cert-manager.enabled | bool | `false` | | | global.defaultDestination.server | string | `"https://kubernetes.default.svc"` | | diff --git a/charts/kubezero-aws-ebs-csi-driver/README.md b/charts/kubezero-aws-ebs-csi-driver/README.md index 1e2f2a3..525ff1b 100644 --- a/charts/kubezero-aws-ebs-csi-driver/README.md +++ b/charts/kubezero-aws-ebs-csi-driver/README.md @@ -1,5 +1,5 @@ -kubezero-aws-ebs -================ +kubezero-aws-ebs-csi-driver +=========================== KubeZero Umbrella Chart for aws-ebs-csi-driver Current chart version is `0.1.0` @@ -12,6 +12,17 @@ Source code can be found [here](https://kubezero.com) |------------|------|---------| | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | +## IAM Role +If you use kiam or kube2iam and restrict access on nodes running this controller please adjust: +``` +podAnnotations: + iam.amazonaws.com/role: +``` + +## Storage Classes +Provides the *ebs-sc-gp2-xfs* storage class for gp2, enrypted and XFS. +This class is also set as default. + ## Chart Values | Key | Type | Default | Description | diff --git a/charts/kubezero-aws-ebs-csi-driver/README.md.gotmpl b/charts/kubezero-aws-ebs-csi-driver/README.md.gotmpl new file mode 100644 index 0000000..787427e --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/README.md.gotmpl @@ -0,0 +1,21 @@ +{{ template "chart.header" . }} +{{ template "chart.description" . }} + +{{ template "chart.versionLine" . }} + +{{ template "chart.sourceLinkLine" . }} + +{{ template "chart.requirementsSection" . }} + +## IAM Role +If you use kiam or kube2iam and restrict access on nodes running this controller please adjust: +``` +podAnnotations: + iam.amazonaws.com/role: +``` + +## Storage Classes +Provides the *ebs-sc-gp2-xfs* storage class for gp2, enrypted and XFS. +This class is also set as default. + +{{ template "chart.valuesSection" . }} -- 2.40.1 From c45c5783dddd7e9f7104fcaf7f56f68a3c9616d8 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 3 Jun 2020 15:45:01 +0100 Subject: [PATCH 21/31] Latest Calico, add QUICKSTART draft --- artifacts/kubezero-calico/README.md | 2 +- artifacts/kubezero-calico/canal.yaml | 162 +++++++++++++++++- .../kubezero-calico/remove-namespace.patch | 50 ++++++ charts/kubezero/Quickstart.md | 43 +++++ 4 files changed, 251 insertions(+), 6 deletions(-) create mode 100644 artifacts/kubezero-calico/remove-namespace.patch create mode 100644 charts/kubezero/Quickstart.md diff --git a/artifacts/kubezero-calico/README.md b/artifacts/kubezero-calico/README.md index ba64f7e..288574d 100644 --- a/artifacts/kubezero-calico/README.md +++ b/artifacts/kubezero-calico/README.md @@ -9,4 +9,4 @@ See eg: `https://github.com/kubernetes-sigs/kustomize/issues/1351` ## Upgrade See: https://docs.projectcalico.org/maintenance/kubernetes-upgrade -`curl https://docs.projectcalico.org/manifests/canal.yaml -O` +`curl https://docs.projectcalico.org/manifests/canal.yaml -O && patch < remove-namespace.patch` diff --git a/artifacts/kubezero-calico/canal.yaml b/artifacts/kubezero-calico/canal.yaml index ef79974..6adfdb5 100644 --- a/artifacts/kubezero-calico/canal.yaml +++ b/artifacts/kubezero-calico/canal.yaml @@ -150,6 +150,8 @@ spec: kind: GlobalNetworkPolicy plural: globalnetworkpolicies singular: globalnetworkpolicy + shortNames: + - gnp --- apiVersion: apiextensions.k8s.io/v1beta1 @@ -238,6 +240,19 @@ spec: --- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition +metadata: + name: kubecontrollersconfigurations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: KubeControllersConfiguration + plural: kubecontrollersconfigurations + singular: kubecontrollersconfiguration +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition metadata: name: networkpolicies.crd.projectcalico.org spec: @@ -267,6 +282,89 @@ spec: --- # Source: calico/templates/rbac.yaml +# Include a clusterrole for the kube-controllers component, +# and bind it to the calico-kube-controllers serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +rules: + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are queried to check for existence. + - apiGroups: [""] + resources: + - pods + verbs: + - get + # IPAM resources are manipulated when nodes are deleted. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + # kube-controllers manages hostendpoints. + - apiGroups: ["crd.projectcalico.org"] + resources: + - hostendpoints + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - create + - update + # KubeControllersConfiguration is where it gets its config + - apiGroups: ["crd.projectcalico.org"] + resources: + - kubecontrollersconfigurations + verbs: + # read its own config + - get + # create a default if none exists + - create + # update status + - update + # watch for changes + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: +- kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system +--- # Include a clusterrole for the calico-node DaemonSet, # and bind it to the calico-node serviceaccount. kind: ClusterRole @@ -479,7 +577,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: calico/cni:v3.13.3 + image: calico/cni:v3.14.1 command: ["/install-cni.sh"] env: # Name of the CNI config file to create. @@ -515,7 +613,7 @@ spec: # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes # to communicate with Felix over the Policy Sync API. - name: flexvol-driver - image: calico/pod2daemon-flexvol:v3.13.3 + image: calico/pod2daemon-flexvol:v3.14.1 volumeMounts: - name: flexvol-driver-host mountPath: /host/driver @@ -526,7 +624,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: calico/node:v3.13.3 + image: calico/node:v3.14.1 env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE @@ -683,10 +781,64 @@ metadata: name: canal --- -# Source: calico/templates/calico-etcd-secrets.yaml +# Source: calico/templates/calico-kube-controllers.yaml +# See https://github.com/projectcalico/kube-controllers +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-kube-controllers + labels: + k8s-app: calico-kube-controllers +spec: + # The controllers can only have a single active instance. + replicas: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + labels: + k8s-app: calico-kube-controllers + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + nodeSelector: + kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical + containers: + - name: calico-kube-controllers + image: calico/kube-controllers:v3.14.1 + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r --- -# Source: calico/templates/calico-kube-controllers.yaml + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + +--- +# Source: calico/templates/calico-etcd-secrets.yaml --- # Source: calico/templates/calico-typha.yaml diff --git a/artifacts/kubezero-calico/remove-namespace.patch b/artifacts/kubezero-calico/remove-namespace.patch new file mode 100644 index 0000000..84b7822 --- /dev/null +++ b/artifacts/kubezero-calico/remove-namespace.patch @@ -0,0 +1,50 @@ +--- canal.yaml.orig 2020-06-03 15:39:41.972295775 +0100 ++++ canal.yaml 2020-06-03 15:39:59.718477177 +0100 +@@ -5,7 +5,6 @@ + apiVersion: v1 + metadata: + name: canal-config +- namespace: kube-system + data: + # Typha is disabled. + typha_service_name: "none" +@@ -536,7 +535,6 @@ + apiVersion: apps/v1 + metadata: + name: canal +- namespace: kube-system + labels: + k8s-app: canal + spec: +@@ -781,7 +779,6 @@ + kind: ServiceAccount + metadata: + name: canal +- namespace: kube-system + + --- + # Source: calico/templates/calico-kube-controllers.yaml +@@ -790,7 +787,6 @@ + kind: Deployment + metadata: + name: calico-kube-controllers +- namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: +@@ -804,7 +800,6 @@ + template: + metadata: + name: calico-kube-controllers +- namespace: kube-system + labels: + k8s-app: calico-kube-controllers + annotations: +@@ -841,7 +836,6 @@ + kind: ServiceAccount + metadata: + name: calico-kube-controllers +- namespace: kube-system + + --- + # Source: calico/templates/calico-etcd-secrets.yaml diff --git a/charts/kubezero/Quickstart.md b/charts/kubezero/Quickstart.md new file mode 100644 index 0000000..67f5baf --- /dev/null +++ b/charts/kubezero/Quickstart.md @@ -0,0 +1,43 @@ +# Quickstart +--- + +# CloudBender + +## Prepare Config +- check config/kube/kube-control-plane.yaml +- check config/kube/kube-workers.yaml + + +## Deploy Control Plane +- cloudbender sync kube-control-plane + +## Get kubectl config +- get admin.conf from S3 and store in your local `~/.kube` folder + +## Verify controller nodes +- Verify all controller nodes have the expected version and are *Ready*, eg via: `kubectl get nodes` + +## Deploy Worker group +- cloudbender sync kube-workers + + +--- +# KubeZero + +## Prepare Config +- check values.yaml + +## Deploy KubeZero Helm chart +`./deploy.sh` + + +## Verify ArgoCD +At this stage we there is no support for any kind of Ingress yet. Therefore in order to reach the Argo API you port forwarding. +`kubectl port-forward svc/argocd-server -n argocd 8080:443` + +Next we to download the argo-cd cli, see https://argoproj.github.io/argo-cd/cli_installation/ + +Finally login into argo-cd via `argocd login localhost:8080` using the *admin* user and the password set in values.yaml earlier. + +# Demo / own apps +- Add your own application to ArgoCD via the cli \ No newline at end of file -- 2.40.1 From 8004cba259d7c64a0453b3a1957c3355b138c9db Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 3 Jun 2020 15:46:25 +0100 Subject: [PATCH 22/31] Move QUICKSTART.md --- charts/kubezero/Quickstart.md => Quickstart.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename charts/kubezero/Quickstart.md => Quickstart.md (100%) diff --git a/charts/kubezero/Quickstart.md b/Quickstart.md similarity index 100% rename from charts/kubezero/Quickstart.md rename to Quickstart.md -- 2.40.1 From d797a1d9a4ed970c0cb51b4ed9e61267eaea11ec Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 3 Jun 2020 16:17:04 +0100 Subject: [PATCH 23/31] Update logo URL --- charts/kubezero-app/Chart.yaml | 2 +- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 2 +- charts/kubezero-cert-manager/Chart.yaml | 2 +- charts/kubezero-kiam/Chart.yaml | 2 +- charts/kubezero-lib/Chart.yaml | 2 +- charts/kubezero/Chart.yaml | 2 +- scripts/publish.sh | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/charts/kubezero-app/Chart.yaml b/charts/kubezero-app/Chart.yaml index adab879..23bbe68 100644 --- a/charts/kubezero-app/Chart.yaml +++ b/charts/kubezero-app/Chart.yaml @@ -4,7 +4,7 @@ description: KubeZero ArgoCD Application - Root chart of the KubeZero type: application version: 0.2.2 home: https://kubezero.com -icon: https://cdn.zero-downtime.net/assets/logo_small.png +icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: - kubezero - argocd diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index 111306d..9ed936b 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -4,7 +4,7 @@ description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application version: 0.1.0 home: https://kubezero.com -icon: https://cdn.zero-downtime.net/assets/logo_small.png +icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver - https://github.com/Zero-Down-Time/kubezero diff --git a/charts/kubezero-cert-manager/Chart.yaml b/charts/kubezero-cert-manager/Chart.yaml index 6fe01b3..0aafe10 100644 --- a/charts/kubezero-cert-manager/Chart.yaml +++ b/charts/kubezero-cert-manager/Chart.yaml @@ -4,7 +4,7 @@ description: KubeZero Umbrella Chart for cert-manager type: application version: 0.3.3 home: https://kubezero.com -icon: https://cdn.zero-downtime.net/assets/logo_small.png +icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: - kubezero - cert-manager diff --git a/charts/kubezero-kiam/Chart.yaml b/charts/kubezero-kiam/Chart.yaml index 1fccd1c..7bc3d65 100644 --- a/charts/kubezero-kiam/Chart.yaml +++ b/charts/kubezero-kiam/Chart.yaml @@ -4,7 +4,7 @@ description: KubeZero Umbrella Chart for Kiam type: application version: 0.2.4 home: https://kubezero.com -icon: https://cdn.zero-downtime.net/assets/logo_small.png +icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: - kubezero - kiam diff --git a/charts/kubezero-lib/Chart.yaml b/charts/kubezero-lib/Chart.yaml index 6ed57f6..5c6e431 100644 --- a/charts/kubezero-lib/Chart.yaml +++ b/charts/kubezero-lib/Chart.yaml @@ -4,7 +4,7 @@ description: KubeZero helm library - common helm functions and blocks type: library version: 0.1.2 home: https://kubezero.com -icon: https://cdn.zero-downtime.net/assets/logo_small.png +icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: - kubezero maintainers: diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index 3194bf9..d6052fe 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -3,7 +3,7 @@ description: KubeZero Helm chart to install Zero Down Time Kuberenetes platform name: kubezero version: 0.2.6 home: https://kubezero.com -icon: https://cdn.zero-downtime.net/assets/logo_small.png +icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: - kubezero - argocd diff --git a/scripts/publish.sh b/scripts/publish.sh index 64c9c57..d94bf44 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -17,7 +17,7 @@ helm repo add uswitch https://uswitch.github.io/kiam-helm-charts/charts/ for dir in $(find $SRCROOT/charts -mindepth 1 -maxdepth 1 -type d); do - rm -rf $dir/charts $dir/Chart.lock + # rm -rf $dir/charts $dir/Chart.lock name=$(basename $dir) -- 2.40.1 From eaef8a643468903553e1114f9930e835fc78faa3 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Wed, 3 Jun 2020 16:18:03 +0100 Subject: [PATCH 24/31] Bump Chart version of kubezero-app due to updated Calico --- charts/kubezero-app/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubezero-app/Chart.yaml b/charts/kubezero-app/Chart.yaml index 23bbe68..764eeb8 100644 --- a/charts/kubezero-app/Chart.yaml +++ b/charts/kubezero-app/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-app description: KubeZero ArgoCD Application - Root chart of the KubeZero type: application -version: 0.2.2 +version: 0.2.3 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: -- 2.40.1 From 748f57899981fd5cb82a05b4e52652cbd6acc549 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 5 Jun 2020 16:38:46 +0100 Subject: [PATCH 25/31] Update Quickstart docs --- Quickstart.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Quickstart.md b/Quickstart.md index 67f5baf..477a532 100644 --- a/Quickstart.md +++ b/Quickstart.md @@ -20,6 +20,9 @@ ## Deploy Worker group - cloudbender sync kube-workers +## Verify all nodes +- Verify all nodes incl. workers have the expected version and are *Ready*, eg via: `kubectl get nodes` + --- # KubeZero @@ -27,17 +30,25 @@ ## Prepare Config - check values.yaml +Easiest way to get the ARNs for various IAM roles is to use the CloudBender output command: +```cloudbender outputs config/kube-control-plane.yaml``` + ## Deploy KubeZero Helm chart `./deploy.sh` ## Verify ArgoCD -At this stage we there is no support for any kind of Ingress yet. Therefore in order to reach the Argo API you port forwarding. -`kubectl port-forward svc/argocd-server -n argocd 8080:443` +At this stage we there is no support for any kind of Ingress yet. To reach the Argo API port forward from localhost via: +`kubectl port-forward svc/kubezero-argocd-server -n argocd 8080:443` -Next we to download the argo-cd cli, see https://argoproj.github.io/argo-cd/cli_installation/ +Next download the argo-cd cli, details for different OS see https://argoproj.github.io/argo-cd/cli_installation/ Finally login into argo-cd via `argocd login localhost:8080` using the *admin* user and the password set in values.yaml earlier. +List all Argo applications via: `argocd app list`. +Currently it is very likely that you need to manually trigger sync runs for `cert-manager`as well as `kiam`. +eg. `argocd app cert-manager sync` + + # Demo / own apps - Add your own application to ArgoCD via the cli \ No newline at end of file -- 2.40.1 From 9893b6b7a1c128f95ab3d07f9f4df84fc0e7326c Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 5 Jun 2020 17:09:42 +0100 Subject: [PATCH 26/31] Update Quickstart docs --- Quickstart.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/Quickstart.md b/Quickstart.md index 477a532..06cce91 100644 --- a/Quickstart.md +++ b/Quickstart.md @@ -31,7 +31,7 @@ - check values.yaml Easiest way to get the ARNs for various IAM roles is to use the CloudBender output command: -```cloudbender outputs config/kube-control-plane.yaml``` +`cloudbender outputs config/kube-control-plane.yaml` ## Deploy KubeZero Helm chart `./deploy.sh` @@ -50,5 +50,27 @@ Currently it is very likely that you need to manually trigger sync runs for `cer eg. `argocd app cert-manager sync` +# Only proceed any further if all Argo Applications show healthy !! + + +## WIP not yet integrated into KubeZero + +### EFS CSI +To deploy the EFS CSI driver the backing EFS filesystem needs to be in place ahead of time. This is easy to do by enabling the EFS functionality in the worker CloudBender stack. + +- retrieve the EFS: `cloudbender outputs config/kube-control-worker.yaml` and look for *EfsFileSystemId* +- update values.yaml in the `aws-efs-csi` artifact folder as well as the efs_pv.yaml +- execute `deploy.sh` + +### Istio +Istio is currently pinned to version 1.4.X as this is the last version supporting installation via helm charts. + +Until Istio is integrated into KubeZero as well as upgraded to 1.6 we have to install manually. + +- adjust values.yaml +- update domain in `ingress-certificate.yaml` +- update.sh +- deploy.sh + # Demo / own apps - Add your own application to ArgoCD via the cli \ No newline at end of file -- 2.40.1 From a476e4d6e064107959fc61f3c72940caa1163bcd Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 5 Jun 2020 17:58:18 +0100 Subject: [PATCH 27/31] Update Quickstart docs --- Quickstart.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Quickstart.md b/Quickstart.md index 06cce91..bd186c6 100644 --- a/Quickstart.md +++ b/Quickstart.md @@ -72,5 +72,16 @@ Until Istio is integrated into KubeZero as well as upgraded to 1.6 we have to in - update.sh - deploy.sh +### Logging +To deploy fluentbit only required adjustment is the `fluentd_host=` in the kustomization.yaml. + +- deploy namespace for logging via deploy.sh +- deploy fluentbit via `kubectl apply -k fluentbit` + +### Prometheus / Grafana +Only adjustment required is the ingress routing config in istio-service.yaml. Adjust as needed before executing: +`deploy.sh` + + # Demo / own apps - Add your own application to ArgoCD via the cli \ No newline at end of file -- 2.40.1 From b82ac1419c6fcc13ae04ab2010cc274044151fc4 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Mon, 8 Jun 2020 15:19:35 +0100 Subject: [PATCH 28/31] Make sure the self-signed resources are applied AFTER cert-manager itself --- charts/kubezero-cert-manager/Chart.yaml | 4 ++-- charts/kubezero-cert-manager/templates/cluster-ca.yaml | 8 ++++++++ .../kubezero-cert-manager/templates/cluster-issuer.yaml | 2 ++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/charts/kubezero-cert-manager/Chart.yaml b/charts/kubezero-cert-manager/Chart.yaml index 0aafe10..855585a 100644 --- a/charts/kubezero-cert-manager/Chart.yaml +++ b/charts/kubezero-cert-manager/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-cert-manager description: KubeZero Umbrella Chart for cert-manager type: application -version: 0.3.3 +version: 0.3.4 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -15,6 +15,6 @@ dependencies: version: ">= 0.1.1" repository: https://zero-down-time.github.io/kubezero/ - name: cert-manager - version: 0.15.0 + version: 0.15.1 repository: https://charts.jetstack.io kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-cert-manager/templates/cluster-ca.yaml b/charts/kubezero-cert-manager/templates/cluster-ca.yaml index f5f70b0..91acb06 100644 --- a/charts/kubezero-cert-manager/templates/cluster-ca.yaml +++ b/charts/kubezero-cert-manager/templates/cluster-ca.yaml @@ -9,6 +9,8 @@ metadata: namespace: kube-system labels: {{ include "kubezero-lib.labels" . | indent 4 }} + annotations: + "helm.sh/hook": "post-install" spec: selfSigned: {} --- @@ -19,6 +21,8 @@ metadata: namespace: kube-system labels: {{ include "kubezero-lib.labels" . | indent 4 }} + annotations: + "helm.sh/hook": "post-install" spec: secretName: kubezero-ca-tls commonName: "kubezero-local-ca" @@ -37,6 +41,8 @@ metadata: namespace: kube-system labels: {{ include "kubezero-lib.labels" . | indent 4 }} + annotations: + "helm.sh/hook": "post-install" data: tls.crt: {{ .Values.localCA.ca.crt | b64enc }} tls.key: {{ .Values.localCA.ca.key | b64enc }} @@ -50,6 +56,8 @@ metadata: namespace: kube-system labels: {{ include "kubezero-lib.labels" . | indent 4 }} + annotations: + "helm.sh/hook": "post-install" spec: ca: secretName: kubezero-ca-tls diff --git a/charts/kubezero-cert-manager/templates/cluster-issuer.yaml b/charts/kubezero-cert-manager/templates/cluster-issuer.yaml index 4861733..918977f 100644 --- a/charts/kubezero-cert-manager/templates/cluster-issuer.yaml +++ b/charts/kubezero-cert-manager/templates/cluster-issuer.yaml @@ -5,6 +5,8 @@ metadata: name: {{ .Values.clusterIssuer.name }} labels: {{ include "kubezero-lib.labels" . | indent 4 }} + annotations: + "helm.sh/hook": "post-install" spec: acme: server: {{ .Values.clusterIssuer.server }} -- 2.40.1 From 6f081c955a306617912e03f2835dc4cd1744d4e8 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Sun, 14 Jun 2020 17:59:56 +0100 Subject: [PATCH 29/31] Revert annotations for cert-manager, enable selfheal for cert-manager to work around bootstrap issues --- charts/kubezero-app/Chart.yaml | 2 +- charts/kubezero-app/README.md | 2 +- charts/kubezero-app/templates/_app.yaml | 4 +++- .../kubezero-app/templates/cert-manager.yaml | 2 +- charts/kubezero-cert-manager/README.md | 16 ++++++++++++--- charts/kubezero-cert-manager/README.md.gotmpl | 20 +++++++++++++++++++ .../templates/cluster-ca.yaml | 8 -------- charts/kubezero-cert-manager/values.yaml | 6 +++--- 8 files changed, 42 insertions(+), 18 deletions(-) create mode 100644 charts/kubezero-cert-manager/README.md.gotmpl diff --git a/charts/kubezero-app/Chart.yaml b/charts/kubezero-app/Chart.yaml index 764eeb8..2522dde 100644 --- a/charts/kubezero-app/Chart.yaml +++ b/charts/kubezero-app/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-app description: KubeZero ArgoCD Application - Root chart of the KubeZero type: application -version: 0.2.3 +version: 0.2.4 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-app/README.md b/charts/kubezero-app/README.md index 479f387..836c8f5 100644 --- a/charts/kubezero-app/README.md +++ b/charts/kubezero-app/README.md @@ -2,7 +2,7 @@ kubezero-app ============ KubeZero ArgoCD Application - Root chart of the KubeZero -Current chart version is `0.2.2` +Current chart version is `0.2.4` Source code can be found [here](https://kubezero.com) diff --git a/charts/kubezero-app/templates/_app.yaml b/charts/kubezero-app/templates/_app.yaml index 1734694..2a59e75 100644 --- a/charts/kubezero-app/templates/_app.yaml +++ b/charts/kubezero-app/templates/_app.yaml @@ -35,5 +35,7 @@ spec: syncPolicy: automated: prune: true - selfHeal: false + {{- if .selfheal }} + selfHeal: true + {{- end }} {{- end }} diff --git a/charts/kubezero-app/templates/cert-manager.yaml b/charts/kubezero-app/templates/cert-manager.yaml index b744db8..afc8981 100644 --- a/charts/kubezero-app/templates/cert-manager.yaml +++ b/charts/kubezero-app/templates/cert-manager.yaml @@ -1,5 +1,5 @@ {{- if index .Values "cert-manager" "enabled" }} -{{ template "kubezero-app.app" dict "root" . "name" "cert-manager" "type" "helm" "namespace" "cert-manager" }} +{{ template "kubezero-app.app" dict "root" . "name" "cert-manager" "type" "helm" "namespace" "cert-manager" "selfheal" "true" }} --- apiVersion: v1 kind: Namespace diff --git a/charts/kubezero-cert-manager/README.md b/charts/kubezero-cert-manager/README.md index 8027bbf..15f0d8d 100644 --- a/charts/kubezero-cert-manager/README.md +++ b/charts/kubezero-cert-manager/README.md @@ -2,7 +2,7 @@ kubezero-cert-manager ===================== KubeZero Umbrella Chart for cert-manager -Current chart version is `0.3.3` +Current chart version is `0.3.4` Source code can be found [here](https://kubezero.com) @@ -10,9 +10,19 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| -| https://charts.jetstack.io | cert-manager | 0.15.0 | +| https://charts.jetstack.io | cert-manager | 0.15.1 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | +## AWS - IAM Role +If you use kiam or kube2iam and restrict access on nodes running cert-manager please adjust: +``` +cert-manager.podAnnotations: + iam.amazonaws.com/role: +``` + +## Resolver Secrets +If your resolvers need additional sercrets like CloudFlare API tokens etc. make sure to provide these secrets separatly matching your defined issuers. + ## Chart Values | Key | Type | Default | Description | @@ -25,7 +35,7 @@ Source code can be found [here](https://kubezero.com) | cert-manager.ingressShim.defaultIssuerName | string | `"letsencrypt-dns-prod"` | | | cert-manager.installCRDs | bool | `true` | | | cert-manager.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | -| cert-manager.podAnnotations."iam.amazonaws.com/role" | string | `""` | IAM role ARN the cert-manager might use via kiam eg."arn:aws:iam::123456789012:role/certManagerRoleArn" | +| cert-manager.podAnnotations | object | `{}` | "iam.amazonaws.com/roleIAM:" role ARN the cert-manager might use via kiam eg."arn:aws:iam::123456789012:role/certManagerRoleArn" | | cert-manager.prometheus.servicemonitor.enabled | bool | `false` | | | cert-manager.tolerations[0].effect | string | `"NoSchedule"` | | | cert-manager.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | diff --git a/charts/kubezero-cert-manager/README.md.gotmpl b/charts/kubezero-cert-manager/README.md.gotmpl new file mode 100644 index 0000000..229df43 --- /dev/null +++ b/charts/kubezero-cert-manager/README.md.gotmpl @@ -0,0 +1,20 @@ +{{ template "chart.header" . }} +{{ template "chart.description" . }} + +{{ template "chart.versionLine" . }} + +{{ template "chart.sourceLinkLine" . }} + +{{ template "chart.requirementsSection" . }} + +## AWS - IAM Role +If you use kiam or kube2iam and restrict access on nodes running cert-manager please adjust: +``` +cert-manager.podAnnotations: + iam.amazonaws.com/role: +``` + +## Resolver Secrets +If your resolvers need additional sercrets like CloudFlare API tokens etc. make sure to provide these secrets separatly matching your defined issuers. + +{{ template "chart.valuesSection" . }} diff --git a/charts/kubezero-cert-manager/templates/cluster-ca.yaml b/charts/kubezero-cert-manager/templates/cluster-ca.yaml index 91acb06..f5f70b0 100644 --- a/charts/kubezero-cert-manager/templates/cluster-ca.yaml +++ b/charts/kubezero-cert-manager/templates/cluster-ca.yaml @@ -9,8 +9,6 @@ metadata: namespace: kube-system labels: {{ include "kubezero-lib.labels" . | indent 4 }} - annotations: - "helm.sh/hook": "post-install" spec: selfSigned: {} --- @@ -21,8 +19,6 @@ metadata: namespace: kube-system labels: {{ include "kubezero-lib.labels" . | indent 4 }} - annotations: - "helm.sh/hook": "post-install" spec: secretName: kubezero-ca-tls commonName: "kubezero-local-ca" @@ -41,8 +37,6 @@ metadata: namespace: kube-system labels: {{ include "kubezero-lib.labels" . | indent 4 }} - annotations: - "helm.sh/hook": "post-install" data: tls.crt: {{ .Values.localCA.ca.crt | b64enc }} tls.key: {{ .Values.localCA.ca.key | b64enc }} @@ -56,8 +50,6 @@ metadata: namespace: kube-system labels: {{ include "kubezero-lib.labels" . | indent 4 }} - annotations: - "helm.sh/hook": "post-install" spec: ca: secretName: kubezero-ca-tls diff --git a/charts/kubezero-cert-manager/values.yaml b/charts/kubezero-cert-manager/values.yaml index e8b6eec..b14b4b9 100644 --- a/charts/kubezero-cert-manager/values.yaml +++ b/charts/kubezero-cert-manager/values.yaml @@ -45,6 +45,6 @@ cert-manager: prometheus: servicemonitor: enabled: false - # cert-manager.podAnnotations."iam.amazonaws.com/role" -- IAM role ARN the cert-manager might use via kiam eg."arn:aws:iam::123456789012:role/certManagerRoleArn" - podAnnotations: - iam.amazonaws.com/role: "" + # cert-manager.podAnnotations -- "iam.amazonaws.com/roleIAM:" role ARN the cert-manager might use via kiam eg."arn:aws:iam::123456789012:role/certManagerRoleArn" + podAnnotations: {} + # iam.amazonaws.com/role: "" -- 2.40.1 From 3e5d9056cbf84f16722c4c970f993ee6cd891fc3 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 19 Jun 2020 12:24:22 +0100 Subject: [PATCH 30/31] Minor cleanup --- README.md | 1 - charts/kubezero-cert-manager/templates/cluster-issuer.yaml | 2 -- charts/kubezero/values.yaml | 1 + 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index dbb55b9..0f0efa7 100644 --- a/README.md +++ b/README.md @@ -14,4 +14,3 @@ This also implements the *umbrella chart* pattern in order to inject custom valu ## Components ### ArgoCD - diff --git a/charts/kubezero-cert-manager/templates/cluster-issuer.yaml b/charts/kubezero-cert-manager/templates/cluster-issuer.yaml index 918977f..4861733 100644 --- a/charts/kubezero-cert-manager/templates/cluster-issuer.yaml +++ b/charts/kubezero-cert-manager/templates/cluster-issuer.yaml @@ -5,8 +5,6 @@ metadata: name: {{ .Values.clusterIssuer.name }} labels: {{ include "kubezero-lib.labels" . | indent 4 }} - annotations: - "helm.sh/hook": "post-install" spec: acme: server: {{ .Values.clusterIssuer.server }} diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 12f853b..ad26707 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -21,6 +21,7 @@ argo-cd: #configs: # secret: + # `htpasswd -nbBC 10 "" $ARGO_PWD | tr -d ':\n' | sed 's/$2y/$2a/'` # argocdServerAdminPassword: "$2a$10$ivKzaXVxMqdeDSfS3nqi1Od3iDbnL7oXrixzDfZFRHlXHnAG6LydG" # argocdServerAdminPasswordMtime: "2020-04-24T15:33:09BST" -- 2.40.1 From 86cfb147e33c3ebb43f7000c7a46f3271ca599f2 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 25 Jun 2020 13:52:07 +0100 Subject: [PATCH 31/31] New cleaned up KubeZero layout --- .helmdocsignore | 2 + charts/kubezero-app/README.md | 27 ------- charts/kubezero-app/values.yaml | 29 -------- .../Chart.yaml | 11 ++- charts/kubezero-argo-cd/README.md | 42 +++++++++++ .../templates/app.yaml | 12 +-- .../templates/istio-service.yaml | 4 +- .../templates/project.yaml | 0 charts/kubezero-argo-cd/values.yaml | 73 +++++++++++++++++++ charts/{kubezero-app => kubezero}/.helmignore | 0 charts/kubezero/Chart.yaml | 9 +-- charts/kubezero/README.md | 30 ++------ .../templates/_app.yaml | 0 .../templates/aws-ebs-csi-driver.yaml | 0 .../templates/calico.yaml | 0 .../templates/cert-manager.yaml | 0 .../templates/kiam.yaml | 0 .../templates/local-volume-provisioner.yaml | 0 .../values-all.yaml | 0 charts/kubezero/values.yaml | 67 +++-------------- 20 files changed, 151 insertions(+), 155 deletions(-) delete mode 100644 charts/kubezero-app/README.md delete mode 100644 charts/kubezero-app/values.yaml rename charts/{kubezero-app => kubezero-argo-cd}/Chart.yaml (57%) create mode 100644 charts/kubezero-argo-cd/README.md rename charts/{kubezero => kubezero-argo-cd}/templates/app.yaml (51%) rename charts/{kubezero => kubezero-argo-cd}/templates/istio-service.yaml (85%) rename charts/{kubezero => kubezero-argo-cd}/templates/project.yaml (100%) create mode 100644 charts/kubezero-argo-cd/values.yaml rename charts/{kubezero-app => kubezero}/.helmignore (100%) rename charts/{kubezero-app => kubezero}/templates/_app.yaml (100%) rename charts/{kubezero-app => kubezero}/templates/aws-ebs-csi-driver.yaml (100%) rename charts/{kubezero-app => kubezero}/templates/calico.yaml (100%) rename charts/{kubezero-app => kubezero}/templates/cert-manager.yaml (100%) rename charts/{kubezero-app => kubezero}/templates/kiam.yaml (100%) rename charts/{kubezero-app => kubezero}/templates/local-volume-provisioner.yaml (100%) rename charts/{kubezero-app => kubezero}/values-all.yaml (100%) diff --git a/.helmdocsignore b/.helmdocsignore index 3a318ae..8fc08f2 100644 --- a/.helmdocsignore +++ b/.helmdocsignore @@ -1,2 +1,4 @@ # Ignore sub-charts charts/*/charts/* +charts/kubezero-lib +deploy diff --git a/charts/kubezero-app/README.md b/charts/kubezero-app/README.md deleted file mode 100644 index 836c8f5..0000000 --- a/charts/kubezero-app/README.md +++ /dev/null @@ -1,27 +0,0 @@ -kubezero-app -============ -KubeZero ArgoCD Application - Root chart of the KubeZero - -Current chart version is `0.2.4` - -Source code can be found [here](https://kubezero.com) - -## Chart Requirements - -| Repository | Name | Version | -|------------|------|---------| -| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | - -## Chart Values - -| Key | Type | Default | Description | -|-----|------|---------|-------------| -| aws-ebs-csi-driver.enabled | bool | `false` | | -| calico.enabled | bool | `false` | | -| cert-manager.enabled | bool | `false` | | -| global.defaultDestination.server | string | `"https://kubernetes.default.svc"` | | -| global.defaultSource.pathPrefix | string | `""` | | -| global.defaultSource.repoURL | string | `"https://github.com/zero-down-time/kubezero"` | | -| global.defaultSource.targetRevision | string | `"HEAD"` | | -| kiam.enabled | bool | `false` | | -| local-volume-provisioner.enabled | bool | `false` | | diff --git a/charts/kubezero-app/values.yaml b/charts/kubezero-app/values.yaml deleted file mode 100644 index c888333..0000000 --- a/charts/kubezero-app/values.yaml +++ /dev/null @@ -1,29 +0,0 @@ -global: - defaultDestination: - server: https://kubernetes.default.svc - - # This repoURL is used a base for all the repoURLs applications - # Setting this to a eg. private git repo incl. the use of pathPrefix allows kubezero to be - # integrated into any repository as a git subtree if for example public internet access is unavailable - defaultSource: - # defaultSource.repoURL -- default repository for argocd applications - repoURL: https://github.com/zero-down-time/kubezero - # defaultSource.targetRevision -- default tracking of repoURL - targetRevision: HEAD - # defaultSource.pathPrefix -- optional path prefix within repoURL to support eg. remote subtrees - pathPrefix: '' - -calico: - enabled: false - -local-volume-provisioner: - enabled: false - -cert-manager: - enabled: false - -kiam: - enabled: false - -aws-ebs-csi-driver: - enabled: false diff --git a/charts/kubezero-app/Chart.yaml b/charts/kubezero-argo-cd/Chart.yaml similarity index 57% rename from charts/kubezero-app/Chart.yaml rename to charts/kubezero-argo-cd/Chart.yaml index 2522dde..758803f 100644 --- a/charts/kubezero-app/Chart.yaml +++ b/charts/kubezero-argo-cd/Chart.yaml @@ -1,8 +1,7 @@ apiVersion: v2 -name: kubezero-app -description: KubeZero ArgoCD Application - Root chart of the KubeZero -type: application -version: 0.2.4 +description: KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application +name: kubezero-argo-cd +version: 0.3.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -11,8 +10,12 @@ keywords: - gitops maintainers: - name: Quarky9 +dependencies: dependencies: - name: kubezero-lib version: ">= 0.1.1" repository: https://zero-down-time.github.io/kubezero/ + - name: argo-cd + version: 2.3.2 + repository: https://argoproj.github.io/argo-helm kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-argo-cd/README.md b/charts/kubezero-argo-cd/README.md new file mode 100644 index 0000000..fb99424 --- /dev/null +++ b/charts/kubezero-argo-cd/README.md @@ -0,0 +1,42 @@ +kubezero-argo-cd +================ +KubeZero ArgoCD Helm chart to install ArgoCD itself and the KubeZero ArgoCD Application + +Current chart version is `0.3.0` + +Source code can be found [here](https://kubezero.com) + +## Chart Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://argoproj.github.io/argo-helm | argo-cd | 2.3.2 | +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | + +## Chart Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| argo-cd.controller.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| argo-cd.controller.tolerations[0].effect | string | `"NoSchedule"` | | +| argo-cd.controller.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| argo-cd.dex.enabled | bool | `false` | | +| argo-cd.installCRDs | bool | `false` | | +| argo-cd.istio.enabled | bool | `false` | Deploy Istio VirtualService to expose ArgoCD | +| argo-cd.istio.gateway | string | `"ingressgateway.istio-system.svc.cluster.local"` | Name of the Istio gateway to add the VirtualService to | +| argo-cd.redis.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| argo-cd.redis.tolerations[0].effect | string | `"NoSchedule"` | | +| argo-cd.redis.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| argo-cd.repoServer.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| argo-cd.repoServer.tolerations[0].effect | string | `"NoSchedule"` | | +| argo-cd.repoServer.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| argo-cd.server.config.url | string | `"argocd.example.com"` | ArgoCD hostname to be exposed via Istio | +| argo-cd.server.extraArgs[0] | string | `"--insecure"` | | +| argo-cd.server.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| argo-cd.server.service.servicePortHttpsName | string | `"grpc"` | | +| argo-cd.server.tolerations[0].effect | string | `"NoSchedule"` | | +| argo-cd.server.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| kubezero.global.defaultDestination | object | `{"server":"https://kubernetes.default.svc"}` | Destination cluster | +| kubezero.global.defaultSource.pathPrefix | string | `""` | optional path prefix within repoURL to support eg. remote subtrees | +| kubezero.global.defaultSource.repoURL | string | `"https://github.com/zero-down-time/kubezero"` | default repository for argocd applications | +| kubezero.global.defaultSource.targetRevision | string | `"HEAD"` | default tracking of repoURL | diff --git a/charts/kubezero/templates/app.yaml b/charts/kubezero-argo-cd/templates/app.yaml similarity index 51% rename from charts/kubezero/templates/app.yaml rename to charts/kubezero-argo-cd/templates/app.yaml index 24b4a83..f997988 100644 --- a/charts/kubezero/templates/app.yaml +++ b/charts/kubezero-argo-cd/templates/app.yaml @@ -8,20 +8,16 @@ metadata: spec: project: kubezero source: - repoURL: {{ .Values.global.defaultSource.repoURL }} - targetRevision: {{ .Values.global.defaultSource.targetRevision }} - path: {{ .Values.global.defaultSource.pathPrefix}}charts/kubezero-app + repoURL: {{ .Values.kubezero.global.defaultSource.repoURL }} + targetRevision: {{ .Values.kubezero.global.defaultSource.targetRevision }} + path: {{ .Values.kubezero.global.defaultSource.pathPrefix}}charts/kubezero helm: values: | - global: -{{- toYaml .Values.global | nindent 10 }} -{{- if .Values.kubezero }} {{- toYaml .Values.kubezero | nindent 8 }} -{{- end }} destination: - server: {{ .Values.global.defaultDestination.server }} + server: {{ .Values.kubezero.global.defaultDestination.server }} namespace: argocd syncPolicy: automated: diff --git a/charts/kubezero/templates/istio-service.yaml b/charts/kubezero-argo-cd/templates/istio-service.yaml similarity index 85% rename from charts/kubezero/templates/istio-service.yaml rename to charts/kubezero-argo-cd/templates/istio-service.yaml index 195494c..03d889c 100644 --- a/charts/kubezero/templates/istio-service.yaml +++ b/charts/kubezero-argo-cd/templates/istio-service.yaml @@ -1,4 +1,4 @@ -{{- if .Values.istio.enabled }} +{{- if index .Values "argo-cd" "istio" "enabled" }} apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: @@ -7,7 +7,7 @@ metadata: {{ include "kubezero-lib.labels" . | indent 4 }} spec: gateways: - - {{ .Values.istio.gateway }} + - {{ index .Values "argo-cd" "istio" "gateway" }} hosts: - {{ index .Values "argo-cd" "server" "config" "url" }} http: diff --git a/charts/kubezero/templates/project.yaml b/charts/kubezero-argo-cd/templates/project.yaml similarity index 100% rename from charts/kubezero/templates/project.yaml rename to charts/kubezero-argo-cd/templates/project.yaml diff --git a/charts/kubezero-argo-cd/values.yaml b/charts/kubezero-argo-cd/values.yaml new file mode 100644 index 0000000..39e382b --- /dev/null +++ b/charts/kubezero-argo-cd/values.yaml @@ -0,0 +1,73 @@ +kubezero: + global: + # kubezero.global.defaultDestination -- Destination cluster + defaultDestination: + server: https://kubernetes.default.svc + + # This repoURL is used a base for all the repoURLs applications + # Setting this to a eg. private git repo incl. the use of pathPrefix allows kubezero to be + # integrated into any repository as a git subtree if for example public internet access is unavailable + defaultSource: + # kubezero.global.defaultSource.repoURL -- default repository for argocd applications + repoURL: https://github.com/zero-down-time/kubezero + # kubezero.global.defaultSource.targetRevision -- default tracking of repoURL + targetRevision: HEAD + # kubezero.global.defaultSource.pathPrefix -- optional path prefix within repoURL to support eg. remote subtrees + pathPrefix: '' + +argo-cd: + installCRDs: false + + #configs: + # secret: + # `htpasswd -nbBC 10 "" $ARGO_PWD | tr -d ':\n' | sed 's/$2y/$2a/'` + # argocdServerAdminPassword: "$2a$10$ivKzaXVxMqdeDSfS3nqi1Od3iDbnL7oXrixzDfZFRHlXHnAG6LydG" + # argocdServerAdminPasswordMtime: "2020-04-24T15:33:09BST" + + # Run Argo on the controllers + controller: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + + repoServer: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + + server: + config: + # argo-cd.server.config.url -- ArgoCD hostname to be exposed via Istio + url: argocd.example.com + + # Rename former https port to grpc, works with istio + insecure + service: + servicePortHttpsName: grpc + + extraArgs: + - --insecure + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + + redis: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + + dex: + enabled: false + + istio: + # argo-cd.istio.enabled -- Deploy Istio VirtualService to expose ArgoCD + enabled: false + # argo-cd.istio.gateway -- Name of the Istio gateway to add the VirtualService to + gateway: ingressgateway.istio-system.svc.cluster.local diff --git a/charts/kubezero-app/.helmignore b/charts/kubezero/.helmignore similarity index 100% rename from charts/kubezero-app/.helmignore rename to charts/kubezero/.helmignore diff --git a/charts/kubezero/Chart.yaml b/charts/kubezero/Chart.yaml index d6052fe..2d141a4 100644 --- a/charts/kubezero/Chart.yaml +++ b/charts/kubezero/Chart.yaml @@ -1,7 +1,8 @@ apiVersion: v2 -description: KubeZero Helm chart to install Zero Down Time Kuberenetes platform name: kubezero -version: 0.2.6 +description: KubeZero ArgoCD Application - Root App of Apps chart of KubeZero +type: application +version: 0.3.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -10,12 +11,8 @@ keywords: - gitops maintainers: - name: Quarky9 -dependencies: dependencies: - name: kubezero-lib version: ">= 0.1.1" repository: https://zero-down-time.github.io/kubezero/ - - name: argo-cd - version: 2.3.2 - repository: https://argoproj.github.io/argo-helm kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero/README.md b/charts/kubezero/README.md index 0de00b9..50d0a04 100644 --- a/charts/kubezero/README.md +++ b/charts/kubezero/README.md @@ -1,8 +1,8 @@ kubezero ======== -KubeZero Helm chart to install Zero Down Time Kuberenetes platform +KubeZero ArgoCD Application - Root App of Apps chart of KubeZero -Current chart version is `0.2.6` +Current chart version is `0.3.0` Source code can be found [here](https://kubezero.com) @@ -10,34 +10,18 @@ Source code can be found [here](https://kubezero.com) | Repository | Name | Version | |------------|------|---------| -| https://argoproj.github.io/argo-helm | argo-cd | 2.3.2 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | ## Chart Values | Key | Type | Default | Description | |-----|------|---------|-------------| -| argo-cd.controller.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | -| argo-cd.controller.tolerations[0].effect | string | `"NoSchedule"` | | -| argo-cd.controller.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | -| argo-cd.dex.enabled | bool | `false` | | -| argo-cd.installCRDs | bool | `false` | | -| argo-cd.redis.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | -| argo-cd.redis.tolerations[0].effect | string | `"NoSchedule"` | | -| argo-cd.redis.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | -| argo-cd.repoServer.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | -| argo-cd.repoServer.tolerations[0].effect | string | `"NoSchedule"` | | -| argo-cd.repoServer.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | -| argo-cd.server.config.url | string | `"argocd.example.com"` | ArgoCD hostname to be exposed via Istio | -| argo-cd.server.extraArgs[0] | string | `"--insecure"` | | -| argo-cd.server.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | -| argo-cd.server.service.servicePortHttpsName | string | `"grpc"` | | -| argo-cd.server.tolerations[0].effect | string | `"NoSchedule"` | | -| argo-cd.server.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| aws-ebs-csi-driver.enabled | bool | `false` | | +| calico.enabled | bool | `false` | | +| cert-manager.enabled | bool | `false` | | | global.defaultDestination.server | string | `"https://kubernetes.default.svc"` | | | global.defaultSource.pathPrefix | string | `""` | | | global.defaultSource.repoURL | string | `"https://github.com/zero-down-time/kubezero"` | | | global.defaultSource.targetRevision | string | `"HEAD"` | | -| istio.enabled | bool | `false` | Deploy Istio VirtualService to expose ArgoCD | -| istio.gateway | string | `"ingressgateway.istio-system.svc.cluster.local"` | Name of the Istio gateway to add the VirtualService to | -| kubezero | object | `{}` | Kubezero configuration, values.yaml please see kubezeroApp | +| kiam.enabled | bool | `false` | | +| local-volume-provisioner.enabled | bool | `false` | | diff --git a/charts/kubezero-app/templates/_app.yaml b/charts/kubezero/templates/_app.yaml similarity index 100% rename from charts/kubezero-app/templates/_app.yaml rename to charts/kubezero/templates/_app.yaml diff --git a/charts/kubezero-app/templates/aws-ebs-csi-driver.yaml b/charts/kubezero/templates/aws-ebs-csi-driver.yaml similarity index 100% rename from charts/kubezero-app/templates/aws-ebs-csi-driver.yaml rename to charts/kubezero/templates/aws-ebs-csi-driver.yaml diff --git a/charts/kubezero-app/templates/calico.yaml b/charts/kubezero/templates/calico.yaml similarity index 100% rename from charts/kubezero-app/templates/calico.yaml rename to charts/kubezero/templates/calico.yaml diff --git a/charts/kubezero-app/templates/cert-manager.yaml b/charts/kubezero/templates/cert-manager.yaml similarity index 100% rename from charts/kubezero-app/templates/cert-manager.yaml rename to charts/kubezero/templates/cert-manager.yaml diff --git a/charts/kubezero-app/templates/kiam.yaml b/charts/kubezero/templates/kiam.yaml similarity index 100% rename from charts/kubezero-app/templates/kiam.yaml rename to charts/kubezero/templates/kiam.yaml diff --git a/charts/kubezero-app/templates/local-volume-provisioner.yaml b/charts/kubezero/templates/local-volume-provisioner.yaml similarity index 100% rename from charts/kubezero-app/templates/local-volume-provisioner.yaml rename to charts/kubezero/templates/local-volume-provisioner.yaml diff --git a/charts/kubezero-app/values-all.yaml b/charts/kubezero/values-all.yaml similarity index 100% rename from charts/kubezero-app/values-all.yaml rename to charts/kubezero/values-all.yaml diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index ad26707..0682f2b 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -13,62 +13,17 @@ global: # defaultSource.pathPrefix -- optional path prefix within repoURL to support eg. remote subtrees pathPrefix: '' -# kubezero -- Kubezero configuration, values.yaml please see kubezeroApp -kubezero: {} +calico: + enabled: true -argo-cd: - installCRDs: false +cert-manager: + enabled: true - #configs: - # secret: - # `htpasswd -nbBC 10 "" $ARGO_PWD | tr -d ':\n' | sed 's/$2y/$2a/'` - # argocdServerAdminPassword: "$2a$10$ivKzaXVxMqdeDSfS3nqi1Od3iDbnL7oXrixzDfZFRHlXHnAG6LydG" - # argocdServerAdminPasswordMtime: "2020-04-24T15:33:09BST" - - # Run Argo on the controllers - controller: - nodeSelector: - node-role.kubernetes.io/master: "" - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - - repoServer: - nodeSelector: - node-role.kubernetes.io/master: "" - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - - server: - config: - # argo-cd.server.config.url -- ArgoCD hostname to be exposed via Istio - url: argocd.example.com - - # Rename former https port to grpc, works with istio + insecure - service: - servicePortHttpsName: grpc - - extraArgs: - - --insecure - nodeSelector: - node-role.kubernetes.io/master: "" - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - - redis: - nodeSelector: - node-role.kubernetes.io/master: "" - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - - dex: - enabled: false - -istio: - # istio.enabled -- Deploy Istio VirtualService to expose ArgoCD +local-volume-provisioner: + enabled: false + +kiam: + enabled: false + +aws-ebs-csi-driver: enabled: false - # istio.gateway -- Name of the Istio gateway to add the VirtualService to - gateway: ingressgateway.istio-system.svc.cluster.local -- 2.40.1