From 7fcdbfc2cd8756f98438d9532430ebbabab8c54e Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Thu, 1 Jul 2021 16:41:31 +0200 Subject: [PATCH] feat: aws-ebs and aws-efs csi driver upgrades, migrate to use service account tokens for AWS access --- charts/kubezero-aws-ebs-csi-driver/Chart.yaml | 8 +- charts/kubezero-aws-ebs-csi-driver/README.md | 21 +- .../charts/aws-ebs-csi-driver/Chart.yaml | 4 +- .../aws-ebs-csi-driver/templates/NOTES.txt | 36 ++++ .../aws-ebs-csi-driver/templates/_helpers.tpl | 15 +- .../templates/clusterrole-attacher.yaml | 24 +-- .../templates/clusterrole-csi-node.yaml | 11 + .../templates/clusterrole-resizer.yaml | 38 ++-- .../clusterrole-snapshot-controller.yaml | 51 +++-- .../templates/clusterrole-snapshotter.yaml | 32 ++- .../clusterrolebinding-csi-node.yaml | 15 ++ .../templates/clusterrolebinding-resizer.yaml | 3 - ...lusterrolebinding-snapshot-controller.yaml | 3 - .../clusterrolebinding-snapshotter.yaml | 3 - .../templates/controller.yaml | 195 ++++++++++-------- .../templates/csidriver.yaml | 2 +- .../aws-ebs-csi-driver/templates/node.yaml | 94 ++++----- .../poddisruptionbudget-controller.yaml | 17 ++ ...ddisruptionbudget-snapshot-controller.yaml | 14 ++ ...le-snapshot-controller-leaderelection.yaml | 9 +- ...ng-snapshot-controller-leaderelection.yaml | 3 - .../serviceaccount-csi-controller.yaml | 3 +- .../templates/serviceaccount-csi-node.yaml | 3 +- .../serviceaccount-snapshot-controller.yaml | 5 +- ...tefulset.yaml => snapshot-controller.yaml} | 39 ++-- .../templates/storageclass.yaml | 8 +- .../charts/aws-ebs-csi-driver/values.yaml | 163 +++++++++------ .../loglevel_leader.patch | 103 --------- charts/kubezero-aws-ebs-csi-driver/patch | 87 ++++++++ charts/kubezero-aws-ebs-csi-driver/update.sh | 4 +- .../kubezero-aws-ebs-csi-driver/values.yaml | 51 +++-- charts/kubezero-aws-ebs-csi-driver/zdt.patch | 75 +++++++ charts/kubezero-aws-efs-csi-driver/Chart.yaml | 6 +- charts/kubezero-aws-efs-csi-driver/README.md | 25 ++- .../affinity_resources.patch | 64 ------ .../charts/aws-efs-csi-driver/CHANGELOG.md | 106 ++++++++++ .../charts/aws-efs-csi-driver/Chart.yaml | 6 +- .../aws-efs-csi-driver/templates/_helpers.tpl | 15 +- .../templates/controller-deployment.yaml | 63 ++++-- .../templates/controller-serviceaccount.yaml | 7 +- .../templates/csidriver.yaml | 2 +- .../templates/node-daemonset.yaml | 51 ++--- .../templates/storageclass.yaml | 13 +- .../charts/aws-efs-csi-driver/values.yaml | 158 +++++++------- .../templates/storage-class.yaml | 13 -- charts/kubezero-aws-efs-csi-driver/update.sh | 4 +- .../kubezero-aws-efs-csi-driver/values.yaml | 54 +++-- charts/kubezero-aws-efs-csi-driver/zdt.patch | 23 +++ 48 files changed, 1041 insertions(+), 708 deletions(-) create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml create mode 100644 charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-snapshot-controller.yaml rename charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/{statefulset.yaml => snapshot-controller.yaml} (62%) delete mode 100644 charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch create mode 100644 charts/kubezero-aws-ebs-csi-driver/patch create mode 100644 charts/kubezero-aws-ebs-csi-driver/zdt.patch delete mode 100644 charts/kubezero-aws-efs-csi-driver/affinity_resources.patch create mode 100644 charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/CHANGELOG.md delete mode 100644 charts/kubezero-aws-efs-csi-driver/templates/storage-class.yaml create mode 100644 charts/kubezero-aws-efs-csi-driver/zdt.patch diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index 14f5d118..44b3e934 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.5.1 -appVersion: 0.10.0 +version: 0.6.0 +appVersion: 1.2.3 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png sources: @@ -18,8 +18,8 @@ maintainers: - name: Quarky9 dependencies: - name: aws-ebs-csi-driver - version: 0.10.0 - repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver + version: 1.2.3 + # repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: kubezero-lib version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ diff --git a/charts/kubezero-aws-ebs-csi-driver/README.md b/charts/kubezero-aws-ebs-csi-driver/README.md index 93b343cc..5c389a20 100644 --- a/charts/kubezero-aws-ebs-csi-driver/README.md +++ b/charts/kubezero-aws-ebs-csi-driver/README.md @@ -1,6 +1,6 @@ # kubezero-aws-ebs-csi-driver -![Version: 0.5.1](https://img.shields.io/badge/Version-0.5.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.10.0](https://img.shields.io/badge/AppVersion-0.10.0-informational?style=flat-square) +![Version: 0.6.0](https://img.shields.io/badge/Version-0.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.3](https://img.shields.io/badge/AppVersion-1.2.3-informational?style=flat-square) KubeZero Umbrella Chart for aws-ebs-csi-driver @@ -23,7 +23,7 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| -| https://kubernetes-sigs.github.io/aws-ebs-csi-driver | aws-ebs-csi-driver | 0.10.0 | +| | aws-ebs-csi-driver | 1.2.3 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## IAM Role @@ -41,17 +41,16 @@ This class is by default also set as default storage class. | Key | Type | Default | Description | |-----|------|---------|-------------| -| aws-ebs-csi-driver.enableVolumeResizing | bool | `true` | | -| aws-ebs-csi-driver.enableVolumeScheduling | bool | `true` | | +| aws-ebs-csi-driver.controller.logLevel | int | `1` | | +| aws-ebs-csi-driver.controller.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| aws-ebs-csi-driver.controller.replicaCount | int | `1` | | +| aws-ebs-csi-driver.controller.resources.limits.memory | string | `"40Mi"` | | +| aws-ebs-csi-driver.controller.resources.requests.cpu | string | `"10m"` | | +| aws-ebs-csi-driver.controller.resources.requests.memory | string | `"24Mi"` | | +| aws-ebs-csi-driver.controller.tolerations[0].effect | string | `"NoSchedule"` | | +| aws-ebs-csi-driver.controller.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | aws-ebs-csi-driver.enableVolumeSnapshot | bool | `true` | | -| aws-ebs-csi-driver.extraVolumeTags | object | `{}` | Optional tags to be added to each EBS volume | -| aws-ebs-csi-driver.logLevel | int | `1` | | | aws-ebs-csi-driver.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | -| aws-ebs-csi-driver.podAnnotations | object | `{}` | iam.amazonaws.com/role: to assume | -| aws-ebs-csi-driver.replicaCount | int | `1` | | -| aws-ebs-csi-driver.resources.limits.memory | string | `"40Mi"` | | -| aws-ebs-csi-driver.resources.requests.cpu | string | `"10m"` | | -| aws-ebs-csi-driver.resources.requests.memory | string | `"24Mi"` | | | aws-ebs-csi-driver.storageClasses[0].allowVolumeExpansion | bool | `true` | | | aws-ebs-csi-driver.storageClasses[0].name | string | `"ebs-sc-gp2-xfs"` | | | aws-ebs-csi-driver.storageClasses[0].parameters."csi.storage.k8s.io/fstype" | string | `"xfs"` | | diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml index d246abe2..b773b00d 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: 0.10.0 +appVersion: 1.1.0 description: A Helm chart for AWS EBS CSI Driver home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver keywords: @@ -15,4 +15,4 @@ maintainers: name: aws-ebs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver -version: 0.10.0 +version: 1.2.3 diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt index 3717647d..8f6d02ee 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt @@ -1,3 +1,39 @@ To verify that aws-ebs-csi-driver has started, run: kubectl get pod -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" + + +WARNING: The following values have been deprecated in favor of moving them into the controller or node groups. They will be removed in a subsequent release. + +affinity: +extraCreateMetadata: +extraVolumeTags: +k8sTagClusterId: +nodeSelector: +podAnnotations: +priorityClassName: +region: +replicaCount: +resources: +tolerations: +topologySpreadConstraints: +volumeAttachLimit: + +are moving to + +controller: + affinity: + extraCreateMetadata: + extraVolumeTags: + k8sTagClusterId: + nodeSelector: + podAnnotations: + priorityClassName: + region: + replicaCount: + resources: + tolerations: + topologySpreadConstraints: + +node: + volumeAttachLimit: diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl index fdc77c4e..ddf7b614 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl @@ -59,11 +59,24 @@ app.kubernetes.io/instance: {{ .Release.Name }} Convert the `--extra-volume-tags` command line arg from a map. */}} {{- define "aws-ebs-csi-driver.extra-volume-tags" -}} +{{- $evt := default .Values.extraVolumeTags .Values.controller.extraVolumeTags }} {{- $result := dict "pairs" (list) -}} -{{- range $key, $value := .Values.extraVolumeTags -}} +{{- range $key, $value := $evt -}} {{- $noop := printf "%s=%s" $key $value | append $result.pairs | set $result "pairs" -}} {{- end -}} {{- if gt (len $result.pairs) 0 -}} {{- printf "%s=%s" "- --extra-volume-tags" (join "," $result.pairs) -}} {{- end -}} {{- end -}} + +{{/* +Handle http proxy env vars +*/}} +{{- define "aws-ebs-csi-driver.http-proxy" -}} +- name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} +- name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} +- name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end -}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml index e0919cec..816fdf66 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml @@ -6,18 +6,18 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "nodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "csi.storage.k8s.io" ] + resources: [ "csinodeinfos" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments" ] + verbs: [ "get", "list", "watch", "update", "patch" ] - apiGroups: [ "storage.k8s.io" ] resources: [ "volumeattachments/status" ] verbs: [ "patch" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml new file mode 100644 index 00000000..3ca368ef --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml @@ -0,0 +1,11 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-node-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml index 9d85b97c..065f3aba 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeResizing }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -12,22 +11,21 @@ rules: # - apiGroups: [""] # resources: ["secrets"] # verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] -{{- end}} + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims/status" ] + verbs: [ "update", "patch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "storageclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "pods" ] + verbs: [ "get", "list", "watch" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml index ffdb1b7d..b5b79788 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -7,29 +6,27 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - -{{- end }} + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims" ] + verbs: [ "get", "list", "watch", "update" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "storageclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents" ] + verbs: [ "create", "get", "list", "watch", "update", "delete" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshots" ] + verbs: [ "get", "list", "watch", "update" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshots/status" ] + verbs: [ "update" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml index 061b5658..5fada8b4 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -7,19 +6,18 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] -{{- end }} + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get", "list" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents" ] + verbs: [ "create", "get", "list", "watch", "update", "delete" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents/status" ] + verbs: [ "update" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml new file mode 100644 index 00000000..63d843e5 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-node-getter-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.node.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: ebs-csi-node-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml index 6fe42d12..768e5891 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeResizing }} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -14,5 +13,3 @@ roleRef: kind: ClusterRole name: ebs-external-resizer-role apiGroup: rbac.authorization.k8s.io - -{{- end}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml index b74484f9..55a634fb 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -14,5 +13,3 @@ roleRef: kind: ClusterRole name: ebs-snapshot-controller-role apiGroup: rbac.authorization.k8s.io - -{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml index cbc1169e..e9f36246 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -14,5 +13,3 @@ roleRef: kind: ClusterRole name: ebs-external-snapshotter-role apiGroup: rbac.authorization.k8s.io - -{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml index 64c92339..48420c8c 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -6,7 +6,7 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} spec: - replicas: {{ .Values.replicaCount }} + replicas: {{ default .Values.replicaCount .Values.controller.replicaCount }} selector: matchLabels: app: ebs-csi-controller @@ -16,40 +16,46 @@ spec: labels: app: ebs-csi-controller {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} - {{- if .Values.podAnnotations }} - annotations: {{ toYaml .Values.podAnnotations | nindent 8 }} + {{- if .Values.controller.podLabels }} + {{- toYaml .Values.controller.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.controller.podAnnotations }} + annotations: + {{- toYaml .Values.controller.podAnnotations | nindent 8 }} + {{- else if .Values.podAnnotations}} + annotations: + {{- toYaml .Values.podAnnotations | nindent 8 }} {{- end }} spec: nodeSelector: kubernetes.io/os: linux - {{- with .Values.nodeSelector }} -{{ toYaml . | indent 8 }} + {{- with default .Values.nodeSelector .Values.controller.nodeSelector }} + {{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ .Values.serviceAccount.controller.name }} - priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} - {{- with .Values.affinity }} - affinity: {{ toYaml . | nindent 8 }} + priorityClassName: {{ default .Values.priorityClassName .Values.controller.priorityClassName }} + {{- with default .Values.affinity .Values.controller.affinity }} + affinity: + {{- toYaml . | nindent 8 }} {{- end }} tolerations: - {{- if .Values.tolerateAllTaints }} - - operator: Exists - {{- else }} - key: CriticalAddonsOnly operator: Exists - operator: Exists effect: NoExecute tolerationSeconds: 300 + {{- with default .Values.tolerations .Values.controller.tolerations }} + {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} -{{ toYaml . | indent 8 }} - {{- end }} -{{- if .Values.topologySpreadConstraints }} -{{- $tscLabelSelector := dict "labelSelector" ( dict "matchLabels" ( dict "app" "ebs-csi-controller" ) ) }} + {{- if or .Values.controller.topologySpreadConstraints .Values.topologySpreadConstraints }} + {{- $tscLabelSelector := dict "labelSelector" ( dict "matchLabels" ( dict "app" "ebs-csi-controller" ) ) }} + {{- $constraints := list }} + {{- range default .Values.topologySpreadConstraints .Values.controller.topologySpreadConstraints }} + {{- $constraints = mustAppend $constraints (mergeOverwrite . $tscLabelSelector) }} + {{- end }} topologySpreadConstraints: - {{- range .Values.topologySpreadConstraints }} - - {{ mergeOverwrite . $tscLabelSelector | toJson }} - {{- end }} -{{- end }} + {{- $constraints | toYaml | nindent 8 }} + {{- end }} containers: - name: ebs-plugin image: {{ .Values.image.repository }}:{{ .Values.image.tag }} @@ -61,17 +67,24 @@ spec: # - {all,controller,node} # specify the driver mode {{- end }} - --endpoint=$(CSI_ENDPOINT) - {{- if .Values.extraVolumeTags }} + {{- if or .Values.controller.extraVolumeTags .Values.extraVolumeTags }} {{- include "aws-ebs-csi-driver.extra-volume-tags" . | nindent 12 }} {{- end }} - {{- if .Values.k8sTagClusterId }} - - --k8s-tag-cluster-id={{ .Values.k8sTagClusterId }} + {{- with default .Values.k8sTagClusterId .Values.controller.k8sTagClusterId }} + - --k8s-tag-cluster-id={{ . }} + {{- end }} + {{- with .Values.controller.httpEndpoint }} + - --http-endpoint={{ . }} {{- end }} - --logtostderr - - --v={{ .Values.logLevel }} + - --v={{ .Values.controller.logLevel }} env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -84,9 +97,9 @@ spec: name: aws-secret key: access_key optional: true - {{- if .Values.region }} + {{- with default .Values.region .Values.controller.region }} - name: AWS_REGION - value: {{ .Values.region }} + value: {{ . }} {{- end }} {{- if .Values.controller.extraVars }} {{- range $key, $val := .Values.controller.extraVars }} @@ -94,17 +107,18 @@ spec: value: "{{ $val }}" {{- end }} {{- end }} -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.controller.env.ebsPlugin }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: aws-token + mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" + readOnly: true ports: - name: healthz containerPort: 9808 @@ -125,111 +139,102 @@ spec: timeoutSeconds: 3 periodSeconds: 10 failureThreshold: 5 - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.ebsPlugin) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - name: csi-provisioner image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} args: - --csi-address=$(ADDRESS) - - --v={{ .Values.logLevel }} - {{- if .Values.enableVolumeScheduling }} + - --v={{ .Values.controller.logLevel }} - --feature-gates=Topology=true - {{- end}} - {{- if .Values.extraCreateMetadata }} + {{- if or .Values.controller.extraCreateMetadata .Values.extraCreateMetadata }} - --extra-create-metadata {{- end}} - - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + - --leader-election=true - --default-fstype=ext4 env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.controller.env.provisioner }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.provisioner) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - name: csi-attacher image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} args: - --csi-address=$(ADDRESS) - - --v={{ .Values.logLevel }} - - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + - --v={{ .Values.controller.logLevel }} + - --leader-election=true env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.controller.env.attacher }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.attacher) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - {{- if .Values.enableVolumeSnapshot }} - name: csi-snapshotter image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} args: - --csi-address=$(ADDRESS) - - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + - --leader-election=true env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.controller.env.snapshotter }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.snapshotter) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - {{- end }} - {{- if .Values.enableVolumeResizing }} - name: csi-resizer image: {{ printf "%s:%s" .Values.sidecars.resizerImage.repository .Values.sidecars.resizerImage.tag }} imagePullPolicy: Always args: - --csi-address=$(ADDRESS) - - --v={{ .Values.logLevel }} + - --v={{ .Values.controller.logLevel }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.controller.env.resizer }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.resizer) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - {{- end }} - name: liveness-probe image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} args: @@ -237,8 +242,9 @@ spec: volumeMounts: - name: socket-dir mountPath: /csi - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.liveness) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: @@ -249,3 +255,10 @@ spec: volumes: - name: socket-dir emptyDir: {} + - name: aws-token + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 86400 + audience: "sts.amazonaws.com" diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml index 1858e39c..6598355a 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml @@ -1,4 +1,4 @@ -apiVersion: storage.k8s.io/v1beta1 +apiVersion: {{ ternary "storage.k8s.io/v1" "storage.k8s.io/v1beta1" (semverCompare ">=1.18.0-0" .Capabilities.KubeVersion.Version) }} kind: CSIDriver metadata: name: ebs.csi.aws.com diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml index 5caeb33d..c816adf7 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml @@ -15,8 +15,12 @@ spec: labels: app: ebs-csi-node {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} - {{- if .Values.node.podAnnotations }} - annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- if .Values.node.podLabels }} + {{- toYaml .Values.node.podLabels | nindent 8 }} + {{- end }} + {{- with .Values.node.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} {{- end }} spec: affinity: @@ -31,11 +35,10 @@ spec: nodeSelector: kubernetes.io/os: linux {{- with .Values.node.nodeSelector }} -{{ toYaml . | indent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} - hostNetwork: true serviceAccountName: {{ .Values.serviceAccount.node.name }} - priorityClassName: {{ .Values.node.priorityClassName | default "system-cluster-critical" }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} tolerations: {{- if .Values.node.tolerateAllTaints }} - operator: Exists @@ -47,7 +50,7 @@ spec: tolerationSeconds: 300 {{- end }} {{- with .Values.node.tolerations }} -{{ toYaml . | indent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} containers: - name: ebs-plugin @@ -57,22 +60,24 @@ spec: args: - node - --endpoint=$(CSI_ENDPOINT) - {{- if .Values.volumeAttachLimit }} - - --volume-attach-limit={{ .Values.volumeAttachLimit }} + {{- with default .Values.volumeAttachLimit .Values.node.volumeAttachLimit }} + - --volume-attach-limit={{ . }} {{- end }} - --logtostderr - - --v={{ .Values.logLevel }} + - --v={{ .Values.node.logLevel }} env: - name: CSI_ENDPOINT value: unix:/csi/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.node.env.ebsPlugin }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet @@ -93,47 +98,35 @@ spec: timeoutSeconds: 3 periodSeconds: 10 failureThreshold: 5 - {{- if .Values.node.resources }} - {{- with .Values.node.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- else }} - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} + {{- with default .Values.resources (default .Values.node.resources .Values.node.containerResources.ebsPlugin) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - name: node-driver-registrar image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v={{ .Values.logLevel }} + - --v={{ .Values.node.logLevel }} env: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.node.env.nodeDriverRegistrar }} + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: plugin-dir mountPath: /csi - name: registration-dir mountPath: /registration - {{- if .Values.node.resources }} - {{- with .Values.node.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- else }} - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} + {{- with default .Values.resources (default .Values.node.resources .Values.node.containerResources.nodeDriverRegistrar) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - name: liveness-probe image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} @@ -142,14 +135,9 @@ spec: volumeMounts: - name: plugin-dir mountPath: /csi - {{- if .Values.node.resources }} - {{- with .Values.node.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} - {{- else }} - {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} - {{- end }} + {{- with default .Values.resources (default .Values.node.resources .Values.node.containerResources.liveness) }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: @@ -160,15 +148,15 @@ spec: volumes: - name: kubelet-dir hostPath: - path: /var/lib/kubelet + path: {{ .Values.node.kubeletPath }} type: Directory - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + path: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} type: DirectoryOrCreate - name: registration-dir hostPath: - path: /var/lib/kubelet/plugins_registry/ + path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} type: Directory - name: device-dir hostPath: diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml new file mode 100644 index 00000000..373974a0 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml @@ -0,0 +1,17 @@ +{{- $replicas := (default .Values.replicaCount .Values.controller.replicaCount) | int }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: ebs-csi-controller + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ebs-csi-controller + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + {{- if le $replicas 2 }} + maxUnavailable: 1 + {{- else }} + minAvailable: 2 +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-snapshot-controller.yaml new file mode 100644 index 00000000..d9383bcb --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-snapshot-controller.yaml @@ -0,0 +1,14 @@ +{{- if .Values.enableVolumeSnapshot }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: ebs-snapshot-controller + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ebs-snapshot-controller + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + maxUnavailable: 1 +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml index 4d09e4ca..2b55a16a 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -7,8 +6,6 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} rules: - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - -{{- end }} + - apiGroups: [ "coordination.k8s.io" ] + resources: [ "leases" ] + verbs: [ "get", "watch", "list", "delete", "update", "create" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml index e8248bd8..74095f38 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -14,5 +13,3 @@ roleRef: kind: Role name: ebs-snapshot-controller-leaderelection apiGroup: rbac.authorization.k8s.io - -{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml index 0490c327..b465c3be 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml @@ -6,7 +6,8 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} {{- with .Values.serviceAccount.controller.annotations }} - annotations: {{ toYaml . | nindent 4 }} + annotations: + {{- toYaml . | nindent 4 }} {{- end }} {{- if eq .Release.Name "kustomize" }} #Enable if EKS IAM for SA is used diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml index 2e93f727..4722b2a9 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml @@ -6,6 +6,7 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} {{- with .Values.serviceAccount.node.annotations }} - annotations: {{ toYaml . | nindent 4 }} + annotations: + {{- toYaml . | nindent 4 }} {{- end }} {{- end -}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml index 19d27cb8..9d2c6811 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml @@ -1,4 +1,3 @@ -{{- if .Values.enableVolumeSnapshot }} {{- if .Values.serviceAccount.snapshot.create }} --- apiVersion: v1 @@ -8,7 +7,7 @@ metadata: labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} {{- with .Values.serviceAccount.snapshot.annotations }} - annotations: {{ toYaml . | nindent 4 }} + annotations: + {{- toYaml . | nindent 4 }} {{- end }} {{- end }} -{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml similarity index 62% rename from charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml rename to charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml index e87aa302..bccfd131 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml @@ -5,7 +5,7 @@ apiVersion: apps/v1 metadata: name: ebs-snapshot-controller labels: - {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} spec: serviceName: ebs-snapshot-controller replicas: 1 @@ -18,41 +18,46 @@ spec: labels: app: ebs-snapshot-controller {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.snapshotController.podLabels }} + {{- toYaml .Values.snapshotController.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.snapshotController.podAnnotations }} + annotations: {{ toYaml .Values.snapshotController.podAnnotations | nindent 8 }} + {{- end }} spec: serviceAccountName: {{ .Values.serviceAccount.snapshot.name }} nodeSelector: kubernetes.io/os: linux {{- with .Values.nodeSelector }} -{{ toYaml . | indent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} {{- with .Values.affinity }} - affinity: {{ toYaml . | nindent 8 }} + affinity: + {{- toYaml . | nindent 8 }} {{- end }} tolerations: - {{- if .Values.tolerateAllTaints }} + - key: CriticalAddonsOnly + operator: Exists - operator: Exists - {{- end }} + effect: NoExecute + tolerationSeconds: 300 {{- with .Values.tolerations }} -{{ toYaml . | indent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} containers: - name: snapshot-controller image: {{ printf "%s:%s" .Values.snapshotController.repository .Values.snapshotController.tag }} {{- with .Values.resources }} - resources: {{ toYaml . | nindent 12 }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} + {{- if .Values.proxy.http_proxy }} env: -{{- if .Values.proxy.http_proxy }} - - name: HTTP_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: HTTPS_PROXY - value: {{ .Values.proxy.http_proxy | quote }} - - name: NO_PROXY - value: {{ .Values.proxy.no_proxy | quote }} -{{- end }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} args: - - --v={{ .Values.logLevel }} + - --v=2 - --leader-election=false {{- if .Values.imagePullSecrets }} imagePullSecrets: @@ -60,4 +65,4 @@ spec: - name: {{ . }} {{- end }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml index 3da90e3d..847f5e21 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml @@ -4,11 +4,11 @@ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: {{ .name }} - {{- if .annotations }} - annotations: {{- .annotations | toYaml | trim | nindent 4 }} + {{- with .annotations }} + annotations: {{- . | toYaml | trim | nindent 4 }} {{- end }} - {{- if .labels }} - labels: {{- .labels | toYaml | trim | nindent 4 }} + {{- with .labels }} + labels: {{- . | toYaml | trim | nindent 4 }} {{- end }} provisioner: ebs.csi.aws.com {{ omit (dict "volumeBindingMode" "WaitForFirstConsumer" | merge .) "name" "annotations" "labels" | toYaml }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml index 0710d187..a33fda9d 100644 --- a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml @@ -2,22 +2,18 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. -replicaCount: 2 - image: repository: k8s.gcr.io/provider-aws/aws-ebs-csi-driver - tag: "v0.10.0" + tag: "v1.1.0" pullPolicy: IfNotPresent -logLevel: 5 - sidecars: provisionerImage: repository: k8s.gcr.io/sig-storage/csi-provisioner - tag: "v2.0.2" + tag: "v2.1.1" attacherImage: repository: k8s.gcr.io/sig-storage/csi-attacher - tag: "v3.0.0" + tag: "v3.1.0" snapshotterImage: repository: k8s.gcr.io/sig-storage/csi-snapshotter tag: "v3.0.3" @@ -29,97 +25,128 @@ sidecars: tag: "v1.0.0" nodeDriverRegistrarImage: repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar - tag: "v2.0.1" + tag: "v2.1.0" snapshotController: repository: k8s.gcr.io/sig-storage/snapshot-controller tag: "v3.0.3" + podAnnotations: {} + podLabels: {} -proxy: {} -# http_proxy: -# no_proxy: +proxy: + http_proxy: + no_proxy: imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -podAnnotations: {} - -# True if enable volume scheduling for dynamic volume provisioning -enableVolumeScheduling: true - -# True if enable volume resizing -enableVolumeResizing: false +nameOverride: +fullnameOverride: # True if enable volume snapshot enableVolumeSnapshot: false -# The "maximum number of attachable volumes" per node -volumeAttachLimit: "" +# Moving to values under controller +affinity: {} +extraCreateMetadata: true +extraVolumeTags: {} +k8sTagClusterId: +nodeSelector: {} +podAnnotations: {} +priorityClassName: "system-cluster-critical" +region: +replicaCount: 2 +resources: {} +tolerations: [] +topologySpreadConstraints: [] -resources: - {} +controller: + affinity: {} + # True if enable volume scheduling for dynamic volume provisioning + env: + ebsPlugin: [] + provisioner: [] + attacher: [] + snapshotter: [] + resizer: [] + # If set, add pv/pvc metadata to plugin create requests as parameters. + extraCreateMetadata: true + # Will be removed in later version in favor of env.ebsPlugin + extraVars: {} + # Extra volume tags to attach to each dynamically provisioned volume. + # --- + # extraVolumeTags: + # key1: value1 + # key2: value2 + extraVolumeTags: {} + httpEndpoint: + # ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional). + k8sTagClusterId: + logLevel: 2 + nodeSelector: {} + podAnnotations: {} + podLabels: {} + priorityClassName: + # AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata + # service. + # --- + # region: us-east-1 + region: + replicaCount: + resources: {} + containerResources: + ebsPlugin: {} + provisioner: {} + attacher: {} + snapshotter: {} + resizer: {} + liveness: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # Note that you will need to set resource requests if you want the cluster autoscaler to + # scale your nodes when you increase/decrease the number of ebs-csi-controller replicas. # limits: # cpu: 100m # memory: 128Mi # requests: # cpu: 100m # memory: 128Mi + tolerations: [] + # TSCs without the label selector stanza + # + # Example: + # + # topologySpreadConstraints: + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: ScheduleAnyway + # - maxSkew: 1 + # topologyKey: kubernetes.io/hostname + # whenUnsatisfiable: ScheduleAnyway + topologySpreadConstraints: [] -priorityClassName: "" -nodeSelector: {} -tolerateAllTaints: false -tolerations: [] -affinity: {} - -# TSCs without the label selector stanza -# -# Example: -# -# topologySpreadConstraints: -# - maxSkew: 1 -# topologyKey: topology.kubernetes.io/zone -# whenUnsatisfiable: ScheduleAnyway -# - maxSkew: 1 -# topologyKey: kubernetes.io/hostname -# whenUnsatisfiable: ScheduleAnyway - -topologySpreadConstraints: [] - -# Extra volume tags to attach to each dynamically provisioned volume. -# --- -# extraVolumeTags: -# key1: value1 -# key2: value2 -extraVolumeTags: {} - -# If set, add pv/pvc metadata to plugin create requests as parameters. -extraCreateMetadata: false - -# ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional). -k8sTagClusterId: "" - -# AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata -# service. -# --- -# region: us-east-1 -region: "" - -# Additonal environment variables for the controller -controller: - extraVars: {} +# Moving to values under node +# The "maximum number of attachable volumes" per node +volumeAttachLimit: node: - priorityClassName: "" + env: + ebsPlugin: [] + nodeDriverRegistrar: [] + kubeletPath: /var/lib/kubelet + logLevel: 2 + priorityClassName: nodeSelector: {} podAnnotations: {} + podLabels: {} tolerateAllTaints: false tolerations: [] resources: {} + containerResources: + ebsPlugin: {} + nodeDriverRegistrar: {} + liveness: {} + volumeAttachLimit: serviceAccount: controller: diff --git a/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch b/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch deleted file mode 100644 index 349eea0a..00000000 --- a/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch +++ /dev/null @@ -1,103 +0,0 @@ -diff -rtubN aws-ebs-csi-driver.orig/templates/controller.yaml aws-ebs-csi-driver/templates/controller.yaml ---- aws-ebs-csi-driver.orig/templates/controller.yaml 2021-03-05 03:10:41.000000000 +0100 -+++ aws-ebs-csi-driver/templates/controller.yaml 2021-03-05 10:29:31.878615411 +0100 -@@ -68,7 +68,7 @@ - - --k8s-tag-cluster-id={{ .Values.k8sTagClusterId }} - {{- end }} - - --logtostderr -- - --v=5 -+ - --v={{ .Values.logLevel }} - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock -@@ -126,14 +126,14 @@ - image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} - args: - - --csi-address=$(ADDRESS) -- - --v=5 -+ - --v={{ .Values.logLevel }} - {{- if .Values.enableVolumeScheduling }} - - --feature-gates=Topology=true - {{- end}} - {{- if .Values.extraCreateMetadata }} - - --extra-create-metadata - {{- end}} -- - --leader-election=true -+ - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} - - --default-fstype=ext4 - env: - - name: ADDRESS -@@ -156,8 +156,8 @@ - image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} - args: - - --csi-address=$(ADDRESS) -- - --v=5 -- - --leader-election=true -+ - --v={{ .Values.logLevel }} -+ - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock -@@ -180,7 +180,7 @@ - image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} - args: - - --csi-address=$(ADDRESS) -- - --leader-election=true -+ - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock -@@ -205,7 +205,7 @@ - imagePullPolicy: Always - args: - - --csi-address=$(ADDRESS) -- - --v=5 -+ - --v={{ .Values.logLevel }} - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock -diff -rtubN aws-ebs-csi-driver.orig/templates/node.yaml aws-ebs-csi-driver/templates/node.yaml ---- aws-ebs-csi-driver.orig/templates/node.yaml 2021-03-05 03:10:41.000000000 +0100 -+++ aws-ebs-csi-driver/templates/node.yaml 2021-03-05 10:30:07.391950366 +0100 -@@ -61,7 +61,7 @@ - - --volume-attach-limit={{ .Values.volumeAttachLimit }} - {{- end }} - - --logtostderr -- - --v=5 -+ - --v={{ .Values.logLevel }} - env: - - name: CSI_ENDPOINT - value: unix:/csi/csi.sock -@@ -107,7 +107,7 @@ - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) -- - --v=5 -+ - --v={{ .Values.logLevel }} - env: - - name: ADDRESS - value: /csi/csi.sock -diff -rtubN aws-ebs-csi-driver.orig/templates/statefulset.yaml aws-ebs-csi-driver/templates/statefulset.yaml ---- aws-ebs-csi-driver.orig/templates/statefulset.yaml 2021-03-05 03:10:41.000000000 +0100 -+++ aws-ebs-csi-driver/templates/statefulset.yaml 2021-03-05 10:29:31.881948744 +0100 -@@ -49,7 +49,7 @@ - value: {{ .Values.proxy.no_proxy | quote }} - {{- end }} - args: -- - --v=5 -+ - --v={{ .Values.logLevel }} - - --leader-election=false - {{- if .Values.imagePullSecrets }} - imagePullSecrets: -diff -rtubN aws-ebs-csi-driver.orig/values.yaml aws-ebs-csi-driver/values.yaml ---- aws-ebs-csi-driver.orig/values.yaml 2021-03-05 03:10:41.000000000 +0100 -+++ aws-ebs-csi-driver/values.yaml 2021-03-05 10:29:31.881948744 +0100 -@@ -9,6 +9,8 @@ - tag: "v0.9.0" - pullPolicy: IfNotPresent - -+logLevel: 5 -+ - sidecars: - provisionerImage: - repository: k8s.gcr.io/sig-storage/csi-provisioner diff --git a/charts/kubezero-aws-ebs-csi-driver/patch b/charts/kubezero-aws-ebs-csi-driver/patch new file mode 100644 index 00000000..71fb0085 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/patch @@ -0,0 +1,87 @@ +diff -rtubN charts/aws-ebs-csi-driver/templates/controller.yaml charts/aws-ebs-csi-driver.zdt/templates/controller.yaml +--- charts/aws-ebs-csi-driver/templates/controller.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/templates/controller.yaml 2021-06-24 16:31:37.042386198 +0200 +@@ -116,6 +116,9 @@ + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ ++ - name: aws-token ++ mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" ++ readOnly: true + ports: + - name: healthz + containerPort: 9808 +@@ -144,7 +147,7 @@ + image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.controller.logLevel }} + - --feature-gates=Topology=true + {{- if or .Values.controller.extraCreateMetadata .Values.extraCreateMetadata }} + - --extra-create-metadata +@@ -171,7 +174,7 @@ + image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.controller.logLevel }} + - --leader-election=true + env: + - name: ADDRESS +@@ -215,7 +218,7 @@ + imagePullPolicy: Always + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.controller.logLevel }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +@@ -252,3 +255,10 @@ + volumes: + - name: socket-dir + emptyDir: {} ++ - name: aws-token ++ projected: ++ sources: ++ - serviceAccountToken: ++ path: token ++ expirationSeconds: 86400 ++ audience: "sts.amazonaws.com" +diff -rtubN charts/aws-ebs-csi-driver/templates/node.yaml charts/aws-ebs-csi-driver.zdt/templates/node.yaml +--- charts/aws-ebs-csi-driver/templates/node.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/templates/node.yaml 2021-06-24 15:03:44.532351851 +0200 +@@ -107,7 +107,7 @@ + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) +- - --v=5 ++ - --v={{ .Values.node.logLevel }} + env: + - name: ADDRESS + value: /csi/csi.sock +diff -rtubN charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml charts/aws-ebs-csi-driver.zdt/templates/snapshot-controller.yaml +--- charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/templates/snapshot-controller.yaml 2021-06-24 15:04:17.999018733 +0200 +@@ -57,7 +57,7 @@ + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + args: +- - --v=5 ++ - --v=2 + - --leader-election=false + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +diff -rtubN charts/aws-ebs-csi-driver/values.yaml charts/aws-ebs-csi-driver.zdt/values.yaml +--- charts/aws-ebs-csi-driver/values.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/values.yaml 2021-06-22 17:08:52.501232371 +0200 +@@ -56,7 +56,7 @@ + replicaCount: 2 + resources: {} + tolerations: [] +-topologySpreadConstraints: [] ++topolk8sTagClusterIdogySpreadConstraints: [] + + controller: + affinity: {} diff --git a/charts/kubezero-aws-ebs-csi-driver/update.sh b/charts/kubezero-aws-ebs-csi-driver/update.sh index 1641bebd..12e4d9d2 100755 --- a/charts/kubezero-aws-ebs-csi-driver/update.sh +++ b/charts/kubezero-aws-ebs-csi-driver/update.sh @@ -1,8 +1,8 @@ #!/bin/bash -VERSION=0.10.0 +VERSION=1.2.3 rm -rf charts/aws-ebs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts -patch -d charts -i ../loglevel_leader.patch -p0 --no-backup-if-mismatch +patch -i zdt.patch -p0 --no-backup-if-mismatch diff --git a/charts/kubezero-aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/values.yaml index 8519805f..b7340b8e 100644 --- a/charts/kubezero-aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/values.yaml @@ -1,33 +1,33 @@ aws-ebs-csi-driver: - replicaCount: 1 - logLevel: 1 - - enableVolumeScheduling: true - enableVolumeResizing: true enableVolumeSnapshot: true - nodeSelector: - node-role.kubernetes.io/master: "" + controller: + replicaCount: 1 + logLevel: 1 - tolerations: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule - # aws-ebs-csi-driver.podAnnotations -- iam.amazonaws.com/role: to assume - podAnnotations: {} - # iam.amazonaws.com/role: '' + # k8sTagClusterId: - # aws-ebs-csi-driver.extraVolumeTags -- Optional tags to be added to each EBS volume - extraVolumeTags: {} - # Name: KubeZero-Cluster - - resources: - requests: - cpu: 10m - memory: 24Mi - limits: - # cpu: 50m - memory: 40Mi + #env: + # ebsPlugin: + #- name: AWS_ROLE_ARN + # value: "" + #- name: AWS_WEB_IDENTITY_TOKEN_FILE + # value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + #- name: AWS_STS_REGIONAL_ENDPOINTS + # value: regional + resources: + requests: + cpu: 10m + memory: 24Mi + limits: + # cpu: 50m + memory: 40Mi storageClasses: - name: ebs-sc-gp2-xfs @@ -46,3 +46,10 @@ aws-ebs-csi-driver: csi.storage.k8s.io/fstype: xfs type: gp3 encrypted: "true" + + # This will probably move to snapshot later on + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule diff --git a/charts/kubezero-aws-ebs-csi-driver/zdt.patch b/charts/kubezero-aws-ebs-csi-driver/zdt.patch new file mode 100644 index 00000000..ecddbcc4 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/zdt.patch @@ -0,0 +1,75 @@ +diff -rtubN charts/aws-ebs-csi-driver/templates/controller.yaml charts/aws-ebs-csi-driver.zdt/templates/controller.yaml +--- charts/aws-ebs-csi-driver/templates/controller.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/templates/controller.yaml 2021-06-24 16:31:37.042386198 +0200 +@@ -116,6 +116,9 @@ + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ ++ - name: aws-token ++ mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" ++ readOnly: true + ports: + - name: healthz + containerPort: 9808 +@@ -144,7 +147,7 @@ + image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.controller.logLevel }} + - --feature-gates=Topology=true + {{- if or .Values.controller.extraCreateMetadata .Values.extraCreateMetadata }} + - --extra-create-metadata +@@ -171,7 +174,7 @@ + image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.controller.logLevel }} + - --leader-election=true + env: + - name: ADDRESS +@@ -215,7 +218,7 @@ + imagePullPolicy: Always + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.controller.logLevel }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +@@ -252,3 +255,10 @@ + volumes: + - name: socket-dir + emptyDir: {} ++ - name: aws-token ++ projected: ++ sources: ++ - serviceAccountToken: ++ path: token ++ expirationSeconds: 86400 ++ audience: "sts.amazonaws.com" +diff -rtubN charts/aws-ebs-csi-driver/templates/node.yaml charts/aws-ebs-csi-driver.zdt/templates/node.yaml +--- charts/aws-ebs-csi-driver/templates/node.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/templates/node.yaml 2021-06-24 15:03:44.532351851 +0200 +@@ -107,7 +107,7 @@ + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) +- - --v=5 ++ - --v={{ .Values.node.logLevel }} + env: + - name: ADDRESS + value: /csi/csi.sock +diff -rtubN charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml charts/aws-ebs-csi-driver.zdt/templates/snapshot-controller.yaml +--- charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml 2021-06-17 22:22:22.000000000 +0200 ++++ charts/aws-ebs-csi-driver.zdt/templates/snapshot-controller.yaml 2021-06-24 15:04:17.999018733 +0200 +@@ -57,7 +57,7 @@ + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + args: +- - --v=5 ++ - --v=2 + - --leader-election=false + {{- if .Values.imagePullSecrets }} + imagePullSecrets: diff --git a/charts/kubezero-aws-efs-csi-driver/Chart.yaml b/charts/kubezero-aws-efs-csi-driver/Chart.yaml index 8bd4a5da..2e00e926 100644 --- a/charts/kubezero-aws-efs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-efs-csi-driver/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: kubezero-aws-efs-csi-driver description: KubeZero Umbrella Chart for aws-efs-csi-driver -version: 0.3.5 -appVersion: 1.2.0 +version: 0.4.0 +appVersion: 1.3.1 kubeVersion: ">=1.18.0-0" home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png @@ -21,5 +21,5 @@ dependencies: version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ - name: aws-efs-csi-driver - version: 1.2.2 + version: 2.1.1 # repository: https://kubernetes-sigs.github.io/aws-efs-csi-driver/ diff --git a/charts/kubezero-aws-efs-csi-driver/README.md b/charts/kubezero-aws-efs-csi-driver/README.md index 6a17d549..01843b5e 100644 --- a/charts/kubezero-aws-efs-csi-driver/README.md +++ b/charts/kubezero-aws-efs-csi-driver/README.md @@ -1,6 +1,6 @@ # kubezero-aws-efs-csi-driver -![Version: 0.3.5](https://img.shields.io/badge/Version-0.3.5-informational?style=flat-square) ![AppVersion: 1.2.0](https://img.shields.io/badge/AppVersion-1.2.0-informational?style=flat-square) +![Version: 0.4.0](https://img.shields.io/badge/Version-0.4.0-informational?style=flat-square) ![AppVersion: 1.3.1](https://img.shields.io/badge/AppVersion-1.3.1-informational?style=flat-square) KubeZero Umbrella Chart for aws-efs-csi-driver @@ -23,7 +23,7 @@ Kubernetes: `>=1.18.0-0` | Repository | Name | Version | |------------|------|---------| -| | aws-efs-csi-driver | 1.2.2 | +| | aws-efs-csi-driver | 2.1.1 | | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 | ## Storage Class @@ -37,11 +37,16 @@ Details also see: [Reserve PV](https://kubernetes.io/docs/concepts/storage/persi | Key | Type | Default | Description | |-----|------|---------|-------------| -| StorageClass.create | bool | `true` | | -| StorageClass.default | bool | `false` | | -| aws-efs-csi-driver.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"node.kubernetes.io/csi.efs.fs"` | | -| aws-efs-csi-driver.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator | string | `"Exists"` | | -| aws-efs-csi-driver.logLevel | int | `1` | | -| aws-efs-csi-driver.resources.limits.memory | string | `"128Mi"` | | -| aws-efs-csi-driver.resources.requests.cpu | string | `"20m"` | | -| aws-efs-csi-driver.resources.requests.memory | string | `"64Mi"` | | +| aws-efs-csi-driver.controller.create | bool | `true` | | +| aws-efs-csi-driver.controller.logLevel | int | `1` | | +| aws-efs-csi-driver.controller.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| aws-efs-csi-driver.controller.tolerations[0].effect | string | `"NoSchedule"` | | +| aws-efs-csi-driver.controller.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| aws-efs-csi-driver.node.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"node.kubernetes.io/csi.efs.fs"` | | +| aws-efs-csi-driver.node.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator | string | `"Exists"` | | +| aws-efs-csi-driver.node.logLevel | int | `1` | | +| aws-efs-csi-driver.node.resources.limits.memory | string | `"128Mi"` | | +| aws-efs-csi-driver.node.resources.requests.cpu | string | `"20m"` | | +| aws-efs-csi-driver.node.resources.requests.memory | string | `"64Mi"` | | +| aws-efs-csi-driver.replicaCount | int | `1` | | +| aws-efs-csi-driver.storageClasses[0].name | string | `"efs-sc"` | | diff --git a/charts/kubezero-aws-efs-csi-driver/affinity_resources.patch b/charts/kubezero-aws-efs-csi-driver/affinity_resources.patch deleted file mode 100644 index 60c546ac..00000000 --- a/charts/kubezero-aws-efs-csi-driver/affinity_resources.patch +++ /dev/null @@ -1,64 +0,0 @@ -diff -tubrN charts/aws-efs-csi-driver/templates/node-daemonset.yaml charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml ---- charts/aws-efs-csi-driver/templates/node-daemonset.yaml 2021-03-23 14:34:03.000000000 +0100 -+++ charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml 2021-04-01 14:31:36.427375803 +0200 -@@ -40,15 +40,10 @@ - {{- with .Values.nodeSelector }} - {{- . | toYaml | nindent 8 }} - {{- end }} -+ {{- with .Values.affinity }} - affinity: -- nodeAffinity: -- requiredDuringSchedulingIgnoredDuringExecution: -- nodeSelectorTerms: -- - matchExpressions: -- - key: eks.amazonaws.com/compute-type -- operator: NotIn -- values: -- - fargate -+ {{- . | toYaml | nindent 8 }} -+ {{- end }} - hostNetwork: true - {{- if .Values.dnsPolicy }} - dnsPolicy: "{{ .Values.dnsPolicy }}" -@@ -99,6 +94,10 @@ - timeoutSeconds: 3 - periodSeconds: 2 - failureThreshold: 5 -+ {{- with .Values.resources }} -+ resources: -+ {{- . | toYaml | nindent 12 }} -+ {{- end }} - - name: csi-driver-registrar - image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} - args: -diff -tubrN charts/aws-efs-csi-driver/values.yaml charts/aws-efs-csi-driver.zdt/values.yaml ---- charts/aws-efs-csi-driver/values.yaml 2021-03-23 14:34:03.000000000 +0100 -+++ charts/aws-efs-csi-driver.zdt/values.yaml 2021-04-01 14:37:21.290724721 +0200 -@@ -11,8 +11,8 @@ - - sidecars: - livenessProbeImage: -- repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe -- tag: v2.1.0-eks-1-18-1 -+ repository: k8s.gcr.io/sig-storage/livenessprobe -+ tag: "v2.2.0" - nodeDriverRegistrarImage: - repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar - tag: v2.0.1-eks-1-18-1 -@@ -43,7 +43,15 @@ - - tolerations: [] - --affinity: {} -+affinity: -+ nodeAffinity: -+ requiredDuringSchedulingIgnoredDuringExecution: -+ nodeSelectorTerms: -+ - matchExpressions: -+ - key: eks.amazonaws.com/compute-type -+ operator: NotIn -+ values: -+ - fargate - - node: - podAnnotations: {} diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/CHANGELOG.md b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/CHANGELOG.md new file mode 100644 index 00000000..a842698d --- /dev/null +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/CHANGELOG.md @@ -0,0 +1,106 @@ +# Helm chart + +# v2.1.1 +* Update app/driver version to `v1.3.0` + +# v2.1.0 + +## New features +* Update app/driver version to `v1.3.0` + +## Bug fixes +* Put comments back in place inside the values file ([#475](https://github.com/kubernetes-sigs/aws-efs-csi-driver/pull/475), [@pierluigilenoci](https://github.com/pierluigilenoci)) + +# v2.0.1 + +## Bug fixes +* Helm chart: fix reclaimPolicy and volumeBindingMode ([#464](https://github.com/kubernetes-sigs/aws-efs-csi-driver/pull/464), [@devinsmith911](https://github.com/devinsmith911)) + + +# v2.0.0 + +## Breaking changes + +Multiple changes in values file at `sidecars`, `controller` and `node` + +--- +```yaml +sidecars: + xxxxxxxxx: + repository: + tag: +``` + +Moving to + +```yaml +sidecars: + xxxxxxxxx: + image: + repository: + tag: +``` + +--- +```yaml +podAnnotations: +resources: +nodeSelector: +tolerations: +affinity: +``` + +Moving to + +```yaml +controller: + podAnnotations: + resources: + nodeSelector: + tolerations: + affinity: +``` + +--- +```yaml +hostAliases: +dnsPolicy: +dnsConfig: +``` + +Moving to + +```yaml +node: + hostAliases: + dnsPolicy: + dnsConfig: +``` + +--- +```yaml +serviceAccount: + controller: +``` + +Moving to + +```yaml +controller: + serviceAccount: +``` + +## New features + +* Chart API `v2` (requires Helm 3) +* Set `resources` and `imagePullPolicy` fields independently for containers +* Set `logLevel`, `affinity`, `nodeSelector`, `podAnnotations` and `tolerations` fields independently +for Controller deployment and Node daemonset +* Set `reclaimPolicy` and `volumeBindingMode` fields in storage class + +## Fixes + +* Fixing Controller deployment using `podAnnotations` and `tolerations` values from Node daemonset +* Let the user define the whole `tolerations` array, default to `- operator: Exists` +* Default `logLevel` lowered from `5` to `2` +* Default `imagePullPolicy` everywhere set to `IfNotPresent` diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml index 4f1b11d0..29c91a78 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ -apiVersion: v1 -appVersion: 1.2.0 +apiVersion: v2 +appVersion: 1.3.1 description: A Helm chart for AWS EFS CSI Driver home: https://github.com/kubernetes-sigs/aws-efs-csi-driver keywords: @@ -15,4 +15,4 @@ maintainers: name: aws-efs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-efs-csi-driver -version: 1.2.2 +version: 2.1.1 diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl index 2b4ed0ba..3a74fef8 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl @@ -49,8 +49,19 @@ Create the name of the service account to use */}} {{- define "aws-efs-csi-driver.serviceAccountName" -}} {{- if .Values.controller.create -}} - {{ default (include "aws-efs-csi-driver.fullname" .) .Values.serviceAccount.controller.name }} + {{ default (include "aws-efs-csi-driver.fullname" .) .Values.controller.serviceAccount.name }} {{- else -}} - {{ default "default" .Values.serviceAccount.controller.name }} + {{ default "default" .Values.controller.serviceAccount.name }} {{- end -}} {{- end -}} + +{{/* +Create a string out of the map for controller tags flag +*/}} +{{- define "aws-efs-csi-driver.tags" -}} +{{- $tags := list -}} +{{ range $key, $val := . }} +{{- $tags = print $key ":" $val | append $tags -}} +{{- end -}} +{{- join " " $tags -}} +{{- end -}} diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml index 122ffe23..67f3adaa 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml @@ -19,10 +19,11 @@ spec: app: efs-csi-controller app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} - {{- if .Values.node.podAnnotations }} - annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- with .Values.controller.podAnnotations }} + annotations: {{- toYaml . | nindent 8 }} {{- end }} spec: + hostNetwork: true {{- if .Values.imagePullSecrets }} imagePullSecrets: {{- range .Values.imagePullSecrets }} @@ -31,37 +32,39 @@ spec: {{- end }} nodeSelector: kubernetes.io/os: linux - {{- with .Values.nodeSelector }} - {{- . | toYaml | nindent 8 }} + {{- with .Values.controller.nodeSelector }} + {{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ include "aws-efs-csi-driver.serviceAccountName" . }} priorityClassName: system-cluster-critical - tolerations: - - operator: Exists - {{- with .Values.node.tolerations }} - {{- . | toYaml | nindent 8 }} - {{- end }} + {{- with .Values.controller.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: efs-plugin securityContext: privileged: true image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: IfNotPresent + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - --endpoint=$(CSI_ENDPOINT) - --logtostderr - - --v=5 - # Uncomment below line to allow access point root directory to be deleted by controller. - #- --delete-access-point-root-dir + {{- if .Values.controller.tags }} + - --tags={{ include "aws-efs-csi-driver.tags" .Values.controller.tags }} + {{- end }} + - --v={{ .Values.controller.logLevel }} + - --delete-access-point-root-dir={{ hasKey .Values.controller "deleteAccessPointRootDir" | ternary .Values.controller.deleteAccessPointRootDir false }} env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: aws-token + mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" ports: - name: healthz - containerPort: 9808 + containerPort: 9909 protocol: TCP livenessProbe: httpGet: @@ -71,28 +74,52 @@ spec: timeoutSeconds: 3 periodSeconds: 10 failureThreshold: 5 + {{- with .Values.controller.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} - name: csi-provisioner - image: {{ printf "%s:%s" .Values.sidecars.csiProvisionerImage.repository .Values.sidecars.csiProvisionerImage.tag }} + image: {{ printf "%s:%s" .Values.sidecars.csiProvisioner.image.repository .Values.sidecars.csiProvisioner.image.tag }} + imagePullPolicy: {{ .Values.sidecars.csiProvisioner.image.pullPolicy }} args: - --csi-address=$(ADDRESS) - - --v=5 + - --v={{ .Values.controller.logLevel }} - --feature-gates=Topology=true - --leader-election env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock + {{- if .Values.controller.extraEnv }} +{{ toYaml .Values.extraEnv | indent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.sidecars.csiProvisioner.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} - name: liveness-probe - image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }} args: - --csi-address=/csi/csi.sock - - --health-port=9808 + - --health-port=9909 volumeMounts: - name: socket-dir mountPath: /csi + {{- with .Values.sidecars.livenessProbe.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} volumes: - name: socket-dir emptyDir: {} + - name: aws-token + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 86400 + audience: "sts.amazonaws.com" + {{- with .Values.controller.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml index 5a08044e..63d45b64 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml @@ -1,11 +1,11 @@ -{{- if .Values.serviceAccount.controller.create }} +{{- if .Values.controller.serviceAccount.create }} apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "aws-efs-csi-driver.serviceAccountName" . }} labels: app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} - {{- with .Values.serviceAccount.controller.annotations }} + {{- with .Values.controller.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} @@ -40,6 +40,9 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get", "watch", "list" ] --- diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/csidriver.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/csidriver.yaml index 33247d5f..e6b4d419 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/csidriver.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/csidriver.yaml @@ -1,4 +1,4 @@ -apiVersion: storage.k8s.io/v1beta1 +apiVersion: {{ ternary "storage.k8s.io/v1" "storage.k8s.io/v1beta1" (semverCompare ">=1.18.0-0" .Capabilities.KubeVersion.Version) }} kind: CSIDriver metadata: name: efs.csi.aws.com diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/node-daemonset.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/node-daemonset.yaml index a8cd9a75..df10a2d6 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/node-daemonset.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/node-daemonset.yaml @@ -21,9 +21,9 @@ spec: annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} {{- end }} spec: - {{- if .Values.hostAliases }} + {{- with .Values.node.hostAliases }} hostAliases: - {{- range $k, $v := .Values.hostAliases }} + {{- range $k, $v := . }} - ip: {{ $v.ip }} hostnames: - {{ $k }}.efs.{{ $v.region }}.amazonaws.com @@ -37,36 +37,32 @@ spec: {{- end }} nodeSelector: beta.kubernetes.io/os: linux - {{- with .Values.nodeSelector }} - {{- . | toYaml | nindent 8 }} + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with .Values.node.affinity }} affinity: {{- . | toYaml | nindent 8 }} {{- end }} hostNetwork: true - {{- if .Values.dnsPolicy }} - dnsPolicy: "{{ .Values.dnsPolicy }}" - {{- end }} - {{- with .Values.dnsConfig }} - dnsConfig: - {{- toYaml . | nindent 8 }} + dnsPolicy: {{ .Values.node.dnsPolicy }} + {{- with .Values.node.dnsConfig }} + dnsConfig: {{- toYaml . | nindent 8 }} {{- end }} priorityClassName: system-node-critical - tolerations: - - operator: Exists - {{- with .Values.node.tolerations }} - {{- . | toYaml | nindent 8 }} - {{- end }} + {{- with .Values.node.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: efs-plugin securityContext: privileged: true image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} args: - --endpoint=$(CSI_ENDPOINT) - --logtostderr - - --v={{ .Values.logLevel }} + - --v={{ .Values.node.logLevel }} env: - name: CSI_ENDPOINT value: unix:/csi/csi.sock @@ -94,16 +90,16 @@ spec: timeoutSeconds: 3 periodSeconds: 2 failureThreshold: 5 - {{- with .Values.resources }} - resources: - {{- . | toYaml | nindent 12 }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} {{- end }} - name: csi-driver-registrar - image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} + imagePullPolicy: {{ .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v={{ .Values.logLevel }} + - --v={{ .Values.node.logLevel }} env: - name: ADDRESS value: /csi/csi.sock @@ -118,15 +114,22 @@ spec: mountPath: /csi - name: registration-dir mountPath: /registration + {{- with .Values.sidecars.nodeDriverRegistrar.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} - name: liveness-probe - image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }} args: - --csi-address=/csi/csi.sock - --health-port=9809 - - --v={{ .Values.logLevel }} + - --v={{ .Values.node.logLevel }} volumeMounts: - name: plugin-dir mountPath: /csi + {{- with .Values.sidecars.livenessProbe.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} volumes: - name: kubelet-dir hostPath: diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/storageclass.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/storageclass.yaml index 39f1fae6..8ce1ec87 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/storageclass.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/storageclass.yaml @@ -3,13 +3,24 @@ kind: StorageClass apiVersion: storage.k8s.io/v1 metadata: name: {{ .name }} + {{- with .annotations }} + annotations: + {{ toYaml . | indent 4 }} + {{- end }} provisioner: efs.csi.aws.com {{- with .mountOptions }} -mountOptions: +mountOptions: {{ toYaml . }} {{- end }} {{- with .parameters }} parameters: {{ toYaml . | indent 2 }} {{- end }} +{{- with .reclaimPolicy }} +reclaimPolicy: {{ . }} {{- end }} +{{- with .volumeBindingMode }} +volumeBindingMode: {{ . }} +{{- end }} +--- +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml index b175d57a..e85d3fb2 100644 --- a/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml +++ b/charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml @@ -2,98 +2,114 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. +nameOverride: "" +fullnameOverride: "" + replicaCount: 2 image: repository: amazon/aws-efs-csi-driver - tag: "v1.2.0" + tag: "v1.3.1" pullPolicy: IfNotPresent sidecars: - livenessProbeImage: - repository: k8s.gcr.io/sig-storage/livenessprobe - tag: "v2.2.0" - nodeDriverRegistrarImage: - repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar - tag: v2.0.1-eks-1-18-1 - csiProvisionerImage: - repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner - tag: v2.0.3-eks-1-18-1 + livenessProbe: + image: + repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe + tag: v2.2.0-eks-1-18-2 + pullPolicy: IfNotPresent + resources: {} + nodeDriverRegistrar: + image: + repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar + tag: v2.1.0-eks-1-18-2 + pullPolicy: IfNotPresent + resources: {} + csiProvisioner: + image: + repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner + tag: v2.1.1-eks-1-18-2 + pullPolicy: IfNotPresent + resources: {} imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" -podAnnotations: {} +## Controller deployment variables -resources: - {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: {} - -tolerations: [] - -affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: eks.amazonaws.com/compute-type - operator: NotIn - values: - - fargate - -node: +controller: + # Specifies whether a deployment should be created + create: true + # Number for the log level verbosity + logLevel: 2 + # Add additional tags to access points + tags: {} + # environment: prod + # region: us-east-1 + # Enable if you want the controller to also delete the + # path on efs when deleteing an access point + deleteAccessPointRootDir: false podAnnotations: {} + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + nodeSelector: {} tolerations: [] - -logLevel: 5 - -hostAliases: - {} - # for cross VPC EFS, you need to poison or overwrite the DNS for the efs volume as per - # https://docs.aws.amazon.com/efs/latest/ug/efs-different-vpc.html#wt6-efs-utils-step3 - # implementing the suggested solution found here: - # https://github.com/kubernetes-sigs/aws-efs-csi-driver/issues/240#issuecomment-676849346 - # EFS Vol ID, IP, Region - # "fs-01234567": - # ip: 10.10.2.2 - # region: us-east-2 - -dnsPolicy: "" -dnsConfig: - {} - # Example config which uses the AWS nameservers - # dnsPolicy: "None" - # dnsConfig: - # nameservers: - # - 169.254.169.253 - -serviceAccount: - controller: - # Specifies whether a service account should be created + affinity: {} + # Specifies whether a service account should be created + serviceAccount: create: true + name: efs-csi-controller-sa annotations: {} ## Enable if EKS IAM for SA is used # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/efs-csi-role - name: efs-csi-controller-sa -controller: - create: true +## Node daemonset variables + +node: + # Number for the log level verbosity + logLevel: 2 + hostAliases: {} + # For cross VPC EFS, you need to poison or overwrite the DNS for the efs volume as per + # https://docs.aws.amazon.com/efs/latest/ug/efs-different-vpc.html#wt6-efs-utils-step3 + # implementing the suggested solution found here: + # https://github.com/kubernetes-sigs/aws-efs-csi-driver/issues/240#issuecomment-676849346 + # EFS Vol ID, IP, Region + # "fs-01234567": + # ip: 10.10.2.2 + # region: us-east-2 + dnsPolicy: ClusterFirst + dnsConfig: {} + # Example config which uses the AWS nameservers + # dnsPolicy: "None" + # dnsConfig: + # nameservers: + # - 169.254.169.253 + podAnnotations: {} + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + nodeSelector: {} + tolerations: + - operator: Exists storageClasses: [] # Add StorageClass resources like: # - name: efs-sc +# annotations: +# # Use that annotation if you want this to your default storageclass +# storageclass.kubernetes.io/is-default-class: "true" # mountOptions: # - tls # parameters: @@ -103,3 +119,5 @@ storageClasses: [] # gidRangeStart: "1000" # gidRangeEnd: "2000" # basePath: "/dynamic_provisioning" +# reclaimPolicy: Delete +# volumeBindingMode: Immediate diff --git a/charts/kubezero-aws-efs-csi-driver/templates/storage-class.yaml b/charts/kubezero-aws-efs-csi-driver/templates/storage-class.yaml deleted file mode 100644 index 7559e4b5..00000000 --- a/charts/kubezero-aws-efs-csi-driver/templates/storage-class.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.StorageClass.create }} -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: efs-sc - labels: -{{ include "kubezero-lib.labels" . | indent 4 }} - {{- if .Values.StorageClass.default }} - annotations: - storageclass.kubernetes.io/is-default-class: "true" - {{- end }} -provisioner: efs.csi.aws.com -{{- end }} diff --git a/charts/kubezero-aws-efs-csi-driver/update.sh b/charts/kubezero-aws-efs-csi-driver/update.sh index 3b19ef3d..a0cc453b 100755 --- a/charts/kubezero-aws-efs-csi-driver/update.sh +++ b/charts/kubezero-aws-efs-csi-driver/update.sh @@ -1,8 +1,8 @@ #!/bin/bash -VERSION=1.2.2 +VERSION=2.1.1 rm -rf charts/aws-efs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-efs-csi-driver/releases/download/helm-chart-aws-efs-csi-driver-${VERSION}/aws-efs-csi-driver-${VERSION}.tgz | tar xfz - -C charts -patch -i affinity_resources.patch -p0 --no-backup-if-mismatch +# patch -i zdt.patch -p0 --no-backup-if-mismatch diff --git a/charts/kubezero-aws-efs-csi-driver/values.yaml b/charts/kubezero-aws-efs-csi-driver/values.yaml index 5ad60883..288a630d 100644 --- a/charts/kubezero-aws-efs-csi-driver/values.yaml +++ b/charts/kubezero-aws-efs-csi-driver/values.yaml @@ -1,24 +1,44 @@ aws-efs-csi-driver: - logLevel: 1 + replicaCount: 1 - resources: - requests: - cpu: 20m - memory: 64Mi - limits: - memory: 128Mi + controller: + create: true + logLevel: 1 - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node.kubernetes.io/csi.efs.fs - operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + nodeSelector: + node-role.kubernetes.io/master: "" -StorageClass: - create: true - default: false + #extraEnv: + #- name: AWS_ROLE_ARN + # value: "" + #- name: AWS_WEB_IDENTITY_TOKEN_FILE + # value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + #- name: AWS_STS_REGIONAL_ENDPOINTS + # value: regional + + node: + logLevel: 1 + + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + memory: 128Mi + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node.kubernetes.io/csi.efs.fs + operator: Exists + + storageClasses: + - name: efs-sc #PersistentVolumes: # - name: example-pv diff --git a/charts/kubezero-aws-efs-csi-driver/zdt.patch b/charts/kubezero-aws-efs-csi-driver/zdt.patch new file mode 100644 index 00000000..36637ef2 --- /dev/null +++ b/charts/kubezero-aws-efs-csi-driver/zdt.patch @@ -0,0 +1,23 @@ +Only in charts/aws-efs-csi-driver.zdt/: .values.yaml.swp +diff -rtub charts/aws-efs-csi-driver/templates/node-daemonset.yaml charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml +--- charts/aws-efs-csi-driver/templates/node-daemonset.yaml 2021-06-14 23:12:20.000000000 +0200 ++++ charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml 2021-06-25 12:29:05.279549688 +0200 +@@ -40,15 +40,10 @@ + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} ++ {{- with .Values.node.affinity }} + affinity: +- nodeAffinity: +- requiredDuringSchedulingIgnoredDuringExecution: +- nodeSelectorTerms: +- - matchExpressions: +- - key: eks.amazonaws.com/compute-type +- operator: NotIn +- values: +- - fargate ++ {{- . | toYaml | nindent 8 }} ++ {{- end }} + hostNetwork: true + dnsPolicy: {{ .Values.node.dnsPolicy }} + {{- with .Values.node.dnsConfig }}