feat: aws-ebs and aws-efs csi driver upgrades, migrate to use service account tokens for AWS access

charts/kubezero-aws-ebs-csi-driver/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: kubezero-aws-ebs-csi-driver
 description: KubeZero Umbrella Chart for aws-ebs-csi-driver
 type: application
-version: 0.5.1
+version: 0.6.0
-appVersion: 0.10.0
+appVersion: 1.2.3
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 sources:
@@ -18,8 +18,8 @@ maintainers:
   - name: Quarky9
 dependencies:
   - name: aws-ebs-csi-driver
-    version: 0.10.0
+    version: 1.2.3
-    repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver
+    # repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver
   - name: kubezero-lib
     version: ">= 0.1.3"
     repository: https://zero-down-time.github.io/kubezero/

charts/kubezero-aws-ebs-csi-driver/README.md

@@ -1,6 +1,6 @@
 # kubezero-aws-ebs-csi-driver
 
-![Version: 0.5.1](https://img.shields.io/badge/Version-0.5.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.10.0](https://img.shields.io/badge/AppVersion-0.10.0-informational?style=flat-square)
+![Version: 0.6.0](https://img.shields.io/badge/Version-0.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.3](https://img.shields.io/badge/AppVersion-1.2.3-informational?style=flat-square)
 
 KubeZero Umbrella Chart for aws-ebs-csi-driver
 
@@ -23,7 +23,7 @@ Kubernetes: `>= 1.18.0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://kubernetes-sigs.github.io/aws-ebs-csi-driver | aws-ebs-csi-driver | 0.10.0 |
+|  | aws-ebs-csi-driver | 1.2.3 |
 | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 |
 
 ## IAM Role
@@ -41,17 +41,16 @@ This class is by default also set as default storage class.
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
-| aws-ebs-csi-driver.enableVolumeResizing | bool | `true` |  |
+| aws-ebs-csi-driver.controller.logLevel | int | `1` |  |
-| aws-ebs-csi-driver.enableVolumeScheduling | bool | `true` |  |
+| aws-ebs-csi-driver.controller.nodeSelector."node-role.kubernetes.io/master" | string | `""` |  |
+| aws-ebs-csi-driver.controller.replicaCount | int | `1` |  |
+| aws-ebs-csi-driver.controller.resources.limits.memory | string | `"40Mi"` |  |
+| aws-ebs-csi-driver.controller.resources.requests.cpu | string | `"10m"` |  |
+| aws-ebs-csi-driver.controller.resources.requests.memory | string | `"24Mi"` |  |
+| aws-ebs-csi-driver.controller.tolerations[0].effect | string | `"NoSchedule"` |  |
+| aws-ebs-csi-driver.controller.tolerations[0].key | string | `"node-role.kubernetes.io/master"` |  |
 | aws-ebs-csi-driver.enableVolumeSnapshot | bool | `true` |  |
-| aws-ebs-csi-driver.extraVolumeTags | object | `{}` | Optional tags to be added to each EBS volume |
-| aws-ebs-csi-driver.logLevel | int | `1` |  |
 | aws-ebs-csi-driver.nodeSelector."node-role.kubernetes.io/master" | string | `""` |  |
-| aws-ebs-csi-driver.podAnnotations | object | `{}` | iam.amazonaws.com/role: <IAM role ARN> to assume |
-| aws-ebs-csi-driver.replicaCount | int | `1` |  |
-| aws-ebs-csi-driver.resources.limits.memory | string | `"40Mi"` |  |
-| aws-ebs-csi-driver.resources.requests.cpu | string | `"10m"` |  |
-| aws-ebs-csi-driver.resources.requests.memory | string | `"24Mi"` |  |
 | aws-ebs-csi-driver.storageClasses[0].allowVolumeExpansion | bool | `true` |  |
 | aws-ebs-csi-driver.storageClasses[0].name | string | `"ebs-sc-gp2-xfs"` |  |
 | aws-ebs-csi-driver.storageClasses[0].parameters."csi.storage.k8s.io/fstype" | string | `"xfs"` |  |

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
-appVersion: 0.10.0
+appVersion: 1.1.0
 description: A Helm chart for AWS EBS CSI Driver
 home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver
 keywords:
@@ -15,4 +15,4 @@ maintainers:
 name: aws-ebs-csi-driver
 sources:
 - https://github.com/kubernetes-sigs/aws-ebs-csi-driver
-version: 0.10.0
+version: 1.2.3

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt

@@ -1,3 +1,39 @@
 To verify that aws-ebs-csi-driver has started, run:
 
     kubectl get pod -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}"
+
+
+WARNING: The following values have been deprecated in favor of moving them into the controller or node groups.  They will be removed in a subsequent release.
+
+affinity:
+extraCreateMetadata:
+extraVolumeTags:
+k8sTagClusterId:
+nodeSelector:
+podAnnotations:
+priorityClassName:
+region:
+replicaCount:
+resources:
+tolerations:
+topologySpreadConstraints:
+volumeAttachLimit:
+
+are moving to
+
+controller:
+  affinity:
+  extraCreateMetadata:
+  extraVolumeTags:
+  k8sTagClusterId:
+  nodeSelector:
+  podAnnotations:
+  priorityClassName:
+  region:
+  replicaCount:
+  resources:
+  tolerations:
+  topologySpreadConstraints:
+
+node:
+  volumeAttachLimit:

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl

@@ -59,11 +59,24 @@ app.kubernetes.io/instance: {{ .Release.Name }}
 Convert the `--extra-volume-tags` command line arg from a map.
 */}}
 {{- define "aws-ebs-csi-driver.extra-volume-tags" -}}
+{{- $evt := default .Values.extraVolumeTags .Values.controller.extraVolumeTags }}
 {{- $result := dict "pairs" (list) -}}
-{{- range $key, $value := .Values.extraVolumeTags -}}
+{{- range $key, $value := $evt -}}
 {{- $noop := printf "%s=%s" $key $value | append $result.pairs | set $result "pairs" -}}
 {{- end -}}
 {{- if gt (len $result.pairs) 0 -}}
 {{- printf "%s=%s" "- --extra-volume-tags" (join "," $result.pairs) -}}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Handle http proxy env vars
+*/}}
+{{- define "aws-ebs-csi-driver.http-proxy" -}}
+- name: HTTP_PROXY
+  value: {{ .Values.proxy.http_proxy | quote }}
+- name: HTTPS_PROXY
+  value: {{ .Values.proxy.http_proxy | quote }}
+- name: NO_PROXY
+  value: {{ .Values.proxy.no_proxy | quote }}
+{{- end -}}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml

@@ -0,0 +1,11 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-node-role
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get"]

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml

@@ -1,4 +1,3 @@
-{{- if .Values.enableVolumeResizing }}
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -30,4 +29,3 @@ rules:
   - apiGroups: [ "" ]
     resources: [ "pods" ]
     verbs: [ "get", "list", "watch" ]
-{{- end}}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml

@@ -1,4 +1,3 @@
-{{- if .Values.enableVolumeSnapshot }}
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -31,5 +30,3 @@ rules:
   - apiGroups: [ "snapshot.storage.k8s.io" ]
     resources: [ "volumesnapshots/status" ]
     verbs: [ "update" ]
-
-{{- end }}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml

@@ -1,4 +1,3 @@
-{{- if .Values.enableVolumeSnapshot }}
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -22,4 +21,3 @@ rules:
   - apiGroups: [ "snapshot.storage.k8s.io" ]
     resources: [ "volumesnapshotcontents/status" ]
     verbs: [ "update" ]
-{{- end }}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml

@@ -0,0 +1,15 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-node-getter-binding
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.serviceAccount.node.name }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: ebs-csi-node-role
+  apiGroup: rbac.authorization.k8s.io

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml

@@ -1,4 +1,3 @@
-{{- if .Values.enableVolumeResizing }}
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -14,5 +13,3 @@ roleRef:
   kind: ClusterRole
   name: ebs-external-resizer-role
   apiGroup: rbac.authorization.k8s.io
-
-{{- end}}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml

@@ -1,4 +1,3 @@
-{{- if .Values.enableVolumeSnapshot }}
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -14,5 +13,3 @@ roleRef:
   kind: ClusterRole
   name: ebs-snapshot-controller-role
   apiGroup: rbac.authorization.k8s.io
-
-{{- end }}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml

@@ -1,4 +1,3 @@
-{{- if .Values.enableVolumeSnapshot }}
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -14,5 +13,3 @@ roleRef:
   kind: ClusterRole
   name: ebs-external-snapshotter-role
   apiGroup: rbac.authorization.k8s.io
-
-{{- end }}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml

@@ -6,7 +6,7 @@ metadata:
   labels:
     {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
 spec:
-  replicas: {{ .Values.replicaCount }}
+  replicas: {{ default .Values.replicaCount .Values.controller.replicaCount }}
   selector:
     matchLabels:
       app: ebs-csi-controller
@@ -16,39 +16,45 @@ spec:
       labels:
         app: ebs-csi-controller
         {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }}
-      {{- if .Values.podAnnotations }}
+        {{- if .Values.controller.podLabels }}
-      annotations: {{ toYaml .Values.podAnnotations | nindent 8 }}
+        {{- toYaml .Values.controller.podLabels | nindent 8 }}
+        {{- end }}
+      {{- if .Values.controller.podAnnotations }}
+      annotations:
+        {{- toYaml .Values.controller.podAnnotations | nindent 8 }}
+      {{- else if .Values.podAnnotations}}
+      annotations:
+        {{- toYaml .Values.podAnnotations | nindent 8 }}
       {{- end }}
     spec:
       nodeSelector:
         kubernetes.io/os: linux
-        {{- with .Values.nodeSelector }}
+        {{- with default .Values.nodeSelector .Values.controller.nodeSelector }}
-{{ toYaml . | indent 8 }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
       serviceAccountName: {{ .Values.serviceAccount.controller.name }}
-      priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }}
+      priorityClassName: {{ default .Values.priorityClassName .Values.controller.priorityClassName }}
-      {{- with .Values.affinity }}
+      {{- with default .Values.affinity .Values.controller.affinity }}
-      affinity: {{ toYaml . | nindent 8 }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
       {{- end }}
       tolerations:
-        {{- if .Values.tolerateAllTaints }}
-        - operator: Exists
-        {{- else }}
         - key: CriticalAddonsOnly
           operator: Exists
         - operator: Exists
           effect: NoExecute
           tolerationSeconds: 300
+        {{- with default .Values.tolerations .Values.controller.tolerations }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
-        {{- with .Values.tolerations }}
+      {{- if or .Values.controller.topologySpreadConstraints .Values.topologySpreadConstraints }}
-{{ toYaml . | indent 8 }}
-        {{- end }}
-{{- if .Values.topologySpreadConstraints }}
       {{- $tscLabelSelector := dict "labelSelector" ( dict "matchLabels" ( dict "app" "ebs-csi-controller" ) ) }}
-      topologySpreadConstraints:
+      {{- $constraints := list }}
-        {{- range .Values.topologySpreadConstraints }}
+      {{- range default .Values.topologySpreadConstraints .Values.controller.topologySpreadConstraints }}
-        - {{ mergeOverwrite . $tscLabelSelector | toJson }}
+        {{- $constraints = mustAppend $constraints (mergeOverwrite . $tscLabelSelector) }}
       {{- end }}
+      topologySpreadConstraints:
+        {{- $constraints | toYaml | nindent 8 }}
       {{- end }}
       containers:
         - name: ebs-plugin
@@ -61,17 +67,24 @@ spec:
             # - {all,controller,node} # specify the driver mode
             {{- end }}
             - --endpoint=$(CSI_ENDPOINT)
-            {{- if .Values.extraVolumeTags }}
+            {{- if or .Values.controller.extraVolumeTags .Values.extraVolumeTags }}
               {{- include "aws-ebs-csi-driver.extra-volume-tags" . | nindent 12 }}
             {{- end }}
-            {{- if .Values.k8sTagClusterId }}
+            {{- with default .Values.k8sTagClusterId .Values.controller.k8sTagClusterId }}
-            - --k8s-tag-cluster-id={{ .Values.k8sTagClusterId }}
+            - --k8s-tag-cluster-id={{ . }}
+            {{- end }}
+            {{- with .Values.controller.httpEndpoint }}
+            - --http-endpoint={{ . }}
             {{- end }}
             - --logtostderr
-            - --v={{ .Values.logLevel }}
+            - --v={{ .Values.controller.logLevel }}
           env:
             - name: CSI_ENDPOINT
               value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: CSI_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
             - name: AWS_ACCESS_KEY_ID
               valueFrom:
                 secretKeyRef:
@@ -84,9 +97,9 @@ spec:
                   name: aws-secret
                   key: access_key
                   optional: true
-            {{- if .Values.region }}
+            {{- with default .Values.region .Values.controller.region }}
             - name: AWS_REGION
-              value: {{ .Values.region }}
+              value: {{ . }}
             {{- end }}
             {{- if .Values.controller.extraVars }}
             {{- range $key, $val :=  .Values.controller.extraVars }}
@@ -95,16 +108,17 @@ spec:
             {{- end }}
             {{- end }}
             {{- if .Values.proxy.http_proxy }}
-            - name: HTTP_PROXY
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- end }}
-            - name: HTTPS_PROXY
+            {{- with .Values.controller.env.ebsPlugin }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- . | toYaml | nindent 12 }}
-            - name: NO_PROXY
-              value: {{ .Values.proxy.no_proxy | quote }}
             {{- end }}
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: aws-token
+              mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/"
+              readOnly: true
           ports:
             - name: healthz
               containerPort: 9808
@@ -125,110 +139,101 @@ spec:
             timeoutSeconds: 3
             periodSeconds: 10
             failureThreshold: 5
-          {{- with .Values.resources }}
+          {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.ebsPlugin) }}
-          resources: {{ toYaml . | nindent 12 }}
+          resources:
+            {{- toYaml . | nindent 12 }}
           {{- end }}
         - name: csi-provisioner
           image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }}
           args:
             - --csi-address=$(ADDRESS)
-            - --v={{ .Values.logLevel }}
+            - --v={{ .Values.controller.logLevel }}
-            {{- if .Values.enableVolumeScheduling }}
             - --feature-gates=Topology=true
-            {{- end}}
+            {{- if or .Values.controller.extraCreateMetadata .Values.extraCreateMetadata }}
-            {{- if .Values.extraCreateMetadata }}
             - --extra-create-metadata
             {{- end}}
-            - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }}
+            - --leader-election=true
             - --default-fstype=ext4
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
             {{- if .Values.proxy.http_proxy }}
-            - name: HTTP_PROXY
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- end }}
-            - name: HTTPS_PROXY
+            {{- with .Values.controller.env.provisioner }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- . | toYaml | nindent 12 }}
-            - name: NO_PROXY
-              value: {{ .Values.proxy.no_proxy | quote }}
             {{- end }}
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
-          {{- with .Values.resources }}
+          {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.provisioner) }}
-          resources: {{ toYaml . | nindent 12 }}
+          resources:
+            {{- toYaml . | nindent 12 }}
           {{- end }}
         - name: csi-attacher
           image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }}
           args:
             - --csi-address=$(ADDRESS)
-            - --v={{ .Values.logLevel }}
+            - --v={{ .Values.controller.logLevel }}
-            - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }}
+            - --leader-election=true
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
             {{- if .Values.proxy.http_proxy }}
-            - name: HTTP_PROXY
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- end }}
-            - name: HTTPS_PROXY
+            {{- with .Values.controller.env.attacher }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- . | toYaml | nindent 12 }}
-            - name: NO_PROXY
-              value: {{ .Values.proxy.no_proxy | quote }}
             {{- end }}
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
-          {{- with .Values.resources }}
+          {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.attacher) }}
-          resources: {{ toYaml . | nindent 12 }}
+          resources:
+            {{- toYaml . | nindent 12 }}
           {{- end }}
-        {{- if .Values.enableVolumeSnapshot }}
         - name: csi-snapshotter
           image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }}
           args:
             - --csi-address=$(ADDRESS)
-            - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }}
+            - --leader-election=true
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
             {{- if .Values.proxy.http_proxy }}
-            - name: HTTP_PROXY
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- end }}
-            - name: HTTPS_PROXY
+            {{- with .Values.controller.env.snapshotter }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- . | toYaml | nindent 12 }}
-            - name: NO_PROXY
-              value: {{ .Values.proxy.no_proxy | quote }}
             {{- end }}
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
-          {{- with .Values.resources }}
+          {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.snapshotter) }}
-          resources: {{ toYaml . | nindent 12 }}
+          resources:
+            {{- toYaml . | nindent 12 }}
           {{- end }}
-        {{- end }}
-        {{- if .Values.enableVolumeResizing }}
         - name: csi-resizer
           image: {{ printf "%s:%s" .Values.sidecars.resizerImage.repository .Values.sidecars.resizerImage.tag }}
           imagePullPolicy: Always
           args:
             - --csi-address=$(ADDRESS)
-            - --v={{ .Values.logLevel }}
+            - --v={{ .Values.controller.logLevel }}
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
             {{- if .Values.proxy.http_proxy }}
-            - name: HTTP_PROXY
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- end }}
-            - name: HTTPS_PROXY
+            {{- with .Values.controller.env.resizer }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- . | toYaml | nindent 12 }}
-            - name: NO_PROXY
-              value: {{ .Values.proxy.no_proxy | quote }}
             {{- end }}
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
-          {{- with .Values.resources }}
+          {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.resizer) }}
-          resources: {{ toYaml . | nindent 12 }}
+          resources:
-          {{- end }}
+            {{- toYaml . | nindent 12 }}
           {{- end }}
         - name: liveness-probe
           image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }}
@@ -237,8 +242,9 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
-          {{- with .Values.resources }}
+          {{- with default .Values.resources (default .Values.controller.resources .Values.controller.containerResources.liveness) }}
-          resources: {{ toYaml . | nindent 12 }}
+          resources:
+            {{- toYaml . | nindent 12 }}
           {{- end }}
       {{- if .Values.imagePullSecrets }}
       imagePullSecrets:
@@ -249,3 +255,10 @@ spec:
       volumes:
         - name: socket-dir
           emptyDir: {}
+        - name: aws-token
+          projected:
+            sources:
+            - serviceAccountToken:
+                path: token
+                expirationSeconds: 86400
+                audience: "sts.amazonaws.com"

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml

@@ -1,4 +1,4 @@
-apiVersion: storage.k8s.io/v1beta1
+apiVersion: {{ ternary "storage.k8s.io/v1" "storage.k8s.io/v1beta1" (semverCompare ">=1.18.0-0" .Capabilities.KubeVersion.Version) }}
 kind: CSIDriver
 metadata:
   name: ebs.csi.aws.com

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml

@@ -15,8 +15,12 @@ spec:
       labels:
         app: ebs-csi-node
         {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }}
-      {{- if .Values.node.podAnnotations }}
+        {{- if .Values.node.podLabels }}
-      annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }}
+        {{- toYaml .Values.node.podLabels | nindent 8 }}
+        {{- end }}
+      {{- with .Values.node.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
       {{- end }}
     spec:
       affinity:
@@ -31,11 +35,10 @@ spec:
       nodeSelector:
         kubernetes.io/os: linux
         {{- with .Values.node.nodeSelector }}
-{{ toYaml . | indent 8 }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
-      hostNetwork: true
       serviceAccountName: {{ .Values.serviceAccount.node.name }}
-      priorityClassName: {{ .Values.node.priorityClassName | default "system-cluster-critical" }}
+      priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }}
       tolerations:
         {{- if .Values.node.tolerateAllTaints }}
         - operator: Exists
@@ -47,7 +50,7 @@ spec:
           tolerationSeconds: 300
         {{- end }}
         {{- with .Values.node.tolerations }}
-{{ toYaml . | indent 8 }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
       containers:
         - name: ebs-plugin
@@ -57,21 +60,23 @@ spec:
           args:
             - node
             - --endpoint=$(CSI_ENDPOINT)
-            {{- if .Values.volumeAttachLimit }}
+            {{- with default .Values.volumeAttachLimit .Values.node.volumeAttachLimit }}
-            - --volume-attach-limit={{ .Values.volumeAttachLimit }}
+            - --volume-attach-limit={{ . }}
             {{- end }}
             - --logtostderr
-            - --v={{ .Values.logLevel }}
+            - --v={{ .Values.node.logLevel }}
           env:
             - name: CSI_ENDPOINT
               value: unix:/csi/csi.sock
+            - name: CSI_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
             {{- if .Values.proxy.http_proxy }}
-            - name: HTTP_PROXY
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- end }}
-            - name: HTTPS_PROXY
+            {{- with .Values.node.env.ebsPlugin }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- . | toYaml | nindent 12 }}
-            - name: NO_PROXY
-              value: {{ .Values.proxy.no_proxy | quote }}
             {{- end }}
           volumeMounts:
             - name: kubelet-dir
@@ -93,47 +98,35 @@ spec:
             timeoutSeconds: 3
             periodSeconds: 10
             failureThreshold: 5
-          {{- if .Values.node.resources }}
+          {{- with default .Values.resources (default .Values.node.resources .Values.node.containerResources.ebsPlugin) }}
-          {{- with .Values.node.resources }}
+          resources:
-          resources: {{ toYaml . | nindent 12 }}
+            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- else }}
-          {{- with .Values.resources }}
-          resources: {{ toYaml . | nindent 12 }}
-          {{- end }}
           {{- end }}
         - name: node-driver-registrar
           image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }}
           args:
             - --csi-address=$(ADDRESS)
             - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
-            - --v={{ .Values.logLevel }}
+            - --v={{ .Values.node.logLevel }}
           env:
             - name: ADDRESS
               value: /csi/csi.sock
             - name: DRIVER_REG_SOCK_PATH
               value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock
             {{- if .Values.proxy.http_proxy }}
-            - name: HTTP_PROXY
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- end }}
-            - name: HTTPS_PROXY
+            {{- with .Values.node.env.nodeDriverRegistrar }}
-              value: {{ .Values.proxy.http_proxy | quote }}
+            {{- . | toYaml | nindent 12 }}
-            - name: NO_PROXY
-              value: {{ .Values.proxy.no_proxy | quote }}
             {{- end }}
           volumeMounts:
             - name: plugin-dir
               mountPath: /csi
             - name: registration-dir
               mountPath: /registration
-          {{- if .Values.node.resources }}
+          {{- with default .Values.resources (default .Values.node.resources .Values.node.containerResources.nodeDriverRegistrar) }}
-          {{- with .Values.node.resources }}
+          resources:
-          resources: {{ toYaml . | nindent 12 }}
+            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- else }}
-          {{- with .Values.resources }}
-          resources: {{ toYaml . | nindent 12 }}
-          {{- end }}
           {{- end }}
         - name: liveness-probe
           image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }}
@@ -142,14 +135,9 @@ spec:
           volumeMounts:
             - name: plugin-dir
               mountPath: /csi
-          {{- if .Values.node.resources }}
+          {{- with default .Values.resources (default .Values.node.resources .Values.node.containerResources.liveness) }}
-          {{- with .Values.node.resources }}
+          resources:
-          resources: {{ toYaml . | nindent 12 }}
+            {{- toYaml . | nindent 12 }}
-          {{- end }}
-          {{- else }}
-          {{- with .Values.resources }}
-          resources: {{ toYaml . | nindent 12 }}
-          {{- end }}
           {{- end }}
       {{- if .Values.imagePullSecrets }}
       imagePullSecrets:
@@ -160,15 +148,15 @@ spec:
       volumes:
         - name: kubelet-dir
           hostPath:
-            path: /var/lib/kubelet
+            path: {{ .Values.node.kubeletPath }}
             type: Directory
         - name: plugin-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/ebs.csi.aws.com/
+            path: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }}
             type: DirectoryOrCreate
         - name: registration-dir
           hostPath:
-            path: /var/lib/kubelet/plugins_registry/
+            path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }}
             type: Directory
         - name: device-dir
           hostPath:

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml

@@ -0,0 +1,17 @@
+{{- $replicas := (default .Values.replicaCount .Values.controller.replicaCount) | int }}
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  name: ebs-csi-controller
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }}
+  {{- if le $replicas 2 }}
+  maxUnavailable: 1
+  {{- else }}
+  minAvailable: 2
+{{- end }}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-snapshot-controller.yaml

@@ -0,0 +1,14 @@
+{{- if .Values.enableVolumeSnapshot }}
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  name: ebs-snapshot-controller
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app: ebs-snapshot-controller
+      {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }}
+  maxUnavailable: 1
+{{- end }}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml

@@ -1,4 +1,3 @@
-{{- if .Values.enableVolumeSnapshot }}
 ---
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
@@ -10,5 +9,3 @@ rules:
   - apiGroups: [ "coordination.k8s.io" ]
     resources: [ "leases" ]
     verbs: [ "get", "watch", "list", "delete", "update", "create" ]
-
-{{- end }}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml

@@ -1,4 +1,3 @@
-{{- if .Values.enableVolumeSnapshot }}
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -14,5 +13,3 @@ roleRef:
   kind: Role
   name: ebs-snapshot-controller-leaderelection
   apiGroup: rbac.authorization.k8s.io
-
-{{- end }}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml

@@ -6,7 +6,8 @@ metadata:
   labels:
     {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
   {{- with .Values.serviceAccount.controller.annotations }}
-  annotations: {{ toYaml . | nindent 4 }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
   {{- end }}
   {{- if eq .Release.Name "kustomize" }}
   #Enable if EKS IAM for SA is used

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml

@@ -6,6 +6,7 @@ metadata:
   labels:
     {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
   {{- with .Values.serviceAccount.node.annotations }}
-  annotations: {{ toYaml . | nindent 4 }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
   {{- end }}
 {{- end -}}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml

@@ -1,4 +1,3 @@
-{{- if .Values.enableVolumeSnapshot }}
 {{- if .Values.serviceAccount.snapshot.create }}
 ---
 apiVersion: v1
@@ -8,7 +7,7 @@ metadata:
   labels:
     {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
   {{- with .Values.serviceAccount.snapshot.annotations }}
-  annotations: {{ toYaml . | nindent 4 }}
+  annotations:
-  {{- end }}
+    {{- toYaml . | nindent 4 }}
   {{- end }}
 {{- end }}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml

@@ -18,41 +18,46 @@ spec:
       labels:
         app: ebs-snapshot-controller
         {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }}
+        {{- if .Values.snapshotController.podLabels }}
+        {{- toYaml .Values.snapshotController.podLabels | nindent 8 }}
+        {{- end }}
+      {{- if .Values.snapshotController.podAnnotations }}
+      annotations: {{ toYaml .Values.snapshotController.podAnnotations | nindent 8 }}
+      {{- end }}
     spec:
       serviceAccountName: {{ .Values.serviceAccount.snapshot.name }}
       nodeSelector:
         kubernetes.io/os: linux
         {{- with .Values.nodeSelector }}
-{{ toYaml . | indent 8 }}
+          {{- toYaml . | nindent 8 }}
         {{- end }}
       priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }}
       {{- with .Values.affinity }}
-      affinity: {{ toYaml . | nindent 8 }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
       {{- end }}
       tolerations:
-        {{- if .Values.tolerateAllTaints }}
+        - key: CriticalAddonsOnly
+          operator: Exists
         - operator: Exists
-        {{- end }}
+          effect: NoExecute
+          tolerationSeconds: 300
         {{- with .Values.tolerations }}
-{{ toYaml . | indent 8 }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
       containers:
         - name: snapshot-controller
           image: {{ printf "%s:%s" .Values.snapshotController.repository .Values.snapshotController.tag }}
           {{- with .Values.resources }}
-          resources: {{ toYaml . | nindent 12 }}
+          resources:
+            {{- toYaml . | nindent 12 }}
           {{- end }}
-          env:
           {{- if .Values.proxy.http_proxy }}
-          - name: HTTP_PROXY
+          env:
-            value: {{ .Values.proxy.http_proxy | quote }}
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
-          - name: HTTPS_PROXY
-            value: {{ .Values.proxy.http_proxy | quote }}
-          - name: NO_PROXY
-            value: {{ .Values.proxy.no_proxy | quote }}
           {{- end }}
           args:
-            - --v={{ .Values.logLevel }}
+            - --v=2
             - --leader-election=false
       {{- if .Values.imagePullSecrets }}
       imagePullSecrets:

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml

@@ -4,11 +4,11 @@ kind: StorageClass
 apiVersion: storage.k8s.io/v1
 metadata:
   name: {{ .name }}
-  {{- if .annotations }}
+  {{- with .annotations }}
-  annotations: {{- .annotations | toYaml | trim | nindent 4 }}
+  annotations: {{- . | toYaml | trim | nindent 4 }}
   {{- end }}
-  {{- if .labels }}
+  {{- with .labels }}
-  labels: {{- .labels | toYaml | trim | nindent 4 }}
+  labels: {{- . | toYaml | trim | nindent 4 }}
   {{- end }}
 provisioner: ebs.csi.aws.com
 {{ omit (dict "volumeBindingMode" "WaitForFirstConsumer" | merge .) "name" "annotations" "labels" | toYaml }}

charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml

@@ -2,22 +2,18 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-replicaCount: 2
-
 image:
   repository: k8s.gcr.io/provider-aws/aws-ebs-csi-driver
-  tag: "v0.10.0"
+  tag: "v1.1.0"
   pullPolicy: IfNotPresent
 
-logLevel: 5
-
 sidecars:
   provisionerImage:
     repository: k8s.gcr.io/sig-storage/csi-provisioner
-    tag: "v2.0.2"
+    tag: "v2.1.1"
   attacherImage:
     repository: k8s.gcr.io/sig-storage/csi-attacher
-    tag: "v3.0.0"
+    tag: "v3.1.0"
   snapshotterImage:
     repository: k8s.gcr.io/sig-storage/csi-snapshotter
     tag: "v3.0.3"
@@ -29,53 +25,93 @@ sidecars:
     tag: "v1.0.0"
   nodeDriverRegistrarImage:
     repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar
-    tag: "v2.0.1"
+    tag: "v2.1.0"
 
 snapshotController:
   repository: k8s.gcr.io/sig-storage/snapshot-controller
   tag: "v3.0.3"
+  podAnnotations: {}
+  podLabels: {}
 
-proxy: {}
+proxy:
-#  http_proxy:
+  http_proxy:
-#  no_proxy:
+  no_proxy:
 
 imagePullSecrets: []
-nameOverride: ""
+nameOverride:
-fullnameOverride: ""
+fullnameOverride:
-
-podAnnotations: {}
-
-# True if enable volume scheduling for dynamic volume provisioning
-enableVolumeScheduling: true
-
-# True if enable volume resizing
-enableVolumeResizing: false
 
 # True if enable volume snapshot
 enableVolumeSnapshot: false
 
-# The "maximum number of attachable volumes" per node
+# Moving to values under controller
-volumeAttachLimit: ""
+affinity: {}
+extraCreateMetadata: true
+extraVolumeTags: {}
+k8sTagClusterId:
+nodeSelector: {}
+podAnnotations: {}
+priorityClassName: "system-cluster-critical"
+region:
+replicaCount: 2
+resources: {}
+tolerations: []
+topologySpreadConstraints: []
 
-resources:
+controller:
-  {}
+  affinity: {}
+  # True if enable volume scheduling for dynamic volume provisioning
+  env:
+    ebsPlugin: []
+    provisioner: []
+    attacher: []
+    snapshotter: []
+    resizer: []
+  # If set, add pv/pvc metadata to plugin create requests as parameters.
+  extraCreateMetadata: true
+  # Will be removed in later version in favor of env.ebsPlugin
+  extraVars: {}
+  # Extra volume tags to attach to each dynamically provisioned volume.
+  # ---
+  # extraVolumeTags:
+  #   key1: value1
+  #   key2: value2
+  extraVolumeTags: {}
+  httpEndpoint:
+  # ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional).
+  k8sTagClusterId:
+  logLevel: 2
+  nodeSelector: {}
+  podAnnotations: {}
+  podLabels: {}
+  priorityClassName:
+  # AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata
+  # service.
+  # ---
+  # region: us-east-1
+  region:
+  replicaCount:
+  resources: {}
+  containerResources:
+    ebsPlugin: {}
+    provisioner: {}
+    attacher: {}
+    snapshotter: {}
+    resizer: {}
+    liveness: {}
   # We usually recommend not to specify default resources and to leave this as a conscious
   # choice for the user. This also increases chances charts run on environments with little
   # resources, such as Minikube. If you do want to specify resources, uncomment the following
   # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # Note that you will need to set resource requests if you want the cluster autoscaler to
+  # scale your nodes when you increase/decrease the number of ebs-csi-controller replicas.
   # limits:
   #   cpu: 100m
   #   memory: 128Mi
   # requests:
   #   cpu: 100m
   #   memory: 128Mi
-
-priorityClassName: ""
-nodeSelector: {}
-tolerateAllTaints: false
   tolerations: []
-affinity: {}
-
   # TSCs without the label selector stanza
   #
   # Example:
@@ -87,39 +123,30 @@ affinity: {}
   #  - maxSkew: 1
   #    topologyKey: kubernetes.io/hostname
   #    whenUnsatisfiable: ScheduleAnyway
-
   topologySpreadConstraints: []
 
-# Extra volume tags to attach to each dynamically provisioned volume.
+# Moving to values under node
-# ---
+# The "maximum number of attachable volumes" per node
-# extraVolumeTags:
+volumeAttachLimit:
-#   key1: value1
-#   key2: value2
-extraVolumeTags: {}
-
-# If set, add pv/pvc metadata to plugin create requests as parameters.
-extraCreateMetadata: false
-
-# ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional).
-k8sTagClusterId: ""
-
-# AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata
-# service.
-# ---
-# region: us-east-1
-region: ""
-
-# Additonal environment variables for the controller
-controller:
-  extraVars: {}
 
 node:
-  priorityClassName: ""
+  env:
+    ebsPlugin: []
+    nodeDriverRegistrar: []
+  kubeletPath: /var/lib/kubelet
+  logLevel: 2
+  priorityClassName:
   nodeSelector: {}
   podAnnotations: {}
+  podLabels: {}
   tolerateAllTaints: false
   tolerations: []
   resources: {}
+  containerResources:
+    ebsPlugin: {}
+    nodeDriverRegistrar: {}
+    liveness: {}
+  volumeAttachLimit:
 
 serviceAccount:
   controller:

charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch

@@ -1,103 +0,0 @@
-diff -rtubN aws-ebs-csi-driver.orig/templates/controller.yaml aws-ebs-csi-driver/templates/controller.yaml
---- aws-ebs-csi-driver.orig/templates/controller.yaml	2021-03-05 03:10:41.000000000 +0100
-+++ aws-ebs-csi-driver/templates/controller.yaml	2021-03-05 10:29:31.878615411 +0100
-@@ -68,7 +68,7 @@
-             - --k8s-tag-cluster-id={{ .Values.k8sTagClusterId }}
-             {{- end }}
-             - --logtostderr
--            - --v=5
-+            - --v={{ .Values.logLevel }}
-           env:
-             - name: CSI_ENDPOINT
-               value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
-@@ -126,14 +126,14 @@
-           image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }}
-           args:
-             - --csi-address=$(ADDRESS)
--            - --v=5
-+            - --v={{ .Values.logLevel }}
-             {{- if .Values.enableVolumeScheduling }}
-             - --feature-gates=Topology=true
-             {{- end}}
-             {{- if .Values.extraCreateMetadata }}
-             - --extra-create-metadata
-             {{- end}}
--            - --leader-election=true
-+            - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }}
-             - --default-fstype=ext4
-           env:
-             - name: ADDRESS
-@@ -156,8 +156,8 @@
-           image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }}
-           args:
-             - --csi-address=$(ADDRESS)
--            - --v=5
--            - --leader-election=true
-+            - --v={{ .Values.logLevel }}
-+            - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }}
-           env:
-             - name: ADDRESS
-               value: /var/lib/csi/sockets/pluginproxy/csi.sock
-@@ -180,7 +180,7 @@
-           image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }}
-           args:
-             - --csi-address=$(ADDRESS)
--            - --leader-election=true
-+            - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }}
-           env:
-             - name: ADDRESS
-               value: /var/lib/csi/sockets/pluginproxy/csi.sock
-@@ -205,7 +205,7 @@
-           imagePullPolicy: Always
-           args:
-             - --csi-address=$(ADDRESS)
--            - --v=5
-+            - --v={{ .Values.logLevel }}
-           env:
-             - name: ADDRESS
-               value: /var/lib/csi/sockets/pluginproxy/csi.sock
-diff -rtubN aws-ebs-csi-driver.orig/templates/node.yaml aws-ebs-csi-driver/templates/node.yaml
---- aws-ebs-csi-driver.orig/templates/node.yaml	2021-03-05 03:10:41.000000000 +0100
-+++ aws-ebs-csi-driver/templates/node.yaml	2021-03-05 10:30:07.391950366 +0100
-@@ -61,7 +61,7 @@
-             - --volume-attach-limit={{ .Values.volumeAttachLimit }}
-             {{- end }}
-             - --logtostderr
--            - --v=5
-+            - --v={{ .Values.logLevel }}
-           env:
-             - name: CSI_ENDPOINT
-               value: unix:/csi/csi.sock
-@@ -107,7 +107,7 @@
-           args:
-             - --csi-address=$(ADDRESS)
-             - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
--            - --v=5
-+            - --v={{ .Values.logLevel }}
-           env:
-             - name: ADDRESS
-               value: /csi/csi.sock
-diff -rtubN aws-ebs-csi-driver.orig/templates/statefulset.yaml aws-ebs-csi-driver/templates/statefulset.yaml
---- aws-ebs-csi-driver.orig/templates/statefulset.yaml	2021-03-05 03:10:41.000000000 +0100
-+++ aws-ebs-csi-driver/templates/statefulset.yaml	2021-03-05 10:29:31.881948744 +0100
-@@ -49,7 +49,7 @@
-             value: {{ .Values.proxy.no_proxy | quote }}
- {{- end }}
-           args:
--            - --v=5
-+            - --v={{ .Values.logLevel }}
-             - --leader-election=false
-       {{- if .Values.imagePullSecrets }}
-       imagePullSecrets:
-diff -rtubN aws-ebs-csi-driver.orig/values.yaml aws-ebs-csi-driver/values.yaml
---- aws-ebs-csi-driver.orig/values.yaml	2021-03-05 03:10:41.000000000 +0100
-+++ aws-ebs-csi-driver/values.yaml	2021-03-05 10:29:31.881948744 +0100
-@@ -9,6 +9,8 @@
-   tag: "v0.9.0"
-   pullPolicy: IfNotPresent
- 
-+logLevel: 5
-+
- sidecars:
-   provisionerImage:
-     repository: k8s.gcr.io/sig-storage/csi-provisioner

charts/kubezero-aws-ebs-csi-driver/patch

@@ -0,0 +1,87 @@
+diff -rtubN charts/aws-ebs-csi-driver/templates/controller.yaml charts/aws-ebs-csi-driver.zdt/templates/controller.yaml
+--- charts/aws-ebs-csi-driver/templates/controller.yaml	2021-06-17 22:22:22.000000000 +0200
++++ charts/aws-ebs-csi-driver.zdt/templates/controller.yaml	2021-06-24 16:31:37.042386198 +0200
+@@ -116,6 +116,9 @@
+           volumeMounts:
+             - name: socket-dir
+               mountPath: /var/lib/csi/sockets/pluginproxy/
++            - name: aws-token
++              mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/"
++              readOnly: true
+           ports:
+             - name: healthz
+               containerPort: 9808
+@@ -144,7 +147,7 @@
+           image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }}
+           args:
+             - --csi-address=$(ADDRESS)
+-            - --v=5
++            - --v={{ .Values.controller.logLevel }}
+             - --feature-gates=Topology=true
+             {{- if or .Values.controller.extraCreateMetadata .Values.extraCreateMetadata }}
+             - --extra-create-metadata
+@@ -171,7 +174,7 @@
+           image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }}
+           args:
+             - --csi-address=$(ADDRESS)
+-            - --v=5
++            - --v={{ .Values.controller.logLevel }}
+             - --leader-election=true
+           env:
+             - name: ADDRESS
+@@ -215,7 +218,7 @@
+           imagePullPolicy: Always
+           args:
+             - --csi-address=$(ADDRESS)
+-            - --v=5
++            - --v={{ .Values.controller.logLevel }}
+           env:
+             - name: ADDRESS
+               value: /var/lib/csi/sockets/pluginproxy/csi.sock
+@@ -252,3 +255,10 @@
+       volumes:
+         - name: socket-dir
+           emptyDir: {}
++        - name: aws-token
++          projected:
++            sources:
++            - serviceAccountToken:
++                path: token
++                expirationSeconds: 86400
++                audience: "sts.amazonaws.com"
+diff -rtubN charts/aws-ebs-csi-driver/templates/node.yaml charts/aws-ebs-csi-driver.zdt/templates/node.yaml
+--- charts/aws-ebs-csi-driver/templates/node.yaml	2021-06-17 22:22:22.000000000 +0200
++++ charts/aws-ebs-csi-driver.zdt/templates/node.yaml	2021-06-24 15:03:44.532351851 +0200
+@@ -107,7 +107,7 @@
+           args:
+             - --csi-address=$(ADDRESS)
+             - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+-            - --v=5
++            - --v={{ .Values.node.logLevel }}
+           env:
+             - name: ADDRESS
+               value: /csi/csi.sock
+diff -rtubN charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml charts/aws-ebs-csi-driver.zdt/templates/snapshot-controller.yaml
+--- charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml	2021-06-17 22:22:22.000000000 +0200
++++ charts/aws-ebs-csi-driver.zdt/templates/snapshot-controller.yaml	2021-06-24 15:04:17.999018733 +0200
+@@ -57,7 +57,7 @@
+             {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
+           {{- end }}
+           args:
+-            - --v=5
++            - --v=2
+             - --leader-election=false
+       {{- if .Values.imagePullSecrets }}
+       imagePullSecrets:
+diff -rtubN charts/aws-ebs-csi-driver/values.yaml charts/aws-ebs-csi-driver.zdt/values.yaml
+--- charts/aws-ebs-csi-driver/values.yaml	2021-06-17 22:22:22.000000000 +0200
++++ charts/aws-ebs-csi-driver.zdt/values.yaml	2021-06-22 17:08:52.501232371 +0200
+@@ -56,7 +56,7 @@
+ replicaCount: 2
+ resources: {}
+ tolerations: []
+-topologySpreadConstraints: []
++topolk8sTagClusterIdogySpreadConstraints: []
+ 
+ controller:
+   affinity: {}

charts/kubezero-aws-ebs-csi-driver/update.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
-VERSION=0.10.0
+VERSION=1.2.3
 
 rm -rf charts/aws-ebs-csi-driver
 curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts
 
-patch -d charts -i ../loglevel_leader.patch -p0 --no-backup-if-mismatch
+patch -i zdt.patch -p0 --no-backup-if-mismatch

charts/kubezero-aws-ebs-csi-driver/values.yaml

@@ -1,26 +1,26 @@
 aws-ebs-csi-driver:
+  enableVolumeSnapshot: true
+
+  controller:
     replicaCount: 1
     logLevel: 1
 
-  enableVolumeScheduling: true
-  enableVolumeResizing: true
-  enableVolumeSnapshot: true
-
     nodeSelector:
       node-role.kubernetes.io/master: ""
-
     tolerations:
     - key: node-role.kubernetes.io/master
       effect: NoSchedule
 
-  # aws-ebs-csi-driver.podAnnotations --  iam.amazonaws.com/role: <IAM role ARN> to assume
+    # k8sTagClusterId: <CLUSTER_NAME>
-  podAnnotations: {}
-  # iam.amazonaws.com/role: ''
-
-  # aws-ebs-csi-driver.extraVolumeTags -- Optional tags to be added to each EBS volume
-  extraVolumeTags: {}
-  #  Name: KubeZero-Cluster
 
+    #env:
+    #  ebsPlugin:
+      #- name: AWS_ROLE_ARN
+      #  value: "<ebs-csi-driver IAM ROLE ARN>"
+      #- name: AWS_WEB_IDENTITY_TOKEN_FILE
+      #  value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token"
+      #- name: AWS_STS_REGIONAL_ENDPOINTS
+      #  value: regional
     resources:
       requests:
         cpu: 10m
@@ -46,3 +46,10 @@ aws-ebs-csi-driver:
       csi.storage.k8s.io/fstype: xfs
       type: gp3
       encrypted: "true"
+
+  # This will probably move to snapshot later on
+  nodeSelector:
+    node-role.kubernetes.io/master: ""
+  tolerations:
+  - key: node-role.kubernetes.io/master
+    effect: NoSchedule

charts/kubezero-aws-ebs-csi-driver/zdt.patch

@@ -0,0 +1,75 @@
+diff -rtubN charts/aws-ebs-csi-driver/templates/controller.yaml charts/aws-ebs-csi-driver.zdt/templates/controller.yaml
+--- charts/aws-ebs-csi-driver/templates/controller.yaml	2021-06-17 22:22:22.000000000 +0200
++++ charts/aws-ebs-csi-driver.zdt/templates/controller.yaml	2021-06-24 16:31:37.042386198 +0200
+@@ -116,6 +116,9 @@
+           volumeMounts:
+             - name: socket-dir
+               mountPath: /var/lib/csi/sockets/pluginproxy/
++            - name: aws-token
++              mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/"
++              readOnly: true
+           ports:
+             - name: healthz
+               containerPort: 9808
+@@ -144,7 +147,7 @@
+           image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }}
+           args:
+             - --csi-address=$(ADDRESS)
+-            - --v=5
++            - --v={{ .Values.controller.logLevel }}
+             - --feature-gates=Topology=true
+             {{- if or .Values.controller.extraCreateMetadata .Values.extraCreateMetadata }}
+             - --extra-create-metadata
+@@ -171,7 +174,7 @@
+           image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }}
+           args:
+             - --csi-address=$(ADDRESS)
+-            - --v=5
++            - --v={{ .Values.controller.logLevel }}
+             - --leader-election=true
+           env:
+             - name: ADDRESS
+@@ -215,7 +218,7 @@
+           imagePullPolicy: Always
+           args:
+             - --csi-address=$(ADDRESS)
+-            - --v=5
++            - --v={{ .Values.controller.logLevel }}
+           env:
+             - name: ADDRESS
+               value: /var/lib/csi/sockets/pluginproxy/csi.sock
+@@ -252,3 +255,10 @@
+       volumes:
+         - name: socket-dir
+           emptyDir: {}
++        - name: aws-token
++          projected:
++            sources:
++            - serviceAccountToken:
++                path: token
++                expirationSeconds: 86400
++                audience: "sts.amazonaws.com"
+diff -rtubN charts/aws-ebs-csi-driver/templates/node.yaml charts/aws-ebs-csi-driver.zdt/templates/node.yaml
+--- charts/aws-ebs-csi-driver/templates/node.yaml	2021-06-17 22:22:22.000000000 +0200
++++ charts/aws-ebs-csi-driver.zdt/templates/node.yaml	2021-06-24 15:03:44.532351851 +0200
+@@ -107,7 +107,7 @@
+           args:
+             - --csi-address=$(ADDRESS)
+             - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+-            - --v=5
++            - --v={{ .Values.node.logLevel }}
+           env:
+             - name: ADDRESS
+               value: /csi/csi.sock
+diff -rtubN charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml charts/aws-ebs-csi-driver.zdt/templates/snapshot-controller.yaml
+--- charts/aws-ebs-csi-driver/templates/snapshot-controller.yaml	2021-06-17 22:22:22.000000000 +0200
++++ charts/aws-ebs-csi-driver.zdt/templates/snapshot-controller.yaml	2021-06-24 15:04:17.999018733 +0200
+@@ -57,7 +57,7 @@
+             {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
+           {{- end }}
+           args:
+-            - --v=5
++            - --v=2
+             - --leader-election=false
+       {{- if .Values.imagePullSecrets }}
+       imagePullSecrets:

charts/kubezero-aws-efs-csi-driver/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v2
 name: kubezero-aws-efs-csi-driver
 description: KubeZero Umbrella Chart for aws-efs-csi-driver
-version: 0.3.5
+version: 0.4.0
-appVersion: 1.2.0
+appVersion: 1.3.1
 kubeVersion: ">=1.18.0-0"
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@@ -21,5 +21,5 @@ dependencies:
     version: ">= 0.1.3"
     repository: https://zero-down-time.github.io/kubezero/
   - name: aws-efs-csi-driver
-    version: 1.2.2
+    version: 2.1.1
     # repository: https://kubernetes-sigs.github.io/aws-efs-csi-driver/

charts/kubezero-aws-efs-csi-driver/README.md

@@ -1,6 +1,6 @@
 # kubezero-aws-efs-csi-driver
 
-![Version: 0.3.5](https://img.shields.io/badge/Version-0.3.5-informational?style=flat-square) ![AppVersion: 1.2.0](https://img.shields.io/badge/AppVersion-1.2.0-informational?style=flat-square)
+![Version: 0.4.0](https://img.shields.io/badge/Version-0.4.0-informational?style=flat-square) ![AppVersion: 1.3.1](https://img.shields.io/badge/AppVersion-1.3.1-informational?style=flat-square)
 
 KubeZero Umbrella Chart for aws-efs-csi-driver
 
@@ -23,7 +23,7 @@ Kubernetes: `>=1.18.0-0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-|  | aws-efs-csi-driver | 1.2.2 |
+|  | aws-efs-csi-driver | 2.1.1 |
 | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 |
 
 ## Storage Class
@@ -37,11 +37,16 @@ Details also see: [Reserve PV](https://kubernetes.io/docs/concepts/storage/persi
 
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
-| StorageClass.create | bool | `true` |  |
+| aws-efs-csi-driver.controller.create | bool | `true` |  |
-| StorageClass.default | bool | `false` |  |
+| aws-efs-csi-driver.controller.logLevel | int | `1` |  |
-| aws-efs-csi-driver.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"node.kubernetes.io/csi.efs.fs"` |  |
+| aws-efs-csi-driver.controller.nodeSelector."node-role.kubernetes.io/master" | string | `""` |  |
-| aws-efs-csi-driver.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator | string | `"Exists"` |  |
+| aws-efs-csi-driver.controller.tolerations[0].effect | string | `"NoSchedule"` |  |
-| aws-efs-csi-driver.logLevel | int | `1` |  |
+| aws-efs-csi-driver.controller.tolerations[0].key | string | `"node-role.kubernetes.io/master"` |  |
-| aws-efs-csi-driver.resources.limits.memory | string | `"128Mi"` |  |
+| aws-efs-csi-driver.node.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"node.kubernetes.io/csi.efs.fs"` |  |
-| aws-efs-csi-driver.resources.requests.cpu | string | `"20m"` |  |
+| aws-efs-csi-driver.node.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator | string | `"Exists"` |  |
-| aws-efs-csi-driver.resources.requests.memory | string | `"64Mi"` |  |
+| aws-efs-csi-driver.node.logLevel | int | `1` |  |
+| aws-efs-csi-driver.node.resources.limits.memory | string | `"128Mi"` |  |
+| aws-efs-csi-driver.node.resources.requests.cpu | string | `"20m"` |  |
+| aws-efs-csi-driver.node.resources.requests.memory | string | `"64Mi"` |  |
+| aws-efs-csi-driver.replicaCount | int | `1` |  |
+| aws-efs-csi-driver.storageClasses[0].name | string | `"efs-sc"` |  |

charts/kubezero-aws-efs-csi-driver/affinity_resources.patch

@@ -1,64 +0,0 @@
-diff -tubrN charts/aws-efs-csi-driver/templates/node-daemonset.yaml charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml
---- charts/aws-efs-csi-driver/templates/node-daemonset.yaml	2021-03-23 14:34:03.000000000 +0100
-+++ charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml	2021-04-01 14:31:36.427375803 +0200
-@@ -40,15 +40,10 @@
-         {{- with .Values.nodeSelector }}
-         {{- . | toYaml | nindent 8 }}
-         {{- end }}
-+      {{- with .Values.affinity }}
-       affinity:
--        nodeAffinity:
--          requiredDuringSchedulingIgnoredDuringExecution:
--            nodeSelectorTerms:
--            - matchExpressions:
--              - key: eks.amazonaws.com/compute-type
--                operator: NotIn
--                values:
--                - fargate
-+        {{- . | toYaml | nindent 8 }}
-+      {{- end }}
-       hostNetwork: true
-       {{- if .Values.dnsPolicy }}
-       dnsPolicy: "{{ .Values.dnsPolicy }}"
-@@ -99,6 +94,10 @@
-             timeoutSeconds: 3
-             periodSeconds: 2
-             failureThreshold: 5
-+          {{- with .Values.resources }}
-+          resources:
-+            {{- . | toYaml | nindent 12 }}
-+          {{- end }}
-         - name: csi-driver-registrar
-           image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }}
-           args:
-diff -tubrN charts/aws-efs-csi-driver/values.yaml charts/aws-efs-csi-driver.zdt/values.yaml
---- charts/aws-efs-csi-driver/values.yaml	2021-03-23 14:34:03.000000000 +0100
-+++ charts/aws-efs-csi-driver.zdt/values.yaml	2021-04-01 14:37:21.290724721 +0200
-@@ -11,8 +11,8 @@
- 
- sidecars:
-   livenessProbeImage:
--    repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe
--    tag: v2.1.0-eks-1-18-1
-+    repository: k8s.gcr.io/sig-storage/livenessprobe
-+    tag: "v2.2.0"
-   nodeDriverRegistrarImage:
-     repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar
-     tag: v2.0.1-eks-1-18-1
-@@ -43,7 +43,15 @@
- 
- tolerations: []
- 
--affinity: {}
-+affinity:
-+  nodeAffinity:
-+    requiredDuringSchedulingIgnoredDuringExecution:
-+      nodeSelectorTerms:
-+      - matchExpressions:
-+        - key: eks.amazonaws.com/compute-type
-+          operator: NotIn
-+          values:
-+          - fargate
- 
- node:
-   podAnnotations: {}

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/CHANGELOG.md

@@ -0,0 +1,106 @@
+# Helm chart
+
+# v2.1.1
+* Update app/driver version to `v1.3.0`
+
+# v2.1.0
+
+## New features
+* Update app/driver version to `v1.3.0`
+
+## Bug fixes
+* Put comments back in place inside the values file ([#475](https://github.com/kubernetes-sigs/aws-efs-csi-driver/pull/475), [@pierluigilenoci](https://github.com/pierluigilenoci))
+
+# v2.0.1
+
+## Bug fixes
+* Helm chart: fix reclaimPolicy and volumeBindingMode ([#464](https://github.com/kubernetes-sigs/aws-efs-csi-driver/pull/464), [@devinsmith911](https://github.com/devinsmith911))
+
+
+# v2.0.0
+
+## Breaking changes
+
+Multiple changes in values file at `sidecars`, `controller` and `node`
+
+---
+```yaml
+sidecars:
+  xxxxxxxxx:
+    repository:
+    tag:
+```
+
+Moving to
+
+```yaml
+sidecars:
+  xxxxxxxxx:
+    image:
+      repository:
+      tag:
+```
+
+---
+```yaml
+podAnnotations:
+resources:
+nodeSelector:
+tolerations:
+affinity:
+```
+
+Moving to
+
+```yaml
+controller:
+  podAnnotations:
+  resources:
+  nodeSelector:
+  tolerations:
+  affinity:
+```
+
+---
+```yaml
+hostAliases:
+dnsPolicy:
+dnsConfig:
+```
+
+Moving to
+
+```yaml
+node:
+  hostAliases:
+  dnsPolicy:
+  dnsConfig:
+```
+
+---
+```yaml
+serviceAccount:
+  controller:
+```
+
+Moving to
+
+```yaml
+controller:
+  serviceAccount:
+```
+
+## New features
+
+* Chart API `v2` (requires Helm 3)
+* Set `resources` and `imagePullPolicy` fields independently for containers
+* Set `logLevel`, `affinity`, `nodeSelector`, `podAnnotations` and `tolerations` fields independently
+for Controller deployment and Node daemonset
+* Set `reclaimPolicy` and `volumeBindingMode` fields in storage class
+
+## Fixes
+
+* Fixing Controller deployment using `podAnnotations` and `tolerations` values from Node daemonset
+* Let the user define the whole `tolerations` array, default to `- operator: Exists`
+* Default `logLevel` lowered from `5` to `2`
+* Default `imagePullPolicy` everywhere set to `IfNotPresent`

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/Chart.yaml

@@ -1,5 +1,5 @@
-apiVersion: v1
+apiVersion: v2
-appVersion: 1.2.0
+appVersion: 1.3.1
 description: A Helm chart for AWS EFS CSI Driver
 home: https://github.com/kubernetes-sigs/aws-efs-csi-driver
 keywords:
@@ -15,4 +15,4 @@ maintainers:
 name: aws-efs-csi-driver
 sources:
 - https://github.com/kubernetes-sigs/aws-efs-csi-driver
-version: 1.2.2
+version: 2.1.1

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/_helpers.tpl

@@ -49,8 +49,19 @@ Create the name of the service account to use
 */}}
 {{- define "aws-efs-csi-driver.serviceAccountName" -}}
 {{- if .Values.controller.create -}}
-    {{ default (include "aws-efs-csi-driver.fullname" .) .Values.serviceAccount.controller.name }}
+    {{ default (include "aws-efs-csi-driver.fullname" .) .Values.controller.serviceAccount.name }}
 {{- else -}}
-    {{ default "default" .Values.serviceAccount.controller.name }}
+    {{ default "default" .Values.controller.serviceAccount.name }}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Create a string out of the map for controller tags flag
+*/}}
+{{- define "aws-efs-csi-driver.tags" -}}
+{{- $tags := list -}}
+{{ range $key, $val := . }}
+{{- $tags = print $key ":" $val | append $tags -}}
+{{- end -}}
+{{- join " " $tags -}}
+{{- end -}}

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-deployment.yaml

@@ -19,10 +19,11 @@ spec:
         app: efs-csi-controller
         app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
-      {{- if .Values.node.podAnnotations }}
+      {{- with .Values.controller.podAnnotations }}
-      annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }}
+      annotations: {{- toYaml . | nindent 8 }}
       {{- end }}
     spec:
+      hostNetwork: true 
       {{- if .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- range .Values.imagePullSecrets }}
@@ -31,37 +32,39 @@ spec:
       {{- end }}
       nodeSelector:
         kubernetes.io/os: linux
-        {{- with .Values.nodeSelector }}
+        {{- with .Values.controller.nodeSelector }}
-        {{- . | toYaml | nindent 8 }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
       serviceAccountName: {{ include "aws-efs-csi-driver.serviceAccountName" . }}
       priorityClassName: system-cluster-critical
-      tolerations:
+      {{- with .Values.controller.tolerations }}
-        - operator: Exists
+      tolerations: {{- toYaml . | nindent 8 }}
-        {{- with .Values.node.tolerations }}
-        {{- . | toYaml | nindent 8 }}
       {{- end }}
       containers:
         - name: efs-plugin
           securityContext:
             privileged: true
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-          imagePullPolicy: IfNotPresent
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
           args:
             - --endpoint=$(CSI_ENDPOINT)
             - --logtostderr
-            - --v=5
+            {{- if .Values.controller.tags }}
-            # Uncomment below line to allow access point root directory to be deleted by controller.
+            - --tags={{ include "aws-efs-csi-driver.tags" .Values.controller.tags }}
-            #- --delete-access-point-root-dir
+            {{- end }}
+            - --v={{ .Values.controller.logLevel }}
+            - --delete-access-point-root-dir={{ hasKey .Values.controller "deleteAccessPointRootDir" | ternary .Values.controller.deleteAccessPointRootDir false }}
           env:
             - name: CSI_ENDPOINT
               value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: aws-token
+              mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/"
           ports:
             - name: healthz
-              containerPort: 9808
+              containerPort: 9909
               protocol: TCP
           livenessProbe:
             httpGet:
@@ -71,28 +74,52 @@ spec:
             timeoutSeconds: 3
             periodSeconds: 10
             failureThreshold: 5
+          {{- with .Values.controller.resources }}
+          resources: {{ toYaml . | nindent 12 }}
+          {{- end }}
         - name: csi-provisioner
-          image: {{ printf "%s:%s" .Values.sidecars.csiProvisionerImage.repository .Values.sidecars.csiProvisionerImage.tag }}
+          image: {{ printf "%s:%s" .Values.sidecars.csiProvisioner.image.repository .Values.sidecars.csiProvisioner.image.tag }}
+          imagePullPolicy: {{ .Values.sidecars.csiProvisioner.image.pullPolicy }}
           args:
             - --csi-address=$(ADDRESS)
-            - --v=5
+            - --v={{ .Values.controller.logLevel }}
             - --feature-gates=Topology=true
             - --leader-election
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          {{- if .Values.controller.extraEnv }}
+{{ toYaml .Values.extraEnv | indent 12 }}
+          {{- end }}
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
+          {{- with .Values.sidecars.csiProvisioner.resources }}
+          resources: {{ toYaml . | nindent 12 }}
+          {{- end }}
         - name: liveness-probe
-          image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }}
+          image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }}
+          imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }}
           args:
             - --csi-address=/csi/csi.sock
-            - --health-port=9808
+            - --health-port=9909
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+          {{- with .Values.sidecars.livenessProbe.resources }}
+          resources: {{ toYaml . | nindent 12 }}
+          {{- end }}
       volumes:
         - name: socket-dir
           emptyDir: {}
+        - name: aws-token
+          projected:
+            sources:
+            - serviceAccountToken:
+                path: token
+                expirationSeconds: 86400
+                audience: "sts.amazonaws.com"
+      {{- with .Values.controller.affinity }}
+      affinity: {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml

@@ -1,11 +1,11 @@
-{{- if .Values.serviceAccount.controller.create }}
+{{- if .Values.controller.serviceAccount.create }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: {{ include "aws-efs-csi-driver.serviceAccountName" . }}
   labels:
     app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
-  {{- with .Values.serviceAccount.controller.annotations }}
+  {{- with .Values.controller.serviceAccount.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}
   {{- end }}
@@ -40,6 +40,9 @@ rules:
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
     verbs: ["get", "watch", "list", "delete", "update", "create"]
+  - apiGroups: [ "" ]
+    resources: [ "secrets" ]
+    verbs: [ "get", "watch", "list" ]
 
 ---
 

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/csidriver.yaml

@@ -1,4 +1,4 @@
-apiVersion: storage.k8s.io/v1beta1
+apiVersion: {{ ternary "storage.k8s.io/v1" "storage.k8s.io/v1beta1" (semverCompare ">=1.18.0-0" .Capabilities.KubeVersion.Version) }}
 kind: CSIDriver
 metadata:
   name: efs.csi.aws.com

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/node-daemonset.yaml

@@ -21,9 +21,9 @@ spec:
       annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }}
       {{- end }}
     spec:
-    {{- if .Values.hostAliases }}
+    {{- with .Values.node.hostAliases }}
       hostAliases:
-      {{- range $k, $v := .Values.hostAliases }}
+      {{- range $k, $v := . }}
         - ip: {{ $v.ip }}
           hostnames:
             - {{ $k }}.efs.{{ $v.region }}.amazonaws.com
@@ -37,36 +37,32 @@ spec:
     {{- end }}
       nodeSelector:
         beta.kubernetes.io/os: linux
-        {{- with .Values.nodeSelector }}
+        {{- with .Values.node.nodeSelector }}
-        {{- . | toYaml | nindent 8 }}
+        {{- toYaml . | nindent 8 }}
         {{- end }}
-      {{- with .Values.affinity }}
+      {{- with .Values.node.affinity }}
       affinity:
         {{- . | toYaml | nindent 8 }}
       {{- end }}
       hostNetwork: true
-      {{- if .Values.dnsPolicy }}
+      dnsPolicy: {{ .Values.node.dnsPolicy }}
-      dnsPolicy: "{{ .Values.dnsPolicy }}"
+      {{- with .Values.node.dnsConfig }}
-      {{- end }}
+      dnsConfig: {{- toYaml . | nindent 8 }}
-      {{- with .Values.dnsConfig }}
-      dnsConfig:
-        {{- toYaml . | nindent 8 }}
       {{- end }}
       priorityClassName: system-node-critical
-      tolerations:
-        - operator: Exists
       {{- with .Values.node.tolerations }}
-        {{- . | toYaml | nindent 8 }}
+      tolerations: {{- toYaml . | nindent 8 }}
       {{- end }}
       containers:
         - name: efs-plugin
           securityContext:
             privileged: true
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
           args:
             - --endpoint=$(CSI_ENDPOINT)
             - --logtostderr
-            - --v={{ .Values.logLevel }}
+            - --v={{ .Values.node.logLevel }}
           env:
             - name: CSI_ENDPOINT
               value: unix:/csi/csi.sock
@@ -94,16 +90,16 @@ spec:
             timeoutSeconds: 3
             periodSeconds: 2
             failureThreshold: 5
-          {{- with .Values.resources }}
+          {{- with .Values.node.resources }}
-          resources:
+          resources: {{ toYaml . | nindent 12 }}
-            {{- . | toYaml | nindent 12 }}
           {{- end }}
         - name: csi-driver-registrar
-          image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }}
+          image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }}
+          imagePullPolicy: {{ .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }}
           args:
             - --csi-address=$(ADDRESS)
             - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
-            - --v={{ .Values.logLevel }}
+            - --v={{ .Values.node.logLevel }}
           env:
             - name: ADDRESS
               value: /csi/csi.sock
@@ -118,15 +114,22 @@ spec:
               mountPath: /csi
             - name: registration-dir
               mountPath: /registration
+          {{- with .Values.sidecars.nodeDriverRegistrar.resources }}
+          resources: {{ toYaml . | nindent 12 }}
+          {{- end }}
         - name: liveness-probe
-          image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }}
+          image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }}
+          imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }}
           args:
             - --csi-address=/csi/csi.sock
             - --health-port=9809
-            - --v={{ .Values.logLevel }}
+            - --v={{ .Values.node.logLevel }}
           volumeMounts:
             - name: plugin-dir
               mountPath: /csi
+          {{- with .Values.sidecars.livenessProbe.resources }}
+          resources: {{ toYaml . | nindent 12 }}
+          {{- end }}
       volumes:
         - name: kubelet-dir
           hostPath:

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/templates/storageclass.yaml

@@ -3,6 +3,10 @@ kind: StorageClass
 apiVersion: storage.k8s.io/v1
 metadata:
   name: {{ .name }}
+  {{- with .annotations }}
+  annotations:
+  {{ toYaml . | indent 4 }}
+  {{- end }}
 provisioner: efs.csi.aws.com
 {{- with .mountOptions }}
 mountOptions:
@@ -12,4 +16,11 @@ mountOptions:
 parameters:
 {{ toYaml . | indent 2 }}
 {{- end }}
+{{- with .reclaimPolicy }}
+reclaimPolicy: {{ . }}
+{{- end }}
+{{- with .volumeBindingMode }}
+volumeBindingMode: {{ . }}
+{{- end }}
+---
 {{- end }}

charts/kubezero-aws-efs-csi-driver/charts/aws-efs-csi-driver/values.yaml

@@ -2,32 +2,54 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
+nameOverride: ""
+fullnameOverride: ""
+
 replicaCount: 2
 
 image:
   repository: amazon/aws-efs-csi-driver
-  tag: "v1.2.0"
+  tag: "v1.3.1"
   pullPolicy: IfNotPresent
 
 sidecars:
-  livenessProbeImage:
+  livenessProbe:
-    repository: k8s.gcr.io/sig-storage/livenessprobe
+    image:
-    tag: "v2.2.0"
+      repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe
-  nodeDriverRegistrarImage:
+      tag: v2.2.0-eks-1-18-2
+      pullPolicy: IfNotPresent
+    resources: {}
+  nodeDriverRegistrar:
+    image:
       repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar
-    tag: v2.0.1-eks-1-18-1
+      tag: v2.1.0-eks-1-18-2
-  csiProvisionerImage:
+      pullPolicy: IfNotPresent
+    resources: {}
+  csiProvisioner:
+    image:
       repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner
-    tag: v2.0.3-eks-1-18-1
+      tag: v2.1.1-eks-1-18-2
+      pullPolicy: IfNotPresent
+    resources: {}
 
 imagePullSecrets: []
-nameOverride: ""
-fullnameOverride: ""
 
+## Controller deployment variables
+
+controller:
+  # Specifies whether a deployment should be created
+  create: true
+  # Number for the log level verbosity
+  logLevel: 2
+  # Add additional tags to access points
+  tags: {}
+    # environment: prod
+    # region: us-east-1
+  # Enable if you want the controller to also delete the
+  # path on efs when deleteing an access point
+  deleteAccessPointRootDir: false
   podAnnotations: {}
-
+  resources: {}
-resources:
-  {}
     # We usually recommend not to specify default resources and to leave this as a conscious
     # choice for the user. This also increases chances charts run on environments with little
     # resources, such as Minikube. If you do want to specify resources, uncomment the following
@@ -38,30 +60,24 @@ resources:
     # requests:
     #   cpu: 100m
     #   memory: 128Mi
-
   nodeSelector: {}
-
   tolerations: []
+  affinity: {}
+  # Specifies whether a service account should be created
+  serviceAccount:
+    create: true
+    name: efs-csi-controller-sa
+    annotations: {}
+    ## Enable if EKS IAM for SA is used
+    #  eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/efs-csi-role
 
-affinity:
+## Node daemonset variables
-  nodeAffinity:
-    requiredDuringSchedulingIgnoredDuringExecution:
-      nodeSelectorTerms:
-      - matchExpressions:
-        - key: eks.amazonaws.com/compute-type
-          operator: NotIn
-          values:
-          - fargate
 
 node:
-  podAnnotations: {}
+  # Number for the log level verbosity
-  tolerations: []
+  logLevel: 2
-
+  hostAliases: {}
-logLevel: 5
+    # For cross VPC EFS, you need to poison or overwrite the DNS for the efs volume as per
-
-hostAliases:
-  {}
-  # for cross VPC EFS, you need to poison or overwrite the DNS for the efs volume as per
     # https://docs.aws.amazon.com/efs/latest/ug/efs-different-vpc.html#wt6-efs-utils-step3
     # implementing the suggested solution found here:
     # https://github.com/kubernetes-sigs/aws-efs-csi-driver/issues/240#issuecomment-676849346
@@ -69,31 +85,31 @@ hostAliases:
     # "fs-01234567":
     #   ip: 10.10.2.2
     #   region: us-east-2
-
+  dnsPolicy: ClusterFirst
-dnsPolicy: ""
+  dnsConfig: {}
-dnsConfig:
-  {}
     # Example config which uses the AWS nameservers
     # dnsPolicy: "None"
     # dnsConfig:
     #   nameservers:
     #     - 169.254.169.253
-
+  podAnnotations: {}
-serviceAccount:
+  resources: {}
-  controller:
+    # limits:
-    # Specifies whether a service account should be created
+    #   cpu: 100m
-    create: true
+    #   memory: 128Mi
-    annotations: {}
+    # requests:
-    ## Enable if EKS IAM for SA is used
+    #   cpu: 100m
-    #  eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/efs-csi-role
+    #   memory: 128Mi
-    name: efs-csi-controller-sa
+  nodeSelector: {}
-
+  tolerations:
-controller:
+    - operator: Exists
-  create: true
 
 storageClasses: []
 # Add StorageClass resources like:
 # - name: efs-sc
+#   annotations:
+#     # Use that annotation if you want this to your default storageclass
+#     storageclass.kubernetes.io/is-default-class: "true"
 #   mountOptions:
 #   - tls
 #   parameters:
@@ -103,3 +119,5 @@ storageClasses: []
 #     gidRangeStart: "1000"
 #     gidRangeEnd: "2000"
 #     basePath: "/dynamic_provisioning"
+#   reclaimPolicy: Delete
+#   volumeBindingMode: Immediate

charts/kubezero-aws-efs-csi-driver/templates/storage-class.yaml

@@ -1,13 +0,0 @@
-{{- if .Values.StorageClass.create }}
-kind: StorageClass
-apiVersion: storage.k8s.io/v1
-metadata:
-  name: efs-sc
-  labels:
-{{ include "kubezero-lib.labels" . | indent 4 }}
-  {{- if .Values.StorageClass.default }}
-  annotations:
-    storageclass.kubernetes.io/is-default-class: "true"
-  {{- end }}
-provisioner: efs.csi.aws.com
-{{- end }}

charts/kubezero-aws-efs-csi-driver/update.sh

@@ -1,8 +1,8 @@
 #!/bin/bash
 
-VERSION=1.2.2
+VERSION=2.1.1
 
 rm -rf charts/aws-efs-csi-driver
 curl -L -s -o - https://github.com/kubernetes-sigs/aws-efs-csi-driver/releases/download/helm-chart-aws-efs-csi-driver-${VERSION}/aws-efs-csi-driver-${VERSION}.tgz | tar xfz - -C charts
 
-patch -i affinity_resources.patch -p0 --no-backup-if-mismatch
+# patch -i zdt.patch -p0 --no-backup-if-mismatch

charts/kubezero-aws-efs-csi-driver/values.yaml

@@ -1,4 +1,25 @@
 aws-efs-csi-driver:
+  replicaCount: 1
+
+  controller:
+    create: true
+    logLevel: 1
+
+    tolerations:
+    - key: node-role.kubernetes.io/master
+      effect: NoSchedule
+    nodeSelector:
+      node-role.kubernetes.io/master: ""
+
+    #extraEnv:
+    #- name: AWS_ROLE_ARN
+    #  value: "<aws-efs-csi-driver IAM ROLE ARN>"
+    #- name: AWS_WEB_IDENTITY_TOKEN_FILE
+    #  value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token"
+    #- name: AWS_STS_REGIONAL_ENDPOINTS
+    #  value: regional
+
+  node:
     logLevel: 1
 
     resources:
@@ -16,9 +37,8 @@ aws-efs-csi-driver:
             - key: node.kubernetes.io/csi.efs.fs
               operator: Exists
 
-StorageClass:
+  storageClasses:
-  create: true
+  - name: efs-sc
-  default: false
 
 #PersistentVolumes:
 #  - name: example-pv

charts/kubezero-aws-efs-csi-driver/zdt.patch

@@ -0,0 +1,23 @@
+Only in charts/aws-efs-csi-driver.zdt/: .values.yaml.swp
+diff -rtub charts/aws-efs-csi-driver/templates/node-daemonset.yaml charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml
+--- charts/aws-efs-csi-driver/templates/node-daemonset.yaml	2021-06-14 23:12:20.000000000 +0200
++++ charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml	2021-06-25 12:29:05.279549688 +0200
+@@ -40,15 +40,10 @@
+         {{- with .Values.node.nodeSelector }}
+         {{- toYaml . | nindent 8 }}
+         {{- end }}
++      {{- with .Values.node.affinity }}
+       affinity:
+-        nodeAffinity:
+-          requiredDuringSchedulingIgnoredDuringExecution:
+-            nodeSelectorTerms:
+-            - matchExpressions:
+-              - key: eks.amazonaws.com/compute-type
+-                operator: NotIn
+-                values:
+-                - fargate
++        {{- . | toYaml | nindent 8 }}
++      {{- end }}
+       hostNetwork: true
+       dnsPolicy: {{ .Values.node.dnsPolicy }}
+       {{- with .Values.node.dnsConfig }}