First version of aws-ebs-csi-driver umbrella chart, updated docs

charts/kubezero-aws-ebs/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/kubezero-aws-ebs/Chart.yaml

@@ -0,0 +1,22 @@
+apiVersion: v2
+name: kubezero-aws-ebs
+description: KubeZero Umbrella Chart for aws-ebs-csi-driver
+type: application
+version: 0.1.0
+home: https://kubezero.com
+icon: https://cdn.zero-downtime.net/assets/logo_small.png
+sources:
+  - https://github.com/kubernetes-sigs/aws-ebs-csi-driver
+  - https://github.com/Zero-Down-Time/kubezero
+keywords:
+  - kubezero
+  - aws
+  - ebs
+  - csi
+maintainers:
+  - name: Quarky9
+dependencies:
+  - name: kubezero-lib
+    version: ">= 0.1.1"
+    repository: https://zero-down-time.github.io/kubezero/
+kubeVersion: ">= 1.16.0"

charts/kubezero-aws-ebs/README.md

@@ -0,0 +1,27 @@
+kubezero-aws-ebs
+================
+KubeZero Umbrella Chart for aws-ebs-csi-driver
+
+Current chart version is `0.1.0`
+
+Source code can be found [here](https://kubezero.com)
+
+## Chart Requirements
+
+| Repository | Name | Version |
+|------------|------|---------|
+| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 |
+
+## Chart Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| aws-ebs-csi-driver.enableVolumeResizing | bool | `false` |  |
+| aws-ebs-csi-driver.enableVolumeScheduling | bool | `true` |  |
+| aws-ebs-csi-driver.enableVolumeSnapshot | bool | `false` |  |
+| aws-ebs-csi-driver.extraVolumeTags | object | `{}` | Optional tags to be added to each EBS volume |
+| aws-ebs-csi-driver.nodeSelector."node-role.kubernetes.io/master" | string | `""` |  |
+| aws-ebs-csi-driver.podAnnotations | object | `{}` |  iam.amazonaws.com/role: <IAM role ARN> to assume |
+| aws-ebs-csi-driver.replicaCount | int | `1` |  |
+| aws-ebs-csi-driver.tolerations[0].effect | string | `"NoSchedule"` |  |
+| aws-ebs-csi-driver.tolerations[0].key | string | `"node-role.kubernetes.io/master"` |  |

charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/Chart.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+appVersion: "0.5.0"
+name: aws-ebs-csi-driver
+description: A Helm chart for AWS EBS CSI Driver
+version: 0.3.0
+kubeVersion: ">=1.13.0-0"
+home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver
+sources:
+  - https://github.com/kubernetes-sigs/aws-ebs-csi-driver
+keywords:
+  - aws
+  - ebs
+  - csi
+maintainers:
+  - name: leakingtapan
+    email: chengpan@amazon.com

charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/NOTES.txt

@@ -0,0 +1,3 @@
+To verify that aws-ebs-csi-driver has started, run:
+
+    kubectl get pod -n kube-system -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}"

charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/_helpers.tpl

@@ -0,0 +1,58 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "aws-ebs-csi-driver.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "aws-ebs-csi-driver.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "aws-ebs-csi-driver.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "aws-ebs-csi-driver.labels" -}}
+app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }}
+helm.sh/chart: {{ include "aws-ebs-csi-driver.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Convert the `--extra-volume-tags` command line arg from a map.
+*/}}
+{{- define "aws-ebs-csi-driver.extra-volume-tags" -}}
+{{- $result := dict "pairs" (list) -}}
+{{- range $key, $value := .Values.extraVolumeTags -}}
+{{- $noop := printf "%s=%s" $key $value | append $result.pairs | set $result "pairs" -}}
+{{- end -}}
+{{- if gt (len $result.pairs) 0 -}}
+- --extra-volume-tags={{- join "," $result.pairs -}}
+{{- end -}}
+{{- end -}}

charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/csidriver.yaml

@@ -0,0 +1,7 @@
+apiVersion: storage.k8s.io/v1beta1
+kind: CSIDriver
+metadata:
+  name: ebs.csi.aws.com
+spec:
+  attachRequired: true
+  podInfoOnMount: false

charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/daemonset.yaml

@@ -0,0 +1,108 @@
+# Node Service
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: ebs-csi-node
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: ebs-csi-node
+      app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: ebs-csi-node
+        app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+      {{- if .Values.node.podAnnotations }}
+      annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }}
+      {{- end }}
+    spec:
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      tolerations:
+        - operator: Exists
+        {{- with .Values.node.tolerations }}
+{{ toYaml . | indent 8 }}
+        {{- end }}
+      containers:
+        - name: ebs-plugin
+          securityContext:
+            privileged: true
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          args:
+            - node
+            - --endpoint=$(CSI_ENDPOINT)
+            - --logtostderr
+            - --v=5
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:/csi/csi.sock
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: plugin-dir
+              mountPath: /csi
+            - name: device-dir
+              mountPath: /dev
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
+        - name: node-driver-registrar
+          image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+            - --v=5
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"]
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: liveness-probe
+          image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }}
+          args:
+            - --csi-address=/csi/csi.sock
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+      volumes:
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/ebs.csi.aws.com/
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: device-dir
+          hostPath:
+            path: /dev
+            type: Directory

charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/deployment.yaml

@@ -0,0 +1,151 @@
+# Controller Service
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: ebs-csi-controller
+  namespace: kube-system
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: ebs-csi-controller
+        app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+      {{- if .Values.podAnnotations }}
+      annotations: {{ toYaml .Values.podAnnotations | nindent 8 }}
+      {{- end }}
+    spec:
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+        {{- with .Values.nodeSelector }}
+{{ toYaml . | indent 8 }}
+        {{- end }}
+      serviceAccountName: ebs-csi-controller-sa
+      priorityClassName: system-cluster-critical
+      {{- with .Values.affinity }}
+      affinity: {{ toYaml . | nindent 8 }}
+      {{- end }}
+      tolerations:
+        - operator: Exists
+      {{- with .Values.tolerations }}
+{{ toYaml . | indent 8 }}
+      {{- end }}
+      containers:
+        - name: ebs-plugin
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            - controller
+            - --endpoint=$(CSI_ENDPOINT)
+            {{ include "aws-ebs-csi-driver.extra-volume-tags" . }}
+            - --logtostderr
+            - --v=5
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: aws-secret
+                  key: key_id
+                  optional: true
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: aws-secret
+                  key: access_key
+                  optional: true
+            {{- if .Values.region }}
+            - name: AWS_REGION
+              value: {{ .Values.region }}
+            {{- end }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
+          {{- with .Values.resources }}
+          resources: {{ toYaml . | nindent 12 }}
+          {{- end }}
+        - name: csi-provisioner
+          image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v=5
+            {{- if .Values.enableVolumeScheduling }}
+            - --feature-gates=Topology=true
+            {{- end}}
+            - --enable-leader-election
+            - --leader-election-type=leases
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-attacher
+          image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v=5
+            - --leader-election=true
+            - --leader-election-type=leases
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        {{- if .Values.enableVolumeSnapshot }}
+        - name: csi-snapshotter
+          image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --leader-election=true
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        {{- end }}
+        {{- if .Values.enableVolumeResizing }}
+        - name: csi-resizer
+          image: {{ printf "%s:%s" .Values.sidecars.resizerImage.repository .Values.sidecars.resizerImage.tag }}
+          imagePullPolicy: Always
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v=5
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        {{- end }}
+        - name: liveness-probe
+          image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }}
+          args:
+            - --csi-address=/csi/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+      volumes:
+        - name: socket-dir
+          emptyDir: {}

charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/rbac.yaml

@@ -0,0 +1,251 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-provisioner-binding
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-provisioner-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-attacher-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["csi.storage.k8s.io"]
+    resources: ["csinodeinfos"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-attacher-binding
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-attacher-role
+  apiGroup: rbac.authorization.k8s.io
+
+{{- if .Values.enableVolumeSnapshot }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-snapshotter-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["create", "list", "watch", "delete"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-snapshotter-binding
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-snapshot-controller-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-snapshot-controller-binding
+subjects:
+  - kind: ServiceAccount
+    name: ebs-snapshot-controller
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-snapshot-controller-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-snapshot-controller-leaderelection
+  namespace: kube-system 
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-leaderelection
+  namespace: kube-system 
+subjects:
+  - kind: ServiceAccount
+    name: ebs-snapshot-controller
+    namespace: kube-system 
+roleRef:
+  kind: Role
+  name: snapshot-controller-leaderelection
+  apiGroup: rbac.authorization.k8s.io
+
+{{- end }}
+
+{{- if .Values.enableVolumeResizing }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-resizer-role
+rules:
+  # The following rule should be uncommented for plugins that require secrets
+  # for provisioning.
+  # - apiGroups: [""]
+  #   resources: ["secrets"]
+  #   verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-resizer-binding
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-resizer-role
+  apiGroup: rbac.authorization.k8s.io
+{{- end}}

charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+  {{- with .Values.serviceAccount.controller.annotations }}
+  annotations: {{ toYaml . | nindent 4 }}
+  {{- end }}
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ebs-snapshot-controller
+  namespace: kube-system
+  {{- with .Values.serviceAccount.snapshot.annotations }}
+  annotations: {{ toYaml . | nindent 4 }}
+  {{- end }}

charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/statefulset.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.enableVolumeSnapshot }}
+#Snapshot controller
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: ebs-snapshot-controller
+  namespace: kube-system
+spec:
+  serviceName: ebs-snapshot-controller
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ebs-snapshot-controller
+  template:
+    metadata:
+      labels:
+        app: ebs-snapshot-controller
+    spec:
+      serviceAccount: ebs-snapshot-controller
+      containers:
+        - name: snapshot-controller
+          image: quay.io/k8scsi/snapshot-controller:v2.0.1
+          args:
+            - --v=5
+            - --leader-election=false
+{{- end }}

charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/values.yaml

@@ -0,0 +1,86 @@
+# Default values for aws-ebs-csi-driver.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 2
+
+image:
+  repository: amazon/aws-ebs-csi-driver
+  tag: "v0.5.0"
+  pullPolicy: IfNotPresent
+
+sidecars:
+  provisionerImage:
+    repository: quay.io/k8scsi/csi-provisioner
+    tag: "v1.5.0"
+  attacherImage:
+    repository: quay.io/k8scsi/csi-attacher
+    tag: "v1.2.0"
+  snapshotterImage:
+    repository: quay.io/k8scsi/csi-snapshotter
+    tag: "v2.0.1"
+  livenessProbeImage:
+    repository: quay.io/k8scsi/livenessprobe
+    tag: "v1.1.0"
+  resizerImage:
+    repository: quay.io/k8scsi/csi-resizer
+    tag: "v0.3.0"
+  nodeDriverRegistrarImage:
+    repository: quay.io/k8scsi/csi-node-driver-registrar
+    tag: "v1.1.0"
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+podAnnotations: {}
+
+# True if enable volume scheduling for dynamic volume provisioning
+enableVolumeScheduling: false
+
+# True if enable volume resizing
+enableVolumeResizing: false
+
+# True if enable volume snapshot
+enableVolumeSnapshot: false
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+# Extra volume tags to attach to each dynamically provisioned volume.
+# ---
+# extraVolumeTags:
+#   key1: value1
+#   key2: value2
+extraVolumeTags: {}
+
+# AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata
+# service.
+# ---
+# region: us-east-1
+region: ""
+
+node:
+  podAnnotations: {}
+  tolerations: []
+
+serviceAccount:
+  controller:
+    annotations: {}
+  snapshot:
+    annotations: {}

charts/kubezero-aws-ebs/templates/snapshot-class.yaml

@@ -0,0 +1,10 @@
+{{- if index .Values "aws-ebs-csi-driver" "enableVolumeSnapshot" }}
+apiVersion: snapshot.storage.k8s.io/v1beta1
+kind: VolumeSnapshotClass
+metadata:
+  name: csi-aws-vsc
+  labels:
+{{ include "kubezero-lib.labels" . | indent 4 }}
+driver: ebs.csi.aws.com
+deletionPolicy: Delete
+{{- end }}

charts/kubezero-aws-ebs/templates/storage-class.yaml

@@ -0,0 +1,41 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: ebs-sc-gp2-xfs
+  labels:
+{{ include "kubezero-lib.labels" . | indent 4 }}
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: ebs.csi.aws.com
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  csi.storage.k8s.io/fstype: xfs
+  type: gp2
+  encrypted: "true"
+{{- if index .Values "aws-ebs-csi-driver" "enableVolumeResizing" }}
+allowVolumeExpansion: true
+{{- end }}
+
+{{- range .Values.storageClassZones }}
+---
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: ebs-sc-gp2-xfs-{{ . }}
+  labels:
+{{ include "kubezero-lib.labels" . | indent 4 }}
+provisioner: ebs.csi.aws.com
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  csi.storage.k8s.io/fstype: xfs
+  type: gp2
+  encrypted: "true"
+{{- if index .Values "aws-ebs-csi-driver" "enableVolumeResizing" }}
+allowVolumeExpansion: true
+{{- end }}
+allowedTopologies:
+- matchLabelExpressions:
+  - key: failure-domain.beta.kubernetes.io/zone
+    values:
+    - {{ . }}
+{{- end }}

charts/kubezero-aws-ebs/update.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -ex
+
+REPO="kubernetes-sigs/aws-ebs-csi-driver"
+LATEST_RELEASE=$(curl -sL -s https://api.github.com/repos/${REPO}/releases | grep '"tag_name":' | cut -d'"' -f4 | grep -v -E "(alpha|beta|rc)" | sort -t"." -k 1,1 -k 2,2 -k 3,3 -k 4,4 | tail -n 1)
+
+URL="https://github.com/${REPO}/releases/download/${LATEST_RELEASE}/helm-chart.tgz"
+
+rm -rf charts/aws-ebs-csi-driver
+curl -sL "$URL" | tar xz -C charts

charts/kubezero-aws-ebs/values.yaml

@@ -0,0 +1,21 @@
+aws-ebs-csi-driver:
+  replicaCount: 1
+
+  enableVolumeScheduling: true
+  enableVolumeResizing: false
+  enableVolumeSnapshot: false
+
+  nodeSelector:
+    node-role.kubernetes.io/master: ""
+
+  tolerations:
+    - key: node-role.kubernetes.io/master
+      effect: NoSchedule
+
+  # aws-ebs-csi-driver.podAnnotations --  iam.amazonaws.com/role: <IAM role ARN> to assume
+  podAnnotations: {}
+  # iam.amazonaws.com/role: ''
+
+  # aws-ebs-csi-driver.extraVolumeTags -- Optional tags to be added to each EBS volume
+  extraVolumeTags: {}
+  #  Name: KubeZero-Cluster

charts/kubezero-cert-manager/README.md

@@ -25,6 +25,7 @@ Source code can be found [here](https://kubezero.com)
 | cert-manager.ingressShim.defaultIssuerName | string | `"letsencrypt-dns-prod"` |  |
 | cert-manager.installCRDs | bool | `true` |  |
 | cert-manager.nodeSelector."node-role.kubernetes.io/master" | string | `""` |  |
+| cert-manager.podAnnotations."iam.amazonaws.com/role" | string | `""` | IAM role ARN the cert-manager might use via kiam eg."arn:aws:iam::123456789012:role/certManagerRoleArn" |
 | cert-manager.prometheus.servicemonitor.enabled | bool | `false` |  |
 | cert-manager.tolerations[0].effect | string | `"NoSchedule"` |  |
 | cert-manager.tolerations[0].key | string | `"node-role.kubernetes.io/master"` |  |

charts/kubezero-cert-manager/values.yaml

@@ -45,5 +45,6 @@ cert-manager:
   prometheus:
     servicemonitor:
       enabled: false
-  #podAnnotations:
+  # cert-manager.podAnnotations."iam.amazonaws.com/role" -- IAM role ARN the cert-manager might use via kiam eg."arn:aws:iam::123456789012:role/certManagerRoleArn"
-  #  iam.amazonaws.com/role: "INSERT_CLOUDFORMATION_OUTPUT_CertManagerRoleArn"
+  podAnnotations:
+    iam.amazonaws.com/role: ""

charts/kubezero-kiam/README.md

@@ -49,7 +49,7 @@ Required for the *csi ebs plugin* and most likely various others assuming basic
 | kiam.agent.tolerations[0].key | string | `"node-role.kubernetes.io/master"` |  |
 | kiam.agent.updateStrategy | string | `"RollingUpdate"` |  |
 | kiam.agent.whiteListRouteRegexp | string | `"^/latest/(meta-data/instance-id|dynamic)"` |  |
-| kiam.server.assumeRoleArn | string | `"arn:aws:iam::123456789012:role/kiam-server-role"` |  kiam server IAM role to assume, required as we run the agents next to the servers normally |
+| kiam.server.assumeRoleArn | string | `""` |  kiam server IAM role to assume, required as we run the agents next to the servers normally, eg. arn:aws:iam::123456789012:role/kiam-server-role |
 | kiam.server.deployment.enabled | bool | `true` |  |
 | kiam.server.deployment.replicas | int | `1` |  |
 | kiam.server.image.tag | string | `"v3.6-rc1"` |  |

charts/kubezero-kiam/values.yaml

@@ -2,8 +2,8 @@ kiam:
   server:
     image:
       tag: "v3.6-rc1"
-    # kiam.server.assumeRoleArn --  kiam server IAM role to assume, required as we run the agents next to the servers normally
+    # kiam.server.assumeRoleArn --  kiam server IAM role to assume, required as we run the agents next to the servers normally, eg. arn:aws:iam::123456789012:role/kiam-server-role
-    assumeRoleArn: arn:aws:iam::123456789012:role/kiam-server-role
+    assumeRoleArn: ''
     useHostNetwork: true
     sslCertHostPath: /etc/ssl/certs
     tlsSecret: kiam-server-tls