From b3fe628d268c3392b21e47d8c395b52b517f152b Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Tue, 19 May 2020 15:13:41 +0100 Subject: [PATCH] First version of aws-ebs-csi-driver umbrella chart, updated docs --- charts/kubezero-aws-ebs/.helmignore | 22 ++ charts/kubezero-aws-ebs/Chart.yaml | 22 ++ charts/kubezero-aws-ebs/README.md | 27 ++ .../charts/aws-ebs-csi-driver/.helmignore | 22 ++ .../charts/aws-ebs-csi-driver/Chart.yaml | 16 ++ .../aws-ebs-csi-driver/templates/NOTES.txt | 3 + .../aws-ebs-csi-driver/templates/_helpers.tpl | 58 ++++ .../templates/csidriver.yaml | 7 + .../templates/daemonset.yaml | 108 ++++++++ .../templates/deployment.yaml | 151 +++++++++++ .../aws-ebs-csi-driver/templates/rbac.yaml | 251 ++++++++++++++++++ .../templates/serviceaccount.yaml | 18 ++ .../templates/statefulset.yaml | 26 ++ .../charts/aws-ebs-csi-driver/values.yaml | 86 ++++++ .../templates/snapshot-class.yaml | 10 + .../templates/storage-class.yaml | 41 +++ charts/kubezero-aws-ebs/update.sh | 10 + charts/kubezero-aws-ebs/values.yaml | 21 ++ charts/kubezero-cert-manager/README.md | 1 + charts/kubezero-cert-manager/values.yaml | 5 +- charts/kubezero-kiam/README.md | 2 +- charts/kubezero-kiam/values.yaml | 4 +- 22 files changed, 906 insertions(+), 5 deletions(-) create mode 100644 charts/kubezero-aws-ebs/.helmignore create mode 100644 charts/kubezero-aws-ebs/Chart.yaml create mode 100644 charts/kubezero-aws-ebs/README.md create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/.helmignore create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/Chart.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/NOTES.txt create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/_helpers.tpl create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/csidriver.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/daemonset.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/deployment.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/rbac.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/statefulset.yaml create mode 100644 charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/values.yaml create mode 100644 charts/kubezero-aws-ebs/templates/snapshot-class.yaml create mode 100644 charts/kubezero-aws-ebs/templates/storage-class.yaml create mode 100755 charts/kubezero-aws-ebs/update.sh create mode 100644 charts/kubezero-aws-ebs/values.yaml diff --git a/charts/kubezero-aws-ebs/.helmignore b/charts/kubezero-aws-ebs/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/kubezero-aws-ebs/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubezero-aws-ebs/Chart.yaml b/charts/kubezero-aws-ebs/Chart.yaml new file mode 100644 index 00000000..2349320c --- /dev/null +++ b/charts/kubezero-aws-ebs/Chart.yaml @@ -0,0 +1,22 @@ +apiVersion: v2 +name: kubezero-aws-ebs +description: KubeZero Umbrella Chart for aws-ebs-csi-driver +type: application +version: 0.1.0 +home: https://kubezero.com +icon: https://cdn.zero-downtime.net/assets/logo_small.png +sources: + - https://github.com/kubernetes-sigs/aws-ebs-csi-driver + - https://github.com/Zero-Down-Time/kubezero +keywords: + - kubezero + - aws + - ebs + - csi +maintainers: + - name: Quarky9 +dependencies: + - name: kubezero-lib + version: ">= 0.1.1" + repository: https://zero-down-time.github.io/kubezero/ +kubeVersion: ">= 1.16.0" diff --git a/charts/kubezero-aws-ebs/README.md b/charts/kubezero-aws-ebs/README.md new file mode 100644 index 00000000..1e2f2a31 --- /dev/null +++ b/charts/kubezero-aws-ebs/README.md @@ -0,0 +1,27 @@ +kubezero-aws-ebs +================ +KubeZero Umbrella Chart for aws-ebs-csi-driver + +Current chart version is `0.1.0` + +Source code can be found [here](https://kubezero.com) + +## Chart Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.1 | + +## Chart Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| aws-ebs-csi-driver.enableVolumeResizing | bool | `false` | | +| aws-ebs-csi-driver.enableVolumeScheduling | bool | `true` | | +| aws-ebs-csi-driver.enableVolumeSnapshot | bool | `false` | | +| aws-ebs-csi-driver.extraVolumeTags | object | `{}` | Optional tags to be added to each EBS volume | +| aws-ebs-csi-driver.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| aws-ebs-csi-driver.podAnnotations | object | `{}` | iam.amazonaws.com/role: to assume | +| aws-ebs-csi-driver.replicaCount | int | `1` | | +| aws-ebs-csi-driver.tolerations[0].effect | string | `"NoSchedule"` | | +| aws-ebs-csi-driver.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/.helmignore b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/Chart.yaml new file mode 100644 index 00000000..df6d0fcb --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +appVersion: "0.5.0" +name: aws-ebs-csi-driver +description: A Helm chart for AWS EBS CSI Driver +version: 0.3.0 +kubeVersion: ">=1.13.0-0" +home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver +sources: + - https://github.com/kubernetes-sigs/aws-ebs-csi-driver +keywords: + - aws + - ebs + - csi +maintainers: + - name: leakingtapan + email: chengpan@amazon.com diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/NOTES.txt b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/NOTES.txt new file mode 100644 index 00000000..34db916b --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/NOTES.txt @@ -0,0 +1,3 @@ +To verify that aws-ebs-csi-driver has started, run: + + kubectl get pod -n kube-system -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/_helpers.tpl b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/_helpers.tpl new file mode 100644 index 00000000..7fa13305 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/_helpers.tpl @@ -0,0 +1,58 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "aws-ebs-csi-driver.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "aws-ebs-csi-driver.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "aws-ebs-csi-driver.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "aws-ebs-csi-driver.labels" -}} +app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} +helm.sh/chart: {{ include "aws-ebs-csi-driver.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Convert the `--extra-volume-tags` command line arg from a map. +*/}} +{{- define "aws-ebs-csi-driver.extra-volume-tags" -}} +{{- $result := dict "pairs" (list) -}} +{{- range $key, $value := .Values.extraVolumeTags -}} +{{- $noop := printf "%s=%s" $key $value | append $result.pairs | set $result "pairs" -}} +{{- end -}} +{{- if gt (len $result.pairs) 0 -}} +- --extra-volume-tags={{- join "," $result.pairs -}} +{{- end -}} +{{- end -}} diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/csidriver.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/csidriver.yaml new file mode 100644 index 00000000..6e427fd0 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/csidriver.yaml @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: ebs.csi.aws.com +spec: + attachRequired: true + podInfoOnMount: false diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/daemonset.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/daemonset.yaml new file mode 100644 index 00000000..1e6e817c --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/daemonset.yaml @@ -0,0 +1,108 @@ +# Node Service +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ebs-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.node.podAnnotations }} + annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- end }} + spec: + nodeSelector: + beta.kubernetes.io/os: linux + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - operator: Exists + {{- with .Values.node.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: ebs-plugin + securityContext: + privileged: true + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + args: + - node + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + - name: node-driver-registrar + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/deployment.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/deployment.yaml new file mode 100644 index 00000000..3316e96c --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/deployment.yaml @@ -0,0 +1,151 @@ +# Controller Service +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ebs-csi-controller + namespace: kube-system +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.podAnnotations }} + annotations: {{ toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} + spec: + nodeSelector: + beta.kubernetes.io/os: linux + {{- with .Values.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + serviceAccountName: ebs-csi-controller-sa + priorityClassName: system-cluster-critical + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + tolerations: + - operator: Exists + {{- with .Values.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: ebs-plugin + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - controller + - --endpoint=$(CSI_ENDPOINT) + {{ include "aws-ebs-csi-driver.extra-volume-tags" . }} + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-secret + key: key_id + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-secret + key: access_key + optional: true + {{- if .Values.region }} + - name: AWS_REGION + value: {{ .Values.region }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: csi-provisioner + image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --v=5 + {{- if .Values.enableVolumeScheduling }} + - --feature-gates=Topology=true + {{- end}} + - --enable-leader-election + - --leader-election-type=leases + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --v=5 + - --leader-election=true + - --leader-election-type=leases + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- if .Values.enableVolumeSnapshot }} + - name: csi-snapshotter + image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- end }} + {{- if .Values.enableVolumeResizing }} + - name: csi-resizer + image: {{ printf "%s:%s" .Values.sidecars.resizerImage.repository .Values.sidecars.resizerImage.tag }} + imagePullPolicy: Always + args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + volumes: + - name: socket-dir + emptyDir: {} diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/rbac.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/rbac.yaml new file mode 100644 index 00000000..464c648f --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/rbac.yaml @@ -0,0 +1,251 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-attacher-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-snapshotter-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-snapshot-controller-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshot-controller-binding +subjects: + - kind: ServiceAccount + name: ebs-snapshot-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-snapshot-controller-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-snapshot-controller-leaderelection + namespace: kube-system +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-leaderelection + namespace: kube-system +subjects: + - kind: ServiceAccount + name: ebs-snapshot-controller + namespace: kube-system +roleRef: + kind: Role + name: snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io + +{{- end }} + +{{- if .Values.enableVolumeResizing }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-resizer-role +rules: + # The following rule should be uncommented for plugins that require secrets + # for provisioning. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-resizer-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-resizer-role + apiGroup: rbac.authorization.k8s.io +{{- end}} diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml new file mode 100644 index 00000000..95396d62 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ebs-csi-controller-sa + namespace: kube-system + {{- with .Values.serviceAccount.controller.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ebs-snapshot-controller + namespace: kube-system + {{- with .Values.serviceAccount.snapshot.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/statefulset.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/statefulset.yaml new file mode 100644 index 00000000..01f36b74 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/templates/statefulset.yaml @@ -0,0 +1,26 @@ +{{- if .Values.enableVolumeSnapshot }} +#Snapshot controller +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: ebs-snapshot-controller + namespace: kube-system +spec: + serviceName: ebs-snapshot-controller + replicas: 1 + selector: + matchLabels: + app: ebs-snapshot-controller + template: + metadata: + labels: + app: ebs-snapshot-controller + spec: + serviceAccount: ebs-snapshot-controller + containers: + - name: snapshot-controller + image: quay.io/k8scsi/snapshot-controller:v2.0.1 + args: + - --v=5 + - --leader-election=false +{{- end }} diff --git a/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/values.yaml new file mode 100644 index 00000000..b8997213 --- /dev/null +++ b/charts/kubezero-aws-ebs/charts/aws-ebs-csi-driver/values.yaml @@ -0,0 +1,86 @@ +# Default values for aws-ebs-csi-driver. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 2 + +image: + repository: amazon/aws-ebs-csi-driver + tag: "v0.5.0" + pullPolicy: IfNotPresent + +sidecars: + provisionerImage: + repository: quay.io/k8scsi/csi-provisioner + tag: "v1.5.0" + attacherImage: + repository: quay.io/k8scsi/csi-attacher + tag: "v1.2.0" + snapshotterImage: + repository: quay.io/k8scsi/csi-snapshotter + tag: "v2.0.1" + livenessProbeImage: + repository: quay.io/k8scsi/livenessprobe + tag: "v1.1.0" + resizerImage: + repository: quay.io/k8scsi/csi-resizer + tag: "v0.3.0" + nodeDriverRegistrarImage: + repository: quay.io/k8scsi/csi-node-driver-registrar + tag: "v1.1.0" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +podAnnotations: {} + +# True if enable volume scheduling for dynamic volume provisioning +enableVolumeScheduling: false + +# True if enable volume resizing +enableVolumeResizing: false + +# True if enable volume snapshot +enableVolumeSnapshot: false + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# Extra volume tags to attach to each dynamically provisioned volume. +# --- +# extraVolumeTags: +# key1: value1 +# key2: value2 +extraVolumeTags: {} + +# AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata +# service. +# --- +# region: us-east-1 +region: "" + +node: + podAnnotations: {} + tolerations: [] + +serviceAccount: + controller: + annotations: {} + snapshot: + annotations: {} diff --git a/charts/kubezero-aws-ebs/templates/snapshot-class.yaml b/charts/kubezero-aws-ebs/templates/snapshot-class.yaml new file mode 100644 index 00000000..1b4a831b --- /dev/null +++ b/charts/kubezero-aws-ebs/templates/snapshot-class.yaml @@ -0,0 +1,10 @@ +{{- if index .Values "aws-ebs-csi-driver" "enableVolumeSnapshot" }} +apiVersion: snapshot.storage.k8s.io/v1beta1 +kind: VolumeSnapshotClass +metadata: + name: csi-aws-vsc + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +driver: ebs.csi.aws.com +deletionPolicy: Delete +{{- end }} diff --git a/charts/kubezero-aws-ebs/templates/storage-class.yaml b/charts/kubezero-aws-ebs/templates/storage-class.yaml new file mode 100644 index 00000000..cd714bb0 --- /dev/null +++ b/charts/kubezero-aws-ebs/templates/storage-class.yaml @@ -0,0 +1,41 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: ebs-sc-gp2-xfs + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer +parameters: + csi.storage.k8s.io/fstype: xfs + type: gp2 + encrypted: "true" +{{- if index .Values "aws-ebs-csi-driver" "enableVolumeResizing" }} +allowVolumeExpansion: true +{{- end }} + +{{- range .Values.storageClassZones }} +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: ebs-sc-gp2-xfs-{{ . }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer +parameters: + csi.storage.k8s.io/fstype: xfs + type: gp2 + encrypted: "true" +{{- if index .Values "aws-ebs-csi-driver" "enableVolumeResizing" }} +allowVolumeExpansion: true +{{- end }} +allowedTopologies: +- matchLabelExpressions: + - key: failure-domain.beta.kubernetes.io/zone + values: + - {{ . }} +{{- end }} diff --git a/charts/kubezero-aws-ebs/update.sh b/charts/kubezero-aws-ebs/update.sh new file mode 100755 index 00000000..c7b5df76 --- /dev/null +++ b/charts/kubezero-aws-ebs/update.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -ex + +REPO="kubernetes-sigs/aws-ebs-csi-driver" +LATEST_RELEASE=$(curl -sL -s https://api.github.com/repos/${REPO}/releases | grep '"tag_name":' | cut -d'"' -f4 | grep -v -E "(alpha|beta|rc)" | sort -t"." -k 1,1 -k 2,2 -k 3,3 -k 4,4 | tail -n 1) + +URL="https://github.com/${REPO}/releases/download/${LATEST_RELEASE}/helm-chart.tgz" + +rm -rf charts/aws-ebs-csi-driver +curl -sL "$URL" | tar xz -C charts diff --git a/charts/kubezero-aws-ebs/values.yaml b/charts/kubezero-aws-ebs/values.yaml new file mode 100644 index 00000000..cac6c1a5 --- /dev/null +++ b/charts/kubezero-aws-ebs/values.yaml @@ -0,0 +1,21 @@ +aws-ebs-csi-driver: + replicaCount: 1 + + enableVolumeScheduling: true + enableVolumeResizing: false + enableVolumeSnapshot: false + + nodeSelector: + node-role.kubernetes.io/master: "" + + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + + # aws-ebs-csi-driver.podAnnotations -- iam.amazonaws.com/role: to assume + podAnnotations: {} + # iam.amazonaws.com/role: '' + + # aws-ebs-csi-driver.extraVolumeTags -- Optional tags to be added to each EBS volume + extraVolumeTags: {} + # Name: KubeZero-Cluster diff --git a/charts/kubezero-cert-manager/README.md b/charts/kubezero-cert-manager/README.md index 756f6301..8027bbf8 100644 --- a/charts/kubezero-cert-manager/README.md +++ b/charts/kubezero-cert-manager/README.md @@ -25,6 +25,7 @@ Source code can be found [here](https://kubezero.com) | cert-manager.ingressShim.defaultIssuerName | string | `"letsencrypt-dns-prod"` | | | cert-manager.installCRDs | bool | `true` | | | cert-manager.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| cert-manager.podAnnotations."iam.amazonaws.com/role" | string | `""` | IAM role ARN the cert-manager might use via kiam eg."arn:aws:iam::123456789012:role/certManagerRoleArn" | | cert-manager.prometheus.servicemonitor.enabled | bool | `false` | | | cert-manager.tolerations[0].effect | string | `"NoSchedule"` | | | cert-manager.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | diff --git a/charts/kubezero-cert-manager/values.yaml b/charts/kubezero-cert-manager/values.yaml index e23fcfbf..e8b6eec0 100644 --- a/charts/kubezero-cert-manager/values.yaml +++ b/charts/kubezero-cert-manager/values.yaml @@ -45,5 +45,6 @@ cert-manager: prometheus: servicemonitor: enabled: false - #podAnnotations: - # iam.amazonaws.com/role: "INSERT_CLOUDFORMATION_OUTPUT_CertManagerRoleArn" + # cert-manager.podAnnotations."iam.amazonaws.com/role" -- IAM role ARN the cert-manager might use via kiam eg."arn:aws:iam::123456789012:role/certManagerRoleArn" + podAnnotations: + iam.amazonaws.com/role: "" diff --git a/charts/kubezero-kiam/README.md b/charts/kubezero-kiam/README.md index b897c6b4..079edc32 100644 --- a/charts/kubezero-kiam/README.md +++ b/charts/kubezero-kiam/README.md @@ -49,7 +49,7 @@ Required for the *csi ebs plugin* and most likely various others assuming basic | kiam.agent.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | | kiam.agent.updateStrategy | string | `"RollingUpdate"` | | | kiam.agent.whiteListRouteRegexp | string | `"^/latest/(meta-data/instance-id|dynamic)"` | | -| kiam.server.assumeRoleArn | string | `"arn:aws:iam::123456789012:role/kiam-server-role"` | kiam server IAM role to assume, required as we run the agents next to the servers normally | +| kiam.server.assumeRoleArn | string | `""` | kiam server IAM role to assume, required as we run the agents next to the servers normally, eg. arn:aws:iam::123456789012:role/kiam-server-role | | kiam.server.deployment.enabled | bool | `true` | | | kiam.server.deployment.replicas | int | `1` | | | kiam.server.image.tag | string | `"v3.6-rc1"` | | diff --git a/charts/kubezero-kiam/values.yaml b/charts/kubezero-kiam/values.yaml index 1f91f825..2affcd8a 100644 --- a/charts/kubezero-kiam/values.yaml +++ b/charts/kubezero-kiam/values.yaml @@ -2,8 +2,8 @@ kiam: server: image: tag: "v3.6-rc1" - # kiam.server.assumeRoleArn -- kiam server IAM role to assume, required as we run the agents next to the servers normally - assumeRoleArn: arn:aws:iam::123456789012:role/kiam-server-role + # kiam.server.assumeRoleArn -- kiam server IAM role to assume, required as we run the agents next to the servers normally, eg. arn:aws:iam::123456789012:role/kiam-server-role + assumeRoleArn: '' useHostNetwork: true sslCertHostPath: /etc/ssl/certs tlsSecret: kiam-server-tls