diff --git a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml index 9cb2ee5a..b2ea4b4f 100644 --- a/charts/kubezero-aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: kubezero-aws-ebs-csi-driver description: KubeZero Umbrella Chart for aws-ebs-csi-driver type: application -version: 0.3.5 -appVersion: 0.8.1 +version: 0.4.0 +appVersion: 0.9.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png sources: @@ -18,9 +18,9 @@ maintainers: - name: Quarky9 dependencies: - name: aws-ebs-csi-driver - version: 0.8.2 + version: 0.9.9 repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: kubezero-lib version: ">= 0.1.3" repository: https://zero-down-time.github.io/kubezero/ -kubeVersion: ">= 1.16.0" +kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/.helmignore b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml new file mode 100644 index 00000000..0910aace --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +appVersion: 0.9.0 +description: A Helm chart for AWS EBS CSI Driver +home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver +keywords: +- aws +- ebs +- csi +kubeVersion: '>=1.17.0-0' +maintainers: +- email: chengpan@amazon.com + name: leakingtapan +- name: krmichel + url: https://github.com/krmichel +name: aws-ebs-csi-driver +sources: +- https://github.com/kubernetes-sigs/aws-ebs-csi-driver +version: 0.9.9 diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt new file mode 100644 index 00000000..34db916b --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/NOTES.txt @@ -0,0 +1,3 @@ +To verify that aws-ebs-csi-driver has started, run: + + kubectl get pod -n kube-system -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl new file mode 100644 index 00000000..fdc77c4e --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/_helpers.tpl @@ -0,0 +1,69 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "aws-ebs-csi-driver.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "aws-ebs-csi-driver.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "aws-ebs-csi-driver.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "aws-ebs-csi-driver.labels" -}} +{{ include "aws-ebs-csi-driver.selectorLabels" . }} +{{- if ne .Release.Name "kustomize" }} +helm.sh/chart: {{ include "aws-ebs-csi-driver.chart" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} +{{- end -}} + +{{/* +Common selector labels +*/}} +{{- define "aws-ebs-csi-driver.selectorLabels" -}} +app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} +{{- if ne .Release.Name "kustomize" }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +{{- end -}} + +{{/* +Convert the `--extra-volume-tags` command line arg from a map. +*/}} +{{- define "aws-ebs-csi-driver.extra-volume-tags" -}} +{{- $result := dict "pairs" (list) -}} +{{- range $key, $value := .Values.extraVolumeTags -}} +{{- $noop := printf "%s=%s" $key $value | append $result.pairs | set $result "pairs" -}} +{{- end -}} +{{- if gt (len $result.pairs) 0 -}} +{{- printf "%s=%s" "- --extra-volume-tags" (join "," $result.pairs) -}} +{{- end -}} +{{- end -}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml new file mode 100644 index 00000000..e0919cec --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml @@ -0,0 +1,23 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-attacher-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments/status" ] + verbs: [ "patch" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml new file mode 100644 index 00000000..0fb7ded0 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml @@ -0,0 +1,38 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-provisioner-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch", "create", "delete" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims" ] + verbs: [ "get", "list", "watch", "update" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "storageclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshots" ] + verbs: [ "get", "list" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents" ] + verbs: [ "get", "list" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "csinodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "nodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "coordination.k8s.io" ] + resources: [ "leases" ] + verbs: [ "get", "watch", "list", "delete", "update", "create" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments" ] + verbs: [ "get", "list", "watch" ] diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml new file mode 100644 index 00000000..9d85b97c --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml @@ -0,0 +1,33 @@ +{{- if .Values.enableVolumeResizing }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-resizer-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + # The following rule should be uncommented for plugins that require secrets + # for provisioning. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +{{- end}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml new file mode 100644 index 00000000..ffdb1b7d --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshot-controller.yaml @@ -0,0 +1,35 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-snapshot-controller-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml new file mode 100644 index 00000000..061b5658 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml @@ -0,0 +1,25 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-snapshotter-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml new file mode 100644 index 00000000..92a8b40f --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-attacher-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller.name }} + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-attacher-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml new file mode 100644 index 00000000..e2478b93 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-provisioner-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller.name }} + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-provisioner-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml new file mode 100644 index 00000000..997dc28e --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml @@ -0,0 +1,18 @@ +{{- if .Values.enableVolumeResizing }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-resizer-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller.name }} + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-resizer-role + apiGroup: rbac.authorization.k8s.io + +{{- end}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml new file mode 100644 index 00000000..cb467309 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshot-controller.yaml @@ -0,0 +1,18 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshot-controller-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.snapshot.name }} + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-snapshot-controller-role + apiGroup: rbac.authorization.k8s.io + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml new file mode 100644 index 00000000..f55c38e4 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml @@ -0,0 +1,18 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshotter-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller.name }} + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml new file mode 100644 index 00000000..7feff6f3 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -0,0 +1,185 @@ +# Controller Service +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ebs-csi-controller + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: ebs-csi-controller + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: ebs-csi-controller + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.podAnnotations }} + annotations: {{ toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} + spec: + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + serviceAccountName: {{ .Values.serviceAccount.controller.name }} + priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + tolerations: + {{- if .Values.tolerateAllTaints }} + - operator: Exists + {{- end }} + {{- with .Values.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: ebs-plugin + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + {{- if ne .Release.Name "kustomize" }} + - controller + {{- else }} + # - {all,controller,node} # specify the driver mode + {{- end }} + - --endpoint=$(CSI_ENDPOINT) + {{- if .Values.extraVolumeTags }} + {{- include "aws-ebs-csi-driver.extra-volume-tags" . | nindent 12 }} + {{- end }} + {{- if .Values.k8sTagClusterId }} + - --k8s-tag-cluster-id={{ .Values.k8sTagClusterId }} + {{- end }} + - --logtostderr + - --v={{ .Values.logLevel }} + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-secret + key: key_id + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-secret + key: access_key + optional: true + {{- if .Values.region }} + - name: AWS_REGION + value: {{ .Values.region }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: csi-provisioner + image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --v={{ .Values.logLevel }} + {{- if .Values.enableVolumeScheduling }} + - --feature-gates=Topology=true + {{- end}} + {{- if .Values.extraCreateMetadata }} + - --extra-create-metadata + {{- end}} + - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: csi-attacher + image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --v={{ .Values.logLevel }} + - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.enableVolumeSnapshot }} + - name: csi-snapshotter + image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.enableVolumeResizing }} + - name: csi-resizer + image: {{ printf "%s:%s" .Values.sidecars.resizerImage.repository .Values.sidecars.resizerImage.tag }} + imagePullPolicy: Always + args: + - --csi-address=$(ADDRESS) + - --v={{ .Values.logLevel }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: socket-dir + emptyDir: {} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml new file mode 100644 index 00000000..1858e39c --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/csidriver.yaml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: ebs.csi.aws.com + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + attachRequired: true + podInfoOnMount: false diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml new file mode 100644 index 00000000..6fd11105 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/node.yaml @@ -0,0 +1,159 @@ +# Node Service +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ebs-csi-node + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.node.podAnnotations }} + annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- end }} + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.node.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + hostNetwork: true + serviceAccountName: {{ .Values.serviceAccount.node.name }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-cluster-critical" }} + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- end }} + {{- with .Values.node.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: ebs-plugin + securityContext: + privileged: true + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + args: + - node + - --endpoint=$(CSI_ENDPOINT) + {{- if .Values.volumeAttachLimit }} + - --volume-attach-limit={{ .Values.volumeAttachLimit }} + {{- end }} + - --logtostderr + - --v={{ .Values.logLevel }} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: node-driver-registrar + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v={{ .Values.logLevel }} + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + {{- if .Values.node.resources }} + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml new file mode 100644 index 00000000..947d241e --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/role-snapshot-controller-leaderelection.yaml @@ -0,0 +1,15 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-snapshot-controller-leaderelection + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml new file mode 100644 index 00000000..0670c705 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/rolebinding-snapshot-controller-leaderelection.yaml @@ -0,0 +1,19 @@ +{{- if .Values.enableVolumeSnapshot }} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-snapshot-controller-leaderelection + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.snapshot.name }} + namespace: kube-system +roleRef: + kind: Role + name: ebs-snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io + +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml new file mode 100644 index 00000000..8ec4c4e0 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml @@ -0,0 +1,17 @@ +{{- if .Values.serviceAccount.controller.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.controller.name }} + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.controller.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} + {{- if eq .Release.Name "kustomize" }} + #Enable if EKS IAM for SA is used + #annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::586565787010:role/ebs-csi-role + {{- end }} +{{- end -}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml new file mode 100644 index 00000000..afe02185 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.node.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.node.name }} + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.node.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml new file mode 100644 index 00000000..3b5ef2bc --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/serviceaccount-snapshot-controller.yaml @@ -0,0 +1,15 @@ +{{- if .Values.enableVolumeSnapshot }} +{{- if .Values.serviceAccount.snapshot.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.snapshot.name }} + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.snapshot.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml new file mode 100644 index 00000000..aeb83514 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/statefulset.yaml @@ -0,0 +1,52 @@ +{{- if .Values.enableVolumeSnapshot }} +#Snapshot controller +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: ebs-snapshot-controller + namespace: kube-system + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + serviceName: ebs-snapshot-controller + replicas: 1 + selector: + matchLabels: + app: ebs-snapshot-controller + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: ebs-snapshot-controller + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + spec: + serviceAccountName: {{ .Values.serviceAccount.snapshot.name }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.nodeSelector }} +{{ toYaml . | indent 8 }} + {{- end }} + priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} + tolerations: + {{- if .Values.tolerateAllTaints }} + - operator: Exists + {{- end }} + {{- with .Values.tolerations }} +{{ toYaml . | indent 8 }} + {{- end }} + containers: + - name: snapshot-controller + image: {{ printf "%s:%s" .Values.snapshotController.repository .Values.snapshotController.tag }} + args: + - --v={{ .Values.logLevel }} + - --leader-election=false + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml new file mode 100644 index 00000000..3da90e3d --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/templates/storageclass.yaml @@ -0,0 +1,15 @@ +{{- range .Values.storageClasses }} +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: {{ .name }} + {{- if .annotations }} + annotations: {{- .annotations | toYaml | trim | nindent 4 }} + {{- end }} + {{- if .labels }} + labels: {{- .labels | toYaml | trim | nindent 4 }} + {{- end }} +provisioner: ebs.csi.aws.com +{{ omit (dict "volumeBindingMode" "WaitForFirstConsumer" | merge .) "name" "annotations" "labels" | toYaml }} +{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml new file mode 100644 index 00000000..fd13fade --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/charts/aws-ebs-csi-driver/values.yaml @@ -0,0 +1,130 @@ +# Default values for aws-ebs-csi-driver. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 2 + +image: + repository: k8s.gcr.io/provider-aws/aws-ebs-csi-driver + tag: "v0.9.0" + pullPolicy: IfNotPresent + +logLevel: 5 + +sidecars: + provisionerImage: + repository: k8s.gcr.io/sig-storage/csi-provisioner + tag: "v2.0.2" + attacherImage: + repository: k8s.gcr.io/sig-storage/csi-attacher + tag: "v3.0.0" + snapshotterImage: + repository: k8s.gcr.io/sig-storage/csi-snapshotter + tag: "v3.0.3" + livenessProbeImage: + repository: k8s.gcr.io/sig-storage/livenessprobe + tag: "v2.1.0" + resizerImage: + repository: k8s.gcr.io/sig-storage/csi-resizer + tag: "v1.0.0" + nodeDriverRegistrarImage: + repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar + tag: "v2.0.1" + +snapshotController: + repository: k8s.gcr.io/sig-storage/snapshot-controller + tag: "v3.0.3" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +podAnnotations: {} + +# True if enable volume scheduling for dynamic volume provisioning +enableVolumeScheduling: true + +# True if enable volume resizing +enableVolumeResizing: false + +# True if enable volume snapshot +enableVolumeSnapshot: false + +# The "maximum number of attachable volumes" per node +volumeAttachLimit: "" + +resources: + {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +priorityClassName: "" +nodeSelector: {} +tolerateAllTaints: true +tolerations: [] +affinity: {} + +# Extra volume tags to attach to each dynamically provisioned volume. +# --- +# extraVolumeTags: +# key1: value1 +# key2: value2 +extraVolumeTags: {} + +# If set, add pv/pvc metadata to plugin create requests as parameters. +extraCreateMetadata: false + +# ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional). +k8sTagClusterId: "" + +# AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata +# service. +# --- +# region: us-east-1 +region: "" + +node: + priorityClassName: "" + nodeSelector: {} + podAnnotations: {} + tolerateAllTaints: true + tolerations: [] + resources: {} + +serviceAccount: + controller: + create: true # A service account will be created for you if set to true. Set to false if you want to use your own. + name: ebs-csi-controller-sa # Name of the service-account to be used/created. + annotations: {} + snapshot: + create: true + name: ebs-snapshot-controller + annotations: {} + node: + create: true + name: ebs-csi-node-sa + annotations: {} + +storageClasses: [] +# Add StorageClass resources like: +# - name: ebs-sc +# # annotation metadata +# annotations: +# storageclass.kubernetes.io/is-default-class: "true" +# # label metadata +# labels: +# my-label-is: supercool +# # defaults to WaitForFirstConsumer +# volumeBindingMode: WaitForFirstConsumer +# # defaults to Delete +# reclaimPolicy: Retain +# parameters: +# encrypted: "true" diff --git a/charts/kubezero-aws-ebs-csi-driver/crds/crd_snapshotter.yaml b/charts/kubezero-aws-ebs-csi-driver/crds/crd_snapshotter.yaml new file mode 100644 index 00000000..aff837ab --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/crds/crd_snapshotter.yaml @@ -0,0 +1,420 @@ +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + preserveUnknownFields: false + validation: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created + through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot + is deleted. Supported values are "Retain" and "Delete". "Retain" means + that the VolumeSnapshotContent and its physical snapshot on underlying + storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + subresources: + status: {} + preserveUnknownFields: false + validation: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. "Delete" + means that the VolumeSnapshotContent and its physical snapshot on + underlying storage system are deleted. In dynamic snapshot creation + case, this field will be filled in with the "DeletionPolicy" field + defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For + pre-existing snapshots, users MUST specify this field when creating + the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be the + same as the name returned by the CSI GetPluginName() call for that + driver. Required. + type: string + source: + description: source specifies from where a snapshot will be created. + This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a + pre-existing snapshot on the underlying storage system. This field + is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume + from which a snapshot should be dynamically taken from. This field + is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass to which this snapshot + belongs. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to + which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be provided + for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an + entire object, this string should contain a valid JSON/Go field + access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen only + to have some well-defined way of referencing a part of an object. + TODO: this design is not final and this field is subject to change + in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is + made, if any. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates the creation time is unknown. The + format of this field is a Unix nanoseconds time encoded as an int64. + On Unix, the command `date +%s%N` returns the current time in nanoseconds + since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the latest observed error during snapshot creation, + if any. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on + the underlying storage system. If not specified, it indicates that + dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + subresources: + status: {} + preserveUnknownFields: false + validation: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested + by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object in the same namespace as the VolumeSnapshot + object where the snapshot should be dynamically taken from. This + field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing + VolumeSnapshotContent object. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. If not specified, the default snapshot + class will be used if one exists. If not specified, and there is no + default snapshot class, dynamic snapshot creation will fail. Empty + string is not allowed for this field. TODO(xiangqian): a webhook validation + on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes' + type: string + required: + - source + type: object + status: + description: 'status represents the current information of a snapshot. NOTE: + status can be modified by sources other than system controllers, and must + not be depended upon for accuracy. Controllers should only use information + from the VolumeSnapshotContent object after verifying that the binding + is accurate and complete.' + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName represents the name of + the VolumeSnapshotContent object to which the VolumeSnapshot object + is bound. If not specified, it indicates that the VolumeSnapshot object + has not been successfully bound to a VolumeSnapshotContent object + yet. NOTE: Specified boundVolumeSnapshotContentName alone does not + mean binding is valid. Controllers MUST always verify bidirectional + binding between VolumeSnapshot and VolumeSnapshotContent to + avoid possible security issues.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot + is taken by the underlying storage system. In dynamic snapshot creation + case, this field will be filled in with the "creation_time" value + returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing + snapshot, this field will be filled with the "creation_time" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. If not specified, it indicates that the creation time of the snapshot + is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be logged, + and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this + field will be filled with the "ready_to_use" value returned from the + CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, + this field will be set to "True". If not specified, it means the readiness + of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be filled + in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + type: string + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch b/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch new file mode 100644 index 00000000..39563e97 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/loglevel_leader.patch @@ -0,0 +1,112 @@ +diff -rtub aws-ebs-csi-driver/templates/controller.yaml /tmp/aws-ebs-csi-driver/templates/controller.yaml +--- aws-ebs-csi-driver/templates/controller.yaml 2021-02-23 18:54:24.000000000 +0100 ++++ /tmp/aws-ebs-csi-driver/templates/controller.yaml 2021-02-24 18:40:00.753541864 +0100 +@@ -56,7 +56,7 @@ + - --k8s-tag-cluster-id={{ .Values.k8sTagClusterId }} + {{- end }} + - --logtostderr +- - --v=5 ++ - --v={{ .Values.logLevel }} + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock +@@ -98,14 +98,14 @@ + image: {{ printf "%s:%s" .Values.sidecars.provisionerImage.repository .Values.sidecars.provisionerImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.logLevel }} + {{- if .Values.enableVolumeScheduling }} + - --feature-gates=Topology=true + {{- end}} + {{- if .Values.extraCreateMetadata }} + - --extra-create-metadata + {{- end}} +- - --leader-election=true ++ - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + - --default-fstype=ext4 + env: + - name: ADDRESS +@@ -120,8 +120,8 @@ + image: {{ printf "%s:%s" .Values.sidecars.attacherImage.repository .Values.sidecars.attacherImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --v=5 +- - --leader-election=true ++ - --v={{ .Values.logLevel }} ++ - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +@@ -136,7 +136,7 @@ + image: {{ printf "%s:%s" .Values.sidecars.snapshotterImage.repository .Values.sidecars.snapshotterImage.tag }} + args: + - --csi-address=$(ADDRESS) +- - --leader-election=true ++ - --leader-election={{ ternary "true" "false" ( gt (.Values.replicaCount|int) 1 ) }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +@@ -153,7 +153,7 @@ + imagePullPolicy: Always + args: + - --csi-address=$(ADDRESS) +- - --v=5 ++ - --v={{ .Values.logLevel }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock +diff -rtub aws-ebs-csi-driver/templates/node.yaml /tmp/aws-ebs-csi-driver/templates/node.yaml +--- aws-ebs-csi-driver/templates/node.yaml 2021-02-23 18:54:24.000000000 +0100 ++++ /tmp/aws-ebs-csi-driver/templates/node.yaml 2021-02-24 18:41:44.630213228 +0100 +@@ -56,7 +56,7 @@ + - --volume-attach-limit={{ .Values.volumeAttachLimit }} + {{- end }} + - --logtostderr +- - --v=5 ++ - --v={{ .Values.logLevel }} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock +@@ -94,7 +94,7 @@ + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) +- - --v=5 ++ - --v={{ .Values.logLevel }} + lifecycle: + preStop: + exec: +diff -rtub aws-ebs-csi-driver/templates/statefulset.yaml /tmp/aws-ebs-csi-driver/templates/statefulset.yaml +--- aws-ebs-csi-driver/templates/statefulset.yaml 2021-02-23 18:54:24.000000000 +0100 ++++ /tmp/aws-ebs-csi-driver/templates/statefulset.yaml 2021-02-24 18:42:07.223547582 +0100 +@@ -41,7 +41,7 @@ + - name: snapshot-controller + image: {{ printf "%s:%s" .Values.snapshotController.repository .Values.snapshotController.tag }} + args: +- - --v=5 ++ - --v={{ .Values.logLevel }} + - --leader-election=false + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +diff -rtub aws-ebs-csi-driver/templates/storageclass.yaml /tmp/aws-ebs-csi-driver/templates/storageclass.yaml +--- aws-ebs-csi-driver/templates/storageclass.yaml 2021-02-23 18:54:24.000000000 +0100 ++++ /tmp/aws-ebs-csi-driver/templates/storageclass.yaml 2021-02-24 17:43:46.866722719 +0100 +@@ -1,4 +1,5 @@ + {{- range .Values.storageClasses }} ++--- + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: +diff -rtub aws-ebs-csi-driver/values.yaml /tmp/aws-ebs-csi-driver/values.yaml +--- aws-ebs-csi-driver/values.yaml 2021-02-23 18:54:24.000000000 +0100 ++++ /tmp/aws-ebs-csi-driver/values.yaml 2021-02-24 18:41:15.513545244 +0100 +@@ -9,6 +9,8 @@ + tag: "v0.9.0" + pullPolicy: IfNotPresent + ++logLevel: 5 ++ + sidecars: + provisionerImage: + repository: k8s.gcr.io/sig-storage/csi-provisioner diff --git a/charts/kubezero-aws-ebs-csi-driver/templates/storage-class.yaml b/charts/kubezero-aws-ebs-csi-driver/templates/storage-class.yaml deleted file mode 100644 index 094d43a1..00000000 --- a/charts/kubezero-aws-ebs-csi-driver/templates/storage-class.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.StorageClass.create }} -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: ebs-sc-gp2-xfs - labels: -{{ include "kubezero-lib.labels" . | indent 4 }} -provisioner: ebs.csi.aws.com -volumeBindingMode: WaitForFirstConsumer -parameters: - csi.storage.k8s.io/fstype: xfs - type: gp2 - encrypted: "true" -{{- if index .Values "aws-ebs-csi-driver" "enableVolumeResizing" }} -allowVolumeExpansion: true -{{- end }} ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: ebs-sc-gp3-xfs - labels: -{{ include "kubezero-lib.labels" . | indent 4 }} - {{- if .Values.StorageClass.default }} - annotations: - storageclass.kubernetes.io/is-default-class: "true" - {{- end }} -provisioner: ebs.csi.aws.com -volumeBindingMode: WaitForFirstConsumer -parameters: - csi.storage.k8s.io/fstype: xfs - type: gp3 - encrypted: "true" -{{- if index .Values "aws-ebs-csi-driver" "enableVolumeResizing" }} -allowVolumeExpansion: true -{{- end }} - -{{- range .Values.StorageClass.Zones }} ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: ebs-sc-gp2-xfs-{{ . }} - labels: -{{ include "kubezero-lib.labels" . | indent 4 }} -provisioner: ebs.csi.aws.com -volumeBindingMode: WaitForFirstConsumer -parameters: - csi.storage.k8s.io/fstype: xfs - type: gp2 - encrypted: "true" -{{- if index .Values "aws-ebs-csi-driver" "enableVolumeResizing" }} -allowVolumeExpansion: true -{{- end }} -allowedTopologies: -- matchLabelExpressions: - - key: failure-domain.beta.kubernetes.io/zone - values: - - {{ . }} -{{- end }} -{{- end }} diff --git a/charts/kubezero-aws-ebs-csi-driver/update.sh b/charts/kubezero-aws-ebs-csi-driver/update.sh new file mode 100755 index 00000000..14f46749 --- /dev/null +++ b/charts/kubezero-aws-ebs-csi-driver/update.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +VERSION=0.9.9 + +rm -rf charts/aws-ebs-csi-driver +curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts + +patch -d charts -i ../loglevel_leader.patch -p0 diff --git a/charts/kubezero-aws-ebs-csi-driver/values.yaml b/charts/kubezero-aws-ebs-csi-driver/values.yaml index 9f8bb5bf..77bbef03 100644 --- a/charts/kubezero-aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-aws-ebs-csi-driver/values.yaml @@ -1,9 +1,10 @@ aws-ebs-csi-driver: replicaCount: 1 + logLevel: 1 enableVolumeScheduling: true - enableVolumeResizing: false - enableVolumeSnapshot: false + enableVolumeResizing: true + enableVolumeSnapshot: true nodeSelector: node-role.kubernetes.io/master: "" @@ -20,6 +21,20 @@ aws-ebs-csi-driver: extraVolumeTags: {} # Name: KubeZero-Cluster -StorageClass: - create: true - default: true + storageClasses: + - name: ebs-sc-gp2-xfs + volumeBindingMode: WaitForFirstConsumer + allowVolumeExpansion: true + parameters: + csi.storage.k8s.io/fstype: xfs + type: gp2 + encrypted: "true" + - name: ebs-sc-gp3-xfs + annotations: + storageclass.kubernetes.io/is-default-class: "true" + volumeBindingMode: WaitForFirstConsumer + allowVolumeExpansion: true + parameters: + csi.storage.k8s.io/fstype: xfs + type: gp3 + encrypted: "true"