From b3d7ecc46463770fd73aebdddd280c9f2804e7fe Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 3 Dec 2021 22:16:52 +0100 Subject: [PATCH] chore: sync commit ... storage WIP ! --- charts/kubezero-storage/Chart.yaml | 12 +- charts/kubezero-storage/README.md | 52 +++- .../charts/aws-ebs-csi-driver/.helmignore | 22 ++ .../charts/aws-ebs-csi-driver/CHANGELOG.md | 54 ++++ .../charts/aws-ebs-csi-driver/Chart.yaml | 22 ++ .../aws-ebs-csi-driver/templates/NOTES.txt | 5 + .../aws-ebs-csi-driver/templates/_helpers.tpl | 81 ++++++ .../templates/clusterrole-attacher.yaml | 23 ++ .../templates/clusterrole-csi-node.yaml | 11 + .../templates/clusterrole-provisioner.yaml | 38 +++ .../templates/clusterrole-resizer.yaml | 31 ++ .../templates/clusterrole-snapshotter.yaml | 23 ++ .../clusterrolebinding-attacher.yaml | 15 + .../clusterrolebinding-csi-node.yaml | 15 + .../clusterrolebinding-provisioner.yaml | 15 + .../templates/clusterrolebinding-resizer.yaml | 15 + .../clusterrolebinding-snapshotter.yaml | 15 + .../templates/controller.yaml | 268 ++++++++++++++++++ .../templates/csidriver.yaml | 9 + .../templates/node-windows.yaml | 180 ++++++++++++ .../aws-ebs-csi-driver/templates/node.yaml | 169 +++++++++++ .../poddisruptionbudget-controller.yaml | 16 ++ .../serviceaccount-csi-controller.yaml | 17 ++ .../templates/serviceaccount-csi-node.yaml | 12 + .../templates/storageclass.yaml | 15 + .../charts/aws-ebs-csi-driver/values.yaml | 170 +++++++++++ .../charts/aws-efs-csi-driver/.helmignore | 22 ++ .../charts/aws-efs-csi-driver/CHANGELOG.md | 118 ++++++++ .../charts/aws-efs-csi-driver/Chart.yaml | 18 ++ .../aws-efs-csi-driver/templates/NOTES.txt | 3 + .../aws-efs-csi-driver/templates/_helpers.tpl | 56 ++++ .../templates/controller-deployment.yaml | 127 +++++++++ .../templates/controller-serviceaccount.yaml | 62 ++++ .../templates/csidriver.yaml | 10 + .../templates/node-daemonset.yaml | 158 +++++++++++ .../templates/node-serviceaccount.yaml | 12 + .../templates/storageclass.yaml | 26 ++ .../charts/aws-efs-csi-driver/values.yaml | 137 +++++++++ .../kubezero-storage/charts/gemini/Chart.yaml | 2 +- charts/kubezero-storage/ebs.patch | 24 ++ charts/kubezero-storage/efs.patch | 62 ++++ charts/kubezero-storage/update.sh | 16 +- charts/kubezero-storage/values.yaml | 118 ++++++++ 43 files changed, 2269 insertions(+), 7 deletions(-) create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/.helmignore create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_helpers.tpl create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/storageclass.yaml create mode 100644 charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml create mode 100644 charts/kubezero-storage/charts/aws-efs-csi-driver/.helmignore create mode 100644 charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md create mode 100644 charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml create mode 100644 charts/kubezero-storage/charts/aws-efs-csi-driver/templates/NOTES.txt create mode 100644 charts/kubezero-storage/charts/aws-efs-csi-driver/templates/_helpers.tpl create mode 100644 charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml create mode 100644 charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml create mode 100644 charts/kubezero-storage/charts/aws-efs-csi-driver/templates/csidriver.yaml create mode 100644 charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml create mode 100644 charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-serviceaccount.yaml create mode 100644 charts/kubezero-storage/charts/aws-efs-csi-driver/templates/storageclass.yaml create mode 100644 charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml create mode 100644 charts/kubezero-storage/ebs.patch create mode 100644 charts/kubezero-storage/efs.patch diff --git a/charts/kubezero-storage/Chart.yaml b/charts/kubezero-storage/Chart.yaml index faad8cc1..449d628f 100644 --- a/charts/kubezero-storage/Chart.yaml +++ b/charts/kubezero-storage/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-storage description: KubeZero umbrella chart for all things storage incl. backup, eg. openEBS-lvm, gemini type: application -version: 0.3.2 +version: 0.4.0 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -14,11 +14,19 @@ maintainers: - name: Quarky9 dependencies: - name: lvm-localpv - version: 0.8.2 + version: 0.8.5 condition: lvm-localpv.enabled # repository: https://openebs.github.io/lvm-localpv - name: gemini version: 0.0.7 condition: gemini.enabled # repository: https://charts.fairwinds.com/stable + - name: aws-ebs-csi-driver + version: 2.3.0 + condition: aws-ebs-csi-driver.enabled + # repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver + - name: aws-efs-csi-driver + version: 2.1.5 + condition: aws-efs-csi-driver.enabled + # repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver kubeVersion: ">= 1.18.0" diff --git a/charts/kubezero-storage/README.md b/charts/kubezero-storage/README.md index 518b22a5..02c9d69f 100644 --- a/charts/kubezero-storage/README.md +++ b/charts/kubezero-storage/README.md @@ -1,6 +1,6 @@ # kubezero-storage -![Version: 0.3.1](https://img.shields.io/badge/Version-0.3.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.4.0](https://img.shields.io/badge/Version-0.4.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero umbrella chart for all things storage incl. backup, eg. openEBS-lvm, gemini @@ -18,26 +18,74 @@ Kubernetes: `>= 1.18.0` | Repository | Name | Version | |------------|------|---------| +| | aws-ebs-csi-driver | 2.3.0 | +| | aws-efs-csi-driver | 2.1.5 | | | gemini | 0.0.7 | -| https://openebs.github.io/lvm-localpv | lvm-localpv | 0.8.2 | +| | lvm-localpv | 0.8.5 | ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| +| aws-ebs-csi-driver.controller.logLevel | int | `2` | | +| aws-ebs-csi-driver.controller.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| aws-ebs-csi-driver.controller.replicaCount | int | `1` | | +| aws-ebs-csi-driver.controller.resources.limits.memory | string | `"40Mi"` | | +| aws-ebs-csi-driver.controller.resources.requests.cpu | string | `"10m"` | | +| aws-ebs-csi-driver.controller.resources.requests.memory | string | `"24Mi"` | | +| aws-ebs-csi-driver.controller.tolerations[0].effect | string | `"NoSchedule"` | | +| aws-ebs-csi-driver.controller.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| aws-ebs-csi-driver.enabled | bool | `false` | | +| aws-ebs-csi-driver.node.tolerations[0].effect | string | `"NoSchedule"` | | +| aws-ebs-csi-driver.node.tolerations[0].key | string | `"kubezero-workergroup"` | | +| aws-ebs-csi-driver.node.tolerations[0].operator | string | `"Exists"` | | +| aws-ebs-csi-driver.storageClasses[0].allowVolumeExpansion | bool | `true` | | +| aws-ebs-csi-driver.storageClasses[0].name | string | `"ebs-sc-gp2-xfs"` | | +| aws-ebs-csi-driver.storageClasses[0].parameters."csi.storage.k8s.io/fstype" | string | `"xfs"` | | +| aws-ebs-csi-driver.storageClasses[0].parameters.encrypted | string | `"true"` | | +| aws-ebs-csi-driver.storageClasses[0].parameters.type | string | `"gp2"` | | +| aws-ebs-csi-driver.storageClasses[0].volumeBindingMode | string | `"WaitForFirstConsumer"` | | +| aws-ebs-csi-driver.storageClasses[1].allowVolumeExpansion | bool | `true` | | +| aws-ebs-csi-driver.storageClasses[1].annotations."storageclass.kubernetes.io/is-default-class" | string | `"true"` | | +| aws-ebs-csi-driver.storageClasses[1].name | string | `"ebs-sc-gp3-xfs"` | | +| aws-ebs-csi-driver.storageClasses[1].parameters."csi.storage.k8s.io/fstype" | string | `"xfs"` | | +| aws-ebs-csi-driver.storageClasses[1].parameters.encrypted | string | `"true"` | | +| aws-ebs-csi-driver.storageClasses[1].parameters.type | string | `"gp3"` | | +| aws-ebs-csi-driver.storageClasses[1].volumeBindingMode | string | `"WaitForFirstConsumer"` | | +| aws-efs-csi-driver.controller.create | bool | `true` | | +| aws-efs-csi-driver.controller.logLevel | int | `2` | | +| aws-efs-csi-driver.controller.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | +| aws-efs-csi-driver.controller.tolerations[0].effect | string | `"NoSchedule"` | | +| aws-efs-csi-driver.controller.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| aws-efs-csi-driver.enabled | bool | `false` | | +| aws-efs-csi-driver.node.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key | string | `"node.kubernetes.io/csi.efs.fs"` | | +| aws-efs-csi-driver.node.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator | string | `"Exists"` | | +| aws-efs-csi-driver.node.logLevel | int | `2` | | +| aws-efs-csi-driver.node.resources.limits.memory | string | `"128Mi"` | | +| aws-efs-csi-driver.node.resources.requests.cpu | string | `"20m"` | | +| aws-efs-csi-driver.node.resources.requests.memory | string | `"64Mi"` | | +| aws-efs-csi-driver.node.tolerations[0].effect | string | `"NoSchedule"` | | +| aws-efs-csi-driver.node.tolerations[0].key | string | `"kubezero-workergroup"` | | +| aws-efs-csi-driver.node.tolerations[0].operator | string | `"Exists"` | | +| aws-efs-csi-driver.replicaCount | int | `1` | | +| aws-efs-csi-driver.storageClasses[0].name | string | `"efs-sc"` | | | gemini.enabled | bool | `false` | | | gemini.resources.limits.cpu | string | `"400m"` | | | gemini.resources.limits.memory | string | `"128Mi"` | | | gemini.resources.requests.cpu | string | `"20m"` | | | gemini.resources.requests.memory | string | `"32Mi"` | | +| lvm-localpv.analytics.enabled | bool | `false` | | | lvm-localpv.enabled | bool | `false` | | +| lvm-localpv.lvmController.logLevel | int | `2` | | | lvm-localpv.lvmController.nodeSelector."node-role.kubernetes.io/master" | string | `""` | | | lvm-localpv.lvmController.tolerations[0].effect | string | `"NoSchedule"` | | | lvm-localpv.lvmController.tolerations[0].key | string | `"node-role.kubernetes.io/master"` | | +| lvm-localpv.lvmNode.logLevel | int | `2` | | | lvm-localpv.lvmNode.nodeSelector."node.kubernetes.io/lvm" | string | `"openebs"` | | | lvm-localpv.lvmNode.tolerations[0].effect | string | `"NoSchedule"` | | | lvm-localpv.lvmNode.tolerations[0].key | string | `"kubezero-workergroup"` | | | lvm-localpv.lvmNode.tolerations[0].operator | string | `"Exists"` | | +| lvm-localpv.storageCapacity | bool | `false` | | | lvm-localpv.storageClass.default | bool | `false` | | | lvm-localpv.storageClass.vgpattern | string | `""` | | diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/.helmignore b/charts/kubezero-storage/charts/aws-ebs-csi-driver/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md new file mode 100644 index 00000000..e9740cbc --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md @@ -0,0 +1,54 @@ +# Helm chart + +## v2.3.0 + +* Support overriding controller `--default-fstype` flag via values + +## v2.2.1 + +* Bump app/driver version to `v1.3.0` + +## v2.2.0 + +* Support setting imagePullPolicy for all containers + +## v2.1.1 + +* Bump app/driver version to `v1.2.1` + +## v2.1.0 + +* Custom `controller.updateStrategy` to set controller deployment strategy. + +## v2.0.4 + +* Use chart app version as default image tag +* Add updateStrategy to daemonsets + +## v2.0.3 + +* Bump app/driver version to `v1.2.0` + +## v2.0.2 + +* Bump app/driver version to `v1.1.3` + +## v2.0.1 + +* Only create Windows daemonset if enableWindows is true +* Update Windows daemonset to align better to the Linux one + +## v2.0.0 + +* Remove support for Helm 2 +* Remove deprecated values +* No longer install snapshot controller or its CRDs +* Reorganize additional values + +[Upgrade instructions](/docs/README.md#upgrading-from-version-1x-to-2x-of-the-helm-chart) + +## v1.2.4 + +* Bump app/driver version to `v1.1.1` +* Install VolumeSnapshotClass, VolumeSnapshotContent, VolumeSnapshot CRDs if enableVolumeSnapshot is true +* Only run csi-snapshotter sidecar if enableVolumeSnapshot is true or if CRDs are already installed diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml new file mode 100644 index 00000000..16042a43 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml @@ -0,0 +1,22 @@ +annotations: + artifacthub.io/changes: | + - kind: added + description: Custom controller.updateStrategy to set controller deployment strategy. +apiVersion: v2 +appVersion: 1.3.0 +description: A Helm chart for AWS EBS CSI Driver +home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver +keywords: +- aws +- ebs +- csi +kubeVersion: '>=1.17.0-0' +maintainers: +- email: chengpan@amazon.com + name: leakingtapan +- name: krmichel + url: https://github.com/krmichel +name: aws-ebs-csi-driver +sources: +- https://github.com/kubernetes-sigs/aws-ebs-csi-driver +version: 2.3.0 diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt new file mode 100644 index 00000000..5d79084e --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt @@ -0,0 +1,5 @@ +To verify that aws-ebs-csi-driver has started, run: + + kubectl get pod -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" + +NOTE: The [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter) controller and CRDs will no longer be installed as part of this chart and moving forward will be a prerequisite of using the snap shotting functionality. \ No newline at end of file diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_helpers.tpl b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_helpers.tpl new file mode 100644 index 00000000..424f7ea1 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_helpers.tpl @@ -0,0 +1,81 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "aws-ebs-csi-driver.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "aws-ebs-csi-driver.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "aws-ebs-csi-driver.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "aws-ebs-csi-driver.labels" -}} +{{ include "aws-ebs-csi-driver.selectorLabels" . }} +{{- if ne .Release.Name "kustomize" }} +helm.sh/chart: {{ include "aws-ebs-csi-driver.chart" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} +{{- end -}} + +{{/* +Common selector labels +*/}} +{{- define "aws-ebs-csi-driver.selectorLabels" -}} +app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }} +{{- if ne .Release.Name "kustomize" }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} +{{- end -}} + +{{/* +Convert the `--extra-volume-tags` command line arg from a map. +*/}} +{{- define "aws-ebs-csi-driver.extra-volume-tags" -}} +{{- $result := dict "pairs" (list) -}} +{{- range $key, $value := .Values.controller.extraVolumeTags -}} +{{- $noop := printf "%s=%v" $key $value | append $result.pairs | set $result "pairs" -}} +{{- end -}} +{{- if gt (len $result.pairs) 0 -}} +{{- printf "%s=%s" "- --extra-volume-tags" (join "," $result.pairs) -}} +{{- end -}} +{{- end -}} + +{{/* +Handle http proxy env vars +*/}} +{{- define "aws-ebs-csi-driver.http-proxy" -}} +- name: HTTP_PROXY + value: {{ .Values.proxy.http_proxy | quote }} +- name: HTTPS_PROXY + value: {{ .Values.proxy.http_proxy | quote }} +- name: NO_PROXY + value: {{ .Values.proxy.no_proxy | quote }} +{{- end -}} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml new file mode 100644 index 00000000..816fdf66 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml @@ -0,0 +1,23 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-attacher-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "nodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "csi.storage.k8s.io" ] + resources: [ "csinodeinfos" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments/status" ] + verbs: [ "patch" ] diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml new file mode 100644 index 00000000..3ca368ef --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml @@ -0,0 +1,11 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-node-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml new file mode 100644 index 00000000..0fb7ded0 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml @@ -0,0 +1,38 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-provisioner-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch", "create", "delete" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims" ] + verbs: [ "get", "list", "watch", "update" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "storageclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshots" ] + verbs: [ "get", "list" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents" ] + verbs: [ "get", "list" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "csinodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "nodes" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "coordination.k8s.io" ] + resources: [ "leases" ] + verbs: [ "get", "watch", "list", "delete", "update", "create" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "volumeattachments" ] + verbs: [ "get", "list", "watch" ] diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml new file mode 100644 index 00000000..065f3aba --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml @@ -0,0 +1,31 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-resizer-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + # The following rule should be uncommented for plugins that require secrets + # for provisioning. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list", "watch"] + - apiGroups: [ "" ] + resources: [ "persistentvolumes" ] + verbs: [ "get", "list", "watch", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "persistentvolumeclaims/status" ] + verbs: [ "update", "patch" ] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "storageclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "pods" ] + verbs: [ "get", "list", "watch" ] diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml new file mode 100644 index 00000000..5fada8b4 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml @@ -0,0 +1,23 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-snapshotter-role + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +rules: + - apiGroups: [ "" ] + resources: [ "events" ] + verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get", "list" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotclasses" ] + verbs: [ "get", "list", "watch" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents" ] + verbs: [ "create", "get", "list", "watch", "update", "delete" ] + - apiGroups: [ "snapshot.storage.k8s.io" ] + resources: [ "volumesnapshotcontents/status" ] + verbs: [ "update" ] diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml new file mode 100644 index 00000000..bb23044b --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-attacher-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.controller.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: ebs-external-attacher-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml new file mode 100644 index 00000000..55231356 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-node-getter-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.node.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: ebs-csi-node-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml new file mode 100644 index 00000000..9d2749ac --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-provisioner-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.controller.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: ebs-external-provisioner-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml new file mode 100644 index 00000000..88cb47de --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-resizer-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.controller.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: ebs-external-resizer-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml new file mode 100644 index 00000000..2d429054 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshotter-binding + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.controller.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: ebs-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml new file mode 100644 index 00000000..dd9bfcbd --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -0,0 +1,268 @@ +# Controller Service +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ebs-csi-controller + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.controller.replicaCount }} + {{- with .Values.controller.updateStrategy }} + strategy: + {{ toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + app: ebs-csi-controller + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + app: ebs-csi-controller + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.controller.podLabels }} + {{- toYaml .Values.controller.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.controller.podAnnotations }} + annotations: + {{- toYaml .Values.controller.podAnnotations | nindent 8 }} + {{- end }} + spec: + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.controller.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.controller.serviceAccount.name }} + priorityClassName: {{ .Values.controller.priorityClassName }} + {{- with default .Values.controller.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 + {{- with .Values.controller.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.controller.topologySpreadConstraints }} + {{- $tscLabelSelector := dict "labelSelector" ( dict "matchLabels" ( dict "app" "ebs-csi-controller" ) ) }} + {{- $constraints := list }} + {{- range .Values.controller.topologySpreadConstraints }} + {{- $constraints = mustAppend $constraints (mergeOverwrite . $tscLabelSelector) }} + {{- end }} + topologySpreadConstraints: + {{- $constraints | toYaml | nindent 8 }} + {{- end }} + containers: + - name: ebs-plugin + image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (.Values.image.tag | toString)) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + {{- if ne .Release.Name "kustomize" }} + - controller + {{- else }} + # - {all,controller,node} # specify the driver mode + {{- end }} + - --endpoint=$(CSI_ENDPOINT) + {{- if .Values.controller.extraVolumeTags }} + {{- include "aws-ebs-csi-driver.extra-volume-tags" . | nindent 12 }} + {{- end }} + {{- with .Values.controller.k8sTagClusterId }} + - --k8s-tag-cluster-id={{ . }} + {{- end }} + {{- with .Values.controller.httpEndpoint }} + - --http-endpoint={{ . }} + {{- end }} + - --logtostderr + - --v={{ .Values.controller.logLevel }} + {{- range .Values.controller.additionalArgs }} + - {{ . }} + {{- end }} + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-secret + key: key_id + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-secret + key: access_key + optional: true + {{- with .Values.controller.region }} + - name: AWS_REGION + value: {{ . }} + {{- end }} + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.controller.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: aws-token + mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" + readOnly: true + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.controller.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: csi-provisioner + image: {{ printf "%s:%s" .Values.sidecars.provisioner.image.repository .Values.sidecars.provisioner.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.provisioner.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --v={{ .Values.sidecars.provisioner.logLevel }} + - --feature-gates=Topology=true + {{- if .Values.controller.extraCreateMetadata }} + - --extra-create-metadata + {{- end}} + - --leader-election=true + - --default-fstype={{ .Values.controller.defaultFsType }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.provisioner.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with default .Values.controller.resources .Values.sidecars.provisioner.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: csi-attacher + image: {{ printf "%s:%s" .Values.sidecars.attacher.image.repository .Values.sidecars.attacher.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.attacher.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --v={{ .Values.sidecars.attacher.logLevel }} + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.attacher.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with default .Values.controller.resources .Values.sidecars.attacher.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Capabilities.APIVersions.Has "snapshot.storage.k8s.io/v1" }} + - name: csi-snapshotter + image: {{ printf "%s:%s" .Values.sidecars.snapshotter.image.repository .Values.sidecars.snapshotter.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.snapshotter.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.snapshotter.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with default .Values.controller.resources .Values.sidecars.snapshotter.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + - name: csi-resizer + image: {{ printf "%s:%s" .Values.sidecars.resizer.image.repository .Values.sidecars.resizer.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.resizer.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --v={{ .Values.sidecars.resizer.logLevel }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.resizer.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with default .Values.controller.resources .Values.sidecars.resizer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + {{- with default .Values.controller.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: socket-dir + emptyDir: {} + - name: aws-token + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 86400 + audience: "sts.amazonaws.com" diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml new file mode 100644 index 00000000..6598355a --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml @@ -0,0 +1,9 @@ +apiVersion: {{ ternary "storage.k8s.io/v1" "storage.k8s.io/v1beta1" (semverCompare ">=1.18.0-0" .Capabilities.KubeVersion.Version) }} +kind: CSIDriver +metadata: + name: ebs.csi.aws.com + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + attachRequired: true + podInfoOnMount: false diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml new file mode 100644 index 00000000..9d0f02d0 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml @@ -0,0 +1,180 @@ +{{- if .Values.node.enableWindows }} +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ebs-csi-node-windows + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + updateStrategy: + {{ toYaml .Values.node.updateStrategy | nindent 4 }} + template: + metadata: + labels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.node.podLabels }} + {{- toYaml .Values.node.podLabels | nindent 8 }} + {{- end }} + {{- with .Values.node.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + nodeSelector: + kubernetes.io/os: windows + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- else }} + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 + {{- end }} + {{- with .Values.node.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: ebs-plugin + image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - node + - --endpoint=$(CSI_ENDPOINT) + {{- with .Values.node.volumeAttachLimit }} + - --volume-attach-limit={{ . }} + {{- end }} + - --logtostderr + - --v={{ .Values.node.logLevel }} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.node.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: C:\var\lib\kubelet + mountPropagation: "None" + - name: plugin-dir + mountPath: C:\csi + - name: csi-proxy-disk-pipe + mountPath: \\.\pipe\csi-proxy-disk-v1 + - name: csi-proxy-volume-pipe + mountPath: \\.\pipe\csi-proxy-volume-v1 + - name: csi-proxy-filesystem-pipe + mountPath: \\.\pipe\csi-proxy-filesystem-v1 + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.node.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: node-driver-registrar + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} + env: + - name: ADDRESS + value: unix:/csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: C:\var\lib\kubelet\plugins\ebs.csi.aws.com\csi.sock + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: plugin-dir + mountPath: C:\csi + - name: registration-dir + mountPath: C:\registration + {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} + args: + - --csi-address=unix:/csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: C:\csi + {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: C:\var\lib\kubelet + type: Directory + - name: plugin-dir + hostPath: + path: C:\var\lib\kubelet\plugins\ebs.csi.aws.com + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: C:\var\lib\kubelet\plugins_registry + type: Directory + - name: csi-proxy-disk-pipe + hostPath: + path: \\.\pipe\csi-proxy-disk-v1 + type: "" + - name: csi-proxy-volume-pipe + hostPath: + path: \\.\pipe\csi-proxy-volume-v1 + type: "" + - name: csi-proxy-filesystem-pipe + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1 + type: "" +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml new file mode 100644 index 00000000..112e79ab --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml @@ -0,0 +1,169 @@ +# Node Service +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ebs-csi-node + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + updateStrategy: + {{- toYaml .Values.node.updateStrategy | nindent 4 }} + template: + metadata: + labels: + app: ebs-csi-node + {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }} + {{- if .Values.node.podLabels }} + {{- toYaml .Values.node.podLabels | nindent 8 }} + {{- end }} + {{- with .Values.node.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} + tolerations: + {{- if .Values.node.tolerateAllTaints }} + - operator: Exists + {{- else }} + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 + {{- end }} + {{- with .Values.node.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: ebs-plugin + securityContext: + privileged: true + image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - node + - --endpoint=$(CSI_ENDPOINT) + {{- with .Values.node.volumeAttachLimit }} + - --volume-attach-limit={{ . }} + {{- end }} + - --logtostderr + - --v={{ .Values.node.logLevel }} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.node.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.node.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: node-driver-registrar + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + {{- if .Values.proxy.http_proxy }} + {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: {{ .Values.node.kubeletPath }} + type: Directory + - name: plugin-dir + hostPath: + path: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml new file mode 100644 index 00000000..f6241fad --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml @@ -0,0 +1,16 @@ +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: ebs-csi-controller + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: ebs-csi-controller + {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + {{- if le (.Values.controller.replicaCount | int) 2 }} + maxUnavailable: 1 + {{- else }} + minAvailable: 2 + {{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml new file mode 100644 index 00000000..d6466e86 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml @@ -0,0 +1,17 @@ +{{- if .Values.controller.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.controller.serviceAccount.name }} + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- with .Values.controller.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- if eq .Release.Name "kustomize" }} + #Enable if EKS IAM for SA is used + #annotations: + # eks.amazonaws.com/role-arn: arn:aws:iam::586565787010:role/ebs-csi-role + {{- end }} +{{- end -}} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml new file mode 100644 index 00000000..90787091 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml @@ -0,0 +1,12 @@ +{{- if .Values.node.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.node.serviceAccount.name }} + labels: + {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} + {{- with .Values.node.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/storageclass.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/storageclass.yaml new file mode 100644 index 00000000..847f5e21 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/storageclass.yaml @@ -0,0 +1,15 @@ +{{- range .Values.storageClasses }} +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: {{ .name }} + {{- with .annotations }} + annotations: {{- . | toYaml | trim | nindent 4 }} + {{- end }} + {{- with .labels }} + labels: {{- . | toYaml | trim | nindent 4 }} + {{- end }} +provisioner: ebs.csi.aws.com +{{ omit (dict "volumeBindingMode" "WaitForFirstConsumer" | merge .) "name" "annotations" "labels" | toYaml }} +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml new file mode 100644 index 00000000..a35fa6e1 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml @@ -0,0 +1,170 @@ +# Default values for aws-ebs-csi-driver. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + repository: k8s.gcr.io/provider-aws/aws-ebs-csi-driver + # Overrides the image tag whose default is v{{ .Chart.AppVersion }} + tag: "" + pullPolicy: IfNotPresent + +sidecars: + provisioner: + env: [] + image: + pullPolicy: IfNotPresent + repository: k8s.gcr.io/sig-storage/csi-provisioner + tag: "v2.1.1" + logLevel: 2 + resources: {} + attacher: + env: [] + image: + pullPolicy: IfNotPresent + repository: k8s.gcr.io/sig-storage/csi-attacher + tag: "v3.1.0" + logLevel: 2 + resources: {} + snapshotter: + env: [] + image: + pullPolicy: IfNotPresent + repository: k8s.gcr.io/sig-storage/csi-snapshotter + tag: "v3.0.3" + logLevel: 2 + resources: {} + livenessProbe: + image: + pullPolicy: IfNotPresent + repository: k8s.gcr.io/sig-storage/livenessprobe + tag: "v2.2.0" + resources: {} + resizer: + env: [] + image: + pullPolicy: IfNotPresent + repository: k8s.gcr.io/sig-storage/csi-resizer + tag: "v1.0.0" + logLevel: 2 + resources: {} + nodeDriverRegistrar: + env: [] + image: + pullPolicy: IfNotPresent + repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar + tag: "v2.1.0" + logLevel: 2 + resources: {} + +proxy: + http_proxy: + no_proxy: + +imagePullSecrets: [] +nameOverride: +fullnameOverride: + +controller: + # If arbitrary args like "--aws-sdk-debug-log=true" need to be passed, use this value + additionalArgs: [] + affinity: {} + # The default filesystem type of the volume to provision when fstype is unspecified in the StorageClass. + # If the default is not set and fstype is unset in the StorageClass, then no fstype will be set + defaultFsType: ext4 + env: [] + # If set, add pv/pvc metadata to plugin create requests as parameters. + extraCreateMetadata: true + # Extra volume tags to attach to each dynamically provisioned volume. + # --- + # extraVolumeTags: + # key1: value1 + # key2: value2 + extraVolumeTags: {} + httpEndpoint: + # ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional). + k8sTagClusterId: + logLevel: 2 + nodeSelector: {} + podAnnotations: {} + podLabels: {} + priorityClassName: system-cluster-critical + # AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata + # service. + # --- + # region: us-east-1 + region: + replicaCount: 2 + updateStrategy: {} + # type: RollingUpdate + # rollingUpdate: + # maxSurge: 0 + # maxUnavailable: 1 + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # Note that you will need to set resource requests if you want the cluster autoscaler to + # scale your nodes when you increase/decrease the number of ebs-csi-controller replicas. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + serviceAccount: + create: true # A service account will be created for you if set to true. Set to false if you want to use your own. + name: ebs-csi-controller-sa # Name of the service-account to be used/created. + annotations: {} + tolerations: [] + # TSCs without the label selector stanza + # + # Example: + # + # topologySpreadConstraints: + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: ScheduleAnyway + # - maxSkew: 1 + # topologyKey: kubernetes.io/hostname + # whenUnsatisfiable: ScheduleAnyway + topologySpreadConstraints: [] + +node: + env: [] + kubeletPath: /var/lib/kubelet + logLevel: 2 + priorityClassName: + nodeSelector: {} + podAnnotations: {} + podLabels: {} + tolerateAllTaints: false + tolerations: [] + resources: {} + serviceAccount: + create: true + name: ebs-csi-node-sa + annotations: {} + enableWindows: false + # The "maximum number of attachable volumes" per node + volumeAttachLimit: + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: "10%" + +storageClasses: [] +# Add StorageClass resources like: +# - name: ebs-sc +# # annotation metadata +# annotations: +# storageclass.kubernetes.io/is-default-class: "true" +# # label metadata +# labels: +# my-label-is: supercool +# # defaults to WaitForFirstConsumer +# volumeBindingMode: WaitForFirstConsumer +# # defaults to Delete +# reclaimPolicy: Retain +# parameters: +# encrypted: "true" diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/.helmignore b/charts/kubezero-storage/charts/aws-efs-csi-driver/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md b/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md new file mode 100644 index 00000000..c73f9e5e --- /dev/null +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md @@ -0,0 +1,118 @@ +# Helm chart + +# v2.1.5 +* Bump app/driver version to `v1.3.3` + +# v2.1.4 +* Add node.serviceAccount values for creating and/or specifying daemonset service account + +# v2.1.3 +* Bump app/driver version to `v1.3.2` + +# v2.1.2 +* Add extra-create-metadata + +# v2.1.1 +* Update app/driver version to `v1.3.1` + +# v2.1.0 + +## New features +* Update app/driver version to `v1.3.0` + +## Bug fixes +* Put comments back in place inside the values file ([#475](https://github.com/kubernetes-sigs/aws-efs-csi-driver/pull/475), [@pierluigilenoci](https://github.com/pierluigilenoci)) + +# v2.0.1 + +## Bug fixes +* Helm chart: fix reclaimPolicy and volumeBindingMode ([#464](https://github.com/kubernetes-sigs/aws-efs-csi-driver/pull/464), [@devinsmith911](https://github.com/devinsmith911)) + + +# v2.0.0 + +## Breaking changes + +Multiple changes in values file at `sidecars`, `controller` and `node` + +--- +```yaml +sidecars: + xxxxxxxxx: + repository: + tag: +``` + +Moving to + +```yaml +sidecars: + xxxxxxxxx: + image: + repository: + tag: +``` + +--- +```yaml +podAnnotations: +resources: +nodeSelector: +tolerations: +affinity: +``` + +Moving to + +```yaml +controller: + podAnnotations: + resources: + nodeSelector: + tolerations: + affinity: +``` + +--- +```yaml +hostAliases: +dnsPolicy: +dnsConfig: +``` + +Moving to + +```yaml +node: + hostAliases: + dnsPolicy: + dnsConfig: +``` + +--- +```yaml +serviceAccount: + controller: +``` + +Moving to + +```yaml +controller: + serviceAccount: +``` + +## New features + +* Chart API `v2` (requires Helm 3) +* Set `resources` and `imagePullPolicy` fields independently for containers +* Set `logLevel`, `affinity`, `nodeSelector`, `podAnnotations` and `tolerations` fields independently +for Controller deployment and Node daemonset +* Set `reclaimPolicy` and `volumeBindingMode` fields in storage class + +## Fixes + +* Fixing Controller deployment using `podAnnotations` and `tolerations` values from Node daemonset +* Let the user define the whole `tolerations` array, default to `- operator: Exists` +* Default `logLevel` lowered from `5` to `2` +* Default `imagePullPolicy` everywhere set to `IfNotPresent` diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml new file mode 100644 index 00000000..95458976 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +appVersion: 1.3.3 +description: A Helm chart for AWS EFS CSI Driver +home: https://github.com/kubernetes-sigs/aws-efs-csi-driver +keywords: +- aws +- efs +- csi +kubeVersion: '>=1.17.0-0' +maintainers: +- name: leakingtapan + url: https://github.com/leakingtapan +- name: krmichel + url: https://github.com/krmichel +name: aws-efs-csi-driver +sources: +- https://github.com/kubernetes-sigs/aws-efs-csi-driver +version: 2.1.5 diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/NOTES.txt b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/NOTES.txt new file mode 100644 index 00000000..66f4d12d --- /dev/null +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/NOTES.txt @@ -0,0 +1,3 @@ +To verify that aws-efs-csi-driver has started, run: + + kubectl get pod -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aws-efs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/_helpers.tpl b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/_helpers.tpl new file mode 100644 index 00000000..ab6b4e0e --- /dev/null +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/_helpers.tpl @@ -0,0 +1,56 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "aws-efs-csi-driver.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "aws-efs-csi-driver.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "aws-efs-csi-driver.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "aws-efs-csi-driver.labels" -}} +app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} +helm.sh/chart: {{ include "aws-efs-csi-driver.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Create a string out of the map for controller tags flag +*/}} +{{- define "aws-efs-csi-driver.tags" -}} +{{- $tags := list -}} +{{ range $key, $val := . }} +{{- $tags = print $key ":" $val | append $tags -}} +{{- end -}} +{{- join " " $tags -}} +{{- end -}} diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml new file mode 100644 index 00000000..d8d49333 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml @@ -0,0 +1,127 @@ +{{- if .Values.controller.create }} +# Controller Service +kind: Deployment +apiVersion: apps/v1 +metadata: + name: efs-csi-controller + labels: + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: efs-csi-controller + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app: efs-csi-controller + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- with .Values.controller.podAnnotations }} + annotations: {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.controller.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.controller.serviceAccount.name }} + priorityClassName: system-cluster-critical + {{- with .Values.controller.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: efs-plugin + securityContext: + privileged: true + image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + {{- if .Values.controller.tags }} + - --tags={{ include "aws-efs-csi-driver.tags" .Values.controller.tags }} + {{- end }} + - --v={{ .Values.controller.logLevel }} + - --delete-access-point-root-dir={{ hasKey .Values.controller "deleteAccessPointRootDir" | ternary .Values.controller.deleteAccessPointRootDir false }} + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + {{- if .Values.controller.extraEnv }} +{{ toYaml .Values.controller.extraEnv | indent 12 }} + {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: aws-token + mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" + ports: + - name: healthz + containerPort: 9909 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.controller.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: csi-provisioner + image: {{ printf "%s:%s" .Values.sidecars.csiProvisioner.image.repository .Values.sidecars.csiProvisioner.image.tag }} + imagePullPolicy: {{ .Values.sidecars.csiProvisioner.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --v={{ .Values.controller.logLevel }} + - --feature-gates=Topology=true + {{- if .Values.controller.extraCreateMetadata }} + - --extra-create-metadata + {{- end }} + - --leader-election + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + {{- with .Values.sidecars.csiProvisioner.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }} + args: + - --csi-address=/csi/csi.sock + - --health-port=9909 + volumeMounts: + - name: socket-dir + mountPath: /csi + {{- with .Values.sidecars.livenessProbe.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: socket-dir + emptyDir: {} + - name: aws-token + projected: + sources: + - serviceAccountToken: + path: token + expirationSeconds: 86400 + audience: "sts.amazonaws.com" + {{- with .Values.controller.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml new file mode 100644 index 00000000..17d1989b --- /dev/null +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml @@ -0,0 +1,62 @@ +{{- if .Values.controller.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.controller.serviceAccount.name }} + labels: + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} + {{- with .Values.controller.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: efs-csi-external-provisioner-role + labels: + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get", "watch", "list" ] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: efs-csi-provisioner-binding + labels: + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} +subjects: + - kind: ServiceAccount + name: {{ .Values.controller.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: efs-csi-external-provisioner-role + apiGroup: rbac.authorization.k8s.io diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/csidriver.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/csidriver.yaml new file mode 100644 index 00000000..e6b4d419 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/csidriver.yaml @@ -0,0 +1,10 @@ +apiVersion: {{ ternary "storage.k8s.io/v1" "storage.k8s.io/v1beta1" (semverCompare ">=1.18.0-0" .Capabilities.KubeVersion.Version) }} +kind: CSIDriver +metadata: + name: efs.csi.aws.com + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/resource-policy": keep +spec: + attachRequired: false diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml new file mode 100644 index 00000000..0352af49 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml @@ -0,0 +1,158 @@ +# Node Service +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: efs-csi-node + labels: + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} +spec: + selector: + matchLabels: + app: efs-csi-node + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app: efs-csi-node + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.node.podAnnotations }} + annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- with .Values.node.hostAliases }} + hostAliases: + {{- range $k, $v := . }} + - ip: {{ $v.ip }} + hostnames: + - {{ $k }}.efs.{{ $v.region }}.amazonaws.com + {{- end }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + nodeSelector: + beta.kubernetes.io/os: linux + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.node.affinity }} + affinity: + {{- . | toYaml | nindent 8 }} + {{- end }} + hostNetwork: true + dnsPolicy: {{ .Values.node.dnsPolicy }} + {{- with .Values.node.dnsConfig }} + dnsConfig: {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + priorityClassName: system-node-critical + {{- with .Values.node.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: efs-plugin + securityContext: + privileged: true + image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v={{ .Values.node.logLevel }} + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: efs-state-dir + mountPath: /var/run/efs + - name: efs-utils-config + mountPath: /var/amazon/efs + - name: efs-utils-config-legacy + mountPath: /etc/amazon/efs-legacy + ports: + - name: healthz + containerPort: 9809 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + failureThreshold: 5 + {{- with .Values.node.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: csi-driver-registrar + image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }} + imagePullPolicy: {{ .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v={{ .Values.node.logLevel }} + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/efs.csi.aws.com/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + {{- with .Values.sidecars.nodeDriverRegistrar.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: {{ printf "%s:%s" .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }} + imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }} + args: + - --csi-address=/csi/csi.sock + - --health-port=9809 + - --v={{ .Values.node.logLevel }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + {{- with .Values.sidecars.livenessProbe.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/efs.csi.aws.com/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: efs-state-dir + hostPath: + path: /var/run/efs + type: DirectoryOrCreate + - name: efs-utils-config + hostPath: + path: /var/amazon/efs + type: DirectoryOrCreate + - name: efs-utils-config-legacy + hostPath: + path: /etc/amazon/efs + type: DirectoryOrCreate diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-serviceaccount.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-serviceaccount.yaml new file mode 100644 index 00000000..9fd3c7a0 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.node.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.node.serviceAccount.name }} + labels: + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} + {{- with .Values.node.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/storageclass.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/storageclass.yaml new file mode 100644 index 00000000..8ce1ec87 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/storageclass.yaml @@ -0,0 +1,26 @@ +{{- range .Values.storageClasses }} +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: {{ .name }} + {{- with .annotations }} + annotations: + {{ toYaml . | indent 4 }} + {{- end }} +provisioner: efs.csi.aws.com +{{- with .mountOptions }} +mountOptions: +{{ toYaml . }} +{{- end }} +{{- with .parameters }} +parameters: +{{ toYaml . | indent 2 }} +{{- end }} +{{- with .reclaimPolicy }} +reclaimPolicy: {{ . }} +{{- end }} +{{- with .volumeBindingMode }} +volumeBindingMode: {{ . }} +{{- end }} +--- +{{- end }} \ No newline at end of file diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml new file mode 100644 index 00000000..67fd2b3c --- /dev/null +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml @@ -0,0 +1,137 @@ +# Default values for aws-efs-csi-driver. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +nameOverride: "" +fullnameOverride: "" + +replicaCount: 2 + +image: + repository: amazon/aws-efs-csi-driver + tag: "v1.3.3" + pullPolicy: IfNotPresent + +sidecars: + livenessProbe: + image: + repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe + tag: v2.2.0-eks-1-18-2 + pullPolicy: IfNotPresent + resources: {} + nodeDriverRegistrar: + image: + repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar + tag: v2.1.0-eks-1-18-2 + pullPolicy: IfNotPresent + resources: {} + csiProvisioner: + image: + repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner + tag: v2.1.1-eks-1-18-2 + pullPolicy: IfNotPresent + resources: {} + +imagePullSecrets: [] + +## Controller deployment variables + +controller: + # Specifies whether a deployment should be created + create: true + # Number for the log level verbosity + logLevel: 2 + # If set, add pv/pvc metadata to plugin create requests as parameters. + extraCreateMetadata: true + # Add additional tags to access points + tags: + {} + # environment: prod + # region: us-east-1 + # Enable if you want the controller to also delete the + # path on efs when deleteing an access point + deleteAccessPointRootDir: false + podAnnotations: {} + resources: + {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + nodeSelector: {} + tolerations: [] + affinity: {} + # Specifies whether a service account should be created + serviceAccount: + create: true + name: efs-csi-controller-sa + annotations: {} + ## Enable if EKS IAM for SA is used + # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/efs-csi-role + +## Node daemonset variables + +node: + # Number for the log level verbosity + logLevel: 2 + hostAliases: + {} + # For cross VPC EFS, you need to poison or overwrite the DNS for the efs volume as per + # https://docs.aws.amazon.com/efs/latest/ug/efs-different-vpc.html#wt6-efs-utils-step3 + # implementing the suggested solution found here: + # https://github.com/kubernetes-sigs/aws-efs-csi-driver/issues/240#issuecomment-676849346 + # EFS Vol ID, IP, Region + # "fs-01234567": + # ip: 10.10.2.2 + # region: us-east-2 + dnsPolicy: ClusterFirst + dnsConfig: + {} + # Example config which uses the AWS nameservers + # dnsPolicy: "None" + # dnsConfig: + # nameservers: + # - 169.254.169.253 + podAnnotations: {} + resources: + {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + nodeSelector: {} + tolerations: + - operator: Exists + # Specifies whether a service account should be created + serviceAccount: + create: true + name: efs-csi-node-sa + annotations: {} + ## Enable if EKS IAM for SA is used + # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/efs-csi-role + +storageClasses: [] +# Add StorageClass resources like: +# - name: efs-sc +# annotations: +# # Use that annotation if you want this to your default storageclass +# storageclass.kubernetes.io/is-default-class: "true" +# mountOptions: +# - tls +# parameters: +# provisioningMode: efs-ap +# fileSystemId: fs-1122aabb +# directoryPerms: "700" +# gidRangeStart: "1000" +# gidRangeEnd: "2000" +# basePath: "/dynamic_provisioning" +# reclaimPolicy: Delete +# volumeBindingMode: Immediate diff --git a/charts/kubezero-storage/charts/gemini/Chart.yaml b/charts/kubezero-storage/charts/gemini/Chart.yaml index bc0a550c..535b27c0 100644 --- a/charts/kubezero-storage/charts/gemini/Chart.yaml +++ b/charts/kubezero-storage/charts/gemini/Chart.yaml @@ -6,4 +6,4 @@ maintainers: - email: robertb@fairwinds.com name: rbren name: gemini -version: 0.0.7 +version: 0.0.8 diff --git a/charts/kubezero-storage/ebs.patch b/charts/kubezero-storage/ebs.patch new file mode 100644 index 00000000..a607f1a0 --- /dev/null +++ b/charts/kubezero-storage/ebs.patch @@ -0,0 +1,24 @@ +diff -tuNr charts/aws-ebs-csi-driver.orig/templates/controller.yaml charts/aws-ebs-csi-driver/templates/controller.yaml +--- charts/aws-ebs-csi-driver.orig/templates/controller.yaml 2021-08-12 10:46:10.000000000 +0200 ++++ charts/aws-ebs-csi-driver/templates/controller.yaml 2021-09-07 15:13:47.497732918 +0200 +@@ -114,6 +114,9 @@ + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ ++ - name: aws-token ++ mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" ++ readOnly: true + ports: + - name: healthz + containerPort: 9808 +@@ -252,3 +255,10 @@ + volumes: + - name: socket-dir + emptyDir: {} ++ - name: aws-token ++ projected: ++ sources: ++ - serviceAccountToken: ++ path: token ++ expirationSeconds: 86400 ++ audience: "sts.amazonaws.com" diff --git a/charts/kubezero-storage/efs.patch b/charts/kubezero-storage/efs.patch new file mode 100644 index 00000000..d5978c83 --- /dev/null +++ b/charts/kubezero-storage/efs.patch @@ -0,0 +1,62 @@ +diff -tubNr charts/aws-efs-csi-driver/templates/controller-deployment.yaml charts/aws-efs-csi-driver.zdt/templates/controller-deployment.yaml +--- charts/aws-efs-csi-driver/templates/controller-deployment.yaml 2021-06-25 22:43:06.000000000 +0200 ++++ charts/aws-efs-csi-driver.zdt/templates/controller-deployment.yaml 2021-07-15 00:37:28.960305011 +0200 +@@ -23,7 +23,6 @@ + annotations: {{- toYaml . | nindent 8 }} + {{- end }} + spec: +- hostNetwork: true + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} +@@ -57,9 +56,14 @@ + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock ++ {{- if .Values.controller.extraEnv }} ++{{ toYaml .Values.controller.extraEnv | indent 12 }} ++ {{- end }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ ++ - name: aws-token ++ mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" + ports: + - name: healthz + containerPort: 9909 +@@ -110,6 +114,13 @@ + volumes: + - name: socket-dir + emptyDir: {} ++ - name: aws-token ++ projected: ++ sources: ++ - serviceAccountToken: ++ path: token ++ expirationSeconds: 86400 ++ audience: "sts.amazonaws.com" + {{- with .Values.controller.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} +diff -tubNr charts/aws-efs-csi-driver/templates/node-daemonset.yaml charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml +--- charts/aws-efs-csi-driver/templates/node-daemonset.yaml 2021-06-25 22:43:06.000000000 +0200 ++++ charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml 2021-07-15 00:14:58.680295564 +0200 +@@ -40,15 +40,10 @@ + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} ++ {{- with .Values.node.affinity }} + affinity: +- nodeAffinity: +- requiredDuringSchedulingIgnoredDuringExecution: +- nodeSelectorTerms: +- - matchExpressions: +- - key: eks.amazonaws.com/compute-type +- operator: NotIn +- values: +- - fargate ++ {{- . | toYaml | nindent 8 }} ++ {{- end }} + hostNetwork: true + dnsPolicy: {{ .Values.node.dnsPolicy }} + {{- with .Values.node.dnsConfig }} diff --git a/charts/kubezero-storage/update.sh b/charts/kubezero-storage/update.sh index 3750e419..48d50327 100755 --- a/charts/kubezero-storage/update.sh +++ b/charts/kubezero-storage/update.sh @@ -1,9 +1,21 @@ #!/bin/bash set -ex -# Gemini +### Gemini rm -rf charts/gemini helm pull fairwinds-stable/gemini --untar --untardir charts - # Patch to run gemini on controller nodes patch -p0 -i gemini.patch --no-backup-if-mismatch + +### EBS +VERSION=$(yq r Chart.yaml dependencies.name==aws-ebs-csi-driver.version) +rm -rf charts/aws-ebs-csi-driver +curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts +patch -i ebs.patch -p0 --no-backup-if-mismatch + + +### EFS +VERSION=$(yq r Chart.yaml dependencies.name==aws-efs-csi-driver.version) +rm -rf charts/aws-efs-csi-driver +curl -L -s -o - https://github.com/kubernetes-sigs/aws-efs-csi-driver/releases/download/helm-chart-aws-efs-csi-driver-${VERSION}/aws-efs-csi-driver-${VERSION}.tgz | tar xfz - -C charts +patch -i efs.patch -p0 --no-backup-if-mismatch diff --git a/charts/kubezero-storage/values.yaml b/charts/kubezero-storage/values.yaml index 551ac5e0..8e46ff2a 100644 --- a/charts/kubezero-storage/values.yaml +++ b/charts/kubezero-storage/values.yaml @@ -25,6 +25,7 @@ lvm-localpv: analytics: enabled: false + gemini: enabled: false # verbosity: 1 @@ -36,3 +37,120 @@ gemini: limits: memory: 128Mi cpu: 400m + + +aws-ebs-csi-driver: + enabled: false + + controller: + replicaCount: 1 + logLevel: 2 + + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + + # k8sTagClusterId: + + #env: + # ebsPlugin: + #- name: AWS_ROLE_ARN + # value: "" + #- name: AWS_WEB_IDENTITY_TOKEN_FILE + # value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + #- name: AWS_STS_REGIONAL_ENDPOINTS + # value: regional + resources: + requests: + cpu: 10m + memory: 24Mi + limits: + # cpu: 50m + memory: 40Mi + + node: + tolerations: + - key: kubezero-workergroup + effect: NoSchedule + operator: Exists + + storageClasses: + - name: ebs-sc-gp2-xfs + volumeBindingMode: WaitForFirstConsumer + allowVolumeExpansion: true + parameters: + csi.storage.k8s.io/fstype: xfs + type: gp2 + encrypted: "true" + - name: ebs-sc-gp3-xfs + annotations: + storageclass.kubernetes.io/is-default-class: "true" + volumeBindingMode: WaitForFirstConsumer + allowVolumeExpansion: true + parameters: + csi.storage.k8s.io/fstype: xfs + type: gp3 + encrypted: "true" + + +aws-efs-csi-driver: + enabled: false + + replicaCount: 1 + + controller: + create: true + logLevel: 2 + + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + nodeSelector: + node-role.kubernetes.io/master: "" + + #extraEnv: + #- name: AWS_ROLE_ARN + # value: "" + #- name: AWS_WEB_IDENTITY_TOKEN_FILE + # value: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" + #- name: AWS_STS_REGIONAL_ENDPOINTS + # value: regional + + node: + logLevel: 2 + + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + memory: 128Mi + + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node.kubernetes.io/csi.efs.fs + operator: Exists + tolerations: + - key: kubezero-workergroup + effect: NoSchedule + operator: Exists + + storageClasses: + - name: efs-sc + +#PersistentVolumes: +# - name: example-pv +# mountOptions: +# - tls +# volumeHandle: ":/path" +# claimRef: +# name: foo-pvc +# namespace: foo +# +# - name: example-pv2 +# volumeHandle: ":/path2"