Minor SQL tweaks

admin/upgrade_cluster.sh

@@ -151,6 +151,7 @@ argo_used && disable_argo
 # Cleanup
 # Remove calico CRDs
 kubectl delete -f https://git.zero-downtime.net/ZeroDownTime/kubezero/raw/tag/v1.23.11/charts/kubezero-network/charts/calico/crds/crds.yaml 2>/dev/null || true
+kubectl delete servicemonitor calico-node -n kube-system || true
 
 # delete old kubelet configs
 for cm in $(kubectl get cm -n kube-system --no-headers |  awk '{if ($1 ~ "kubelet-config-1*") print $1}'); do kubectl delete cm $cm -n kube-system; done

charts/kubezero-sql/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-sql
 description: KubeZero umbrella chart for SQL databases, Percona XtraDB Cluster
 type: application
-version: 0.3.1
+version: 0.3.2
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -15,14 +15,14 @@ maintainers:
     email: stefan@zero-downtime.net
 dependencies:
   - name: kubezero-lib
-    version: ">= 0.1.5"
+    version: ">= 0.1.6"
     repository: https://cdn.zero-downtime.net/charts/
   - name: pxc-operator
-    version: 1.12.0
+    version: 1.12.1
     repository: https://percona.github.io/percona-helm-charts/
     condition: pxc-operator.enabled
   - name: mariadb-galera
     version: 7.4.7
     repository: https://charts.bitnami.com/bitnami
     condition: mariadb-galera.enabled
-kubeVersion: ">= 1.20.0"
+kubeVersion: ">= 1.24.0"

charts/kubezero-storage/charts/aws-ebs-csi-driver/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md

@@ -0,0 +1,249 @@
+# Helm chart
+
+## v2.18.0
+### Urgent Upgrade Notes
+*(No, really, you MUST read this before you upgrade)*
+
+The Helm chart now defaults to using specific releases of the EKS-D sidecars, rather than the `-latest` versions. This is done so the chart will specify an exact container image, as well as for consistency with the EKS Addons version of the driver.
+
+The new sidecar tags are:
+* csi-provisioner: `v3.4.1-eks-1-26-7`
+* csi-attacher: `v4.2.0-eks-1-26-7`
+* csi-snapshotter: `v6.2.1-eks-1-26-7`
+* livenessprobe: `v2.9.0-eks-1-26-7`
+* csi-resizer: `v1.7.0-eks-1-26-7`
+* node-driver-registrar: `v2.7.0-eks-1-26-7`
+
+### Improvements
+* Bump driver version to `v1.18.0`
+* Increase speed and reliability of `helm test` ([#1533](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1533), [@torredil](https://github.com/torredil))
+* Support `VolumeSnapshotClass` in helm chart ([#1540](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1540), [@hanyuel](https://github.com/hanyuel))
+
+## v2.17.2
+* Bump driver version to `v1.17.0`
+* Bump `external-resizer` version to `v4.2.0`
+* All other sidecars have been updated to the latest rebuild (without an associated version change)
+
+## v2.17.1
+* Bump driver version to `v1.16.1`
+
+## v2.17.0
+* Bump driver version to `v1.16.0`
+* Add support for JSON logging ([#1467](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1467), [@torredil](https://github.com/torredil))
+    * `--logging-format` flag has been added to set the log format. Valid values are `text` and `json`. The default value is `text`.
+    * `--logtostderr` is deprecated.
+    * Long arguments prefixed with `-` are no longer supported, and must be prefixed with `--`. For example, `--volume-attach-limit` instead of `-volume-attach-limit`.
+* The sidecars have been updated. The new versions are:
+    - csi-provisioner: `v3.4.0`
+    - csi-attacher: `v4.1.0`
+    - csi-snapshotter: `v6.2.1`
+    - livenessprobe: `v2.9.0`
+    - csi-resizer: `v1.7.0`
+    - node-driver-registrar: `v2.7.0`
+
+
+## v2.16.0
+* Bump driver version to `v1.15.0`
+* Change default sidecars to EKS-D ([#1475](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1475), [@ConnorJC3](https://github.com/ConnorJC3), [@torredil](https://github.com/torredil))
+* The sidecars have been updated. The new versions are:
+    - csi-provisioner: `v3.3.0`
+    - csi-attacher: `v4.0.0`
+    - csi-snapshotter: `v6.1.0`
+    - livenessprobe: `v2.8.0`
+    - csi-resizer: `v1.6.0`
+    - node-driver-registrar: `v2.6.2`
+
+## v2.15.1
+* Bugfix: Prevent deployment of testing resources during normal installation by adding `helm.sh/hook: test` annotation.
+
+## v2.15.0
+* Set sensible default resource requests/limits
+* Add sensible default update strategy
+* Add podAntiAffinity so controller pods prefer scheduling on separate nodes if possible
+* Add container registry parameter
+
+## v2.14.2
+* Bump driver version to `v1.14.1`
+
+## v2.14.1
+* Add `controller.sdkDebugLog` parameter
+
+## v2.14.0
+* Bump driver version to `v1.14.0`
+
+## v2.13.0
+* Bump app/driver to version `v1.13.0`
+* Expose volumes and volumeMounts for the ebs-csi-controller deployment ([#1400](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1436), [@cnmcavoy](https://github.com/cnmcavoy))
+* refactor: Move the default controller tolerations in the helm chart values ([#1427](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1427), [@cnmcavoy](https://github.com/Linutux42))
+* Add serviceMonitor.labels parameter ([#1419](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1419), [@torredil](https://github.com/torredil))
+* Add parameter to force enable snapshotter sidecar ([#1418](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1418), [@ConnorJC3](https://github.com/ConnorJC3))
+
+## v2.12.1
+* Bump app/driver to version `v1.12.1`
+
+## v2.12.0
+* Bump app/driver to version `v1.12.0`
+* Move default toleration to values.yaml so it can be overriden if desired by users ([#1400](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1400), [@cnmcavoy](https://github.com/cnmcavoy))
+* Add enableMetrics configuration ([#1380](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1380), [@torredil](https://github.com/torredil))
+* add initContainer to the controller's template ([#1379](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1379), [@InsomniaCoder](https://github.com/InsomniaCoder))
+* Add controller nodeAffinity to prefer EC2 over Fargate ([#1360](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1360), [@torredil](https://github.com/torredil))
+
+## v2.11.1
+* Add `useOldCSIDriver` parameter to use old `CSIDriver` object.
+
+## v2.11.0
+
+**Important Notice:** This version updates the `CSIDriver` object in order to fix [a bug with static volumes and the `fsGroup` parameter](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1365). This upgrade will fail on existing clusters because the associated field in `CSIDriver` is immutable.
+
+Users upgrading to this version should pre-delete the existing `CSIDriver` object (example: `kubectl delete csidriver ebs.csi.aws.com`). This will not affect any existing volumes, but will cause the EBS CSI Driver to be unavailable to handle future requests, and should be immediately followed by an upgrade. For users that cannot delete the `CSIDriver` object, v2.11.1 implements a new parameter `useOldCSIDriver` that will use the previous `CSIDriver`.
+
+* Bump app/driver to version `v1.11.3`
+* Add support for leader election tuning for `csi-provisioner` and `csi-attacher` ([#1371](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1371), [@moogzy](https://github.com/moogzy))
+* Change `fsGroupPolicy` to `File` ([#1377](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1377), [@ConnorJC3](https://github.com/ConnorJC3))
+* Allow all taint for `csi-node` by default ([#1381](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1381), [@gtxu](https://github.com/gtxu))
+
+## v2.10.1
+* Bump app/driver to version `v1.11.2`
+
+## v2.10.0
+* Implement securityContext for containers
+* Add securityContext for node pod
+* Utilize more secure defaults for securityContext
+
+## v2.9.0
+* Bump app/driver to version `v1.10.0`
+* Feature: Reference `configMaps` across multiple resources using `envFrom` ([#1312](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1312), [@jebbens](https://github.com/jebbens))
+
+## v2.8.1
+* Bump app/driver to version `v1.9.0`
+* Update livenessprobe to version `v2.6.0`
+
+## v2.8.0
+* Feature: Support custom affinity definition on node daemon set ([#1277](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1277), [@vauchok](https://github.com/vauchok))
+
+## v2.7.1
+* Bump app/driver to version `v1.8.0`
+
+## v2.7.0
+* Support optional ec2 endpoint configuration.
+* Fix node driver registrar socket path.
+* Fix hardcoded kubelet path.
+
+## v2.6.11
+* Bump app/driver to version `v1.7.0`
+* Set handle-volume-inuse-error to `false`
+
+## v2.6.10
+
+* Add quotes around the `extra-tags` argument in order to prevent special characters such as `":"` from breaking the manifest YAML after template rendering.
+
+## v2.6.9
+
+* Update csi-snapshotter to version `v6.0.1`
+* Update external-attacher to version `v3.4.0`
+* Update external-resizer to version `v1.4.0`
+* Update external-provisioner to version `v3.1.0`
+* Update node-driver-registrar to version `v2.5.1`
+* Update livenessprobe to version `v2.5.0`
+
+## v2.6.8
+
+* Bump app/driver to version `v1.6.2`
+* Bump sidecar version for nodeDriverRegistrar, provisioner to be consistent with EKS CSI Driver Add-on
+
+## v2.6.7
+
+* Bump app/driver to version `v1.6.1`
+
+## v2.6.6
+
+* Bump app/driver to version `v1.6.0`
+
+## v2.6.5
+
+* Bump app/driver to version `v1.5.3`
+
+## v2.6.4
+
+* Remove exposure all secrets to external-snapshotter-role
+
+## v2.6.3
+
+* Bump app/driver to version `v1.5.1`
+
+## v2.6.2
+
+* Update csi-resizer version to v1.1.0
+
+## v2.6.1
+
+* Add securityContext support for controller Deployment
+
+## v2.5.0
+
+* Bump app/driver version to `v1.5.0`
+
+## v2.4.1
+
+* Replace deprecated arg `--extra-volume-tags` by `--extra-tags`
+
+## v2.4.0
+
+* Bump app/driver version to `v1.4.0`
+
+## v2.3.1
+
+* Bump app/driver version to `v1.3.1`
+
+## v2.3.0
+
+* Support overriding controller `--default-fstype` flag via values
+
+## v2.2.1
+
+* Bump app/driver version to `v1.3.0`
+
+## v2.2.0
+
+* Support setting imagePullPolicy for all containers
+
+## v2.1.1
+
+* Bump app/driver version to `v1.2.1`
+
+## v2.1.0
+
+* Custom `controller.updateStrategy` to set controller deployment strategy.
+
+## v2.0.4
+
+* Use chart app version as default image tag
+* Add updateStrategy to daemonsets
+
+## v2.0.3
+
+* Bump app/driver version to `v1.2.0`
+
+## v2.0.2
+
+* Bump app/driver version to `v1.1.3`
+
+## v2.0.1
+
+* Only create Windows daemonset if enableWindows is true
+* Update Windows daemonset to align better to the Linux one
+
+## v2.0.0
+
+* Remove support for Helm 2
+* Remove deprecated values
+* No longer install snapshot controller or its CRDs
+* Reorganize additional values
+
+[Upgrade instructions](/docs/README.md#upgrading-from-version-1x-to-2x-of-the-helm-chart)
+
+## v1.2.4
+
+* Bump app/driver version to `v1.1.1`
+* Install VolumeSnapshotClass, VolumeSnapshotContent, VolumeSnapshot CRDs if enableVolumeSnapshot is true
+* Only run csi-snapshotter sidecar if enableVolumeSnapshot is true or if CRDs are already installed

charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml

@@ -0,0 +1,16 @@
+apiVersion: v2
+appVersion: 1.18.0
+description: A Helm chart for AWS EBS CSI Driver
+home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver
+keywords:
+- aws
+- ebs
+- csi
+kubeVersion: '>=1.17.0-0'
+maintainers:
+- name: Kubernetes Authors
+  url: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/
+name: aws-ebs-csi-driver
+sources:
+- https://github.com/kubernetes-sigs/aws-ebs-csi-driver
+version: 2.18.0

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt

@@ -0,0 +1,5 @@
+To verify that aws-ebs-csi-driver has started, run:
+
+    kubectl get pod -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}"
+
+NOTE: The [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter) controller and CRDs will no longer be installed as part of this chart and moving forward will be a prerequisite of using the snap shotting functionality.

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_helpers.tpl

@@ -0,0 +1,85 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "aws-ebs-csi-driver.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "aws-ebs-csi-driver.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "aws-ebs-csi-driver.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "aws-ebs-csi-driver.labels" -}}
+{{ include "aws-ebs-csi-driver.selectorLabels" . }}
+{{- if ne .Release.Name "kustomize" }}
+helm.sh/chart: {{ include "aws-ebs-csi-driver.chart" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/component: csi-driver
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+{{- if .Values.customLabels }}
+{{ toYaml .Values.customLabels }}
+{{- end }}
+{{- end -}}
+
+{{/*
+Common selector labels
+*/}}
+{{- define "aws-ebs-csi-driver.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "aws-ebs-csi-driver.name" . }}
+{{- if ne .Release.Name "kustomize" }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+{{- end -}}
+
+{{/*
+Convert the `--extra-tags` command line arg from a map.
+*/}}
+{{- define "aws-ebs-csi-driver.extra-volume-tags" -}}
+{{- $result := dict "pairs" (list) -}}
+{{- range $key, $value := .Values.controller.extraVolumeTags -}}
+{{- $noop := printf "%s=%v" $key $value | append $result.pairs | set $result "pairs" -}}
+{{- end -}}
+{{- if gt (len $result.pairs) 0 -}}
+{{- printf "- \"--extra-tags=%s\"" (join "," $result.pairs) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Handle http proxy env vars
+*/}}
+{{- define "aws-ebs-csi-driver.http-proxy" -}}
+- name: HTTP_PROXY
+  value: {{ .Values.proxy.http_proxy | quote }}
+- name: HTTPS_PROXY
+  value: {{ .Values.proxy.http_proxy | quote }}
+- name: NO_PROXY
+  value: {{ .Values.proxy.no_proxy | quote }}
+{{- end -}}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml

@@ -0,0 +1,23 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-attacher-role
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [ "" ]
+    resources: [ "persistentvolumes" ]
+    verbs: [ "get", "list", "watch", "update", "patch" ]
+  - apiGroups: [ "" ]
+    resources: [ "nodes" ]
+    verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "csi.storage.k8s.io" ]
+    resources: [ "csinodeinfos" ]
+    verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "volumeattachments" ]
+    verbs: [ "get", "list", "watch", "update", "patch" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "volumeattachments/status" ]
+    verbs: [ "patch" ]

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml

@@ -0,0 +1,11 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-node-role
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get"]

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml

@@ -0,0 +1,38 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-provisioner-role
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [ "" ]
+    resources: [ "persistentvolumes" ]
+    verbs: [ "get", "list", "watch", "create", "delete" ]
+  - apiGroups: [ "" ]
+    resources: [ "persistentvolumeclaims" ]
+    verbs: [ "get", "list", "watch", "update" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "storageclasses" ]
+    verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "" ]
+    resources: [ "events" ]
+    verbs: [ "list", "watch", "create", "update", "patch" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources: [ "volumesnapshots" ]
+    verbs: [ "get", "list" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources: [ "volumesnapshotcontents" ]
+    verbs: [ "get", "list" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "csinodes" ]
+    verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "" ]
+    resources: [ "nodes" ]
+    verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "coordination.k8s.io" ]
+    resources: [ "leases" ]
+    verbs: [ "get", "watch", "list", "delete", "update", "create" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "volumeattachments" ]
+    verbs: [ "get", "list", "watch" ]

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml

@@ -0,0 +1,31 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-resizer-role
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+rules:
+  # The following rule should be uncommented for plugins that require secrets
+  # for provisioning.
+  # - apiGroups: [""]
+  #   resources: ["secrets"]
+  #   verbs: ["get", "list", "watch"]
+  - apiGroups: [ "" ]
+    resources: [ "persistentvolumes" ]
+    verbs: [ "get", "list", "watch", "update", "patch" ]
+  - apiGroups: [ "" ]
+    resources: [ "persistentvolumeclaims" ]
+    verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "" ]
+    resources: [ "persistentvolumeclaims/status" ]
+    verbs: [ "update", "patch" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "storageclasses" ]
+    verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "" ]
+    resources: [ "events" ]
+    verbs: [ "list", "watch", "create", "update", "patch" ]
+  - apiGroups: [ "" ]
+    resources: [ "pods" ]
+    verbs: [ "get", "list", "watch" ]

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml

@@ -0,0 +1,27 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-snapshotter-role
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [ "" ]
+    resources: [ "events" ]
+    verbs: [ "list", "watch", "create", "update", "patch" ]
+  # Secret permission is optional.
+  # Enable it if your driver needs secret.
+  # For example, `csi.storage.k8s.io/snapshotter-secret-name` is set in VolumeSnapshotClass.
+  # See https://kubernetes-csi.github.io/docs/secrets-and-credentials.html for more details.
+  # - apiGroups: [ "" ]
+  #   resources: [ "secrets" ]
+  #   verbs: [ "get", "list" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources: [ "volumesnapshotclasses" ]
+    verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources: [ "volumesnapshotcontents" ]
+    verbs: [ "create", "get", "list", "watch", "update", "delete", "patch" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources: [ "volumesnapshotcontents/status" ]
+    verbs: [ "update" ]

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml

@@ -0,0 +1,15 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-attacher-binding
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.controller.serviceAccount.name }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-attacher-role
+  apiGroup: rbac.authorization.k8s.io

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-csi-node.yaml

@@ -0,0 +1,15 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-node-getter-binding
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.node.serviceAccount.name }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: ebs-csi-node-role
+  apiGroup: rbac.authorization.k8s.io

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml

@@ -0,0 +1,15 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-provisioner-binding
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.controller.serviceAccount.name }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-provisioner-role
+  apiGroup: rbac.authorization.k8s.io

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml

@@ -0,0 +1,15 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-resizer-binding
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.controller.serviceAccount.name }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-resizer-role
+  apiGroup: rbac.authorization.k8s.io

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml

@@ -0,0 +1,15 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-snapshotter-binding
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.controller.serviceAccount.name }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml

@@ -0,0 +1,361 @@
+# Controller Service
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: ebs-csi-controller
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.controller.replicaCount }}
+  {{- with .Values.controller.updateStrategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        app: ebs-csi-controller
+        {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }}
+        {{- if .Values.controller.podLabels }}
+        {{- toYaml .Values.controller.podLabels | nindent 8 }}
+        {{- end }}
+      {{- if .Values.controller.podAnnotations }}
+      annotations:
+        {{- toYaml .Values.controller.podAnnotations | nindent 8 }}
+      {{- end }}
+    spec:
+      nodeSelector:
+        kubernetes.io/os: linux
+        {{- with .Values.controller.nodeSelector }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      serviceAccountName: {{ .Values.controller.serviceAccount.name }}
+      priorityClassName: {{ .Values.controller.priorityClassName }}
+      {{- with default .Values.controller.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      tolerations:
+        {{- with .Values.controller.tolerations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- if .Values.controller.topologySpreadConstraints }}
+      {{- $tscLabelSelector := dict "labelSelector" ( dict "matchLabels" ( dict "app" "ebs-csi-controller" ) ) }}
+      {{- $constraints := list }}
+      {{- range .Values.controller.topologySpreadConstraints }}
+        {{- $constraints = mustAppend $constraints (mergeOverwrite . $tscLabelSelector) }}
+      {{- end }}
+      topologySpreadConstraints:
+        {{- $constraints | toYaml | nindent 8 }}
+      {{- end }}
+      {{- with .Values.controller.securityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.controller.initContainers }}
+      initContainers:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: ebs-plugin
+          image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (.Values.image.tag | toString)) }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            {{- if ne .Release.Name "kustomize" }}
+            - controller
+            {{- else }}
+            # - {all,controller,node} # specify the driver mode
+            {{- end }}
+            - --endpoint=$(CSI_ENDPOINT)
+            {{- if .Values.controller.extraVolumeTags }}
+              {{- include "aws-ebs-csi-driver.extra-volume-tags" . | nindent 12 }}
+            {{- end }}
+            {{- with .Values.controller.k8sTagClusterId }}
+            - --k8s-tag-cluster-id={{ . }}
+            {{- end }}
+            {{- if and (.Values.controller.enableMetrics) (not .Values.controller.httpEndpoint) }}
+            - --http-endpoint=0.0.0.0:3301
+            {{- end}}
+            {{- with .Values.controller.httpEndpoint }}
+            - --http-endpoint={{ . }}
+            {{- end }}
+            {{- if .Values.controller.sdkDebugLog }}
+            - --aws-sdk-debug-log=true
+            {{- end}}
+            {{- with .Values.controller.loggingFormat }}
+            - --logging-format={{ . }}
+            {{- end }}
+            - --v={{ .Values.controller.logLevel }}
+            {{- range .Values.controller.additionalArgs }}
+            - {{ . }}
+            {{- end }}
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: CSI_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: aws-secret
+                  key: key_id
+                  optional: true
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: aws-secret
+                  key: access_key
+                  optional: true
+            - name: AWS_EC2_ENDPOINT
+              valueFrom:
+                configMapKeyRef:
+                  name: aws-meta
+                  key: endpoint
+                  optional: true
+            {{- with .Values.controller.region }}
+            - name: AWS_REGION
+              value: {{ . }}
+            {{- end }}
+            {{- if .Values.proxy.http_proxy }}
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
+            {{- end }}
+            {{- with .Values.controller.env }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          envFrom:
+            {{- with .Values.controller.envFrom }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          {{- with .Values.controller.volumeMounts }}
+          {{- toYaml . | nindent 12 }}
+          {{- end }}
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+            {{- if .Values.controller.enableMetrics }}
+            - name: metrics
+              containerPort: 3301
+              protocol: TCP
+            {{- end}}
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
+          {{- with .Values.controller.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.controller.containerSecurityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        - name: csi-provisioner
+          image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.provisioner.image.repository .Values.sidecars.provisioner.image.tag }}
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.provisioner.image.pullPolicy }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v={{ .Values.sidecars.provisioner.logLevel }}
+            - --feature-gates=Topology=true
+            {{- if .Values.controller.extraCreateMetadata }}
+            - --extra-create-metadata
+            {{- end}}
+            - --leader-election={{ .Values.sidecars.provisioner.leaderElection.enabled | required "leader election state for csi-provisioner is required, must be set to true || false." }}
+            {{- if .Values.sidecars.provisioner.leaderElection.enabled }}
+            {{- if .Values.sidecars.provisioner.leaderElection.leaseDuration }}
+            - --leader-election-lease-duration={{ .Values.sidecars.provisioner.leaderElection.leaseDuration }}
+            {{- end }}
+            {{- if .Values.sidecars.provisioner.leaderElection.renewDeadline}}
+            - --leader-election-renew-deadline={{ .Values.sidecars.provisioner.leaderElection.renewDeadline }}
+            {{- end }}
+            {{- if .Values.sidecars.provisioner.leaderElection.retryPeriod }}
+            - --leader-election-retry-period={{ .Values.sidecars.provisioner.leaderElection.retryPeriod }}
+            {{- end }}
+            {{- end }}
+            - --default-fstype={{ .Values.controller.defaultFsType }}
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+            {{- if .Values.proxy.http_proxy }}
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
+            {{- end }}
+            {{- with .Values.sidecars.provisioner.env }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          envFrom:
+            {{- with .Values.controller.envFrom }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          {{- with default .Values.controller.resources .Values.sidecars.provisioner.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.sidecars.provisioner.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        - name: csi-attacher
+          image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.attacher.image.repository .Values.sidecars.attacher.image.tag }}
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.attacher.image.pullPolicy }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v={{ .Values.sidecars.attacher.logLevel }}
+            - --leader-election={{ .Values.sidecars.attacher.leaderElection.enabled | required "leader election state for csi-attacher is required, must be set to true || false." }}
+            {{- if .Values.sidecars.attacher.leaderElection.enabled }}
+            {{- if .Values.sidecars.attacher.leaderElection.leaseDuration }}
+            - --leader-election-lease-duration={{ .Values.sidecars.attacher.leaderElection.leaseDuration }}
+            {{- end }}
+            {{- if .Values.sidecars.attacher.leaderElection.renewDeadline}}
+            - --leader-election-renew-deadline={{ .Values.sidecars.attacher.leaderElection.renewDeadline }}
+            {{- end }}
+            {{- if .Values.sidecars.attacher.leaderElection.retryPeriod }}
+            - --leader-election-retry-period={{ .Values.sidecars.attacher.leaderElection.retryPeriod }}
+            {{- end }}
+            {{- end }}
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+            {{- if .Values.proxy.http_proxy }}
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
+            {{- end }}
+            {{- with .Values.sidecars.attacher.env }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          envFrom:
+            {{- with .Values.controller.envFrom }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          {{- with default .Values.controller.resources .Values.sidecars.attacher.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.sidecars.attacher.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- if or .Values.sidecars.snapshotter.forceEnable (.Capabilities.APIVersions.Has "snapshot.storage.k8s.io/v1beta1") (.Capabilities.APIVersions.Has "snapshot.storage.k8s.io/v1") }}
+        - name: csi-snapshotter
+          image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.snapshotter.image.repository .Values.sidecars.snapshotter.image.tag }}
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.snapshotter.image.pullPolicy }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --leader-election=true
+            {{- if .Values.controller.extraCreateMetadata }}
+            - --extra-create-metadata
+            {{- end}}
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+            {{- if .Values.proxy.http_proxy }}
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
+            {{- end }}
+            {{- with .Values.sidecars.snapshotter.env }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          envFrom:
+            {{- with .Values.controller.envFrom }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          {{- with default .Values.controller.resources .Values.sidecars.snapshotter.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.sidecars.snapshotter.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        {{- end }}
+        - name: csi-resizer
+          image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.resizer.image.repository .Values.sidecars.resizer.image.tag }}
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.resizer.image.pullPolicy }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v={{ .Values.sidecars.resizer.logLevel }}
+            - --handle-volume-inuse-error=false
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+            {{- if .Values.proxy.http_proxy }}
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
+            {{- end }}
+            {{- with .Values.sidecars.resizer.env }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          envFrom:
+            {{- with .Values.controller.envFrom }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          {{- with default .Values.controller.resources .Values.sidecars.resizer.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.sidecars.resizer.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        - name: liveness-probe
+          image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }}
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }}
+          args:
+            - --csi-address=/csi/csi.sock
+          envFrom:
+            {{- with .Values.controller.envFrom }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+          {{- with default .Values.controller.resources .Values.sidecars.livenessProbe.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.sidecars.livenessProbe.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- if .Values.imagePullSecrets }}
+      imagePullSecrets:
+      {{- range .Values.imagePullSecrets }}
+        - name: {{ . }}
+      {{- end }}
+      {{- end }}
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        {{- with .Values.controller.volumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml

@@ -0,0 +1,12 @@
+apiVersion: {{ ternary "storage.k8s.io/v1" "storage.k8s.io/v1beta1" (semverCompare ">=1.18.0-0" .Capabilities.KubeVersion.Version) }}
+kind: CSIDriver
+metadata:
+  name: ebs.csi.aws.com
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+spec:
+  attachRequired: true
+  podInfoOnMount: false
+  {{- if not .Values.useOldCSIDriver }}
+  fsGroupPolicy: File
+  {{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/metrics.yaml

@@ -0,0 +1,42 @@
+{{- if .Values.controller.enableMetrics -}}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ebs-csi-controller
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: ebs-csi-controller
+spec:
+  selector:
+    app: ebs-csi-controller
+  ports:
+    - name: metrics
+      port: 3301
+      targetPort: 3301
+  type: ClusterIP
+---
+{{- if or .Values.controller.serviceMonitor.forceEnable (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: ebs-csi-controller
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: ebs-csi-controller
+    {{- if .Values.controller.serviceMonitor.labels }}
+    {{- toYaml .Values.controller.serviceMonitor.labels | nindent 4 }}
+    {{- end }}
+spec:
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  endpoints:
+    - targetPort: 3301
+      path: /metrics
+      interval: 15s
+{{- end }}
+{{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml

@@ -0,0 +1,184 @@
+{{- if .Values.node.enableWindows }}
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: ebs-csi-node-windows
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app: ebs-csi-node
+      {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }}
+  updateStrategy:
+    {{ toYaml .Values.node.updateStrategy | nindent 4 }}
+  template:
+    metadata:
+      labels:
+        app: ebs-csi-node
+        {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }}
+        {{- if .Values.node.podLabels }}
+        {{- toYaml .Values.node.podLabels | nindent 8 }}
+        {{- end }}
+      {{- with .Values.node.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      {{- with .Values.node.affinity }}
+      affinity: {{- toYaml . | nindent 8 }}
+      {{- end }}
+      nodeSelector:
+        kubernetes.io/os: windows
+        {{- with .Values.node.nodeSelector }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      serviceAccountName: {{ .Values.node.serviceAccount.name }}
+      priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }}
+      tolerations:
+        {{- if .Values.node.tolerateAllTaints }}
+        - operator: Exists
+        {{- else }}
+        {{- with .Values.node.tolerations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- end }}
+      containers:
+        - name: ebs-plugin
+          image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            - node
+            - --endpoint=$(CSI_ENDPOINT)
+            {{- with .Values.node.volumeAttachLimit }}
+            - --volume-attach-limit={{ . }}
+            {{- end }}
+            {{- with .Values.node.loggingFormat }}
+            - --logging-format={{ . }}
+            {{- end }}
+            - --v={{ .Values.node.logLevel }}
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:/csi/csi.sock
+            - name: CSI_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            {{- if .Values.proxy.http_proxy }}
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
+            {{- end }}
+            {{- with .Values.node.env }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: C:\var\lib\kubelet
+              mountPropagation: "None"
+            - name: plugin-dir
+              mountPath: C:\csi
+            - name: csi-proxy-disk-pipe
+              mountPath: \\.\pipe\csi-proxy-disk-v1
+            - name: csi-proxy-volume-pipe
+              mountPath: \\.\pipe\csi-proxy-volume-v1
+            - name: csi-proxy-filesystem-pipe
+              mountPath: \\.\pipe\csi-proxy-filesystem-v1
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
+          {{- with .Values.node.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        - name: node-driver-registrar
+          image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }}
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+            - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }}
+          env:
+            - name: ADDRESS
+              value: unix:/csi/csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: C:\var\lib\kubelet\plugins\ebs.csi.aws.com\csi.sock
+            {{- if .Values.proxy.http_proxy }}
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
+            {{- end }}
+            {{- with .Values.sidecars.nodeDriverRegistrar.env }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          livenessProbe:
+            exec:
+              command:
+                - /csi-node-driver-registrar.exe
+                - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+                - --mode=kubelet-registration-probe
+            initialDelaySeconds: 30
+            timeoutSeconds: 15
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: C:\csi
+            - name: registration-dir
+              mountPath: C:\registration
+            - name: probe-dir
+              mountPath: C:\var\lib\kubelet\plugins\ebs.csi.aws.com
+          {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        - name: liveness-probe
+          image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }}
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }}
+          args:
+            - --csi-address=unix:/csi/csi.sock
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: C:\csi
+          {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- if .Values.imagePullSecrets }}
+      imagePullSecrets:
+      {{- range .Values.imagePullSecrets }}
+        - name: {{ . }}
+      {{- end }}
+      {{- end }}
+      volumes:
+        - name: kubelet-dir
+          hostPath:
+            path: C:\var\lib\kubelet
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: C:\var\lib\kubelet\plugins\ebs.csi.aws.com
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: C:\var\lib\kubelet\plugins_registry
+            type: Directory
+        - name: csi-proxy-disk-pipe
+          hostPath:
+            path: \\.\pipe\csi-proxy-disk-v1
+            type: ""
+        - name: csi-proxy-volume-pipe
+          hostPath:
+            path: \\.\pipe\csi-proxy-volume-v1
+            type: ""
+        - name: csi-proxy-filesystem-pipe
+          hostPath:
+            path: \\.\pipe\csi-proxy-filesystem-v1
+            type: ""
+        - name: probe-dir
+          emptyDir: {}
+{{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml

@@ -0,0 +1,199 @@
+# Node Service
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: ebs-csi-node
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app: ebs-csi-node
+      {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }}
+  updateStrategy:
+    {{- toYaml .Values.node.updateStrategy | nindent 4 }}
+  template:
+    metadata:
+      labels:
+        app: ebs-csi-node
+        {{- include "aws-ebs-csi-driver.labels" . | nindent 8 }}
+        {{- if .Values.node.podLabels }}
+        {{- toYaml .Values.node.podLabels | nindent 8 }}
+        {{- end }}
+      {{- with .Values.node.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      {{- with .Values.node.affinity }}
+      affinity: {{- toYaml . | nindent 8 }}
+      {{- end }}
+      nodeSelector:
+        kubernetes.io/os: linux
+        {{- with .Values.node.nodeSelector }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      serviceAccountName: {{ .Values.node.serviceAccount.name }}
+      priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }}
+      tolerations:
+        {{- if .Values.node.tolerateAllTaints }}
+        - operator: Exists
+        {{- else }}
+        {{- with .Values.node.tolerations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- end }}
+      {{- with .Values.node.securityContext }}
+      securityContext:  
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: ebs-plugin
+          image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            - node
+            - --endpoint=$(CSI_ENDPOINT)
+            {{- with .Values.node.volumeAttachLimit }}
+            - --volume-attach-limit={{ . }}
+            {{- end }}
+            {{- with .Values.node.loggingFormat }}
+            - --logging-format={{ . }}
+            {{- end }}
+            - --v={{ .Values.node.logLevel }}
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:/csi/csi.sock
+            - name: CSI_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            {{- if .Values.proxy.http_proxy }}
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
+            {{- end }}
+            {{- with .Values.node.env }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          envFrom:
+            {{- with .Values.controller.envFrom }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: {{ .Values.node.kubeletPath }}
+              mountPropagation: "Bidirectional"
+            - name: plugin-dir
+              mountPath: /csi
+            - name: device-dir
+              mountPath: /dev
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
+          {{- with .Values.node.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.node.containerSecurityContext }}
+          securityContext:  
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        - name: node-driver-registrar
+          image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.nodeDriverRegistrar.image.repository .Values.sidecars.nodeDriverRegistrar.image.tag }}
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+            - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }}
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: {{ printf "%s/plugins/ebs.csi.aws.com/csi.sock" (trimSuffix "/" .Values.node.kubeletPath) }}
+            {{- if .Values.proxy.http_proxy }}
+            {{- include "aws-ebs-csi-driver.http-proxy" . | nindent 12 }}
+            {{- end }}
+            {{- with .Values.sidecars.nodeDriverRegistrar.env }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          envFrom:
+            {{- with .Values.controller.envFrom }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          livenessProbe:
+            exec:
+              command:
+                - /csi-node-driver-registrar
+                - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+                - --mode=kubelet-registration-probe
+            initialDelaySeconds: 30
+            timeoutSeconds: 15
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+            - name: probe-dir
+              mountPath: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }}
+          {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.sidecars.nodeDriverRegistrar.securityContext }}
+          securityContext:  
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+        - name: liveness-probe
+          image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.sidecars.livenessProbe.image.repository .Values.sidecars.livenessProbe.image.tag }}
+          imagePullPolicy: {{ default .Values.image.pullPolicy .Values.sidecars.livenessProbe.image.pullPolicy }}
+          args:
+            - --csi-address=/csi/csi.sock
+          envFrom:
+            {{- with .Values.controller.envFrom }}
+            {{- . | toYaml | nindent 12 }}
+            {{- end }}
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+          {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.sidecars.livenessProbe.securityContext }}
+          securityContext:  
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- if .Values.imagePullSecrets }}
+      imagePullSecrets:
+      {{- range .Values.imagePullSecrets }}
+        - name: {{ . }}
+      {{- end }}
+      {{- end }}
+      volumes:
+        - name: kubelet-dir
+          hostPath:
+            path: {{ .Values.node.kubeletPath }}
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: {{ printf "%s/plugins/ebs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }}
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }}
+            type: Directory
+        - name: device-dir
+          hostPath:
+            path: /dev
+            type: Directory
+        - name: probe-dir
+          emptyDir: {}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml

@@ -0,0 +1,21 @@
+{{- if .Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget" }}
+apiVersion: policy/v1
+{{- else }}
+apiVersion: policy/v1beta1
+{{- end }}
+kind: PodDisruptionBudget
+metadata:
+  name: ebs-csi-controller
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }}
+  {{- if le (.Values.controller.replicaCount | int) 2 }}
+  maxUnavailable: 1
+  {{- else }}
+  minAvailable: 2
+  {{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.controller.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.controller.serviceAccount.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+  {{- with .Values.controller.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- if eq .Release.Name "kustomize" }}
+  #Enable if EKS IAM roles for service accounts (IRSA) is used. See https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html for details.
+  #annotations:
+  #  eks.amazonaws.com/role-arn: arn:<partition>:iam::<account>:role/ebs-csi-role
+  {{- end }}
+{{- end -}}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-node.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.node.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.node.serviceAccount.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+  {{- with .Values.node.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end -}}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/storageclass.yaml

@@ -0,0 +1,15 @@
+{{- range .Values.storageClasses }}
+---
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: {{ .name }}
+  {{- with .annotations }}
+  annotations: {{- . | toYaml | trim | nindent 4 }}
+  {{- end }}
+  {{- with .labels }}
+  labels: {{- . | toYaml | trim | nindent 4 }}
+  {{- end }}
+provisioner: ebs.csi.aws.com
+{{ omit (dict "volumeBindingMode" "WaitForFirstConsumer" | merge .) "name" "annotations" "labels" | toYaml }}
+{{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/tests/helm-tester.yaml

@@ -0,0 +1,223 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ebs-csi-driver-test
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-driver-test
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
+rules:
+  - apiGroups: [ "" ]
+    resources:
+      - events
+      - nodes
+      - pods
+      - replicationcontrollers
+      - serviceaccounts
+      - configmaps
+      - persistentvolumes
+      - persistentvolumeclaims
+    verbs: [ "list" ]
+  - apiGroups: [ "" ]
+    resources:
+      - services
+      - nodes
+      - nodes/proxy
+      - persistentvolumes
+      - persistentvolumeclaims
+      - pods
+      - pods/log
+    verbs: [ "get" ]
+  - apiGroups: [ "" ]
+    resources:
+      - namespaces
+      - persistentvolumes
+      - persistentvolumeclaims
+      - pods
+      - pods/exec
+    verbs: [ "create" ]
+  - apiGroups: [ "" ]
+    resources:
+      - namespaces
+      - persistentvolumes
+      - persistentvolumeclaims
+      - pods
+    verbs: [ "delete" ]
+  - apiGroups: [ "" ]
+    resources:
+      - persistentvolumeclaims
+    verbs: [ "update" ]
+  - apiGroups: [ "" ]
+    resources:
+      - pods/ephemeralcontainers
+    verbs: [ "patch" ]
+  - apiGroups: [ "" ]
+    resources:
+      - serviceaccounts
+      - configmaps
+    verbs: [ "watch" ]
+  - apiGroups: [ "apps" ]
+    resources:
+      - replicasets
+      - daemonsets
+    verbs: [ "list" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources:
+      - storageclasses
+    verbs: [ "create" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources:
+      - storageclasses
+      - csinodes
+    verbs: [ "get" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources:
+      - storageclasses
+    verbs: [ "delete" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources:
+      - volumesnapshots
+      - volumesnapshotclasses
+      - volumesnapshotcontents
+    verbs: [ "create" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources:
+      - volumesnapshots
+      - volumesnapshotclasses
+      - volumesnapshotcontents
+    verbs: [ "get" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources:
+      - volumesnapshotcontents
+    verbs: [ "update" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources:
+      - volumesnapshots
+      - volumesnapshotclasses
+      - volumesnapshotcontents
+    verbs: [ "delete" ]
+  - apiGroups: [ "authorization.k8s.io" ]
+    resources:
+      - clusterroles
+    verbs: [ "list" ]
+  - apiGroups: [ "authorization.k8s.io" ]
+    resources:
+      - subjectaccessreviews
+    verbs: [ "create" ]
+  - apiGroups: [ "rbac.authorization.k8s.io" ]
+    resources:
+      - clusterroles
+    verbs: [ "list" ]
+  - apiGroups: [ "rbac.authorization.k8s.io" ]
+    resources:
+      - clusterrolebindings
+    verbs: [ "create" ]
+  - apiGroups: [ "apiextensions.k8s.io" ]
+    resources:
+      - customresourcedefinitions
+    verbs: [ "get" ]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-driver-test
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-driver-test
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-csi-driver-test
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: ConfigMap
+data:
+  manifests.yaml: |
+    ShortName: ebs
+    StorageClass:
+      FromFile: storageclass.yaml
+    SnapshotClass:
+      FromName: true
+    DriverInfo:
+      Name: ebs.csi.aws.com
+      SupportedSizeRange:
+        Min: 1Gi
+        Max: 16Ti
+      SupportedFsType:
+        xfs: {}
+        ext4: {}
+      SupportedMountOption:
+        dirsync: {}
+      TopologyKeys: ["topology.ebs.csi.aws.com/zone"]
+      Capabilities:
+        persistence: true
+        fsGroup: true
+        block: true
+        exec: true
+        snapshotDataSource: true
+        pvcDataSource: false
+        multipods: true
+        controllerExpansion: true
+        nodeExpansion: true
+        volumeLimits: true
+        topology: true
+  storageclass.yaml: |
+    kind: StorageClass
+    apiVersion: storage.k8s.io/v1
+    metadata:
+      name: ebs.csi.aws.com
+    provisioner: ebs.csi.aws.com
+    volumeBindingMode: WaitForFirstConsumer
+metadata:
+  name: ebs-csi-driver-test
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ebs-csi-driver-test
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
+spec:
+  containers:
+    - name: kubetest2
+      image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20220624-1a63fdd9f2-master
+      command: [ "/bin/sh", "-c" ]
+      args:
+        - |
+          cp /etc/config/storageclass.yaml /workspace/storageclass.yaml
+          go install sigs.k8s.io/kubetest2/...@latest
+          kubectl config set-cluster cluster --server=https://kubernetes.default --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+          kubectl config set-context kubetest2 --cluster=cluster
+          kubectl config set-credentials sa --token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+          kubectl config set-context kubetest2 --user=sa && kubectl config use-context kubetest2
+          kubectl get crd volumesnapshots.snapshot.storage.k8s.io
+          if [ $? -eq 0 ]; then 
+            SNAPSHOTS="|snapshot fields"
+          fi
+          export FOCUS_REGEX="\bebs.csi.aws.com\b.+(validate content|resize volume|offline PVC|AllowedTopologies|store data$SNAPSHOTS)"
+          kubetest2 noop --run-id='e2e-kubernetes' --test=ginkgo -- --test-package-version=$(curl https://storage.googleapis.com/kubernetes-release/release/stable-1.25.txt) --skip-regex='\[Disruptive\]|\[Serial\]' --focus-regex="$FOCUS_REGEX" --parallel=25 --test-args='-storage.testdriver=/etc/config/manifests.yaml'
+      volumeMounts:
+      - name: config-vol
+        mountPath: /etc/config
+  serviceAccountName: ebs-csi-driver-test
+  volumes:
+    - name: config-vol
+      configMap:
+        name: ebs-csi-driver-test
+  restartPolicy: Never

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/volumesnapshotclass.yaml

@@ -0,0 +1,17 @@
+{{- if or .Values.sidecars.snapshotter.forceEnable (.Capabilities.APIVersions.Has "snapshot.storage.k8s.io/v1beta1") (.Capabilities.APIVersions.Has "snapshot.storage.k8s.io/v1") }}
+{{- range .Values.volumeSnapshotClasses }}
+---
+kind: VolumeSnapshotClass
+apiVersion: snapshot.storage.k8s.io/v1
+metadata:
+  name: {{ .name }}
+  {{- with .annotations }}
+  annotations: {{- . | toYaml | trim | nindent 4 }}
+  {{- end }}
+driver: ebs.csi.aws.com
+deletionPolicy: {{ .deletionPolicy }}
+{{- with .parameters }}
+parameters: {{- . | toYaml | trim | nindent 2 }}
+{{- end }}
+{{- end }}
+{{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml

@@ -0,0 +1,321 @@
+# Default values for aws-ebs-csi-driver.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+image:
+  repository: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver
+  # Overrides the image tag whose default is v{{ .Chart.AppVersion }}
+  tag: ""
+  pullPolicy: IfNotPresent
+
+# -- Custom labels to add into metadata
+customLabels:
+  {}
+  # k8s-app: aws-ebs-csi-driver
+
+sidecars:
+  provisioner:
+    env: []
+    image:
+      pullPolicy: IfNotPresent
+      repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner
+      tag: "v3.4.1-eks-1-26-7"
+    logLevel: 2
+    resources: {}
+    # Tune leader lease election for csi-provisioner.
+    # Leader election is on by default.
+    leaderElection:
+      enabled: true
+      # Optional values to tune lease behavior.
+      # The arguments provided must be in an acceptable time.ParseDuration format.
+      # Ref: https://pkg.go.dev/flag#Duration
+      # leaseDuration: "15s"
+      # renewDeadline: "10s"
+      # retryPeriod: "5s"
+    securityContext:
+      readOnlyRootFilesystem: true
+      allowPrivilegeEscalation: false
+  attacher:
+    env: []
+    image:
+      pullPolicy: IfNotPresent
+      repository: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher
+      tag: "v4.2.0-eks-1-26-7"
+    # Tune leader lease election for csi-attacher.
+    # Leader election is on by default.
+    leaderElection:
+      enabled: true
+      # Optional values to tune lease behavior.
+      # The arguments provided must be in an acceptable time.ParseDuration format.
+      # Ref: https://pkg.go.dev/flag#Duration
+      # leaseDuration: "15s"
+      # renewDeadline: "10s"
+      # retryPeriod: "5s"
+    logLevel: 2
+    resources: {}
+    securityContext:
+      readOnlyRootFilesystem: true
+      allowPrivilegeEscalation: false
+  snapshotter:
+    # Enables the snapshotter sidecar even if the snapshot CRDs are not installed
+    forceEnable: false
+    env: []
+    image:
+      pullPolicy: IfNotPresent
+      repository: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter
+      tag: "v6.2.1-eks-1-26-7"
+    logLevel: 2
+    resources: {}
+    securityContext:
+      readOnlyRootFilesystem: true
+      allowPrivilegeEscalation: false
+  livenessProbe:
+    image:
+      pullPolicy: IfNotPresent
+      repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe
+      tag: "v2.9.0-eks-1-26-7"
+    resources: {}
+    securityContext:
+      readOnlyRootFilesystem: true
+      allowPrivilegeEscalation: false
+  resizer:
+    env: []
+    image:
+      pullPolicy: IfNotPresent
+      repository: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer
+      tag: "v1.7.0-eks-1-26-7"
+    logLevel: 2
+    resources: {}
+    securityContext:
+      readOnlyRootFilesystem: true
+      allowPrivilegeEscalation: false
+  nodeDriverRegistrar:
+    env: []
+    image:
+      pullPolicy: IfNotPresent
+      repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar
+      tag: "v2.7.0-eks-1-26-7"
+    logLevel: 2
+    resources: {}
+    securityContext:
+      readOnlyRootFilesystem: true
+      allowPrivilegeEscalation: false
+
+proxy:
+  http_proxy:
+  no_proxy:
+
+imagePullSecrets: []
+nameOverride:
+fullnameOverride:
+
+controller:
+  additionalArgs: []
+  sdkDebugLog: false
+  loggingFormat: text
+  affinity:
+    nodeAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        preference:
+          matchExpressions:
+          - key: eks.amazonaws.com/compute-type
+            operator: NotIn
+            values:
+            - fargate
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+            - key: app
+              operator: In
+              values:
+              - ebs-csi-controller
+          topologyKey: kubernetes.io/hostname
+        weight: 100
+  # The default filesystem type of the volume to provision when fstype is unspecified in the StorageClass.
+  # If the default is not set and fstype is unset in the StorageClass, then no fstype will be set
+  defaultFsType: ext4
+  env: []
+  # Use envFrom to reference ConfigMaps and Secrets across all containers in the deployment
+  envFrom: []
+  # If set, add pv/pvc metadata to plugin create requests as parameters.
+  extraCreateMetadata: true
+  # Extra volume tags to attach to each dynamically provisioned volume.
+  # ---
+  # extraVolumeTags:
+  #   key1: value1
+  #   key2: value2
+  extraVolumeTags: {}
+  httpEndpoint:
+  # (deprecated) The TCP network address where the prometheus metrics endpoint
+  # will run (example: `:8080` which corresponds to port 8080 on local host).
+  # The default is empty string, which means metrics endpoint is disabled.
+  # ---
+  enableMetrics: false
+  serviceMonitor:
+    # Enables the ServiceMonitor resource even if the prometheus-operator CRDs are not installed
+    forceEnable: false
+    # Additional labels for ServiceMonitor object
+    labels:
+      release: prometheus
+  # If set to true, AWS API call metrics will be exported to the following
+  # TCP endpoint: "0.0.0.0:3301"
+  # ---
+  # ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional).
+  k8sTagClusterId:
+  logLevel: 2
+  nodeSelector: {}
+  podAnnotations: {}
+  podLabels: {}
+  priorityClassName: system-cluster-critical
+  # AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata
+  # service.
+  # ---
+  # region: us-east-1
+  region:
+  replicaCount: 2
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  # type: RollingUpdate
+  # rollingUpdate:
+  #   maxSurge: 0
+  #   maxUnavailable: 1
+  resources:
+    requests:
+      cpu: 10m
+      memory: 40Mi
+    limits:
+      cpu: 100m
+      memory: 256Mi
+  serviceAccount:
+  # A service account will be created for you if set to true. Set to false if you want to use your own.
+    create: true
+    name: ebs-csi-controller-sa
+    annotations: {}
+  tolerations:
+    - key: CriticalAddonsOnly
+      operator: Exists
+    - effect: NoExecute
+      operator: Exists
+      tolerationSeconds: 300
+  # TSCs without the label selector stanza
+  #
+  # Example:
+  #
+  # topologySpreadConstraints:
+  #  - maxSkew: 1
+  #    topologyKey: topology.kubernetes.io/zone
+  #    whenUnsatisfiable: ScheduleAnyway
+  #  - maxSkew: 1
+  #    topologyKey: kubernetes.io/hostname
+  #    whenUnsatisfiable: ScheduleAnyway
+  topologySpreadConstraints: []
+  # securityContext on the controller pod
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 1000
+    runAsGroup: 1000
+    fsGroup: 1000
+  volumes: []
+  volumeMounts: []
+  # securityContext on the controller container (see sidecars for securityContext on sidecar containers)
+  containerSecurityContext:
+    readOnlyRootFilesystem: true
+    allowPrivilegeEscalation: false
+  initContainers: []
+  # containers to be run before the controller's container starts.
+  #
+  # Example:
+  #
+  # - name: wait
+  #   image: busybox
+  #   command: [ 'sh', '-c', "sleep 20" ]
+
+node:
+  env: []
+  envFrom: []
+  kubeletPath: /var/lib/kubelet
+  loggingFormat: text
+  logLevel: 2
+  priorityClassName:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: eks.amazonaws.com/compute-type
+            operator: NotIn
+            values:
+            - fargate
+  nodeSelector: {}
+  podAnnotations: {}
+  podLabels: {}
+  tolerateAllTaints: true
+  tolerations:
+  - operator: Exists
+    effect: NoExecute
+    tolerationSeconds: 300
+  resources:
+    requests:
+      cpu: 10m
+      memory: 40Mi
+    limits:
+      cpu: 100m
+      memory: 256Mi
+  serviceAccount:
+    create: true
+    name: ebs-csi-node-sa
+    annotations: {}
+  enableWindows: false
+  # The "maximum number of attachable volumes" per node
+  volumeAttachLimit:
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: "10%"
+  # securityContext on the node pod
+  securityContext:
+    # The node pod must be run as root to bind to the registration/driver sockets
+    runAsNonRoot: false
+    runAsUser: 0
+    runAsGroup: 0
+    fsGroup: 0
+  # securityContext on the node container (see sidecars for securityContext on sidecar containers)
+  containerSecurityContext:
+    readOnlyRootFilesystem: true
+    privileged: true
+
+storageClasses: []
+# Add StorageClass resources like:
+# - name: ebs-sc
+#   # annotation metadata
+#   annotations:
+#     storageclass.kubernetes.io/is-default-class: "true"
+#   # label metadata
+#   labels:
+#     my-label-is: supercool
+#   # defaults to WaitForFirstConsumer
+#   volumeBindingMode: WaitForFirstConsumer
+#   # defaults to Delete
+#   reclaimPolicy: Retain
+#   parameters:
+#     encrypted: "true"
+
+volumeSnapshotClasses: []
+# Add VolumeSnapshotClass resources like:
+# - name: ebs-vsc
+#   # annotation metadata
+#   annotations:
+#     snapshot.storage.kubernetes.io/is-default-class: "true"
+#   # deletionPolicy must be specified
+#   deletionPolicy: Delete
+#   parameters:
+
+# Use old CSIDriver without an fsGroupPolicy set
+# Intended for use with older clusters that cannot easily replace the CSIDriver object
+# This parameter should always be false for new installations
+useOldCSIDriver: false

charts/kubezero-storage/jsonnet/jsonnetfile.lock.json

@@ -38,7 +38,7 @@
           "subdir": "grafana-builder"
         }
       },
-      "version": "af3ca2c3fae4096002b0c0c921f18ca7da8d361f",
+      "version": "f3644000f0bbca4511945fd61e6a55be16478d4e",
       "sum": "tDR6yT2GVfw0wTU12iZH+m01HrbIr6g/xN+/8nzNkU0="
     },
     {

charts/kubezero-storage/update.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -ex
 
+helm dependencies update
+
 ### Gemini
 rm -rf charts/gemini
 helm pull fairwinds-stable/gemini --untar --untardir charts
@@ -20,10 +22,10 @@ mv charts/lvm-localpv/crds/volumesnapshot* crds
 patch -i lvm.patch -p0 --no-backup-if-mismatch
 
 ### EBS
-#VERSION=$(yq eval '.dependencies[] | select(.name=="aws-ebs-csi-driver") | .version' Chart.yaml)
+VERSION=$(yq eval '.dependencies[] | select(.name=="aws-ebs-csi-driver") | .version' Chart.yaml)
-#rm -rf charts/aws-ebs-csi-driver
+rm -rf charts/aws-ebs-csi-driver
-#curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts
+curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts
-#patch -i ebs.patch -p0 --no-backup-if-mismatch
+rm -rf charts/aws-ebs-csi-driver/templates/test
 
 ### EFS
 VERSION=$(yq eval '.dependencies[] | select(.name=="aws-efs-csi-driver") | .version' Chart.yaml)
@@ -35,6 +37,4 @@ patch -i efs.patch -p0 --no-backup-if-mismatch
 cd jsonnet
 make render
 
-helm dependencies update
-
 helm-docs