feat: upgrade addons, storage and network module as part of v1.28

charts/kubeadm/templates/ClusterConfiguration.yaml

@@ -73,6 +73,7 @@ apiServer:
     {{- end }}
     {{- if .Values.api.awsIamAuth.enabled }}
     authentication-token-webhook-config-file: /etc/kubernetes/apiserver/aws-iam-authenticator.yaml
+    authentication-token-webhook-cache-ttl: 3600s
     {{- end }}
     feature-gates: {{ include "kubeadm.featuregates" ( dict "return" "csv" ) | trimSuffix "," | quote }}
     enable-admission-plugins: DenyServiceExternalIPs,NodeRestriction,EventRateLimit,ExtendedResourceToleration

charts/kubeadm/templates/resources/51-aws-iam-authenticator-deployment.yaml

@@ -117,7 +117,7 @@ spec:
 
       containers:
       - name: aws-iam-authenticator
-        image: public.ecr.aws/zero-downtime/aws-iam-authenticator:v0.6.11
+        image: public.ecr.aws/zero-downtime/aws-iam-authenticator:v0.6.14
         args:
         - server
         - --backend-mode=CRD,MountedFile

charts/kubezero-addons/Chart.yaml

@@ -3,7 +3,7 @@ name: kubezero-addons
 description: KubeZero umbrella chart for various optional cluster addons
 type: application
 version: 0.8.5
-appVersion: v1.27
+appVersion: v1.28
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:

charts/kubezero-addons/README.md

@@ -1,6 +1,6 @@
 # kubezero-addons
 
-![Version: 0.8.4](https://img.shields.io/badge/Version-0.8.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.27](https://img.shields.io/badge/AppVersion-v1.27-informational?style=flat-square)
+![Version: 0.8.5](https://img.shields.io/badge/Version-0.8.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.28](https://img.shields.io/badge/AppVersion-v1.28-informational?style=flat-square)
 
 KubeZero umbrella chart for various optional cluster addons
 
@@ -18,12 +18,12 @@ Kubernetes: `>= 1.26.0`
 
 | Repository | Name | Version |
 |------------|------|---------|
-| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.13.2 |
+| https://bitnami-labs.github.io/sealed-secrets | sealed-secrets | 2.15.1 |
-| https://kubernetes-sigs.github.io/external-dns/ | external-dns | 1.13.1 |
+| https://kubernetes-sigs.github.io/external-dns/ | external-dns | 1.14.3 |
-| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.29.5 |
+| https://kubernetes.github.io/autoscaler | cluster-autoscaler | 9.36.0 |
-| https://nvidia.github.io/k8s-device-plugin | nvidia-device-plugin | 0.14.2 |
+| https://nvidia.github.io/k8s-device-plugin | nvidia-device-plugin | 0.14.5 |
 | https://twin.github.io/helm-charts | aws-eks-asg-rolling-update-handler | 1.5.0 |
-| oci://public.ecr.aws/aws-ec2/helm | aws-node-termination-handler | 0.22.0 |
+| oci://public.ecr.aws/aws-ec2/helm | aws-node-termination-handler | 0.23.0 |
 
 # MetalLB   
    
@@ -63,7 +63,7 @@ Device plugin for [AWS Neuron](https://aws.amazon.com/machine-learning/neuron/)
 | aws-eks-asg-rolling-update-handler.environmentVars[8].name | string | `"AWS_STS_REGIONAL_ENDPOINTS"` |  |
 | aws-eks-asg-rolling-update-handler.environmentVars[8].value | string | `"regional"` |  |
 | aws-eks-asg-rolling-update-handler.image.repository | string | `"twinproduction/aws-eks-asg-rolling-update-handler"` |  |
-| aws-eks-asg-rolling-update-handler.image.tag | string | `"v1.8.2"` |  |
+| aws-eks-asg-rolling-update-handler.image.tag | string | `"v1.8.3"` |  |
 | aws-eks-asg-rolling-update-handler.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` |  |
 | aws-eks-asg-rolling-update-handler.resources.limits.memory | string | `"128Mi"` |  |
 | aws-eks-asg-rolling-update-handler.resources.requests.cpu | string | `"10m"` |  |
@@ -102,7 +102,7 @@ Device plugin for [AWS Neuron](https://aws.amazon.com/machine-learning/neuron/)
 | aws-node-termination-handler.useProviderId | bool | `true` |  |
 | awsNeuron.enabled | bool | `false` |  |
 | awsNeuron.image.name | string | `"public.ecr.aws/neuron/neuron-device-plugin"` |  |
-| awsNeuron.image.tag | string | `"2.12.5.0"` |  |
+| awsNeuron.image.tag | string | `"2.19.16.0"` |  |
 | cluster-autoscaler.autoDiscovery.clusterName | string | `""` |  |
 | cluster-autoscaler.awsRegion | string | `"us-west-2"` |  |
 | cluster-autoscaler.enabled | bool | `false` |  |
@@ -111,7 +111,7 @@ Device plugin for [AWS Neuron](https://aws.amazon.com/machine-learning/neuron/)
 | cluster-autoscaler.extraArgs.scan-interval | string | `"30s"` |  |
 | cluster-autoscaler.extraArgs.skip-nodes-with-local-storage | bool | `false` |  |
 | cluster-autoscaler.image.repository | string | `"registry.k8s.io/autoscaling/cluster-autoscaler"` |  |
-| cluster-autoscaler.image.tag | string | `"v1.27.3"` |  |
+| cluster-autoscaler.image.tag | string | `"v1.28.2"` |  |
 | cluster-autoscaler.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` |  |
 | cluster-autoscaler.podDisruptionBudget | bool | `false` |  |
 | cluster-autoscaler.prometheusRule.enabled | bool | `false` |  |

charts/kubezero-addons/charts/aws-node-termination-handler/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 1.20.0
+appVersion: 1.21.0
 description: A Helm chart for the AWS Node Termination Handler.
 home: https://github.com/aws/aws-node-termination-handler/
 icon: https://raw.githubusercontent.com/aws/eks-charts/master/docs/logo/aws.png
@@ -21,4 +21,4 @@ name: aws-node-termination-handler
 sources:
 - https://github.com/aws/aws-node-termination-handler/
 type: application
-version: 0.22.0
+version: 0.23.0

charts/kubezero-addons/charts/aws-node-termination-handler/README.md

@@ -119,7 +119,7 @@ The configuration in this table applies to AWS Node Termination Handler in queue
 | `checkASGTagBeforeDraining`  | [DEPRECATED](Use `checkTagBeforeDraining` instead) If `true`, check that the instance is tagged with the `managedAsgTag` before draining the node. If `false`, disables calls ASG API.                                                                          | `true`                                 |
 | `managedAsgTag`              | [DEPRECATED](Use `managedTag` instead) The node tag to check if `checkASGTagBeforeDraining` is `true`.     
 | `useProviderId`              | If `true`, fetch node name through Kubernetes node spec ProviderID instead of AWS event PrivateDnsHostname.                                                               | `false`                                |
-
+| `topologySpreadConstraints`  | [Topology Spread Constraints](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/) for pod scheduling. Useful with a highly available deployment to reduce the risk of running multiple replicas on the same Node      | `[]`                                   |
 ### IMDS Mode Configuration
 
 The configuration in this table applies to AWS Node Termination Handler in IMDS mode.
@@ -174,6 +174,6 @@ The configuration in this table applies to AWS Node Termination Handler testing
 
 ## Metrics Endpoint Considerations
 
-AWS Node Termination HAndler in IMDS mode runs as a DaemonSet with `useHostNetwork: true` by default. If the Prometheus server is enabled with `enablePrometheusServer: true` nothing else will be able to bind to the configured port (by default `prometheusServerPort: 9092`) in the root network namespace. Therefore, it will need to have a firewall/security group configured on the nodes to block access to the `/metrics` endpoint.
+AWS Node Termination Handler in IMDS mode runs as a DaemonSet with `useHostNetwork: true` by default. If the Prometheus server is enabled with `enablePrometheusServer: true` nothing else will be able to bind to the configured port (by default `prometheusServerPort: 9092`) in the root network namespace. Therefore, it will need to have a firewall/security group configured on the nodes to block access to the `/metrics` endpoint.
 
 You can switch NTH in IMDS mode to run w/ `useHostNetwork: false`, but you will need to make sure that IMDSv1 is enabled or IMDSv2 IP hop count will need to be incremented to 2 (see the [IMDSv2 documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html).

charts/kubezero-addons/charts/aws-node-termination-handler/templates/deployment.yaml

@@ -220,4 +220,8 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

charts/kubezero-addons/charts/aws-node-termination-handler/values.yaml

@@ -52,6 +52,8 @@ affinity: {}
 
 tolerations: []
 
+topologySpreadConstraints: []
+
 # Extra environment variables
 extraEnv: []
 

charts/kubezero-addons/templates/cluster-backup/cronjob.yaml

@@ -24,21 +24,22 @@ spec:
             volumeMounts:
             - name: host
               mountPath: /host
+              readOnly: true
             - name: workdir
               mountPath: /tmp
-            env: 
+            env:
             - name: DEBUG
               value: ""
             - name: RESTIC_REPOSITORY
-              valueFrom: 
+              valueFrom:
-                secretKeyRef: 
+                secretKeyRef:
                   name: kubezero-backup-restic
                   key: repository
             - name: RESTIC_PASSWORD
-              valueFrom: 
+              valueFrom:
-                secretKeyRef: 
+                secretKeyRef:
                   name: kubezero-backup-restic
-                  key: password 
+                  key: password
             {{- with .Values.clusterBackup.extraEnv }}
               {{- toYaml . | nindent 12 }}
             {{- end }}

charts/kubezero-addons/values.yaml

@@ -107,7 +107,6 @@ aws-node-termination-handler:
 
   fullnameOverride: "aws-node-termination-handler"
 
-  checkASGTagBeforeDraining: false
   # -- "zdt:kubezero:nth:${ClusterName}"
   managedTag: "zdt:kubezero:nth:${ClusterName}"
 
@@ -161,7 +160,7 @@ awsNeuron:
 
   image:
     name: public.ecr.aws/neuron/neuron-device-plugin
-    tag: 2.12.5.0
+    tag: 2.19.16.0
 
 nvidia-device-plugin:
   enabled: false
@@ -201,7 +200,7 @@ cluster-autoscaler:
 
   image:
     repository: registry.k8s.io/autoscaling/cluster-autoscaler
-    tag: v1.27.3
+    tag: v1.28.2
 
   autoDiscovery:
     clusterName: ""
@@ -236,7 +235,7 @@ cluster-autoscaler:
   # On AWS enable Projected Service Accounts to assume IAM role
   #extraEnv:
   # AWS_ROLE_ARN: <IamArn>
-  # AWS_WEB_IDENTITY_TOKEN_FILE: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token" 
+  # AWS_WEB_IDENTITY_TOKEN_FILE: "/var/run/secrets/sts.amazonaws.com/serviceaccount/token"
   # AWS_STS_REGIONAL_ENDPOINTS: "regional"
 
   #extraVolumes:

charts/kubezero-network/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-network
 description: KubeZero umbrella chart for all things network
 type: application
-version: 0.4.7
+version: 0.5.0
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:

charts/kubezero-network/README.md

@@ -1,6 +1,6 @@
 # kubezero-network
 
-![Version: 0.4.6](https://img.shields.io/badge/Version-0.4.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+![Version: 0.5.0](https://img.shields.io/badge/Version-0.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
 
 KubeZero umbrella chart for all things network
 
@@ -19,8 +19,8 @@ Kubernetes: `>= 1.26.0`
 | Repository | Name | Version |
 |------------|------|---------|
 | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 |
-| https://helm.cilium.io/ | cilium | 1.14.4 |
+| https://helm.cilium.io/ | cilium | 1.15.2 |
-| https://metallb.github.io/metallb | metallb | 0.13.12 |
+| https://metallb.github.io/metallb | metallb | 0.14.3 |
 
 ## Values
 
@@ -56,7 +56,7 @@ Kubernetes: `>= 1.26.0`
 | cilium.resources.limits.memory | string | `"1024Mi"` |  |
 | cilium.resources.requests.cpu | string | `"10m"` |  |
 | cilium.resources.requests.memory | string | `"256Mi"` |  |
-| cilium.tunnel | string | `"geneve"` |  |
+| cilium.tunnelProtocol | string | `"geneve"` |  |
 | metallb.controller.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` |  |
 | metallb.controller.tolerations[0].effect | string | `"NoSchedule"` |  |
 | metallb.controller.tolerations[0].key | string | `"node-role.kubernetes.io/control-plane"` |  |

charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md

@@ -1,4 +1,48 @@
 # Helm chart
+## v2.29.0
+### Urgent Upgrade Notes
+*(No, really, you MUST read this before you upgrade)*
+
+The EBS CSI Driver Helm chart no longer supports upgrading with `--reuse-values`. This chart will not test for `--reuse-values` compatibility and upgrading with `--reuse-values` will likely fail. Users of `--reuse-values` are strongly encouraged to migrate to `--reset-then-reuse-values`.
+
+For more information see [the deprecation announcement](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1864).
+
+### Other Changes
+* Bump driver version to `v1.29.0` and sidecars to latest versions
+* Add helm-tester enabled flag ([#1954](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1954), [@nunodomingues-td](https://github.com/nunodomingues-td))
+
+## v2.28.1
+* Add `reservedVolumeAttachments` that overrides heuristic-determined reserved attachments via  `--reserved-volume-attachments` CLI option from [PR #1919](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1919) through Helm ([#1939](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1939), [@AndrewSirenko](https://github.com/AndrewSirenko)) 
+* Add `additionalArgs` parameter to node daemonSet ([#1939](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1939), [@AndrewSirenko](https://github.com/AndrewSirenko))
+
+## v2.28.0
+### Urgent Upgrade Notes
+*(No, really, you MUST read this before you upgrade)*
+
+This is the last minor version of the EBS CSI Driver Helm chart to support upgrading with `--reuse-values`. Future versions of the chart (starting with `v2.29.0`) will not test for `--reuse-values` compatibility and upgrading with `--reuse-values` will likely fail. Users of `--reuse-values` are strongly encouraged to migrate to `--reset-then-reuse-values`.
+
+For more information see [the deprecation announcement](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1864).
+
+### Other Changes
+* Bump driver version to `v1.28.0` and sidecars to latest versions
+* Add labels to leases role used by EBS CSI controller ([#1914](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1914), [@cHiv0rz](https://github.com/cHiv0rz))
+* Enforce `linux` and `amd64` node affinity for helm tester pod ([#1922](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1922), [@AndrewSirenko](https://github.com/AndrewSirenko))
+* Add configuration for `DaemonSet` annotations ([#1923](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1923), [@AndrewSirenko](https://github.com/AndrewSirenko))
+* Incorporate KubeLinter recommended best practices for chart tester pod ([#1924](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1924), [@torredil](https://github.com/torredil))
+* Add configuration for chart tester pod image ([#1928](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1928), [@AndrewSirenko](https://github.com/AndrewSirenko))
+
+## v2.27.0
+* Bump driver version to `v1.27.0`
+* Add parameters for tuning revisionHistoryLimit and emptyDir volumes ([#1840](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1840), [@bodgit](https://github.com/bodgit))
+
+## v2.26.1
+* Bump driver version to `v1.26.1`
+* Bump sidecar container versions to fix [restart bug in external attacher, provisioner, resizer, snapshotter, and node-driver-registrar](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1875) ([#1886](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1886), [@AndrewSirenko](https://github.com/AndrewSirenko))
+
+## v2.26.0
+* Bump driver version to `v1.26.0`
+* Bump sidecar container versions ([#1867](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1867), [@AndrewSirenko](https://github.com/AndrewSirenko)) 
+* Add warning about --reuse-values deprecation to NOTES.txt ([#1865](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1865), [@ConnorJC3](https://github.com/ConnorJC3))
 
 ## v2.25.0
 * Bump driver version to `v1.25.0`

charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 1.25.0
+appVersion: 1.29.0
 description: A Helm chart for AWS EBS CSI Driver
 home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver
 keywords:
@@ -13,4 +13,4 @@ maintainers:
 name: aws-ebs-csi-driver
 sources:
 - https://github.com/kubernetes-sigs/aws-ebs-csi-driver
-version: 2.25.0
+version: 2.29.0

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt

@@ -2,4 +2,6 @@ To verify that aws-ebs-csi-driver has started, run:
 
     kubectl get pod -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}"
 
-NOTE: The [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter) controller and CRDs will no longer be installed as part of this chart and moving forward will be a prerequisite of using the snap shotting functionality.
+NOTE: The [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter) controller and CRDs will no longer be installed as part of this chart and moving forward will be a prerequisite of using the snap shotting functionality.
+
+WARNING: Upgrading the EBS CSI Driver Helm chart with --reuse-values will no longer be supported in a future release. For more information, see https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1864

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node-windows.tpl

@@ -9,6 +9,9 @@ metadata:
   labels:
     {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
 spec:
+  {{- if or (kindIs "float64" .Values.node.revisionHistoryLimit) (kindIs "int64" .Values.node.revisionHistoryLimit) }}
+  revisionHistoryLimit: {{ .Values.node.revisionHistoryLimit }}
+  {{- end }}
   selector:
     matchLabels:
       app: {{ .NodeName }}
@@ -199,6 +202,10 @@ spec:
             path: \\.\pipe\csi-proxy-filesystem-v1
             type: ""
         - name: probe-dir
+          {{- if .Values.node.probeDirVolume }}
+          {{- toYaml .Values.node.probeDirVolume | nindent 10 }}
+          {{- else }}
           emptyDir: {}
+          {{- end }}
 {{- end }}
 {{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node.tpl

@@ -8,7 +8,14 @@ metadata:
   namespace: {{ .Release.Namespace }}
   labels:
     {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+  {{- with .Values.node.daemonSetAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
+  {{- if or (kindIs "float64" .Values.node.revisionHistoryLimit) (kindIs "int64" .Values.node.revisionHistoryLimit) }}
+  revisionHistoryLimit: {{ .Values.node.revisionHistoryLimit }}
+  {{- end }}
   selector:
     matchLabels:
       app: {{ .NodeName }}
@@ -60,6 +67,9 @@ spec:
           args:
             - node
             - --endpoint=$(CSI_ENDPOINT)
+            {{- with .Values.node.reservedVolumeAttachments }}
+            - --reserved-volume-attachments={{ . }}
+            {{- end }}
             {{- with .Values.node.volumeAttachLimit }}
             - --volume-attach-limit={{ . }}
             {{- end }}
@@ -70,6 +80,9 @@ spec:
             {{- if .Values.node.otelTracing }}
             - --enable-otel-tracing=true
             {{- end}}
+            {{- range .Values.node.additionalArgs }}
+            - {{ . }}
+            {{- end }}
           env:
             - name: CSI_ENDPOINT
               value: unix:/csi/csi.sock
@@ -219,7 +232,11 @@ spec:
             path: /dev
             type: Directory
         - name: probe-dir
+          {{- if .Values.node.probeDirVolume }}
+          {{- toYaml .Values.node.probeDirVolume | nindent 10 }}
+          {{- else }}
           emptyDir: {}
+          {{- end }}
         {{- with .Values.node.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-csi-node.yaml

@@ -11,4 +11,7 @@ rules:
     verbs: ["get", "patch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["volumeattachments"]
-    verbs: ["list"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get"]

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml

@@ -33,6 +33,9 @@ rules:
   - apiGroups: [ "storage.k8s.io" ]
     resources: [ "volumeattachments" ]
     verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "volumeattributesclasses" ]
+    verbs: [ "get" ]
   {{- with .Values.sidecars.provisioner.additionalClusterRoleRules }}
     {{- . | toYaml | nindent 2 }}
   {{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml

@@ -29,6 +29,9 @@ rules:
   - apiGroups: [ "" ]
     resources: [ "pods" ]
     verbs: [ "get", "list", "watch" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources: [ "volumeattributesclasses" ]
+    verbs: [ "get", "list", "watch" ]
   {{- with .Values.sidecars.resizer.additionalClusterRoleRules }}
     {{- . | toYaml | nindent 2 }}
   {{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml

@@ -6,8 +6,15 @@ metadata:
   namespace: {{ .Release.Namespace }}
   labels:
     {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
+  {{- with .Values.controller.deploymentAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 spec:
   replicas: {{ .Values.controller.replicaCount }}
+  {{- if or (kindIs "float64" .Values.controller.revisionHistoryLimit) (kindIs "int64" .Values.controller.revisionHistoryLimit) }}
+  revisionHistoryLimit: {{ .Values.controller.revisionHistoryLimit }}
+  {{- end }}
   {{- with .Values.controller.updateStrategy }}
   strategy:
     {{- toYaml . | nindent 4 }}
@@ -486,7 +493,11 @@ spec:
       {{- end }}
       volumes:
         - name: socket-dir
+          {{- if .Values.controller.socketDirVolume }}
+          {{- toYaml .Values.controller.socketDirVolume | nindent 10 }}
+          {{- else }}
           emptyDir: {}
+          {{- end }}
         {{- with .Values.controller.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/role-leases.yaml

@@ -3,6 +3,8 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   namespace: {{ .Release.Namespace }}
   name: ebs-csi-leases-role
+  labels:
+    {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }}
 rules:
 - apiGroups: ["coordination.k8s.io"]
   resources: ["leases"]

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/tests/helm-tester.yaml

@@ -0,0 +1,235 @@
+{{- if .Values.helmTester.enabled -}}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ebs-csi-driver-test
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-driver-test
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
+rules:
+  - apiGroups: [ "" ]
+    resources:
+      - events
+      - nodes
+      - pods
+      - replicationcontrollers
+      - serviceaccounts
+      - configmaps
+      - persistentvolumes
+      - persistentvolumeclaims
+    verbs: [ "list" ]
+  - apiGroups: [ "" ]
+    resources:
+      - services
+      - nodes
+      - nodes/proxy
+      - persistentvolumes
+      - persistentvolumeclaims
+      - pods
+      - pods/log
+    verbs: [ "get" ]
+  - apiGroups: [ "" ]
+    resources:
+      - namespaces
+      - persistentvolumes
+      - persistentvolumeclaims
+      - pods
+      - pods/exec
+    verbs: [ "create" ]
+  - apiGroups: [ "" ]
+    resources:
+      - namespaces
+      - persistentvolumes
+      - persistentvolumeclaims
+      - pods
+    verbs: [ "delete" ]
+  - apiGroups: [ "" ]
+    resources:
+      - persistentvolumeclaims
+    verbs: [ "update" ]
+  - apiGroups: [ "" ]
+    resources:
+      - pods/ephemeralcontainers
+    verbs: [ "patch" ]
+  - apiGroups: [ "" ]
+    resources:
+      - serviceaccounts
+      - configmaps
+    verbs: [ "watch" ]
+  - apiGroups: [ "apps" ]
+    resources:
+      - replicasets
+      - daemonsets
+    verbs: [ "list" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources:
+      - storageclasses
+    verbs: [ "create" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources:
+      - storageclasses
+      - csinodes
+    verbs: [ "get" ]
+  - apiGroups: [ "storage.k8s.io" ]
+    resources:
+      - storageclasses
+    verbs: [ "delete" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources:
+      - volumesnapshots
+      - volumesnapshotclasses
+      - volumesnapshotcontents
+    verbs: [ "create" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources:
+      - volumesnapshots
+      - volumesnapshotclasses
+      - volumesnapshotcontents
+    verbs: [ "get" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources:
+      - volumesnapshotcontents
+    verbs: [ "update" ]
+  - apiGroups: [ "snapshot.storage.k8s.io" ]
+    resources:
+      - volumesnapshots
+      - volumesnapshotclasses
+      - volumesnapshotcontents
+    verbs: [ "delete" ]
+  - apiGroups: [ "authorization.k8s.io" ]
+    resources:
+      - clusterroles
+    verbs: [ "list" ]
+  - apiGroups: [ "authorization.k8s.io" ]
+    resources:
+      - subjectaccessreviews
+    verbs: [ "create" ]
+  - apiGroups: [ "rbac.authorization.k8s.io" ]
+    resources:
+      - clusterroles
+    verbs: [ "list" ]
+  - apiGroups: [ "rbac.authorization.k8s.io" ]
+    resources:
+      - clusterrolebindings
+    verbs: [ "create" ]
+  - apiGroups: [ "apiextensions.k8s.io" ]
+    resources:
+      - customresourcedefinitions
+    verbs: [ "get" ]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-driver-test
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-driver-test
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-csi-driver-test
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: ConfigMap
+data:
+  manifests.yaml: |
+    ShortName: ebs
+    StorageClass:
+      FromFile: storageclass.yaml
+    SnapshotClass:
+      FromName: true
+    DriverInfo:
+      Name: ebs.csi.aws.com
+      SupportedSizeRange:
+        Min: 1Gi
+        Max: 16Ti
+      SupportedFsType:
+        xfs: {}
+        ext4: {}
+      SupportedMountOption:
+        dirsync: {}
+      TopologyKeys: ["topology.ebs.csi.aws.com/zone"]
+      Capabilities:
+        persistence: true
+        fsGroup: true
+        block: true
+        exec: true
+        snapshotDataSource: true
+        pvcDataSource: false
+        multipods: true
+        controllerExpansion: true
+        nodeExpansion: true
+        volumeLimits: true
+        topology: true
+  storageclass.yaml: |
+    kind: StorageClass
+    apiVersion: storage.k8s.io/v1
+    metadata:
+      name: ebs.csi.aws.com
+    provisioner: ebs.csi.aws.com
+    volumeBindingMode: WaitForFirstConsumer
+metadata:
+  name: ebs-csi-driver-test
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ebs-csi-driver-test
+  annotations:
+    "helm.sh/hook": test
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
+    "ignore-check.kube-linter.io/run-as-non-root": "kubetest2 image runs as root"
+    "ignore-check.kube-linter.io/no-read-only-root-fs": "test pod requires privileged access"
+spec:
+  containers:
+    - name: kubetest2
+      image: {{ .Values.helmTester.image }}
+      resources:
+        requests:
+          cpu: 2000m
+          memory: 4Gi
+        limits:
+          memory: 4Gi
+      command: [ "/bin/sh", "-c" ]
+      args:
+        - |
+          cp /etc/config/storageclass.yaml /workspace/storageclass.yaml
+          kubectl config set-cluster cluster --server=https://kubernetes.default --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+          kubectl config set-context kubetest2 --cluster=cluster
+          kubectl config set-credentials sa --token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+          kubectl config set-context kubetest2 --user=sa && kubectl config use-context kubetest2
+          export FOCUS_REGEX='\bebs.csi.aws.com\b.+(validate content|resize volume|offline PVC|AllowedTopologies|store data$SNAPSHOTS)'
+          if kubectl get crd volumesnapshots.snapshot.storage.k8s.io; then
+            FORCUS_REGEX="${FOCUS_REGEX}|snapshot fields"
+          fi
+          kubetest2 noop --run-id='e2e-kubernetes' --test=ginkgo -- --test-package-version="$(curl -L https://dl.k8s.io/release/stable-1.29.txt)" --skip-regex='[Disruptive]|[Serial]' --focus-regex="$FOCUS_REGEX" --parallel=25 --test-args='-storage.testdriver=/etc/config/manifests.yaml'
+      volumeMounts:
+      - name: config-vol
+        mountPath: /etc/config
+  # kubekins-e2e v1 image is linux amd64 only.
+  nodeSelector:
+    kubernetes.io/os: linux
+    kubernetes.io/arch: amd64
+  serviceAccountName: ebs-csi-driver-test
+  volumes:
+    - name: config-vol
+      configMap:
+        name: ebs-csi-driver-test
+  restartPolicy: Never
+{{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml

@@ -19,7 +19,7 @@ sidecars:
     image:
       pullPolicy: IfNotPresent
       repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner
-      tag: "v3.6.2-eks-1-28-9"
+      tag: "v4.0.0-eks-1-29-7"
     logLevel: 2
     # Additional parameters provided by external-provisioner.
     additionalArgs: []
@@ -44,7 +44,7 @@ sidecars:
     image:
       pullPolicy: IfNotPresent
       repository: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher
-      tag: "v4.4.2-eks-1-28-9"
+      tag: "v4.5.0-eks-1-29-7"
     # Tune leader lease election for csi-attacher.
     # Leader election is on by default.
     leaderElection:
@@ -71,7 +71,7 @@ sidecars:
     image:
       pullPolicy: IfNotPresent
       repository: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter
-      tag: "v6.3.2-eks-1-28-9"
+      tag: "v7.0.1-eks-1-29-7"
     logLevel: 2
     # Additional parameters provided by csi-snapshotter.
     additionalArgs: []
@@ -85,7 +85,7 @@ sidecars:
     image:
       pullPolicy: IfNotPresent
       repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe
-      tag: "v2.11.0-eks-1-28-9"
+      tag: "v2.12.0-eks-1-29-7"
     # Additional parameters provided by livenessprobe.
     additionalArgs: []
     resources: {}
@@ -97,7 +97,7 @@ sidecars:
     image:
       pullPolicy: IfNotPresent
       repository: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer
-      tag: "v1.9.2-eks-1-28-9"
+      tag: "v1.10.0-eks-1-29-7"
     # Tune leader lease election for csi-resizer.
     # Leader election is on by default.
     leaderElection:
@@ -122,7 +122,7 @@ sidecars:
     image:
       pullPolicy: IfNotPresent
       repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar
-      tag: "v2.9.1-eks-1-28-9"
+      tag: "v2.10.0-eks-1-29-7"
     logLevel: 2
     # Additional parameters provided by node-driver-registrar.
     additionalArgs: []
@@ -144,7 +144,7 @@ sidecars:
     image:
       pullPolicy: IfNotPresent
       repository: public.ecr.aws/ebs-csi-driver/volume-modifier-for-k8s
-      tag: "v0.1.3"
+      tag: "v0.2.1"
     leaderElection:
       enabled: true
       # Optional values to tune lease behavior.
@@ -237,6 +237,7 @@ controller:
   logLevel: 2
   userAgentExtra: "helm"
   nodeSelector: {}
+  deploymentAnnotations: {}
   podAnnotations: {}
   podLabels: {}
   priorityClassName: system-cluster-critical
@@ -246,6 +247,9 @@ controller:
   # region: us-east-1
   region:
   replicaCount: 2
+  revisionHistoryLimit: 10
+  socketDirVolume:
+    emptyDir: {}
   updateStrategy:
     type: RollingUpdate
     rollingUpdate:
@@ -328,6 +332,7 @@ node:
   loggingFormat: text
   logLevel: 2
   priorityClassName:
+  additionalArgs: []
   affinity:
     nodeAffinity:
       requiredDuringSchedulingIgnoredDuringExecution:
@@ -346,6 +351,7 @@ node:
             - a1.2xlarge
             - a1.4xlarge
   nodeSelector: {}
+  daemonSetAnnotations: {}
   podAnnotations: {}
   podLabels: {}
   tolerateAllTaints: true
@@ -359,6 +365,9 @@ node:
       memory: 40Mi
     limits:
       memory: 256Mi
+  revisionHistoryLimit: 10
+  probeDirVolume:
+    emptyDir: {}
   serviceAccount:
     create: true
     name: ebs-csi-node-sa
@@ -369,7 +378,12 @@ node:
   # Enable the linux daemonset creation
   enableLinux: true
   enableWindows: false
+  # The number of attachment slots to reserve for system use (and not to be used for CSI volumes)
+  # When this parameter is not specified (or set to -1), the EBS CSI Driver will attempt to determine the number of reserved slots via heuristic
+  # Cannot be specified at the same time as `node.volumeAttachLimit`
+  reservedVolumeAttachments:
   # The "maximum number of attachable volumes" per node
+  # Cannot be specified at the same time as `node.reservedVolumeAttachments`
   volumeAttachLimit:
   updateStrategy:
     type: RollingUpdate
@@ -449,3 +463,8 @@ volumeSnapshotClasses: []
 # Intended for use with older clusters that cannot easily replace the CSIDriver object
 # This parameter should always be false for new installations
 useOldCSIDriver: false
+
+helmTester:
+  enabled: true
+  # Supply a custom image to the ebs-csi-driver-test pod in helm-tester.yaml
+  image: "gcr.io/k8s-staging-test-infra/kubekins-e2e:v20240311-b09cdeb92c-master"

charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md

@@ -1,4 +1,14 @@
 # Helm chart
+# v2.5.6
+* Bump app/driver version to `v1.7.6`
+# v2.5.5
+* Bump app/driver version to `v1.7.5`
+# v2.5.4
+* Bump app/driver version to `v1.7.4`
+# v2.5.3
+* Bump app/driver version to `v1.7.3`
+# v2.5.2
+* Bump app/driver version to `v1.7.2`
 # v2.5.1
 * Bump app/driver version to `v1.7.1`
 # v2.5.0

charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 1.7.1
+appVersion: 1.7.6
 description: A Helm chart for AWS EFS CSI Driver
 home: https://github.com/kubernetes-sigs/aws-efs-csi-driver
 keywords:
@@ -15,4 +15,4 @@ maintainers:
 name: aws-efs-csi-driver
 sources:
 - https://github.com/kubernetes-sigs/aws-efs-csi-driver
-version: 2.5.1
+version: 2.5.6

charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml

@@ -6,6 +6,9 @@ metadata:
   name: efs-csi-controller
   labels:
     app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
+    {{- with .Values.controller.additionalLabels }}
+    {{ toYaml . | nindent 4 }}
+    {{- end }}
 spec:
   replicas: {{ .Values.replicaCount }}
   selector:
@@ -23,10 +26,16 @@ spec:
         app: efs-csi-controller
         app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
+        {{- with .Values.controller.podLabels }}
+        {{ toYaml . | nindent 8 }}
+        {{- end }}
       {{- with .Values.controller.podAnnotations }}
       annotations: {{- toYaml . | nindent 8 }}
       {{- end }}
     spec:
+      {{- if hasKey .Values.controller "hostNetwork" }}
+      hostNetwork: {{ .Values.controller.hostNetwork }}
+      {{- end }}
       {{- if .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- range .Values.imagePullSecrets }}
@@ -39,7 +48,7 @@ spec:
         {{- toYaml . | nindent 8 }}
         {{- end }}
       serviceAccountName: {{ .Values.controller.serviceAccount.name }}
-      priorityClassName: system-cluster-critical
+      priorityClassName: {{ .Values.controller.priorityClassName | default "system-cluster-critical" }}
       {{- with .Values.controller.tolerations }}
       tolerations: {{- toYaml . | nindent 8 }}
       {{- end }}
@@ -47,10 +56,18 @@ spec:
       securityContext:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- if .Values.controller.dnsPolicy }}
+      dnsPolicy: {{ .Values.controller.dnsPolicy }}
+      {{- end }}
+      {{- with .Values.controller.dnsConfig }}
+      dnsConfig: {{- toYaml . | nindent 8 }}
+      {{- end }}
       containers:
         - name: efs-plugin
+          {{- with .Values.controller.containerSecurityContext }}
           securityContext:
-            privileged: true
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           args:
@@ -110,6 +127,12 @@ spec:
             - --extra-create-metadata
             {{- end }}
             - --leader-election
+            {{- if hasKey .Values.controller "leaderElectionRenewDeadline" }}
+            - --leader-election-renew-deadline={{ .Values.controller.leaderElectionRenewDeadline }}
+            {{- end }}
+            {{- if hasKey .Values.controller "leaderElectionLeaseDuration" }}
+            - --leader-election-lease-duration={{ .Values.controller.leaderElectionLeaseDuration }}
+            {{- end }}
           env:
             - name: ADDRESS
               value: /var/lib/csi/sockets/pluginproxy/csi.sock

charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml

@@ -40,12 +40,19 @@ rules:
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
     verbs: ["get", "watch", "list", "delete", "update", "create"]
-  # - apiGroups: [ "" ]
-  #   resources: [ "secrets" ]
-  #   verbs: [ "get", "watch", "list" ]
-
 ---
-
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: efs-csi-external-provisioner-role-describe-secrets
+  labels:
+    app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
+rules:
+  - apiGroups: [ "" ]
+    resources: [ "secrets" ]
+    resourceNames: ["x-account"]
+    verbs: [ "get", "watch", "list" ]
+---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -60,3 +67,20 @@ roleRef:
   kind: ClusterRole
   name: efs-csi-external-provisioner-role
   apiGroup: rbac.authorization.k8s.io
+---
+# We use a RoleBinding to restrict Secret access to the namespace that the 
+# RoleBinding is created in (typically kube-system)
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: efs-csi-provisioner-binding-describe-secrets
+  labels:
+    app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.controller.serviceAccount.name }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: efs-csi-external-provisioner-role-describe-secrets
+  apiGroup: rbac.authorization.k8s.io

charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-serviceaccount.yaml

@@ -20,7 +20,7 @@ metadata:
 rules:
   - apiGroups: [""]
     resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
+    verbs: ["get", "list", "watch", "patch"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml

@@ -11,14 +11,14 @@ useFIPS: false
 
 image:
   repository: amazon/aws-efs-csi-driver
-  tag: "v1.7.1"
+  tag: "v1.7.6"
   pullPolicy: IfNotPresent
 
 sidecars:
   livenessProbe:
     image:
       repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe
-      tag: v2.10.0-eks-1-27-3
+      tag: v2.11.0-eks-1-29-2
       pullPolicy: IfNotPresent
     resources: {}
     securityContext:
@@ -27,7 +27,7 @@ sidecars:
   nodeDriverRegistrar:
     image:
       repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar
-      tag: v2.8.0-eks-1-27-3
+      tag: v2.9.3-eks-1-29-2
       pullPolicy: IfNotPresent
     resources: {}
     securityContext:
@@ -36,7 +36,7 @@ sidecars:
   csiProvisioner:
     image:
       repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner
-      tag: v3.5.0-eks-1-27-3
+      tag: v3.6.3-eks-1-29-2
       pullPolicy: IfNotPresent
     resources: {}
     securityContext:
@@ -63,6 +63,12 @@ controller:
   # path on efs when deleteing an access point
   deleteAccessPointRootDir: false
   podAnnotations: {}
+  podLabel: {}
+  hostNetwork: false
+  priorityClassName: system-cluster-critical
+  dnsPolicy: ClusterFirst
+  dnsConfig: {}
+  additionalLabels: {}
   resources:
     {}
     # We usually recommend not to specify default resources and to leave this as a conscious
@@ -80,6 +86,8 @@ controller:
   tolerations:
     - key: CriticalAddonsOnly
       operator: Exists
+    - key: efs.csi.aws.com/agent-not-ready
+      operator: Exists
   affinity: {}
   # Specifies whether a service account should be created
   serviceAccount:
@@ -96,6 +104,12 @@ controller:
     runAsUser: 0
     runAsGroup: 0
     fsGroup: 0
+  # securityContext on the controller container
+  # Setting privileged=false will cause the "delete-access-point-root-dir" controller option to fail
+  containerSecurityContext:
+    privileged: true
+  leaderElectionRenewDeadline: 10s
+  leaderElectionLeaseDuration: 15s
 
 
 ## Node daemonset variables

charts/kubezero-storage/charts/lvm-localpv/Chart.yaml

@@ -1,5 +1,9 @@
 apiVersion: v2
-appVersion: 1.3.0
+appVersion: 1.5.0
+dependencies:
+- name: crds
+  repository: ""
+  version: 1.5.0
 description: CSI Driver for dynamic provisioning of LVM Persistent Local Volumes.
 home: https://openebs.io/
 icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/openebs/icon/color/openebs-icon-color.png
@@ -20,4 +24,4 @@ maintainers:
 name: lvm-localpv
 sources:
 - https://github.com/openebs/lvm-localpv
-version: 1.3.0
+version: 1.5.0

charts/kubezero-storage/charts/lvm-localpv/README.md

@@ -47,10 +47,10 @@ $ helm install [RELEASE_NAME] openebs-lvmlocalpv/lvm-localpv --namespace [NAMESP
 
 
 **Note:** If moving from the operator to helm
-- Make sure the namespace provided in the helm install command is same as `LVM_NAMESPACE` (by default it is `openebs`) env in the controller statefulset.
+- Make sure the namespace provided in the helm install command is same as `LVM_NAMESPACE` (by default it is `openebs`) env in the controller deployment.
-- Before installing, clean up the stale statefulset and daemonset from `kube-system` namespace using the below commands
+- Before installing, clean up the stale deployment and daemonset from `kube-system` namespace using the below commands
 ```sh
-kubectl delete sts openebs-lvm-controller -n kube-system
+kubectl delete deployment openebs-lvm-controller -n kube-system
 kubectl delete ds openebs-lvm-node -n kube-system
 ```
 
@@ -96,7 +96,8 @@ helm install openebs-lvmlocalpv openebs-lvmlocalpv/lvm-localpv --namespace opene
 
 | Parameter                                           | Description                                                                      | Default                                 |
 |-----------------------------------------------------|----------------------------------------------------------------------------------|-----------------------------------------|
-| `imagePullSecrets`                                  | Provides image pull secrect                                                      | `""`                                    |
+| `crds.csi.volumeSnapshots.enabled`                  | Enable/Disable installation of VolumeSnapshot-related CRDs                       | `true`                                   |
+| `imagePullSecrets`                                  | Provides image pull secret                                                       | `""`                                    |
 | `lvmPlugin.image.registry`                          | Registry for openebs-lvm-plugin image                                            | `""`                                    |
 | `lvmPlugin.image.repository`                        | Image repository for openebs-lvm-plugin                                          | `openebs/lvm-driver`                    |
 | `lvmPlugin.image.pullPolicy`                        | Image pull policy for openebs-lvm-plugin                                         | `IfNotPresent`                          |
@@ -133,15 +134,16 @@ helm install openebs-lvmlocalpv openebs-lvmlocalpv/lvm-localpv --namespace opene
 | `lvmController.provisioner.image.repository`        | Image repository for csi-provisioner                                             | `sig-storage/csi-provisioner`           |
 | `lvmController.provisioner.image.pullPolicy`        | Image pull policy for csi-provisioner                                            | `IfNotPresent`                          |
 | `lvmController.provisioner.image.tag`               | Image tag for csi-provisioner                                                    | `v3.5.0`                                |
-| `lvmController.updateStrategy.type`                 | Update strategy for lvm localpv controller statefulset                           | `RollingUpdate`                         |
+| `lvmController.updateStrategy.type`                 | Update strategy for lvm localpv controller deployment                            | `RollingUpdate`                         |
-| `lvmController.annotations`                         | Annotations for lvm localpv controller statefulset metadata                      | `""`                                    |
+| `lvmController.annotations`                         | Annotations for lvm localpv controller deployment metadata                       | `""`                                    |
-| `lvmController.podAnnotations`                      | Annotations for lvm localpv controller statefulset's pods metadata               | `""`                                    |
+| `lvmController.podAnnotations`                      | Annotations for lvm localpv controller deployment's pods metadata                | `""`                                    |
-| `lvmController.resources`                           | Resource and request and limit for lvm localpv controller statefulset containers | `""`                                    |
+| `lvmController.resources`                           | Resource and request and limit for lvm localpv controller deployment containers  | `""`                                    |
-| `lvmController.labels`                              | Labels for lvm localpv controller statefulset metadata                           | `""`                                    |
+| `lvmController.labels`                              | Labels for lvm localpv controller deployment metadata                            | `""`                                    |
-| `lvmController.podLabels`                           | Appends labels to the lvm localpv controller statefulset pods                    | `""`                                    |
+| `lvmController.podLabels`                           | Appends labels to the lvm localpv controller deployment pods                     | `""`                                    |
-| `lvmController.nodeSelector`                        | Nodeselector for lvm localpv controller statefulset pods                         | `""`                                    |
+| `lvmController.nodeSelector`                        | Nodeselector for lvm localpv controller deployment pods                          | `""`                                    |
-| `lvmController.tolerations`                         | lvm localpv controller statefulset's pod toleration values                       | `""`                                    |
+| `lvmController.tolerations`                         | lvm localpv controller deployment's pod toleration values                        | `""`                                    |
-| `lvmController.securityContext`                     | Seurity context for lvm localpv controller statefulset container                 | `""`                                    |
+| `lvmController.topologySpreadConstraints`           | lvm localpv controller deployment's pod topologySpreadConstraints values         | `""`                                    |
+| `lvmController.securityContext`                     | Security context for lvm localpv controller deployment container                 | `""`                                    |
 | `rbac.pspEnabled`                                   | Enable PodSecurityPolicy                                                         | `false`                                 |
 | `serviceAccount.lvmNode.create`                     | Create a service account for lvmnode or not                                      | `true`                                  |
 | `serviceAccount.lvmNode.name`                       | Name for the lvmnode service account                                             | `openebs-lvm-node-sa`                   |

charts/kubezero-storage/charts/lvm-localpv/charts/crds/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/kubezero-storage/charts/lvm-localpv/charts/crds/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v2
+description: A Helm chart that collects CustomResourceDefinitions (CRDs) from lvm-localpv.
+name: crds
+version: 1.5.0

charts/kubezero-storage/charts/lvm-localpv/charts/crds/templates/_helpers.tpl

@@ -0,0 +1,17 @@
+{{/*
+    This returns a "1" if the CRD is absent in the cluster
+    Usage:
+      {{- if (include "crdIsAbsent" (list <crd-name>)) -}}
+      # CRD Yaml
+      {{- end -}}
+*/}}
+{{- define "crdIsAbsent" -}}
+    {{- $crdName := index . 0 -}}
+    {{- $crd := lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition" "" $crdName -}}
+    {{- $output := "1" -}}
+    {{- if $crd -}}
+        {{- $output = "" -}}
+    {{- end -}}
+
+    {{- $output -}}
+{{- end -}}

charts/kubezero-storage/charts/lvm-localpv/charts/crds/templates/csi-volume-snapshot-class.yaml

@@ -0,0 +1,152 @@
+{{- if .Values.csi.volumeSnapshots.enabled -}}
+{{- $crdName := "volumesnapshotclasses.snapshot.storage.k8s.io" -}}
+{{- if (include "crdIsAbsent" (list $crdName)) -}}
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/814
+    controller-gen.kubebuilder.io/version: v0.11.3
+  creationTimestamp: null
+  name: volumesnapshotclasses.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotClass
+    listKind: VolumeSnapshotClassList
+    plural: volumesnapshotclasses
+    shortNames:
+      - vsclass
+      - vsclasses
+    singular: volumesnapshotclass
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - jsonPath: .driver
+          name: Driver
+          type: string
+        - description: Determines whether a VolumeSnapshotContent created through the
+            VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted.
+          jsonPath: .deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotClass specifies parameters that a underlying storage
+            system uses when creating a volume snapshot. A specific VolumeSnapshotClass
+            is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
+            are non-namespaced
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            deletionPolicy:
+              description: deletionPolicy determines whether a VolumeSnapshotContent
+                created through the VolumeSnapshotClass should be deleted when its bound
+                VolumeSnapshot is deleted. Supported values are "Retain" and "Delete".
+                "Retain" means that the VolumeSnapshotContent and its physical snapshot
+                on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent
+                and its physical snapshot on underlying storage system are deleted.
+                Required.
+              enum:
+                - Delete
+                - Retain
+              type: string
+            driver:
+              description: driver is the name of the storage driver that handles this
+                VolumeSnapshotClass. Required.
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            parameters:
+              additionalProperties:
+                type: string
+              description: parameters is a key-value map with storage driver specific
+                parameters for creating snapshots. These values are opaque to Kubernetes.
+              type: object
+          required:
+            - deletionPolicy
+            - driver
+          type: object
+      served: true
+      storage: true
+      subresources: {}
+    - additionalPrinterColumns:
+        - jsonPath: .driver
+          name: Driver
+          type: string
+        - description: Determines whether a VolumeSnapshotContent created through the
+            VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted.
+          jsonPath: .deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      deprecated: true
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated;
+        use snapshot.storage.k8s.io/v1 VolumeSnapshotClass
+      name: v1beta1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotClass specifies parameters that a underlying storage
+            system uses when creating a volume snapshot. A specific VolumeSnapshotClass
+            is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
+            are non-namespaced
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            deletionPolicy:
+              description: deletionPolicy determines whether a VolumeSnapshotContent
+                created through the VolumeSnapshotClass should be deleted when its bound
+                VolumeSnapshot is deleted. Supported values are "Retain" and "Delete".
+                "Retain" means that the VolumeSnapshotContent and its physical snapshot
+                on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent
+                and its physical snapshot on underlying storage system are deleted.
+                Required.
+              enum:
+                - Delete
+                - Retain
+              type: string
+            driver:
+              description: driver is the name of the storage driver that handles this
+                VolumeSnapshotClass. Required.
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            parameters:
+              additionalProperties:
+                type: string
+              description: parameters is a key-value map with storage driver specific
+                parameters for creating snapshots. These values are opaque to Kubernetes.
+              type: object
+          required:
+            - deletionPolicy
+            - driver
+          type: object
+      served: false
+      storage: false
+      subresources: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+{{- end -}}
+{{- end -}}

charts/kubezero-storage/charts/lvm-localpv/charts/crds/templates/csi-volume-snapshot-content.yaml

@@ -0,0 +1,490 @@
+{{- if .Values.csi.volumeSnapshots.enabled -}}
+{{- $crdName := "volumesnapshotcontents.snapshot.storage.k8s.io" -}}
+{{- if (include "crdIsAbsent" (list $crdName)) -}}
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/814
+    controller-gen.kubebuilder.io/version: v0.11.3
+  creationTimestamp: null
+  name: volumesnapshotcontents.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotContent
+    listKind: VolumeSnapshotContentList
+    plural: volumesnapshotcontents
+    shortNames:
+      - vsc
+      - vscs
+    singular: volumesnapshotcontent
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: Represents the complete size of the snapshot in bytes
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: integer
+        - description: Determines whether this VolumeSnapshotContent and its physical
+            snapshot on the underlying storage system should be deleted when its bound
+            VolumeSnapshot is deleted.
+          jsonPath: .spec.deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - description: Name of the CSI driver used to create the physical snapshot on
+            the underlying storage system.
+          jsonPath: .spec.driver
+          name: Driver
+          type: string
+        - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: VolumeSnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.name
+          name: VolumeSnapshot
+          type: string
+        - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.namespace
+          name: VolumeSnapshotNamespace
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotContent represents the actual "on-disk" snapshot
+            object in the underlying storage system
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: spec defines properties of a VolumeSnapshotContent created
+                by the underlying storage system. Required.
+              properties:
+                deletionPolicy:
+                  description: deletionPolicy determines whether this VolumeSnapshotContent
+                    and its physical snapshot on the underlying storage system should
+                    be deleted when its bound VolumeSnapshot is deleted. Supported values
+                    are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
+                    and its physical snapshot on underlying storage system are kept.
+                    "Delete" means that the VolumeSnapshotContent and its physical snapshot
+                    on underlying storage system are deleted. For dynamically provisioned
+                    snapshots, this field will automatically be filled in by the CSI
+                    snapshotter sidecar with the "DeletionPolicy" field defined in the
+                    corresponding VolumeSnapshotClass. For pre-existing snapshots, users
+                    MUST specify this field when creating the VolumeSnapshotContent
+                    object. Required.
+                  enum:
+                    - Delete
+                    - Retain
+                  type: string
+                driver:
+                  description: driver is the name of the CSI driver used to create the
+                    physical snapshot on the underlying storage system. This MUST be
+                    the same as the name returned by the CSI GetPluginName() call for
+                    that driver. Required.
+                  type: string
+                source:
+                  description: source specifies whether the snapshot is (or should be)
+                    dynamically provisioned or already exists, and just requires a Kubernetes
+                    object representation. This field is immutable after creation. Required.
+                  oneOf:
+                    - required:
+                        - snapshotHandle
+                    - required:
+                        - volumeHandle
+                  properties:
+                    snapshotHandle:
+                      description: snapshotHandle specifies the CSI "snapshot_id" of
+                        a pre-existing snapshot on the underlying storage system for
+                        which a Kubernetes object representation was (or should be)
+                        created. This field is immutable.
+                      type: string
+                    volumeHandle:
+                      description: volumeHandle specifies the CSI "volume_id" of the
+                        volume from which a snapshot should be dynamically taken from.
+                        This field is immutable.
+                      type: string
+                  type: object
+                sourceVolumeMode:
+                  description: SourceVolumeMode is the mode of the volume whose snapshot
+                    is taken. Can be either “Filesystem” or “Block”. If not specified,
+                    it indicates the source volume's mode is unknown. This field is
+                    immutable. This field is an alpha field.
+                  type: string
+                volumeSnapshotClassName:
+                  description: name of the VolumeSnapshotClass from which this snapshot
+                    was (or will be) created. Note that after provisioning, the VolumeSnapshotClass
+                    may be deleted or recreated with different set of values, and as
+                    such, should not be referenced post-snapshot creation.
+                  type: string
+                volumeSnapshotRef:
+                  description: volumeSnapshotRef specifies the VolumeSnapshot object
+                    to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
+                    field must reference to this VolumeSnapshotContent's name for the
+                    bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
+                    object, name and namespace of the VolumeSnapshot object MUST be
+                    provided for binding to happen. This field is immutable after creation.
+                    Required.
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                      an entire object, this string should contain a valid JSON/Go
+                      field access statement, such as desiredState.manifest.containers[2].
+                      For example, if the object reference is to a container within
+                      a pod, this would take on a value like: "spec.containers{name}"
+                      (where "name" refers to the name of the container that triggered
+                      the event) or if no container name is specified "spec.containers[2]"
+                      (container with index 2 in this pod). This syntax is chosen
+                      only to have some well-defined way of referencing a part of
+                      an object. TODO: this design is not final and this field is
+                      subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                      is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+                  x-kubernetes-map-type: atomic
+              required:
+                - deletionPolicy
+                - driver
+                - source
+                - volumeSnapshotRef
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+              properties:
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic snapshot
+                    creation case, this field will be filled in by the CSI snapshotter
+                    sidecar with the "creation_time" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. If not specified, it indicates
+                    the creation time is unknown. The format of this field is a Unix
+                    nanoseconds time encoded as an int64. On Unix, the command `date
+                    +%s%N` returns the current time in nanoseconds since 1970-01-01
+                    00:00:00 UTC.
+                  format: int64
+                  type: integer
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. Upon success after retry, this error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered error
+                      during snapshot creation if specified. NOTE: message may be
+                      logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if a snapshot is ready to be used
+                    to restore a volume. In dynamic snapshot creation case, this field
+                    will be filled in by the CSI snapshotter sidecar with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver supports
+                    it, otherwise, this field will be set to "True". If not specified,
+                    it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  description: restoreSize represents the complete size of the snapshot
+                    in bytes. In dynamic snapshot creation case, this field will be
+                    filled in by the CSI snapshotter sidecar with the "size_bytes" value
+                    returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "size_bytes" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver supports
+                    it. When restoring a volume from this snapshot, the size of the
+                    volume MUST NOT be smaller than the restoreSize if it is specified,
+                    otherwise the restoration will fail. If not specified, it indicates
+                    that the size is unknown.
+                  format: int64
+                  minimum: 0
+                  type: integer
+                snapshotHandle:
+                  description: snapshotHandle is the CSI "snapshot_id" of a snapshot
+                    on the underlying storage system. If not specified, it indicates
+                    that dynamic snapshot creation has either failed or it is still
+                    in progress.
+                  type: string
+                volumeGroupSnapshotContentName:
+                  description: VolumeGroupSnapshotContentName is the name of the VolumeGroupSnapshotContent
+                    of which this VolumeSnapshotContent is a part of.
+                  type: string
+              type: object
+          required:
+            - spec
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: Represents the complete size of the snapshot in bytes
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: integer
+        - description: Determines whether this VolumeSnapshotContent and its physical
+            snapshot on the underlying storage system should be deleted when its bound
+            VolumeSnapshot is deleted.
+          jsonPath: .spec.deletionPolicy
+          name: DeletionPolicy
+          type: string
+        - description: Name of the CSI driver used to create the physical snapshot on
+            the underlying storage system.
+          jsonPath: .spec.driver
+          name: Driver
+          type: string
+        - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: VolumeSnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.name
+          name: VolumeSnapshot
+          type: string
+        - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent
+            object is bound.
+          jsonPath: .spec.volumeSnapshotRef.namespace
+          name: VolumeSnapshotNamespace
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      deprecated: true
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated;
+        use snapshot.storage.k8s.io/v1 VolumeSnapshotContent
+      name: v1beta1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshotContent represents the actual "on-disk" snapshot
+            object in the underlying storage system
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: spec defines properties of a VolumeSnapshotContent created
+                by the underlying storage system. Required.
+              properties:
+                deletionPolicy:
+                  description: deletionPolicy determines whether this VolumeSnapshotContent
+                    and its physical snapshot on the underlying storage system should
+                    be deleted when its bound VolumeSnapshot is deleted. Supported values
+                    are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
+                    and its physical snapshot on underlying storage system are kept.
+                    "Delete" means that the VolumeSnapshotContent and its physical snapshot
+                    on underlying storage system are deleted. For dynamically provisioned
+                    snapshots, this field will automatically be filled in by the CSI
+                    snapshotter sidecar with the "DeletionPolicy" field defined in the
+                    corresponding VolumeSnapshotClass. For pre-existing snapshots, users
+                    MUST specify this field when creating the  VolumeSnapshotContent
+                    object. Required.
+                  enum:
+                    - Delete
+                    - Retain
+                  type: string
+                driver:
+                  description: driver is the name of the CSI driver used to create the
+                    physical snapshot on the underlying storage system. This MUST be
+                    the same as the name returned by the CSI GetPluginName() call for
+                    that driver. Required.
+                  type: string
+                source:
+                  description: source specifies whether the snapshot is (or should be)
+                    dynamically provisioned or already exists, and just requires a Kubernetes
+                    object representation. This field is immutable after creation. Required.
+                  properties:
+                    snapshotHandle:
+                      description: snapshotHandle specifies the CSI "snapshot_id" of
+                        a pre-existing snapshot on the underlying storage system for
+                        which a Kubernetes object representation was (or should be)
+                        created. This field is immutable.
+                      type: string
+                    volumeHandle:
+                      description: volumeHandle specifies the CSI "volume_id" of the
+                        volume from which a snapshot should be dynamically taken from.
+                        This field is immutable.
+                      type: string
+                  type: object
+                volumeSnapshotClassName:
+                  description: name of the VolumeSnapshotClass from which this snapshot
+                    was (or will be) created. Note that after provisioning, the VolumeSnapshotClass
+                    may be deleted or recreated with different set of values, and as
+                    such, should not be referenced post-snapshot creation.
+                  type: string
+                volumeSnapshotRef:
+                  description: volumeSnapshotRef specifies the VolumeSnapshot object
+                    to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
+                    field must reference to this VolumeSnapshotContent's name for the
+                    bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
+                    object, name and namespace of the VolumeSnapshot object MUST be
+                    provided for binding to happen. This field is immutable after creation.
+                    Required.
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                      an entire object, this string should contain a valid JSON/Go
+                      field access statement, such as desiredState.manifest.containers[2].
+                      For example, if the object reference is to a container within
+                      a pod, this would take on a value like: "spec.containers{name}"
+                      (where "name" refers to the name of the container that triggered
+                      the event) or if no container name is specified "spec.containers[2]"
+                      (container with index 2 in this pod). This syntax is chosen
+                      only to have some well-defined way of referencing a part of
+                      an object. TODO: this design is not final and this field is
+                      subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                      is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+              required:
+                - deletionPolicy
+                - driver
+                - source
+                - volumeSnapshotRef
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+              properties:
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic snapshot
+                    creation case, this field will be filled in by the CSI snapshotter
+                    sidecar with the "creation_time" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. If not specified, it indicates
+                    the creation time is unknown. The format of this field is a Unix
+                    nanoseconds time encoded as an int64. On Unix, the command `date
+                    +%s%N` returns the current time in nanoseconds since 1970-01-01
+                    00:00:00 UTC.
+                  format: int64
+                  type: integer
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. Upon success after retry, this error field will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered error
+                      during snapshot creation if specified. NOTE: message may be
+                      logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if a snapshot is ready to be used
+                    to restore a volume. In dynamic snapshot creation case, this field
+                    will be filled in by the CSI snapshotter sidecar with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver supports
+                    it, otherwise, this field will be set to "True". If not specified,
+                    it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  description: restoreSize represents the complete size of the snapshot
+                    in bytes. In dynamic snapshot creation case, this field will be
+                    filled in by the CSI snapshotter sidecar with the "size_bytes" value
+                    returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "size_bytes" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver supports
+                    it. When restoring a volume from this snapshot, the size of the
+                    volume MUST NOT be smaller than the restoreSize if it is specified,
+                    otherwise the restoration will fail. If not specified, it indicates
+                    that the size is unknown.
+                  format: int64
+                  minimum: 0
+                  type: integer
+                snapshotHandle:
+                  description: snapshotHandle is the CSI "snapshot_id" of a snapshot
+                    on the underlying storage system. If not specified, it indicates
+                    that dynamic snapshot creation has either failed or it is still
+                    in progress.
+                  type: string
+              type: object
+          required:
+            - spec
+          type: object
+      served: false
+      storage: false
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+{{- end -}}
+{{- end -}}

charts/kubezero-storage/charts/lvm-localpv/charts/crds/templates/csi-volume-snapshot.yaml

@@ -0,0 +1,392 @@
+{{- if .Values.csi.volumeSnapshots.enabled -}}
+{{- $crdName := "volumesnapshots.snapshot.storage.k8s.io" -}}
+{{- if (include "crdIsAbsent" (list $crdName)) -}}
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    api-approved.kubernetes.io: https://github.com/kubernetes-csi/external-snapshotter/pull/814
+    controller-gen.kubebuilder.io/version: v0.11.3
+  creationTimestamp: null
+  name: volumesnapshots.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshot
+    listKind: VolumeSnapshotList
+    plural: volumesnapshots
+    shortNames:
+      - vs
+    singular: volumesnapshot
+  scope: Namespaced
+  versions:
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: If a new snapshot needs to be created, this contains the name of
+            the source PVC from which this snapshot was (or will be) created.
+          jsonPath: .spec.source.persistentVolumeClaimName
+          name: SourcePVC
+          type: string
+        - description: If a snapshot already exists, this contains the name of the existing
+            VolumeSnapshotContent object representing the existing snapshot.
+          jsonPath: .spec.source.volumeSnapshotContentName
+          name: SourceSnapshotContent
+          type: string
+        - description: Represents the minimum size of volume required to rehydrate from
+            this snapshot.
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: string
+        - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: SnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot
+            object intends to bind to. Please note that verification of binding actually
+            requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure
+            both are pointing at each other. Binding MUST be verified prior to usage of
+            this object.
+          jsonPath: .status.boundVolumeSnapshotContentName
+          name: SnapshotContent
+          type: string
+        - description: Timestamp when the point-in-time snapshot was taken by the underlying
+            storage system.
+          jsonPath: .status.creationTime
+          name: CreationTime
+          type: date
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshot is a user's request for either creating a point-in-time
+            snapshot of a persistent volume, or binding to a pre-existing snapshot.
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: 'spec defines the desired characteristics of a snapshot requested
+              by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
+              Required.'
+              properties:
+                source:
+                  description: source specifies where a snapshot will be created from.
+                    This field is immutable after creation. Required.
+                  oneOf:
+                    - required:
+                        - persistentVolumeClaimName
+                    - required:
+                        - volumeSnapshotContentName
+                  properties:
+                    persistentVolumeClaimName:
+                      description: persistentVolumeClaimName specifies the name of the
+                        PersistentVolumeClaim object representing the volume from which
+                        a snapshot should be created. This PVC is assumed to be in the
+                        same namespace as the VolumeSnapshot object. This field should
+                        be set if the snapshot does not exists, and needs to be created.
+                        This field is immutable.
+                      type: string
+                    volumeSnapshotContentName:
+                      description: volumeSnapshotContentName specifies the name of a
+                        pre-existing VolumeSnapshotContent object representing an existing
+                        volume snapshot. This field should be set if the snapshot already
+                        exists and only needs a representation in Kubernetes. This field
+                        is immutable.
+                      type: string
+                  type: object
+                volumeSnapshotClassName:
+                  description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass
+                  requested by the VolumeSnapshot. VolumeSnapshotClassName may be
+                  left nil to indicate that the default SnapshotClass should be used.
+                  A given cluster may have multiple default Volume SnapshotClasses:
+                  one default per CSI Driver. If a VolumeSnapshot does not specify
+                  a SnapshotClass, VolumeSnapshotSource will be checked to figure
+                  out what the associated CSI Driver is, and the default VolumeSnapshotClass
+                  associated with that CSI Driver will be used. If more than one VolumeSnapshotClass
+                  exist for a given CSI Driver and more than one have been marked
+                  as default, CreateSnapshot will fail and generate an event. Empty
+                  string is not allowed for this field.'
+                  type: string
+              required:
+                - source
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+                Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent
+                point at each other) before using this object.
+              properties:
+                boundVolumeSnapshotContentName:
+                  description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent
+                  object to which this VolumeSnapshot object intends to bind to. If
+                  not specified, it indicates that the VolumeSnapshot object has not
+                  been successfully bound to a VolumeSnapshotContent object yet. NOTE:
+                  To avoid possible security issues, consumers must verify binding
+                  between VolumeSnapshot and VolumeSnapshotContent objects is successful
+                  (by validating that both VolumeSnapshot and VolumeSnapshotContent
+                  point at each other) before using this object.'
+                  type: string
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic snapshot
+                    creation case, this field will be filled in by the snapshot controller
+                    with the "creation_time" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. If not specified, it may indicate
+                    that the creation time of the snapshot is unknown.
+                  format: date-time
+                  type: string
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. This field could be helpful to upper level controllers(i.e.,
+                    application controller) to decide whether they should continue on
+                    waiting for the snapshot to be created based on the type of error
+                    reported. The snapshot controller will keep retrying when an error
+                    occurs during the snapshot creation. Upon success, this error field
+                    will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered error
+                      during snapshot creation if specified. NOTE: message may be
+                      logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if the snapshot is ready to be used
+                    to restore a volume. In dynamic snapshot creation case, this field
+                    will be filled in by the snapshot controller with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver supports
+                    it, otherwise, this field will be set to "True". If not specified,
+                    it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  description: restoreSize represents the minimum size of volume required
+                    to create a volume from this snapshot. In dynamic snapshot creation
+                    case, this field will be filled in by the snapshot controller with
+                    the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call.
+                    For a pre-existing snapshot, this field will be filled with the
+                    "size_bytes" value returned from the CSI "ListSnapshots" gRPC call
+                    if the driver supports it. When restoring a volume from this snapshot,
+                    the size of the volume MUST NOT be smaller than the restoreSize
+                    if it is specified, otherwise the restoration will fail. If not
+                    specified, it indicates that the size is unknown.
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  type: string
+                  x-kubernetes-int-or-string: true
+                volumeGroupSnapshotName:
+                  description: VolumeGroupSnapshotName is the name of the VolumeGroupSnapshot
+                    of which this VolumeSnapshot is a part of.
+                  type: string
+              type: object
+          required:
+            - spec
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
+    - additionalPrinterColumns:
+        - description: Indicates if the snapshot is ready to be used to restore a volume.
+          jsonPath: .status.readyToUse
+          name: ReadyToUse
+          type: boolean
+        - description: If a new snapshot needs to be created, this contains the name of
+            the source PVC from which this snapshot was (or will be) created.
+          jsonPath: .spec.source.persistentVolumeClaimName
+          name: SourcePVC
+          type: string
+        - description: If a snapshot already exists, this contains the name of the existing
+            VolumeSnapshotContent object representing the existing snapshot.
+          jsonPath: .spec.source.volumeSnapshotContentName
+          name: SourceSnapshotContent
+          type: string
+        - description: Represents the minimum size of volume required to rehydrate from
+            this snapshot.
+          jsonPath: .status.restoreSize
+          name: RestoreSize
+          type: string
+        - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+          jsonPath: .spec.volumeSnapshotClassName
+          name: SnapshotClass
+          type: string
+        - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot
+            object intends to bind to. Please note that verification of binding actually
+            requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure
+            both are pointing at each other. Binding MUST be verified prior to usage of
+            this object.
+          jsonPath: .status.boundVolumeSnapshotContentName
+          name: SnapshotContent
+          type: string
+        - description: Timestamp when the point-in-time snapshot was taken by the underlying
+            storage system.
+          jsonPath: .status.creationTime
+          name: CreationTime
+          type: date
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      deprecated: true
+      deprecationWarning: snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated;
+        use snapshot.storage.k8s.io/v1 VolumeSnapshot
+      name: v1beta1
+      schema:
+        openAPIV3Schema:
+          description: VolumeSnapshot is a user's request for either creating a point-in-time
+            snapshot of a persistent volume, or binding to a pre-existing snapshot.
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            spec:
+              description: 'spec defines the desired characteristics of a snapshot requested
+              by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
+              Required.'
+              properties:
+                source:
+                  description: source specifies where a snapshot will be created from.
+                    This field is immutable after creation. Required.
+                  properties:
+                    persistentVolumeClaimName:
+                      description: persistentVolumeClaimName specifies the name of the
+                        PersistentVolumeClaim object representing the volume from which
+                        a snapshot should be created. This PVC is assumed to be in the
+                        same namespace as the VolumeSnapshot object. This field should
+                        be set if the snapshot does not exists, and needs to be created.
+                        This field is immutable.
+                      type: string
+                    volumeSnapshotContentName:
+                      description: volumeSnapshotContentName specifies the name of a
+                        pre-existing VolumeSnapshotContent object representing an existing
+                        volume snapshot. This field should be set if the snapshot already
+                        exists and only needs a representation in Kubernetes. This field
+                        is immutable.
+                      type: string
+                  type: object
+                volumeSnapshotClassName:
+                  description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass
+                  requested by the VolumeSnapshot. VolumeSnapshotClassName may be
+                  left nil to indicate that the default SnapshotClass should be used.
+                  A given cluster may have multiple default Volume SnapshotClasses:
+                  one default per CSI Driver. If a VolumeSnapshot does not specify
+                  a SnapshotClass, VolumeSnapshotSource will be checked to figure
+                  out what the associated CSI Driver is, and the default VolumeSnapshotClass
+                  associated with that CSI Driver will be used. If more than one VolumeSnapshotClass
+                  exist for a given CSI Driver and more than one have been marked
+                  as default, CreateSnapshot will fail and generate an event. Empty
+                  string is not allowed for this field.'
+                  type: string
+              required:
+                - source
+              type: object
+            status:
+              description: status represents the current information of a snapshot.
+                Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent
+                objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent
+                point at each other) before using this object.
+              properties:
+                boundVolumeSnapshotContentName:
+                  description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent
+                  object to which this VolumeSnapshot object intends to bind to. If
+                  not specified, it indicates that the VolumeSnapshot object has not
+                  been successfully bound to a VolumeSnapshotContent object yet. NOTE:
+                  To avoid possible security issues, consumers must verify binding
+                  between VolumeSnapshot and VolumeSnapshotContent objects is successful
+                  (by validating that both VolumeSnapshot and VolumeSnapshotContent
+                  point at each other) before using this object.'
+                  type: string
+                creationTime:
+                  description: creationTime is the timestamp when the point-in-time
+                    snapshot is taken by the underlying storage system. In dynamic snapshot
+                    creation case, this field will be filled in by the snapshot controller
+                    with the "creation_time" value returned from CSI "CreateSnapshot"
+                    gRPC call. For a pre-existing snapshot, this field will be filled
+                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                    gRPC call if the driver supports it. If not specified, it may indicate
+                    that the creation time of the snapshot is unknown.
+                  format: date-time
+                  type: string
+                error:
+                  description: error is the last observed error during snapshot creation,
+                    if any. This field could be helpful to upper level controllers(i.e.,
+                    application controller) to decide whether they should continue on
+                    waiting for the snapshot to be created based on the type of error
+                    reported. The snapshot controller will keep retrying when an error
+                    occurs during the snapshot creation. Upon success, this error field
+                    will be cleared.
+                  properties:
+                    message:
+                      description: 'message is a string detailing the encountered error
+                      during snapshot creation if specified. NOTE: message may be
+                      logged, and it should not contain sensitive information.'
+                      type: string
+                    time:
+                      description: time is the timestamp when the error was encountered.
+                      format: date-time
+                      type: string
+                  type: object
+                readyToUse:
+                  description: readyToUse indicates if the snapshot is ready to be used
+                    to restore a volume. In dynamic snapshot creation case, this field
+                    will be filled in by the snapshot controller with the "ready_to_use"
+                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                    snapshot, this field will be filled with the "ready_to_use" value
+                    returned from the CSI "ListSnapshots" gRPC call if the driver supports
+                    it, otherwise, this field will be set to "True". If not specified,
+                    it means the readiness of a snapshot is unknown.
+                  type: boolean
+                restoreSize:
+                  description: restoreSize represents the minimum size of volume required
+                    to create a volume from this snapshot. In dynamic snapshot creation
+                    case, this field will be filled in by the snapshot controller with
+                    the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call.
+                    For a pre-existing snapshot, this field will be filled with the
+                    "size_bytes" value returned from the CSI "ListSnapshots" gRPC call
+                    if the driver supports it. When restoring a volume from this snapshot,
+                    the size of the volume MUST NOT be smaller than the restoreSize
+                    if it is specified, otherwise the restoration will fail. If not
+                    specified, it indicates that the size is unknown.
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  type: string
+                  x-kubernetes-int-or-string: true
+              type: object
+          required:
+            - spec
+          type: object
+      served: false
+      storage: false
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+{{- end -}}
+{{- end -}}

charts/kubezero-storage/charts/lvm-localpv/charts/crds/templates/lvmnode.yaml

@@ -1,5 +1,6 @@
-
+{{- if .Values.lvmLocalPv.enabled -}}
-
+{{- $crdName := "lvmnodes.local.openebs.io" -}}
+{{- if (include "crdIsAbsent" (list $crdName)) -}}
 ##############################################
 ###########                       ############
 ###########     LVMNode CRD       ############
@@ -175,3 +176,5 @@ status:
     plural: ""
   conditions: []
   storedVersions: []
+{{- end -}}
+{{- end -}}

charts/kubezero-storage/charts/lvm-localpv/charts/crds/templates/lvmsnapshot.yaml

@@ -1,5 +1,6 @@
-
+{{- if .Values.lvmLocalPv.enabled -}}
-
+{{- $crdName := "lvmsnapshots.local.openebs.io" -}}
+{{- if (include "crdIsAbsent" (list $crdName)) -}}
 ##############################################
 ###########                       ############
 ###########   LVMSnapshot CRD     ############
@@ -83,3 +84,5 @@ status:
     plural: ""
   conditions: []
   storedVersions: []
+{{- end -}}
+{{- end -}}

charts/kubezero-storage/charts/lvm-localpv/charts/crds/templates/lvmvolume.yaml

@@ -1,5 +1,6 @@
-
+{{- if .Values.lvmLocalPv.enabled -}}
-
+{{- $crdName := "lvmvolumes.local.openebs.io" -}}
+{{- if (include "crdIsAbsent" (list $crdName)) -}}
 ##############################################
 ###########                       ############
 ###########   LVMVolume CRD       ############
@@ -151,3 +152,5 @@ status:
     plural: ""
   conditions: []
   storedVersions: []
+{{- end -}}
+{{- end -}}

charts/kubezero-storage/charts/lvm-localpv/charts/crds/values.yaml

@@ -0,0 +1,8 @@
+lvmLocalPv:
+  # Install lvm-localpv CRDs
+  enabled: true
+
+csi:
+  volumeSnapshots:
+    # Install Volume Snapshot CRDs
+    enabled: true

charts/kubezero-storage/charts/lvm-localpv/templates/lvm-controller.yaml

@@ -1,5 +1,5 @@
 apiVersion: apps/v1
-kind: StatefulSet
+kind: Deployment
 metadata:
   name: {{ template "lvmlocalpv.fullname" . }}-controller
   {{- with .Values.lvmController.annotations }}
@@ -11,7 +11,6 @@ spec:
   selector:
     matchLabels:
       {{- include "lvmlocalpv.lvmController.matchLabels" . | nindent 6 }}
-  serviceName: "{{ .Values.lvmController.serviceName }}"
   replicas: {{ .Values.lvmController.replicas }}
   template:
     metadata:
@@ -147,3 +146,7 @@ spec:
       tolerations:
 {{ toYaml .Values.lvmController.tolerations | indent 8 }}
 {{- end }}
+{{- if .Values.lvmController.topologySpreadConstraints }}
+      topologySpreadConstraints:
+{{ toYaml .Values.lvmController.topologySpreadConstraints | indent 8 }}
+{{- end }}

charts/kubezero-storage/charts/lvm-localpv/templates/lvm-node.yaml

@@ -30,7 +30,7 @@ spec:
       priorityClassName: {{ template "lvmlocalpv.lvmNode.priorityClassName" . }}
 {{- end }}
       serviceAccountName: {{ .Values.serviceAccount.lvmNode.name }}
-      hostNetwork: true
+      hostNetwork: {{ .Values.lvmNode.hostNetwork }}
       containers:
         - name: {{ .Values.lvmNode.driverRegistrar.name }}
           image: "{{ .Values.lvmNode.driverRegistrar.image.registry }}{{ .Values.lvmNode.driverRegistrar.image.repository }}:{{ .Values.lvmNode.driverRegistrar.image.tag }}"

charts/kubezero-storage/charts/lvm-localpv/templates/psp.yaml

@@ -10,7 +10,7 @@ spec:
   allowPrivilegeEscalation: true
   allowedCapabilities: ['*']
   volumes: ['*']
-  hostNetwork: true
+  hostNetwork: {{ .Values.lvmNode.hostNetwork}}
   hostIPC: true
   hostPID: true
   runAsUser:

charts/kubezero-storage/charts/lvm-localpv/templates/rbac.yaml

@@ -14,9 +14,6 @@ metadata:
   labels:
     {{- include "lvmlocalpv.lvmController.labels" . | nindent 4 }}
 rules:
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "list"]
   - apiGroups: [""]
     resources: ["namespaces"]
     verbs: ["get", "list", "watch"]
@@ -123,7 +120,8 @@ roleRef:
   apiGroup: rbac.authorization.k8s.io
 ---
 {{- end }}
-{{- if .Values.serviceAccount.lvmNode.create -}}
+
+{{- if .Values.serviceAccount.lvmNode.create }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:

charts/kubezero-storage/charts/lvm-localpv/values.yaml

@@ -2,7 +2,7 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 release:
-  version: "1.3.0"
+  version: "1.5.0"
 
 imagePullSecrets:
 # - name: "image-pull-secret"
@@ -61,14 +61,15 @@ lvmNode:
     # Configure the maximum number of queries allowed after
     # accounting for rolled over qps from previous seconds.
     burst: 0
+  # Disable or enable the use of hostNetwork for the lvm node daemonset.
+  hostNetwork: false
 
 
 # lvmController contains the configurables for
-# the lvm controller statefulset
+# the lvm controller deployment
 lvmController:
   componentName: openebs-lvm-controller
   replicas: 1
-  serviceName: openebs-lvm
   logLevel: 5
   resizer:
     name: "csi-resizer"
@@ -126,6 +127,7 @@ lvmController:
     name: openebs-lvm-controller
   nodeSelector: {}
   tolerations: []
+  topologySpreadConstraints: []
   securityContext: {}
   priorityClass:
     create: true
@@ -139,7 +141,7 @@ lvmController:
     burst: 0
 
 # lvmPlugin is the common csi container used by the
-# controller statefulset and node daemonset
+# controller deployment and node daemonset
 lvmPlugin:
   name: "openebs-lvm-plugin"
   image:
@@ -149,7 +151,7 @@ lvmPlugin:
     repository: openebs/lvm-driver
     pullPolicy: IfNotPresent
     # Overrides the image tag whose default is the chart appVersion.
-    tag: 1.3.0
+    tag: 1.5.0
   ioLimits:
     enabled: false
     containerRuntime: containerd
@@ -164,12 +166,6 @@ lvmPlugin:
 
 role: openebs-lvm
 
-crd:
-  enableInstall: true
-  # Specify installation of the kubernetes-csi volume snapshot CRDs if your Kubernetes distribution
-  # or another storage operator already manages them.
-  volumeSnapshot: true
-
 serviceAccount:
   lvmController:
     # Specifies whether a service account should be created
@@ -186,3 +182,12 @@ serviceAccount:
 
 analytics:
   enabled: true
+
+crds:
+  lvmLocalPv:
+    # Install lvm-localpv CRDs
+    enabled: true
+  csi:
+    volumeSnapshots:
+      # Install Volume Snapshot CRDs
+      enabled: true

charts/kubezero-storage/jsonnet/jsonnetfile.lock.json

@@ -18,7 +18,7 @@
           "subdir": "contrib/mixin"
         }
       },
-      "version": "7851295966ae3dd5308c37079b5df58440d1fb36",
+      "version": "9359aef3e3dd39b7bbf57cab4b6899a238af3144",
       "sum": "xuUBd2vqF7asyVDe5CE08uPT/RxAdy8O75EjFJoMXXU="
     },
     {
@@ -51,6 +51,16 @@
       "version": "a1d61cce1da59c71409b99b5c7568511fec661ea",
       "sum": "gCtR9s/4D5fxU9aKXg0Bru+/njZhA0YjLjPiASc61FM="
     },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/grafonnet.git",
+          "subdir": "gen/grafonnet-latest"
+        }
+      },
+      "version": "6ac1593ca787638da223380ff4a3fd0f96e953e1",
+      "sum": "GxEO83uxgsDclLp/fmlUJZDbSGpeUZY6Ap3G2cgdL1g="
+    },
     {
       "source": {
         "git": {
@@ -58,8 +68,18 @@
           "subdir": "gen/grafonnet-v10.0.0"
         }
       },
-      "version": "a1b14991306adebdb0107ea9aa74870bf86c346e",
+      "version": "6ac1593ca787638da223380ff4a3fd0f96e953e1",
-      "sum": "gj/20VIGucG2vDGjG7YdHLC4yUUfrpuaneUYaRmymOM="
+      "sum": "W7sLuAvMSJPkC7Oo31t45Nz/cUdJV7jzNSJTd3F1daM="
+    },
+    {
+      "source": {
+        "git": {
+          "remote": "https://github.com/grafana/grafonnet.git",
+          "subdir": "gen/grafonnet-v10.4.0"
+        }
+      },
+      "version": "6ac1593ca787638da223380ff4a3fd0f96e953e1",
+      "sum": "ZSmDT7i/qU9P8ggmuPuJT+jonq1ZEsBRCXycW/H5L/A="
     },
     {
       "source": {
@@ -68,8 +88,8 @@
           "subdir": "grafana-builder"
         }
       },
-      "version": "931f6b1139bb3694b06f2261279ba3dc01aca5b8",
+      "version": "7561fd330312538d22b00e0c7caecb4ba66321ea",
-      "sum": "VmOxvg9FuY9UYr3lN6ZJe2HhuIErJoWimPybQr3S3yQ="
+      "sum": "+z5VY+bPBNqXcmNAV8xbJcbsRA+pro1R3IM7aIY8OlU="
     },
     {
       "source": {
@@ -78,8 +98,8 @@
           "subdir": "doc-util"
         }
       },
-      "version": "503e5c8fe96d6b55775037713ac10b184709ad93",
+      "version": "6ac6c69685b8c29c54515448eaca583da2d88150",
-      "sum": "BY4u0kLF3Qf/4IB4HnX9S5kEQIpHb4MUrppp6WLDtlU="
+      "sum": "BrAL/k23jq+xy9oA7TWIhUx07dsA/QLm3g7ktCwe//U="
     },
     {
       "source": {
@@ -88,8 +108,8 @@
           "subdir": ""
         }
       },
-      "version": "c1a315a7dbead0335a5e0486acc5583395b22a24",
+      "version": "fc2e57a8839902ed4ba6cab5a99d642500f7102b",
-      "sum": "UVdL+uuFI8BSQgLfMJEJk2WDKsQXNT3dRHcr2Ti9rLI="
+      "sum": "43waffw1QzvpY4rKcWoo3L7Vpee+DCYexwLDd5cPG0M="
     },
     {
       "source": {
@@ -98,8 +118,8 @@
           "subdir": ""
         }
       },
-      "version": "2dbe4f9625a811b8b89f0495e74509c74779da82",
+      "version": "a1c276d7a46c4b06fa5d8b4a64441939d398efe5",
-      "sum": "Fe7bN9E6qeKNUdENjQvYttgf4S1DDqXRVB80wdmQgHQ="
+      "sum": "b/mEai1MvVnZ22YvZlXEO4jWDZledrtJg8eOS1ZUj0M="
     },
     {
       "source": {
@@ -108,8 +128,8 @@
           "subdir": "jsonnet/kube-state-metrics"
         }
       },
-      "version": "c707af4c2d84193a3480729b3525b0fc3d686e73",
+      "version": "9ba1c3702142918e09e8eb5ca530e15198624259",
-      "sum": "+dOzAK+fwsFf97uZpjcjTcEJEC1H8hh/j8f5uIQK/5g="
+      "sum": "msMZyUvcebzRILLzNlTIiSOwa1XgQKtP7jbZTkiqwM0="
     },
     {
       "source": {
@@ -118,7 +138,7 @@
           "subdir": "jsonnet/kube-state-metrics-mixin"
         }
       },
-      "version": "c707af4c2d84193a3480729b3525b0fc3d686e73",
+      "version": "9ba1c3702142918e09e8eb5ca530e15198624259",
       "sum": "qclI7LwucTjBef3PkGBkKxF0mfZPbHnn4rlNWKGtR4c="
     },
     {
@@ -138,8 +158,8 @@
           "subdir": "jsonnet/kube-prometheus"
         }
       },
-      "version": "035b09f42441d4630b3a3de4e4a490d19b1ba5e4",
+      "version": "76f2e1ef95be0df752037baa040781c5219e1fb3",
-      "sum": "bp+cUUcoQjREBPigCP2S1xIvrh7HDQeYqCcrHCuDnUQ="
+      "sum": "IgpAgyyBZ7VT2vr9kSYQP/lkZUNQnbqpGh2sYCtUKs0="
     },
     {
       "source": {
@@ -148,8 +168,8 @@
           "subdir": "jsonnet/mixin"
         }
       },
-      "version": "0d918323945ce87f0094c05c153075c0a6edc8de",
+      "version": "8f8464b41775e13c71c2700799352a3dcd82f528",
-      "sum": "n3flMIzlADeyygb0uipZ4KPp2uNSjdtkrwgHjTC7Ca4=",
+      "sum": "gi+knjdxs2T715iIQIntrimbHRgHnpM8IFBJDD1gYfs=",
       "name": "prometheus-operator-mixin"
     },
     {
@@ -159,8 +179,8 @@
           "subdir": "jsonnet/prometheus-operator"
         }
       },
-      "version": "0d918323945ce87f0094c05c153075c0a6edc8de",
+      "version": "8f8464b41775e13c71c2700799352a3dcd82f528",
-      "sum": "1X9mGAj+nRaBAgNRG19mYtDc+ZLVIeAiK5M3h0Tpu7A="
+      "sum": "/xycwh6lbet/dMzqZHJjSv6AfBEAQPAgk+1usi3d3W4="
     },
     {
       "source": {
@@ -169,7 +189,7 @@
           "subdir": "doc/alertmanager-mixin"
         }
       },
-      "version": "83486834deb4f886b4828cad3dbbe42d141d951d",
+      "version": "14cbe6301c732658d6fe877ec55ad5b738abcf06",
       "sum": "IpF46ZXsm+0wJJAPtAre8+yxTNZA57mBqGpBP/r7/kw=",
       "name": "alertmanager"
     },
@@ -180,8 +200,8 @@
           "subdir": "docs/node-mixin"
         }
       },
-      "version": "9666d002487039ac66b20287998945461eefe746",
+      "version": "6425f079d162ebd22d4c6c4e4d7e4a36ebbe2239",
-      "sum": "QZwFBpulndqo799gkR5rP2/WdcQKQkNnaBwhaOI8Jeg="
+      "sum": "vWhHvFqV7+fxrQddTeGVKi1e4EzB3VWtNyD8TjSmevY="
     },
     {
       "source": {
@@ -190,8 +210,8 @@
           "subdir": "documentation/prometheus-mixin"
         }
       },
-      "version": "2ae84f980f981a004143c8239f4f20a35547ef04",
+      "version": "bfaa0a319ceca0814b076072a61cc1640e6a4f36",
-      "sum": "rNvddVTMNfaguOGzEGoeKjUsfhlXJBUImC+SIFNNCiM=",
+      "sum": "u/Fpz2MPkezy71/q+c7mF0vc3hE9fWt2W/YbvF0LP/8=",
       "name": "prometheus"
     },
     {
@@ -212,7 +232,7 @@
           "subdir": "mixin"
         }
       },
-      "version": "e7aecb401f54bec52540900d455a9c226c5791ff",
+      "version": "4a2a4555d24665a52c3ed43e007301dd492af9b3",
       "sum": "HhSSbGGCNHCMy1ee5jElYDm0yS9Vesa7QB2/SHKdjsY=",
       "name": "thanos-mixin"
     }

charts/kubezero-storage/templates/snapshot-controller/controller.yaml

@@ -22,7 +22,7 @@ spec:
   # the snapshot controller won't be marked as ready if the v1 CRDs are unavailable
   # in #504 the snapshot-controller will exit after around 7.5 seconds if it
   # can't find the v1 CRDs so this value should be greater than that
-  minReadySeconds: 15
+  minReadySeconds: 35
   strategy:
     rollingUpdate:
       maxSurge: 0
@@ -33,7 +33,7 @@ spec:
       labels:
         app: snapshot-controller
     spec:
-      affinity:          
+      affinity:
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
           - labelSelector:

charts/kubezero-storage/templates/snapshot-controller/rbac-snapshot-controller.yaml

@@ -0,0 +1,103 @@
+# RBAC file for the snapshot controller.
+#
+# The snapshot controller implements the control loop for CSI snapshot functionality.
+# It should be installed as part of the base Kubernetes distribution in an appropriate
+# namespace for components implementing base system functionality. For installing with
+# Vanilla Kubernetes, kube-system makes sense for the namespace.
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: snapshot-controller
+  namespace: kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update", "patch", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update", "patch"]
+
+  - apiGroups: ["groupsnapshot.storage.k8s.io"]
+    resources: ["volumegroupsnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["groupsnapshot.storage.k8s.io"]
+    resources: ["volumegroupsnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: ["groupsnapshot.storage.k8s.io"]
+    resources: ["volumegroupsnapshotcontents/status"]
+    verbs: ["patch"]
+  - apiGroups: ["groupsnapshot.storage.k8s.io"]
+    resources: ["volumegroupsnapshots"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["groupsnapshot.storage.k8s.io"]
+    resources: ["volumegroupsnapshots/status"]
+    verbs: ["update", "patch"]
+
+  # Enable this RBAC rule only when using distributed snapshotting, i.e. when the enable-distributed-snapshotting flag is set to true
+  # - apiGroups: [""]
+  #   resources: ["nodes"]
+  #   verbs: ["get", "list", "watch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-role
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: snapshot-controller-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-leaderelection
+  namespace: kube-system
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-leaderelection
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+roleRef:
+  kind: Role
+  name: snapshot-controller-leaderelection
+  apiGroup: rbac.authorization.k8s.io

charts/kubezero-storage/templates/snapshot-controller/volumesnapshotclasses-crd.yaml

@@ -1,11 +1,11 @@
-{{- if .Values.crd.volumeSnapshot }}
+{{- if .Values.snapshotController.enabled }}
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.3
     api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814"
+    controller-gen.kubebuilder.io/version: v0.12.0
   creationTimestamp: null
   name: volumesnapshotclasses.snapshot.storage.k8s.io
 spec:
@@ -15,118 +15,120 @@ spec:
     listKind: VolumeSnapshotClassList
     plural: volumesnapshotclasses
     shortNames:
-      - vsclass
+    - vsclass
-      - vsclasses
+    - vsclasses
     singular: volumesnapshotclass
   scope: Cluster
   versions:
-    - additionalPrinterColumns:
+  - additionalPrinterColumns:
-        - jsonPath: .driver
+    - jsonPath: .driver
-          name: Driver
+      name: Driver
-          type: string
+      type: string
-        - description: Determines whether a VolumeSnapshotContent created through the
+    - description: Determines whether a VolumeSnapshotContent created through the
-            VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted.
+        VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted.
-          jsonPath: .deletionPolicy
+      jsonPath: .deletionPolicy
-          name: DeletionPolicy
+      name: DeletionPolicy
-          type: string
+      type: string
-        - jsonPath: .metadata.creationTimestamp
+    - jsonPath: .metadata.creationTimestamp
-          name: Age
+      name: Age
-          type: date
+      type: date
-      name: v1
+    name: v1
-      schema:
+    schema:
-        openAPIV3Schema:
+      openAPIV3Schema:
-          description: VolumeSnapshotClass specifies parameters that a underlying storage
+        description: VolumeSnapshotClass specifies parameters that a underlying storage
-            system uses when creating a volume snapshot. A specific VolumeSnapshotClass
+          system uses when creating a volume snapshot. A specific VolumeSnapshotClass
-            is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
+          is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
-            are non-namespaced
+          are non-namespaced
-          properties:
+        properties:
-            apiVersion:
+          apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation
+            description: 'APIVersion defines the versioned schema of this representation
               of an object. Servers should convert recognized schemas to the latest
               internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-              type: string
+            type: string
-            deletionPolicy:
+          deletionPolicy:
-              description: deletionPolicy determines whether a VolumeSnapshotContent
+            description: deletionPolicy determines whether a VolumeSnapshotContent
-                created through the VolumeSnapshotClass should be deleted when its bound
+              created through the VolumeSnapshotClass should be deleted when its bound
-                VolumeSnapshot is deleted. Supported values are "Retain" and "Delete".
+              VolumeSnapshot is deleted. Supported values are "Retain" and "Delete".
-                "Retain" means that the VolumeSnapshotContent and its physical snapshot
+              "Retain" means that the VolumeSnapshotContent and its physical snapshot
-                on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent
+              on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent
-                and its physical snapshot on underlying storage system are deleted.
+              and its physical snapshot on underlying storage system are deleted.
-                Required.
+              Required.
-              enum:
+            enum:
-                - Delete
+            - Delete
-                - Retain
+            - Retain
-              type: string
+            type: string
-            driver:
+          driver:
-              description: driver is the name of the storage driver that handles this
+            description: driver is the name of the storage driver that handles this
-                VolumeSnapshotClass. Required.
+              VolumeSnapshotClass. Required.
-              type: string
+            type: string
-            kind:
+          kind:
-              description: 'Kind is a string value representing the REST resource this
+            description: 'Kind is a string value representing the REST resource this
               object represents. Servers may infer this from the endpoint the client
               submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          parameters:
+            additionalProperties:
               type: string
-            parameters:
+            description: parameters is a key-value map with storage driver specific
-              additionalProperties:
+              parameters for creating snapshots. These values are opaque to Kubernetes.
-                type: string
+            type: object
-              description: parameters is a key-value map with storage driver specific
+        required:
-                parameters for creating snapshots. These values are opaque to Kubernetes.
+        - deletionPolicy
-              type: object
+        - driver
-          required:
+        type: object
-            - deletionPolicy
+    served: true
-            - driver
+    storage: true
-          type: object
+    subresources: {}
-      served: true
+  - additionalPrinterColumns:
-      storage: true
+    - jsonPath: .driver
-      subresources: {}
+      name: Driver
-    - additionalPrinterColumns:
+      type: string
-        - jsonPath: .driver
+    - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted.
-          name: Driver
+      jsonPath: .deletionPolicy
-          type: string
+      name: DeletionPolicy
-        - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted.
+      type: string
-          jsonPath: .deletionPolicy
+    - jsonPath: .metadata.creationTimestamp
-          name: DeletionPolicy
+      name: Age
-          type: string
+      type: date
-        - jsonPath: .metadata.creationTimestamp
+    name: v1beta1
-          name: Age
+    # This indicates the v1beta1 version of the custom resource is deprecated.
-          type: date
+    # API requests to this version receive a warning in the server response.
-      name: v1beta1
+    deprecated: true
-      # This indicates the v1beta1 version of the custom resource is deprecated.
+    # This overrides the default warning returned to clients making v1beta1 API requests.
-      # API requests to this version receive a warning in the server response.
+    deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass"
-      deprecated: true
+    schema:
-      # This overrides the default warning returned to clients making v1beta1 API requests.
+      openAPIV3Schema:
-      deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass"
+        description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced
-      schema:
+        properties:
-        openAPIV3Schema:
+          apiVersion:
-          description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-          properties:
+            type: string
-            apiVersion:
+          deletionPolicy:
-              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required.
+            enum:
+            - Delete
+            - Retain
+            type: string
+          driver:
+            description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required.
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          parameters:
+            additionalProperties:
               type: string
-            deletionPolicy:
+            description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes.
-              description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required.
+            type: object
-              enum:
+        required:
-                - Delete
+        - deletionPolicy
-                - Retain
+        - driver
-              type: string
+        type: object
-            driver:
+    served: false
-              description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required.
+    storage: false
-              type: string
+    subresources: {}
-            kind:
-              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-              type: string
-            parameters:
-              additionalProperties:
-                type: string
-              description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes.
-              type: object
-          required:
-            - deletionPolicy
-            - driver
-          type: object
-      served: false
-      storage: false
-      subresources: {}
 status:
   acceptedNames:
     kind: ""

charts/kubezero-storage/templates/snapshot-controller/volumesnapshotcontents-crd.yaml

@@ -1,11 +1,11 @@
-{{- if .Values.crd.volumeSnapshot }}
+{{- if .Values.snapshotController.enabled }}
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.3
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/955"
-    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814"
+    controller-gen.kubebuilder.io/version: v0.12.0
   creationTimestamp: null
   name: volumesnapshotcontents.snapshot.storage.k8s.io
 spec:
@@ -15,389 +15,391 @@ spec:
     listKind: VolumeSnapshotContentList
     plural: volumesnapshotcontents
     shortNames:
-      - vsc
+    - vsc
-      - vscs
+    - vscs
     singular: volumesnapshotcontent
   scope: Cluster
   versions:
-    - additionalPrinterColumns:
+  - additionalPrinterColumns:
-        - description: Indicates if the snapshot is ready to be used to restore a volume.
+    - description: Indicates if the snapshot is ready to be used to restore a volume.
-          jsonPath: .status.readyToUse
+      jsonPath: .status.readyToUse
-          name: ReadyToUse
+      name: ReadyToUse
-          type: boolean
+      type: boolean
-        - description: Represents the complete size of the snapshot in bytes
+    - description: Represents the complete size of the snapshot in bytes
-          jsonPath: .status.restoreSize
+      jsonPath: .status.restoreSize
-          name: RestoreSize
+      name: RestoreSize
-          type: integer
+      type: integer
-        - description: Determines whether this VolumeSnapshotContent and its physical
+    - description: Determines whether this VolumeSnapshotContent and its physical
-            snapshot on the underlying storage system should be deleted when its bound
+        snapshot on the underlying storage system should be deleted when its bound
-            VolumeSnapshot is deleted.
+        VolumeSnapshot is deleted.
-          jsonPath: .spec.deletionPolicy
+      jsonPath: .spec.deletionPolicy
-          name: DeletionPolicy
+      name: DeletionPolicy
-          type: string
+      type: string
-        - description: Name of the CSI driver used to create the physical snapshot on
+    - description: Name of the CSI driver used to create the physical snapshot on
-            the underlying storage system.
+        the underlying storage system.
-          jsonPath: .spec.driver
+      jsonPath: .spec.driver
-          name: Driver
+      name: Driver
-          type: string
+      type: string
-        - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+    - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
-          jsonPath: .spec.volumeSnapshotClassName
+      jsonPath: .spec.volumeSnapshotClassName
-          name: VolumeSnapshotClass
+      name: VolumeSnapshotClass
-          type: string
+      type: string
-        - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
+    - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent
-            object is bound.
+        object is bound.
-          jsonPath: .spec.volumeSnapshotRef.name
+      jsonPath: .spec.volumeSnapshotRef.name
-          name: VolumeSnapshot
+      name: VolumeSnapshot
-          type: string
+      type: string
-        - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.
+    - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.
-          jsonPath: .spec.volumeSnapshotRef.namespace
+      jsonPath: .spec.volumeSnapshotRef.namespace
-          name: VolumeSnapshotNamespace
+      name: VolumeSnapshotNamespace
-          type: string
+      type: string
-        - jsonPath: .metadata.creationTimestamp
+    - jsonPath: .metadata.creationTimestamp
-          name: Age
+      name: Age
-          type: date
+      type: date
-      name: v1
+    name: v1
-      schema:
+    schema:
-        openAPIV3Schema:
+      openAPIV3Schema:
-          description: VolumeSnapshotContent represents the actual "on-disk" snapshot
+        description: VolumeSnapshotContent represents the actual "on-disk" snapshot
-            object in the underlying storage system
+          object in the underlying storage system
-          properties:
+        properties:
-            apiVersion:
+          apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation
+            description: 'APIVersion defines the versioned schema of this representation
-                of an object. Servers should convert recognized schemas to the latest
+              of an object. Servers should convert recognized schemas to the latest
-                internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-              type: string
+            type: string
-            kind:
+          kind:
-              description: 'Kind is a string value representing the REST resource this
+            description: 'Kind is a string value representing the REST resource this
-                object represents. Servers may infer this from the endpoint the client
+              object represents. Servers may infer this from the endpoint the client
-                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-              type: string
+            type: string
-            spec:
+          metadata:
-              description: spec defines properties of a VolumeSnapshotContent created
+            type: object
-                by the underlying storage system. Required.
+          spec:
-              properties:
+            description: spec defines properties of a VolumeSnapshotContent created
-                deletionPolicy:
+              by the underlying storage system. Required.
-                  description: deletionPolicy determines whether this VolumeSnapshotContent
+            properties:
-                    and its physical snapshot on the underlying storage system should
+              deletionPolicy:
-                    be deleted when its bound VolumeSnapshot is deleted. Supported values
+                description: deletionPolicy determines whether this VolumeSnapshotContent
-                    are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
+                  and its physical snapshot on the underlying storage system should
-                    and its physical snapshot on underlying storage system are kept.
+                  be deleted when its bound VolumeSnapshot is deleted. Supported values
-                    "Delete" means that the VolumeSnapshotContent and its physical snapshot
+                  are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
-                    on underlying storage system are deleted. For dynamically provisioned
+                  and its physical snapshot on underlying storage system are kept.
-                    snapshots, this field will automatically be filled in by the CSI
+                  "Delete" means that the VolumeSnapshotContent and its physical snapshot
-                    snapshotter sidecar with the "DeletionPolicy" field defined in the
+                  on underlying storage system are deleted. For dynamically provisioned
-                    corresponding VolumeSnapshotClass. For pre-existing snapshots, users
+                  snapshots, this field will automatically be filled in by the CSI
-                    MUST specify this field when creating the VolumeSnapshotContent
+                  snapshotter sidecar with the "DeletionPolicy" field defined in the
-                    object. Required.
+                  corresponding VolumeSnapshotClass. For pre-existing snapshots, users
-                  enum:
+                  MUST specify this field when creating the VolumeSnapshotContent
-                    - Delete
+                  object. Required.
-                    - Retain
+                enum:
-                  type: string
+                - Delete
-                driver:
+                - Retain
-                  description: driver is the name of the CSI driver used to create the
+                type: string
-                    physical snapshot on the underlying storage system. This MUST be
+              driver:
-                    the same as the name returned by the CSI GetPluginName() call for
+                description: driver is the name of the CSI driver used to create the
-                    that driver. Required.
+                  physical snapshot on the underlying storage system. This MUST be
-                  type: string
+                  the same as the name returned by the CSI GetPluginName() call for
-                source:
+                  that driver. Required.
-                  description: source specifies whether the snapshot is (or should be)
+                type: string
-                    dynamically provisioned or already exists, and just requires a Kubernetes
+              source:
-                    object representation. This field is immutable after creation. Required.
+                description: source specifies whether the snapshot is (or should be)
-                  properties:
+                  dynamically provisioned or already exists, and just requires a Kubernetes
-                    snapshotHandle:
+                  object representation. This field is immutable after creation. Required.
-                      description: snapshotHandle specifies the CSI "snapshot_id" of
+                properties:
-                        a pre-existing snapshot on the underlying storage system for
+                  snapshotHandle:
-                        which a Kubernetes object representation was (or should be)
+                    description: snapshotHandle specifies the CSI "snapshot_id" of
-                        created. This field is immutable.
+                      a pre-existing snapshot on the underlying storage system for
-                      type: string
+                      which a Kubernetes object representation was (or should be)
-                    volumeHandle:
+                      created. This field is immutable.
-                      description: volumeHandle specifies the CSI "volume_id" of the
+                    type: string
-                        volume from which a snapshot should be dynamically taken from.
+                  volumeHandle:
-                        This field is immutable.
+                    description: volumeHandle specifies the CSI "volume_id" of the
-                      type: string
+                      volume from which a snapshot should be dynamically taken from.
-                  type: object
+                      This field is immutable.
-                  oneOf:
+                    type: string
-                    - required: [ "snapshotHandle" ]
+                type: object
-                    - required: [ "volumeHandle" ]
+                oneOf:
-                sourceVolumeMode:
+                - required: ["snapshotHandle"]
-                  description: SourceVolumeMode is the mode of the volume whose snapshot
+                - required: ["volumeHandle"]
-                    is taken. Can be either “Filesystem” or “Block”. If not specified,
+              sourceVolumeMode:
-                    it indicates the source volume's mode is unknown. This field is
+                description: SourceVolumeMode is the mode of the volume whose snapshot
-                    immutable. This field is an alpha field.
+                  is taken. Can be either “Filesystem” or “Block”. If not specified,
-                  type: string
+                  it indicates the source volume's mode is unknown. This field is
-                volumeSnapshotClassName:
+                  immutable. This field is an alpha field.
-                  description: name of the VolumeSnapshotClass from which this snapshot
+                type: string
-                    was (or will be) created. Note that after provisioning, the VolumeSnapshotClass
+              volumeSnapshotClassName:
-                    may be deleted or recreated with different set of values, and as
+                description: name of the VolumeSnapshotClass from which this snapshot
-                    such, should not be referenced post-snapshot creation.
+                  was (or will be) created. Note that after provisioning, the VolumeSnapshotClass
-                  type: string
+                  may be deleted or recreated with different set of values, and as
-                volumeSnapshotRef:
+                  such, should not be referenced post-snapshot creation.
-                  description: volumeSnapshotRef specifies the VolumeSnapshot object
+                type: string
-                    to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
+              volumeSnapshotRef:
-                    field must reference to this VolumeSnapshotContent's name for the
+                description: volumeSnapshotRef specifies the VolumeSnapshot object
-                    bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
+                  to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
-                    object, name and namespace of the VolumeSnapshot object MUST be
+                  field must reference to this VolumeSnapshotContent's name for the
-                    provided for binding to happen. This field is immutable after creation.
+                  bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
-                    Required.
+                  object, name and namespace of the VolumeSnapshot object MUST be
-                  properties:
+                  provided for binding to happen. This field is immutable after creation.
-                    apiVersion:
+                  Required.
-                      description: API version of the referent.
+                properties:
-                      type: string
+                  apiVersion:
-                    fieldPath:
+                    description: API version of the referent.
-                      description: 'If referring to a piece of an object instead of
+                    type: string
-                        an entire object, this string should contain a valid JSON/Go
+                  fieldPath:
-                        field access statement, such as desiredState.manifest.containers[2].
+                    description: 'If referring to a piece of an object instead of
-                        For example, if the object reference is to a container within
+                      an entire object, this string should contain a valid JSON/Go
-                        a pod, this would take on a value like: "spec.containers{name}"
+                      field access statement, such as desiredState.manifest.containers[2].
-                        (where "name" refers to the name of the container that triggered
+                      For example, if the object reference is to a container within
-                        the event) or if no container name is specified "spec.containers[2]"
+                      a pod, this would take on a value like: "spec.containers{name}"
-                        (container with index 2 in this pod). This syntax is chosen
+                      (where "name" refers to the name of the container that triggered
-                        only to have some well-defined way of referencing a part of
+                      the event) or if no container name is specified "spec.containers[2]"
-                        an object. TODO: this design is not final and this field is
+                      (container with index 2 in this pod). This syntax is chosen
-                        subject to change in the future.'
+                      only to have some well-defined way of referencing a part of
-                      type: string
+                      an object. TODO: this design is not final and this field is
-                    kind:
+                      subject to change in the future.'
-                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                    type: string
-                      type: string
+                  kind:
-                    name:
+                    description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                    type: string
-                      type: string
+                  name:
-                    namespace:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
-                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                    type: string
-                      type: string
+                  namespace:
-                    resourceVersion:
+                    description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
-                      description: 'Specific resourceVersion to which this reference
+                    type: string
-                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                  resourceVersion:
-                      type: string
+                    description: 'Specific resourceVersion to which this reference
-                    uid:
+                      is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
-                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                    type: string
-                      type: string
+                  uid:
-                  type: object
+                    description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
-                  x-kubernetes-map-type: atomic
+                    type: string
-              required:
+                type: object
-                - deletionPolicy
+                x-kubernetes-map-type: atomic
-                - driver
+            required:
-                - source
+            - deletionPolicy
-                - volumeSnapshotRef
+            - driver
-              type: object
+            - source
-            status:
+            - volumeSnapshotRef
-              description: status represents the current information of a snapshot.
+            type: object
-              properties:
+          status:
-                creationTime:
+            description: status represents the current information of a snapshot.
-                  description: creationTime is the timestamp when the point-in-time
+            properties:
-                    snapshot is taken by the underlying storage system. In dynamic snapshot
+              creationTime:
-                    creation case, this field will be filled in by the CSI snapshotter
+                description: creationTime is the timestamp when the point-in-time
-                    sidecar with the "creation_time" value returned from CSI "CreateSnapshot"
+                  snapshot is taken by the underlying storage system. In dynamic snapshot
-                    gRPC call. For a pre-existing snapshot, this field will be filled
+                  creation case, this field will be filled in by the CSI snapshotter
-                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                  sidecar with the "creation_time" value returned from CSI "CreateSnapshot"
-                    gRPC call if the driver supports it. If not specified, it indicates
+                  gRPC call. For a pre-existing snapshot, this field will be filled
-                    the creation time is unknown. The format of this field is a Unix
+                  with the "creation_time" value returned from the CSI "ListSnapshots"
-                    nanoseconds time encoded as an int64. On Unix, the command `date
+                  gRPC call if the driver supports it. If not specified, it indicates
-                    +%s%N` returns the current time in nanoseconds since 1970-01-01
+                  the creation time is unknown. The format of this field is a Unix
-                    00:00:00 UTC.
+                  nanoseconds time encoded as an int64. On Unix, the command `date
-                  format: int64
+                  +%s%N` returns the current time in nanoseconds since 1970-01-01
-                  type: integer
+                  00:00:00 UTC.
-                error:
+                format: int64
-                  description: error is the last observed error during snapshot creation,
+                type: integer
-                    if any. Upon success after retry, this error field will be cleared.
+              error:
-                  properties:
+                description: error is the last observed error during snapshot creation,
-                    message:
+                  if any. Upon success after retry, this error field will be cleared.
-                      description: 'message is a string detailing the encountered error
+                properties:
-                        during snapshot creation if specified. NOTE: message may be
+                  message:
-                        logged, and it should not contain sensitive information.'
+                    description: 'message is a string detailing the encountered error
-                      type: string
+                      during snapshot creation if specified. NOTE: message may be
-                    time:
+                      logged, and it should not contain sensitive information.'
-                      description: time is the timestamp when the error was encountered.
+                    type: string
-                      format: date-time
+                  time:
-                      type: string
+                    description: time is the timestamp when the error was encountered.
-                  type: object
+                    format: date-time
-                readyToUse:
+                    type: string
-                  description: readyToUse indicates if a snapshot is ready to be used
+                type: object
-                    to restore a volume. In dynamic snapshot creation case, this field
+              readyToUse:
-                    will be filled in by the CSI snapshotter sidecar with the "ready_to_use"
+                description: readyToUse indicates if a snapshot is ready to be used
-                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                  to restore a volume. In dynamic snapshot creation case, this field
-                    snapshot, this field will be filled with the "ready_to_use" value
+                  will be filled in by the CSI snapshotter sidecar with the "ready_to_use"
-                    returned from the CSI "ListSnapshots" gRPC call if the driver supports
+                  value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
-                    it, otherwise, this field will be set to "True". If not specified,
+                  snapshot, this field will be filled with the "ready_to_use" value
-                    it means the readiness of a snapshot is unknown.
+                  returned from the CSI "ListSnapshots" gRPC call if the driver supports
-                  type: boolean
+                  it, otherwise, this field will be set to "True". If not specified,
-                restoreSize:
+                  it means the readiness of a snapshot is unknown.
-                  description: restoreSize represents the complete size of the snapshot
+                type: boolean
-                    in bytes. In dynamic snapshot creation case, this field will be
+              restoreSize:
-                    filled in by the CSI snapshotter sidecar with the "size_bytes" value
+                description: restoreSize represents the complete size of the snapshot
-                    returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                  in bytes. In dynamic snapshot creation case, this field will be
-                    snapshot, this field will be filled with the "size_bytes" value
+                  filled in by the CSI snapshotter sidecar with the "size_bytes" value
-                    returned from the CSI "ListSnapshots" gRPC call if the driver supports
+                  returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
-                    it. When restoring a volume from this snapshot, the size of the
+                  snapshot, this field will be filled with the "size_bytes" value
-                    volume MUST NOT be smaller than the restoreSize if it is specified,
+                  returned from the CSI "ListSnapshots" gRPC call if the driver supports
-                    otherwise the restoration will fail. If not specified, it indicates
+                  it. When restoring a volume from this snapshot, the size of the
-                    that the size is unknown.
+                  volume MUST NOT be smaller than the restoreSize if it is specified,
-                  format: int64
+                  otherwise the restoration will fail. If not specified, it indicates
-                  minimum: 0
+                  that the size is unknown.
-                  type: integer
+                format: int64
-                snapshotHandle:
+                minimum: 0
-                  description: snapshotHandle is the CSI "snapshot_id" of a snapshot
+                type: integer
-                    on the underlying storage system. If not specified, it indicates
+              snapshotHandle:
-                    that dynamic snapshot creation has either failed or it is still
+                description: snapshotHandle is the CSI "snapshot_id" of a snapshot
-                    in progress.
+                  on the underlying storage system. If not specified, it indicates
-                  type: string
+                  that dynamic snapshot creation has either failed or it is still
-                volumeGroupSnapshotContentName:
+                  in progress.
-                  description: VolumeGroupSnapshotContentName is the name of the VolumeGroupSnapshotContent
+                type: string
-                    of which this VolumeSnapshotContent is a part of.
+              volumeGroupSnapshotHandle:
-                  type: string
+                description: VolumeGroupSnapshotHandle is the CSI "group_snapshot_id"
-              type: object
+                  of a group snapshot on the underlying storage system.
-          required:
+                type: string
-            - spec
+            type: object
-          type: object
+        required:
-      served: true
+        - spec
-      storage: true
+        type: object
-      subresources:
+    served: true
-        status: { }
+    storage: true
-    - additionalPrinterColumns:
+    subresources:
-        - description: Indicates if the snapshot is ready to be used to restore a volume.
+      status: {}
-          jsonPath: .status.readyToUse
+  - additionalPrinterColumns:
-          name: ReadyToUse
+    - description: Indicates if the snapshot is ready to be used to restore a volume.
-          type: boolean
+      jsonPath: .status.readyToUse
-        - description: Represents the complete size of the snapshot in bytes
+      name: ReadyToUse
-          jsonPath: .status.restoreSize
+      type: boolean
-          name: RestoreSize
+    - description: Represents the complete size of the snapshot in bytes
-          type: integer
+      jsonPath: .status.restoreSize
-        - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted.
+      name: RestoreSize
-          jsonPath: .spec.deletionPolicy
+      type: integer
-          name: DeletionPolicy
+    - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted.
-          type: string
+      jsonPath: .spec.deletionPolicy
-        - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system.
+      name: DeletionPolicy
-          jsonPath: .spec.driver
+      type: string
-          name: Driver
+    - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system.
-          type: string
+      jsonPath: .spec.driver
-        - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+      name: Driver
-          jsonPath: .spec.volumeSnapshotClassName
+      type: string
-          name: VolumeSnapshotClass
+    - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
-          type: string
+      jsonPath: .spec.volumeSnapshotClassName
-        - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.
+      name: VolumeSnapshotClass
-          jsonPath: .spec.volumeSnapshotRef.name
+      type: string
-          name: VolumeSnapshot
+    - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.
-          type: string
+      jsonPath: .spec.volumeSnapshotRef.name
-        - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.
+      name: VolumeSnapshot
-          jsonPath: .spec.volumeSnapshotRef.namespace
+      type: string
-          name: VolumeSnapshotNamespace
+    - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.
-          type: string
+      jsonPath: .spec.volumeSnapshotRef.namespace
-        - jsonPath: .metadata.creationTimestamp
+      name: VolumeSnapshotNamespace
-          name: Age
+      type: string
-          type: date
+    - jsonPath: .metadata.creationTimestamp
-      name: v1beta1
+      name: Age
-      # This indicates the v1beta1 version of the custom resource is deprecated.
+      type: date
-      # API requests to this version receive a warning in the server response.
+    name: v1beta1
-      deprecated: true
+    # This indicates the v1beta1 version of the custom resource is deprecated.
-      # This overrides the default warning returned to clients making v1beta1 API requests.
+    # API requests to this version receive a warning in the server response.
-      deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent"
+    deprecated: true
-      schema:
+    # This overrides the default warning returned to clients making v1beta1 API requests.
-        openAPIV3Schema:
+    deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent"
-          description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system
+    schema:
-          properties:
+      openAPIV3Schema:
-            apiVersion:
+        description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system
-              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+        properties:
-              type: string
+          apiVersion:
-            kind:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
-              type: string
+          kind:
-            spec:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-              description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required.
+            type: string
-              properties:
+          spec:
-                deletionPolicy:
+            description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required.
-                  description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the  VolumeSnapshotContent object. Required.
+            properties:
-                  enum:
+              deletionPolicy:
-                    - Delete
+                description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the  VolumeSnapshotContent object. Required.
-                    - Retain
+                enum:
-                  type: string
+                - Delete
-                driver:
+                - Retain
-                  description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required.
+                type: string
-                  type: string
+              driver:
-                source:
+                description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required.
-                  description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required.
+                type: string
-                  properties:
+              source:
-                    snapshotHandle:
+                description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required.
-                      description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable.
+                properties:
-                      type: string
+                  snapshotHandle:
-                    volumeHandle:
+                    description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable.
-                      description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable.
+                    type: string
-                      type: string
+                  volumeHandle:
-                  type: object
+                    description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable.
-                volumeSnapshotClassName:
+                    type: string
-                  description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation.
+                type: object
-                  type: string
+              volumeSnapshotClassName:
-                volumeSnapshotRef:
+                description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation.
-                  description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required.
+                type: string
-                  properties:
+              volumeSnapshotRef:
-                    apiVersion:
+                description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required.
-                      description: API version of the referent.
+                properties:
-                      type: string
+                  apiVersion:
-                    fieldPath:
+                    description: API version of the referent.
-                      description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.'
+                    type: string
-                      type: string
+                  fieldPath:
-                    kind:
+                    description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.'
-                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                    type: string
-                      type: string
+                  kind:
-                    name:
+                    description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                    type: string
-                      type: string
+                  name:
-                    namespace:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
-                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                    type: string
-                      type: string
+                  namespace:
-                    resourceVersion:
+                    description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
-                      description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                    type: string
-                      type: string
+                  resourceVersion:
-                    uid:
+                    description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
-                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                    type: string
-                      type: string
+                  uid:
-                  type: object
+                    description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
-              required:
+                    type: string
-                - deletionPolicy
+                type: object
-                - driver
+            required:
-                - source
+            - deletionPolicy
-                - volumeSnapshotRef
+            - driver
-              type: object
+            - source
-            status:
+            - volumeSnapshotRef
-              description: status represents the current information of a snapshot.
+            type: object
-              properties:
+          status:
-                creationTime:
+            description: status represents the current information of a snapshot.
-                  description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC.
+            properties:
-                  format: int64
+              creationTime:
-                  type: integer
+                description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC.
-                error:
+                format: int64
-                  description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared.
+                type: integer
-                  properties:
+              error:
-                    message:
+                description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared.
-                      description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'
+                properties:
-                      type: string
+                  message:
-                    time:
+                    description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'
-                      description: time is the timestamp when the error was encountered.
+                    type: string
-                      format: date-time
+                  time:
-                      type: string
+                    description: time is the timestamp when the error was encountered.
-                  type: object
+                    format: date-time
-                readyToUse:
+                    type: string
-                  description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown.
+                type: object
-                  type: boolean
+              readyToUse:
-                restoreSize:
+                description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown.
-                  description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.
+                type: boolean
-                  format: int64
+              restoreSize:
-                  minimum: 0
+                description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.
-                  type: integer
+                format: int64
-                snapshotHandle:
+                minimum: 0
-                  description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress.
+                type: integer
-                  type: string
+              snapshotHandle:
-              type: object
+                description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress.
-          required:
+                type: string
-            - spec
+            type: object
-          type: object
+        required:
-      served: false
+        - spec
-      storage: false
+        type: object
-      subresources:
+    served: false
-        status: { }
+    storage: false
+    subresources:
+      status: {}
 status:
   acceptedNames:
     kind: ""
     plural: ""
-  conditions: [ ]
+  conditions: []
-  storedVersions: [ ]
+  storedVersions: []
 {{- end }}

charts/kubezero-storage/templates/snapshot-controller/volumesnapshots-crd.yaml

@@ -1,11 +1,11 @@
-{{- if .Values.crd.volumeSnapshot }}
+{{- if .Values.snapshotController.enabled }}
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.3
     api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814"
+    controller-gen.kubebuilder.io/version: v0.12.0
   creationTimestamp: null
   name: volumesnapshots.snapshot.storage.k8s.io
 spec:
@@ -15,96 +15,98 @@ spec:
     listKind: VolumeSnapshotList
     plural: volumesnapshots
     shortNames:
-      - vs
+    - vs
     singular: volumesnapshot
   scope: Namespaced
   versions:
-    - additionalPrinterColumns:
+  - additionalPrinterColumns:
-        - description: Indicates if the snapshot is ready to be used to restore a volume.
+    - description: Indicates if the snapshot is ready to be used to restore a volume.
-          jsonPath: .status.readyToUse
+      jsonPath: .status.readyToUse
-          name: ReadyToUse
+      name: ReadyToUse
-          type: boolean
+      type: boolean
-        - description: If a new snapshot needs to be created, this contains the name of
+    - description: If a new snapshot needs to be created, this contains the name of
-            the source PVC from which this snapshot was (or will be) created.
+        the source PVC from which this snapshot was (or will be) created.
-          jsonPath: .spec.source.persistentVolumeClaimName
+      jsonPath: .spec.source.persistentVolumeClaimName
-          name: SourcePVC
+      name: SourcePVC
-          type: string
+      type: string
-        - description: If a snapshot already exists, this contains the name of the existing
+    - description: If a snapshot already exists, this contains the name of the existing
-            VolumeSnapshotContent object representing the existing snapshot.
+        VolumeSnapshotContent object representing the existing snapshot.
-          jsonPath: .spec.source.volumeSnapshotContentName
+      jsonPath: .spec.source.volumeSnapshotContentName
-          name: SourceSnapshotContent
+      name: SourceSnapshotContent
-          type: string
+      type: string
-        - description: Represents the minimum size of volume required to rehydrate from
+    - description: Represents the minimum size of volume required to rehydrate from
-            this snapshot.
+        this snapshot.
-          jsonPath: .status.restoreSize
+      jsonPath: .status.restoreSize
-          name: RestoreSize
+      name: RestoreSize
-          type: string
+      type: string
-        - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+    - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
-          jsonPath: .spec.volumeSnapshotClassName
+      jsonPath: .spec.volumeSnapshotClassName
-          name: SnapshotClass
+      name: SnapshotClass
-          type: string
+      type: string
-        - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot
+    - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot
-            object intends to bind to. Please note that verification of binding actually
+        object intends to bind to. Please note that verification of binding actually
-            requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure
+        requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure
-            both are pointing at each other. Binding MUST be verified prior to usage of
+        both are pointing at each other. Binding MUST be verified prior to usage of
-            this object.
+        this object.
-          jsonPath: .status.boundVolumeSnapshotContentName
+      jsonPath: .status.boundVolumeSnapshotContentName
-          name: SnapshotContent
+      name: SnapshotContent
-          type: string
+      type: string
-        - description: Timestamp when the point-in-time snapshot was taken by the underlying
+    - description: Timestamp when the point-in-time snapshot was taken by the underlying
-            storage system.
+        storage system.
-          jsonPath: .status.creationTime
+      jsonPath: .status.creationTime
-          name: CreationTime
+      name: CreationTime
-          type: date
+      type: date
-        - jsonPath: .metadata.creationTimestamp
+    - jsonPath: .metadata.creationTimestamp
-          name: Age
+      name: Age
-          type: date
+      type: date
-      name: v1
+    name: v1
-      schema:
+    schema:
-        openAPIV3Schema:
+      openAPIV3Schema:
-          description: VolumeSnapshot is a user's request for either creating a point-in-time
+        description: VolumeSnapshot is a user's request for either creating a point-in-time
-            snapshot of a persistent volume, or binding to a pre-existing snapshot.
+          snapshot of a persistent volume, or binding to a pre-existing snapshot.
-          properties:
+        properties:
-            apiVersion:
+          apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation
+            description: 'APIVersion defines the versioned schema of this representation
               of an object. Servers should convert recognized schemas to the latest
               internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-              type: string
+            type: string
-            kind:
+          kind:
-              description: 'Kind is a string value representing the REST resource this
+            description: 'Kind is a string value representing the REST resource this
               object represents. Servers may infer this from the endpoint the client
               submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-              type: string
+            type: string
-            spec:
+          metadata:
-              description: 'spec defines the desired characteristics of a snapshot requested
+            type: object
+          spec:
+            description: 'spec defines the desired characteristics of a snapshot requested
               by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
               Required.'
-              properties:
+            properties:
-                source:
+              source:
-                  description: source specifies where a snapshot will be created from.
+                description: source specifies where a snapshot will be created from.
-                    This field is immutable after creation. Required.
+                  This field is immutable after creation. Required.
-                  properties:
+                properties:
-                    persistentVolumeClaimName:
+                  persistentVolumeClaimName:
-                      description: persistentVolumeClaimName specifies the name of the
+                    description: persistentVolumeClaimName specifies the name of the
-                        PersistentVolumeClaim object representing the volume from which
+                      PersistentVolumeClaim object representing the volume from which
-                        a snapshot should be created. This PVC is assumed to be in the
+                      a snapshot should be created. This PVC is assumed to be in the
-                        same namespace as the VolumeSnapshot object. This field should
+                      same namespace as the VolumeSnapshot object. This field should
-                        be set if the snapshot does not exists, and needs to be created.
+                      be set if the snapshot does not exists, and needs to be created.
-                        This field is immutable.
+                      This field is immutable.
-                      type: string
+                    type: string
-                    volumeSnapshotContentName:
+                  volumeSnapshotContentName:
-                      description: volumeSnapshotContentName specifies the name of a
+                    description: volumeSnapshotContentName specifies the name of a
-                        pre-existing VolumeSnapshotContent object representing an existing
+                      pre-existing VolumeSnapshotContent object representing an existing
-                        volume snapshot. This field should be set if the snapshot already
+                      volume snapshot. This field should be set if the snapshot already
-                        exists and only needs a representation in Kubernetes. This field
+                      exists and only needs a representation in Kubernetes. This field
-                        is immutable.
+                      is immutable.
-                      type: string
+                    type: string
-                  type: object
+                type: object
-                  oneOf:
+                oneOf:
-                    - required: ["persistentVolumeClaimName"]
+                - required: ["persistentVolumeClaimName"]
-                    - required: ["volumeSnapshotContentName"]
+                - required: ["volumeSnapshotContentName"]
-                volumeSnapshotClassName:
+              volumeSnapshotClassName:
-                  description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass
+                description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass
                   requested by the VolumeSnapshot. VolumeSnapshotClassName may be
                   left nil to indicate that the default SnapshotClass should be used.
                   A given cluster may have multiple default Volume SnapshotClasses:
@@ -115,18 +117,18 @@ spec:
                   exist for a given CSI Driver and more than one have been marked
                   as default, CreateSnapshot will fail and generate an event. Empty
                   string is not allowed for this field.'
-                  type: string
+                type: string
-              required:
+            required:
-                - source
+            - source
-              type: object
+            type: object
-            status:
+          status:
-              description: status represents the current information of a snapshot.
+            description: status represents the current information of a snapshot.
-                Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent
+              Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent
-                objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent
+              objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent
-                point at each other) before using this object.
+              point at each other) before using this object.
-              properties:
+            properties:
-                boundVolumeSnapshotContentName:
+              boundVolumeSnapshotContentName:
-                  description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent
+                description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent
                   object to which this VolumeSnapshot object intends to bind to. If
                   not specified, it indicates that the VolumeSnapshot object has not
                   been successfully bound to a VolumeSnapshotContent object yet. NOTE:
@@ -134,177 +136,177 @@ spec:
                   between VolumeSnapshot and VolumeSnapshotContent objects is successful
                   (by validating that both VolumeSnapshot and VolumeSnapshotContent
                   point at each other) before using this object.'
-                  type: string
+                type: string
-                creationTime:
+              creationTime:
-                  description: creationTime is the timestamp when the point-in-time
+                description: creationTime is the timestamp when the point-in-time
-                    snapshot is taken by the underlying storage system. In dynamic snapshot
+                  snapshot is taken by the underlying storage system. In dynamic snapshot
-                    creation case, this field will be filled in by the snapshot controller
+                  creation case, this field will be filled in by the snapshot controller
-                    with the "creation_time" value returned from CSI "CreateSnapshot"
+                  with the "creation_time" value returned from CSI "CreateSnapshot"
-                    gRPC call. For a pre-existing snapshot, this field will be filled
+                  gRPC call. For a pre-existing snapshot, this field will be filled
-                    with the "creation_time" value returned from the CSI "ListSnapshots"
+                  with the "creation_time" value returned from the CSI "ListSnapshots"
-                    gRPC call if the driver supports it. If not specified, it may indicate
+                  gRPC call if the driver supports it. If not specified, it may indicate
-                    that the creation time of the snapshot is unknown.
+                  that the creation time of the snapshot is unknown.
-                  format: date-time
+                format: date-time
-                  type: string
+                type: string
-                error:
+              error:
-                  description: error is the last observed error during snapshot creation,
+                description: error is the last observed error during snapshot creation,
-                    if any. This field could be helpful to upper level controllers(i.e.,
+                  if any. This field could be helpful to upper level controllers(i.e.,
-                    application controller) to decide whether they should continue on
+                  application controller) to decide whether they should continue on
-                    waiting for the snapshot to be created based on the type of error
+                  waiting for the snapshot to be created based on the type of error
-                    reported. The snapshot controller will keep retrying when an error
+                  reported. The snapshot controller will keep retrying when an error
-                    occurs during the snapshot creation. Upon success, this error field
+                  occurs during the snapshot creation. Upon success, this error field
-                    will be cleared.
+                  will be cleared.
-                  properties:
+                properties:
-                    message:
+                  message:
-                      description: 'message is a string detailing the encountered error
+                    description: 'message is a string detailing the encountered error
                       during snapshot creation if specified. NOTE: message may be
                       logged, and it should not contain sensitive information.'
-                      type: string
+                    type: string
-                    time:
+                  time:
-                      description: time is the timestamp when the error was encountered.
+                    description: time is the timestamp when the error was encountered.
-                      format: date-time
+                    format: date-time
-                      type: string
+                    type: string
-                  type: object
+                type: object
-                readyToUse:
+              readyToUse:
-                  description: readyToUse indicates if the snapshot is ready to be used
+                description: readyToUse indicates if the snapshot is ready to be used
-                    to restore a volume. In dynamic snapshot creation case, this field
+                  to restore a volume. In dynamic snapshot creation case, this field
-                    will be filled in by the snapshot controller with the "ready_to_use"
+                  will be filled in by the snapshot controller with the "ready_to_use"
-                    value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
+                  value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing
-                    snapshot, this field will be filled with the "ready_to_use" value
+                  snapshot, this field will be filled with the "ready_to_use" value
-                    returned from the CSI "ListSnapshots" gRPC call if the driver supports
+                  returned from the CSI "ListSnapshots" gRPC call if the driver supports
-                    it, otherwise, this field will be set to "True". If not specified,
+                  it, otherwise, this field will be set to "True". If not specified,
-                    it means the readiness of a snapshot is unknown.
+                  it means the readiness of a snapshot is unknown.
-                  type: boolean
+                type: boolean
-                restoreSize:
+              restoreSize:
-                  type: string
+                type: string
-                  description: restoreSize represents the minimum size of volume required
+                description: restoreSize represents the minimum size of volume required
-                    to create a volume from this snapshot. In dynamic snapshot creation
+                  to create a volume from this snapshot. In dynamic snapshot creation
-                    case, this field will be filled in by the snapshot controller with
+                  case, this field will be filled in by the snapshot controller with
-                    the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call.
+                  the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call.
-                    For a pre-existing snapshot, this field will be filled with the
+                  For a pre-existing snapshot, this field will be filled with the
-                    "size_bytes" value returned from the CSI "ListSnapshots" gRPC call
+                  "size_bytes" value returned from the CSI "ListSnapshots" gRPC call
-                    if the driver supports it. When restoring a volume from this snapshot,
+                  if the driver supports it. When restoring a volume from this snapshot,
-                    the size of the volume MUST NOT be smaller than the restoreSize
+                  the size of the volume MUST NOT be smaller than the restoreSize
-                    if it is specified, otherwise the restoration will fail. If not
+                  if it is specified, otherwise the restoration will fail. If not
-                    specified, it indicates that the size is unknown.
+                  specified, it indicates that the size is unknown.
-                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
-                  x-kubernetes-int-or-string: true
+                x-kubernetes-int-or-string: true
-                volumeGroupSnapshotName:
+              volumeGroupSnapshotName:
-                  description: VolumeGroupSnapshotName is the name of the VolumeGroupSnapshot
+                description: VolumeGroupSnapshotName is the name of the VolumeGroupSnapshot
-                    of which this VolumeSnapshot is a part of.
+                  of which this VolumeSnapshot is a part of.
-                  type: string
+                type: string
-              type: object
+            type: object
-          required:
+        required:
-            - spec
+        - spec
-          type: object
+        type: object
-      served: true
+    served: true
-      storage: true
+    storage: true
-      subresources:
+    subresources:
-        status: {}
+      status: {}
-    - additionalPrinterColumns:
+  - additionalPrinterColumns:
-        - description: Indicates if the snapshot is ready to be used to restore a volume.
+    - description: Indicates if the snapshot is ready to be used to restore a volume.
-          jsonPath: .status.readyToUse
+      jsonPath: .status.readyToUse
-          name: ReadyToUse
+      name: ReadyToUse
-          type: boolean
+      type: boolean
-        - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created.
+    - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created.
-          jsonPath: .spec.source.persistentVolumeClaimName
+      jsonPath: .spec.source.persistentVolumeClaimName
-          name: SourcePVC
+      name: SourcePVC
-          type: string
+      type: string
-        - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot.
+    - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot.
-          jsonPath: .spec.source.volumeSnapshotContentName
+      jsonPath: .spec.source.volumeSnapshotContentName
-          name: SourceSnapshotContent
+      name: SourceSnapshotContent
-          type: string
+      type: string
-        - description: Represents the minimum size of volume required to rehydrate from this snapshot.
+    - description: Represents the minimum size of volume required to rehydrate from this snapshot.
-          jsonPath: .status.restoreSize
+      jsonPath: .status.restoreSize
-          name: RestoreSize
+      name: RestoreSize
-          type: string
+      type: string
-        - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+    - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
-          jsonPath: .spec.volumeSnapshotClassName
+      jsonPath: .spec.volumeSnapshotClassName
-          name: SnapshotClass
+      name: SnapshotClass
-          type: string
+      type: string
-        - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object.
+    - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object.
-          jsonPath: .status.boundVolumeSnapshotContentName
+      jsonPath: .status.boundVolumeSnapshotContentName
-          name: SnapshotContent
+      name: SnapshotContent
-          type: string
+      type: string
-        - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system.
+    - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system.
-          jsonPath: .status.creationTime
+      jsonPath: .status.creationTime
-          name: CreationTime
+      name: CreationTime
-          type: date
+      type: date
-        - jsonPath: .metadata.creationTimestamp
+    - jsonPath: .metadata.creationTimestamp
-          name: Age
+      name: Age
-          type: date
+      type: date
-      name: v1beta1
+    name: v1beta1
-      # This indicates the v1beta1 version of the custom resource is deprecated.
+    # This indicates the v1beta1 version of the custom resource is deprecated.
-      # API requests to this version receive a warning in the server response.
+    # API requests to this version receive a warning in the server response.
-      deprecated: true
+    deprecated: true
-      # This overrides the default warning returned to clients making v1beta1 API requests.
+    # This overrides the default warning returned to clients making v1beta1 API requests.
-      deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot"
+    deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot"
-      schema:
+    schema:
-        openAPIV3Schema:
+      openAPIV3Schema:
-          description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot.
+        description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot.
-          properties:
+        properties:
-            apiVersion:
+          apiVersion:
-              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-              type: string
+            type: string
-            kind:
+          kind:
-              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-              type: string
+            type: string
-            spec:
+          spec:
-              description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.'
+            description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.'
-              properties:
+            properties:
-                source:
+              source:
-                  description: source specifies where a snapshot will be created from. This field is immutable after creation. Required.
+                description: source specifies where a snapshot will be created from. This field is immutable after creation. Required.
-                  properties:
+                properties:
-                    persistentVolumeClaimName:
+                  persistentVolumeClaimName:
-                      description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable.
+                    description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable.
-                      type: string
+                    type: string
-                    volumeSnapshotContentName:
+                  volumeSnapshotContentName:
-                      description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable.
+                    description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable.
-                      type: string
+                    type: string
-                  type: object
+                type: object
-                volumeSnapshotClassName:
+              volumeSnapshotClassName:
-                  description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.'
+                description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.'
-                  type: string
+                type: string
-              required:
+            required:
-                - source
+            - source
-              type: object
+            type: object
-            status:
+          status:
-              description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.
+            description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.
-              properties:
+            properties:
-                boundVolumeSnapshotContentName:
+              boundVolumeSnapshotContentName:
-                  description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.'
+                description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.'
-                  type: string
+                type: string
-                creationTime:
+              creationTime:
-                  description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown.
+                description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown.
-                  format: date-time
+                format: date-time
-                  type: string
+                type: string
-                error:
+              error:
-                  description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared.
+                description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared.
-                  properties:
+                properties:
-                    message:
+                  message:
-                      description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'
+                    description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'
-                      type: string
+                    type: string
-                    time:
+                  time:
-                      description: time is the timestamp when the error was encountered.
+                    description: time is the timestamp when the error was encountered.
-                      format: date-time
+                    format: date-time
-                      type: string
+                    type: string
-                  type: object
+                type: object
-                readyToUse:
+              readyToUse:
-                  description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown.
+                description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown.
-                  type: boolean
+                type: boolean
-                restoreSize:
+              restoreSize:
-                  type: string
+                type: string
-                  description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.
+                description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.
-                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
-                  x-kubernetes-int-or-string: true
+                x-kubernetes-int-or-string: true
-              type: object
+            type: object
-          required:
+        required:
-            - spec
+        - spec
-          type: object
+        type: object
-      served: false
+    served: false
-      storage: false
+    storage: false
-      subresources:
+    subresources:
-        status: {}
+      status: {}
 status:
   acceptedNames:
     kind: ""

charts/kubezero-storage/update.sh

@@ -6,16 +6,25 @@ set -ex
 #login_ecr_public
 update_helm
 
-patch_chart gemini
-
 patch_chart aws-ebs-csi-driver
-rm -rf charts/aws-ebs-csi-driver/templates/tests
 
 patch_chart aws-efs-csi-driver
 
 patch_chart lvm-localpv
-# move snapshotclasses/content from lvm-localpv to toplevel
+
-mv charts/lvm-localpv/templates/*crd.yaml templates/snapshot-controller
+patch_chart gemini
+
+# snapshotter
+# https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml
+# https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml
+
+for crd in volumesnapshotclasses volumesnapshotcontents volumesnapshots; do
+  _f="templates/snapshot-controller/${crd}-crd.yaml"
+  echo "{{- if .Values.snapshotController.enabled }}" > $_f
+  curl -L -s https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/client/config/crd/snapshot.storage.k8s.io_${crd}.yaml >> $_f
+  echo "{{- end }}" >> $_f
+done
+
 
 # k8up - CRDs
 VERSION=$(yq eval '.dependencies[] | select(.name=="k8up") | .version' Chart.yaml)

charts/kubezero-storage/values.yaml

@@ -1,12 +1,9 @@
-crd:
-  volumeSnapshot: true
-
 snapshotController:
   enabled: false
 
   image:
     name: registry.k8s.io/sig-storage/snapshot-controller
-    tag: v6.3.0
+    tag: v7.0.1
 
   replicas: 1
   logLevel: 2
@@ -28,6 +25,11 @@ snapshotController:
 lvm-localpv:
   enabled: false
 
+  crds:
+    csi:
+      volumeSnapshots:
+        enabled: false
+
   lvmNode:
     logLevel: 2
     nodeSelector:
@@ -190,6 +192,8 @@ aws-ebs-csi-driver:
       type: gp3
       encrypted: "true"
 
+  helmTester:
+    enabled: false
 
 aws-efs-csi-driver:
   enabled: false

charts/kubezero/templates/network.yaml

@@ -37,12 +37,6 @@ metallb:
 
 
 {{- define "network-argo" }}
-  # Metallb
-  ignoreDifferences:
-  - group: apiextensions.k8s.io
-    kind: CustomResourceDefinition
-    jsonPointers:
-    - /spec/conversion/webhook/clientConfig/caBundle
 {{- end }}
 
 {{ include "kubezero-app.app" . }}

charts/kubezero/values.yaml

@@ -11,7 +11,7 @@ global:
 
 addons:
   enabled: true
-  targetRevision: 0.8.4
+  targetRevision: 0.8.5
   external-dns:
     enabled: false
   forseti:
@@ -30,7 +30,7 @@ addons:
 network:
   enabled: true
   retain: true
-  targetRevision: 0.4.6
+  targetRevision: 0.5.0
   cilium:
     cluster: {}
 
@@ -41,7 +41,7 @@ cert-manager:
 
 storage:
   enabled: false
-  targetRevision: 0.8.4
+  targetRevision: 0.8.5
   lvm-localpv:
     enabled: false
   aws-ebs-csi-driver: