diff --git a/charts/kubezero-storage/Chart.yaml b/charts/kubezero-storage/Chart.yaml index bea71dce..3a7a6b39 100644 --- a/charts/kubezero-storage/Chart.yaml +++ b/charts/kubezero-storage/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-storage description: KubeZero umbrella chart for all things storage incl. AWS EBS/EFS, openEBS-lvm, gemini type: application -version: 0.4.3 +version: 0.5.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -20,15 +20,15 @@ dependencies: version: ">= 0.1.4" repository: https://cdn.zero-downtime.net/charts/ - name: lvm-localpv - version: 0.8.5 + version: 0.8.6 condition: lvm-localpv.enabled - repository: https://openebs.github.io/lvm-localpv + # repository: https://openebs.github.io/lvm-localpv - name: gemini version: 0.0.8 condition: gemini.enabled # repository: https://charts.fairwinds.com/stable - name: aws-ebs-csi-driver - version: 2.5.1 + version: 2.6.2 condition: aws-ebs-csi-driver.enabled # repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: aws-efs-csi-driver diff --git a/charts/kubezero-storage/README.md b/charts/kubezero-storage/README.md index 922ccbd7..f79ece7d 100644 --- a/charts/kubezero-storage/README.md +++ b/charts/kubezero-storage/README.md @@ -1,6 +1,6 @@ # kubezero-storage -![Version: 0.4.3](https://img.shields.io/badge/Version-0.4.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.5.2](https://img.shields.io/badge/Version-0.5.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero umbrella chart for all things storage incl. AWS EBS/EFS, openEBS-lvm, gemini @@ -18,11 +18,11 @@ Kubernetes: `>= 1.20.0` | Repository | Name | Version | |------------|------|---------| -| | aws-ebs-csi-driver | 2.5.0 | +| | aws-ebs-csi-driver | 2.6.2 | | | aws-efs-csi-driver | 2.2.0 | | | gemini | 0.0.8 | +| | lvm-localpv | 0.8.6 | | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.4 | -| https://openebs.github.io/lvm-localpv | lvm-localpv | 0.8.5 | ## Values @@ -92,6 +92,15 @@ Kubernetes: `>= 1.20.0` | lvm-localpv.lvmNode.tolerations[0].operator | string | `"Exists"` | | | lvm-localpv.storageClass.default | bool | `false` | | | lvm-localpv.storageClass.vgpattern | string | `""` | | +| snapshotController.enabled | bool | `true` | | +| snapshotController.logLevel | int | `2` | | +| snapshotController.replicas | int | `1` | | +| snapshotController.resources.limits.cpu | string | `"100m"` | | +| snapshotController.resources.limits.memory | string | `"64Mi"` | | +| snapshotController.resources.requests.cpu | string | `"20m"` | | +| snapshotController.resources.requests.memory | string | `"16Mi"` | | ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0) +# Snapshotter +- https://kubernetes-csi.github.io/docs/snapshot-controller.html#deployment + +## Resources diff --git a/charts/kubezero-storage/README.md.gotmpl b/charts/kubezero-storage/README.md.gotmpl new file mode 100644 index 00000000..f62a1f79 --- /dev/null +++ b/charts/kubezero-storage/README.md.gotmpl @@ -0,0 +1,21 @@ +{{ template "chart.header" . }} +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} + +# Snapshotter +- https://kubernetes-csi.github.io/docs/snapshot-controller.html#deployment + +## Resources diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md index f7e66a3c..6127fb83 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md @@ -1,10 +1,17 @@ # Helm chart +## v2.6.2 + +* Update csi-resizer version to v1.1.0 + +## v2.6.1 + +* Add securityContext support for controller Deployment + ## v2.5.0 * Bump app/driver version to `v1.5.0` - ## v2.4.1 * Replace deprecated arg `--extra-volume-tags` by `--extra-tags` diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml index 22128794..9bdd84b2 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml @@ -19,4 +19,4 @@ maintainers: name: aws-ebs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver -version: 2.5.1 +version: 2.6.2 diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml index 3777eb91..ef84c12b 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -57,6 +57,10 @@ spec: topologySpreadConstraints: {{- $constraints | toYaml | nindent 8 }} {{- end }} + {{- with .Values.controller.securityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: ebs-plugin image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (.Values.image.tag | toString)) }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml index 71d6f6eb..c5d3c2ac 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml @@ -9,7 +9,8 @@ image: pullPolicy: IfNotPresent # -- Custom labels to add into metadata -customLabels: {} +customLabels: + {} # k8s-app: aws-ebs-csi-driver sidecars: @@ -48,7 +49,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: k8s.gcr.io/sig-storage/csi-resizer - tag: "v1.0.0" + tag: "v1.1.0" logLevel: 2 resources: {} nodeDriverRegistrar: @@ -117,8 +118,8 @@ controller: # cpu: 100m # memory: 128Mi serviceAccount: - create: true # A service account will be created for you if set to true. Set to false if you want to use your own. - name: ebs-csi-controller-sa # Name of the service-account to be used/created. + create: true # A service account will be created for you if set to true. Set to false if you want to use your own. + name: ebs-csi-controller-sa # Name of the service-account to be used/created. annotations: {} tolerations: [] # TSCs without the label selector stanza @@ -133,6 +134,9 @@ controller: # topologyKey: kubernetes.io/hostname # whenUnsatisfiable: ScheduleAnyway topologySpreadConstraints: [] + securityContext: {} + # AWS EKS /var/run/secrets/eks.amazonaws.com/serviceaccount/token FS group is nogroup (65534) - required for Kubernetes 1.18.x and below + # fsGroup: 65534 node: env: [] diff --git a/charts/kubezero-storage/charts/lvm-localpv/.helmignore b/charts/kubezero-storage/charts/lvm-localpv/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/kubezero-storage/charts/lvm-localpv/Chart.yaml b/charts/kubezero-storage/charts/lvm-localpv/Chart.yaml new file mode 100644 index 00000000..1ff285f7 --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +appVersion: 0.8.3 +description: CSI Driver for dynamic provisioning of LVM Persistent Local Volumes. +home: http://www.openebs.io/ +icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/openebs/icon/color/openebs-icon-color.png +keywords: +- cloud-native-storage +- block-storage +- filesystem +- LVM +- Local Persistent Volumes +- storage +maintainers: +- email: prateek.pandey@mayadata.io + name: prateekpandey14 +- email: pawan@mayadata.io + name: pawanpraka1 +- email: yashpal.c1995@gmail.com + name: iyashu +name: lvm-localpv +sources: +- https://github.com/openebs/lvm-localpv +version: 0.8.6 diff --git a/charts/kubezero-storage/charts/lvm-localpv/README.md b/charts/kubezero-storage/charts/lvm-localpv/README.md new file mode 100644 index 00000000..f792b1d6 --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/README.md @@ -0,0 +1,160 @@ + +# OpenEBS LocalPV Provisioner + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +![Chart Lint and Test](https://github.com/openebs/lvm-localpv/workflows/Chart%20Lint%20and%20Test/badge.svg) +![Release Charts](https://github.com/openebs/lvm-localpv/workflows/Release%20Charts/badge.svg?branch=develop) + +A Helm chart for openebs lvm localpv provisioner. This chart bootstraps OpenEBS LVM LocalPV provisioner deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| pawanpraka1 | pawan@mayadata.io | | +| prateekpandey14 | prateek.pandey@mayadata.io | | +| iyashu | yashpal.c1995@gmail.com | | + + +## Get Repo Info + +```console +helm repo add openebs-lvmlocalpv https://openebs.github.io/lvm-localpv +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +Please visit the [link](https://openebs.github.io/lvm-localpv/) for install instructions via helm3. + +```console +# Helm +$ helm install [RELEASE_NAME] openebs-lvmlocalpv/lvm-localpv --namespace [NAMESPACE] +``` + +
+ Click here if you're using MicroK8s. + + ```console + microk8s helm3 install [RELEASE_NAME] openebs-lvmlocalpv/lvm-localpv --namespace [NAMESPACE] --set-string lvmNode.kubeletDir="/var/snap/microk8s/common/var/lib/kubelet/" + ``` +
+ + +**Note:** If moving from the operator to helm +- Make sure the namespace provided in the helm install command is same as `LVM_NAMESPACE` (by default it is `openebs`) env in the controller statefulset. +- Before installing, clean up the stale statefulset and daemonset from `kube-system` namespace using the below commands +```sh +kubectl delete sts openebs-lvm-controller -n kube-system +kubectl delete ds openebs-lvm-node -n kube-system +``` + + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +# Helm +$ helm uninstall [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +# Helm +$ helm upgrade [RELEASE_NAME] [CHART] --install --namespace [NAMESPACE] +``` + +## Configuration + +The following table lists the configurable parameters of the OpenEBS LVM Localpv chart and their default values. + +```console +helm install openebs-lvmlocalpv openebs-lvmlocalpv/lvm-localpv --namespace openebs --create-namespace +``` +
+ Click here if you're using MicroK8s. + + If you are using MicroK8s, it is necessary to add the following flag: + + ```console + --set-string lvmNode.kubeletDir="/var/snap/microk8s/common/var/lib/kubelet/" + ``` +
+ +| Parameter| Description| Default| +| -| -| -| +| `imagePullSecrets`| Provides image pull secrect| `""`| +| `lvmPlugin.image.registry`| Registry for openebs-lvm-plugin image| `""`| +| `lvmPlugin.image.repository`| Image repository for openebs-lvm-plugin| `openebs/lvm-driver`| +| `lvmPlugin.image.pullPolicy`| Image pull policy for openebs-lvm-plugin| `IfNotPresent`| +| `lvmPlugin.image.tag`| Image tag for openebs-lvm-plugin| `0.8.3`| +| `lvmPlugin.metricsPort`| The TCP port number used for exposing lvm-metrics | `9500`| +| `lvmPlugin.allowedTopologies`| The comma seperated list of allowed node topologies | `kubernetes.io/hostname,`| +| `lvmNode.driverRegistrar.image.registry`| Registry for csi-node-driver-registrar image| `k8s.gcr.io/`| +| `lvmNode.driverRegistrar.image.repository`| Image repository for csi-node-driver-registrar| `sig-storage/csi-node-driver-registrar`| +| `lvmNode.driverRegistrar.image.pullPolicy`| Image pull policy for csi-node-driver-registrar| `IfNotPresent`| +| `lvmNode.driverRegistrar.image.tag`| Image tag for csi-node-driver-registrar| `v2.3.0`| +| `lvmNode.updateStrategy.type`| Update strategy for lvmnode daemonset | `RollingUpdate` | +| `lvmNode.kubeletDir`| Kubelet mount point for lvmnode daemonset| `"/var/lib/kubelet/"` | +| `lvmNode.annotations` | Annotations for lvmnode daemonset metadata| `""`| +| `lvmNode.podAnnotations`| Annotations for lvmnode daemonset's pods metadata | `""`| +| `lvmNode.resources`| Resource and request and limit for lvmnode daemonset containers | `""`| +| `lvmNode.labels`| Labels for lvmnode daemonset metadata | `""`| +| `lvmNode.podLabels`| Appends labels to the lvmnode daemonset pods| `""`| +| `lvmNode.nodeSelector`| Nodeselector for lvmnode daemonset pods| `""`| +| `lvmNode.tolerations` | lvmnode daemonset's pod toleration values | `""`| +| `lvmNode.securityContext` | Security context for lvmnode daemonset container | `""`| +| `lvmController.resizer.image.registry`| Registry for csi-resizer image| `k8s.gcr.io/`| +| `lvmController.resizer.image.repository`| Image repository for csi-resizer| `sig-storage/csi-resizer`| +| `lvmController.resizer.image.pullPolicy`| Image pull policy for csi-resizer| `IfNotPresent`| +| `lvmController.resizer.image.tag`| Image tag for csi-resizer| `v1.2.0`| +| `lvmController.snapshotter.image.registry`| Registry for csi-snapshotter image| `k8s.gcr.io/`| +| `lvmController.snapshotter.image.repository`| Image repository for csi-snapshotter| `sig-storage/csi-snapshotter`| +| `lvmController.snapshotter.image.pullPolicy`| Image pull policy for csi-snapshotter| `IfNotPresent`| +| `lvmController.snapshotter.image.tag`| Image tag for csi-snapshotter| `v4.0.0`| +| `lvmController.snapshotController.image.registry`| Registry for snapshot-controller image| `k8s.gcr.io/`| +| `lvmController.snapshotController.image.repository`| Image repository for snapshot-controller| `sig-storage/snapshot-controller`| +| `lvmController.snapshotController.image.pullPolicy`| Image pull policy for snapshot-controller| `IfNotPresent`| +| `lvmController.snapshotController.image.tag`| Image tag for snapshot-controller| `v4.0.0`| +| `lvmController.provisioner.image.registry`| Registry for csi-provisioner image| `k8s.gcr.io/`| +| `lvmController.provisioner.image.repository`| Image repository for csi-provisioner| `sig-storage/csi-provisioner`| +| `lvmController.provisioner.image.pullPolicy`| Image pull policy for csi-provisioner| `IfNotPresent`| +| `lvmController.provisioner.image.tag`| Image tag for csi-provisioner| `v2.3.0`| +| `lvmController.updateStrategy.type`| Update strategy for lvm localpv controller statefulset | `RollingUpdate` | +| `lvmController.annotations` | Annotations for lvm localpv controller statefulset metadata| `""`| +| `lvmController.podAnnotations`| Annotations for lvm localpv controller statefulset's pods metadata | `""`| +| `lvmController.resources`| Resource and request and limit for lvm localpv controller statefulset containers | `""`| +| `lvmController.labels`| Labels for lvm localpv controller statefulset metadata | `""`| +| `lvmController.podLabels`| Appends labels to the lvm localpv controller statefulset pods| `""`| +| `lvmController.nodeSelector`| Nodeselector for lvm localpv controller statefulset pods| `""`| +| `lvmController.tolerations` | lvm localpv controller statefulset's pod toleration values | `""`| +| `lvmController.securityContext` | Seurity context for lvm localpv controller statefulset container | `""`| +| `rbac.pspEnabled` | Enable PodSecurityPolicy | `false` | +| `serviceAccount.lvmNode.create` | Create a service account for lvmnode or not| `true`| +| `serviceAccount.lvmNode.name` | Name for the lvmnode service account| `openebs-lvm-node-sa`| +| `serviceAccount.lvmController.create` | Create a service account for lvm localpv controller or not| `true`| +| `serviceAccount.lvmController.name` | Name for the lvm localpv controller service account| `openebs-lvm-controller-sa`| +| `analytics.enabled` | Enable or Disable google analytics for the controller| `true`| + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, + +```bash +helm install -f values.yaml openebs/lvm-localpv +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) diff --git a/charts/kubezero-storage/charts/lvm-localpv/crds/lvmnode.yaml b/charts/kubezero-storage/charts/lvm-localpv/crds/lvmnode.yaml new file mode 100644 index 00000000..cf321392 --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/crds/lvmnode.yaml @@ -0,0 +1,177 @@ + + +############################################## +########### ############ +########### LVMNode CRD ############ +########### ############ +############################################## + +# LVMNode CRD is autogenerated via `make manifests` command. +# Do the modification in the code and run the `make manifests` command +# to generate the CRD definition + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + creationTimestamp: null + name: lvmnodes.local.openebs.io +spec: + group: local.openebs.io + names: + kind: LVMNode + listKind: LVMNodeList + plural: lvmnodes + shortNames: + - lvmnode + singular: lvmnode + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: LVMNode records information about all lvm volume groups available + in a node. In general, the openebs node-agent creates the LVMNode object + & periodically synchronizing the volume groups available in the node. LVMNode + has an owner reference pointing to the corresponding node object. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + volumeGroups: + items: + description: VolumeGroup specifies attributes of a given vg exists on + node. + properties: + allocationPolicy: + description: 'AllocationPolicy indicates the volume group allocation + policy. AllocationPolicy has the following mapping between int + and string for its value: [-1: "", 0: "normal", 1: "contiguous", + 2: "cling", 3: "anywhere", 4: "inherited"]' + type: integer + free: + anyOf: + - type: integer + - type: string + description: Free specifies the available capacity of volume group. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + lvCount: + description: LVCount denotes total number of logical volumes in + volume group. + format: int32 + minimum: 0 + type: integer + maxLv: + description: MaxLV denotes maximum number of logical volumes allowed + in volume group or 0 if unlimited. + format: int32 + type: integer + maxPv: + description: MaxPV denotes maximum number of physical volumes allowed + in volume group or 0 if unlimited. + format: int32 + type: integer + metadataCount: + description: MetadataCount denotes number of metadata areas on the + volume group. + format: int32 + type: integer + metadataFree: + anyOf: + - type: integer + - type: string + description: MetadataFree specifies the available metadata area + space for the volume group + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + metadataSize: + anyOf: + - type: integer + - type: string + description: MetadataSize specifies size of smallest metadata area + for the volume group + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + metadataUsedCount: + description: MetadataUsedCount denotes number of used metadata areas + in volume group + format: int32 + type: integer + missingPvCount: + description: MissingPVCount denotes number of physical volumes in + volume group which are missing. + format: int32 + type: integer + name: + description: Name of the lvm volume group. + minLength: 1 + type: string + permissions: + description: 'Permission indicates the volume group permission which + can be writable or read-only. Permission has the following mapping + between int and string for its value: [-1: "", 0: "writeable", + 1: "read-only"]' + type: integer + pvCount: + description: PVCount denotes total number of physical volumes constituting + the volume group. + format: int32 + minimum: 0 + type: integer + size: + anyOf: + - type: integer + - type: string + description: Size specifies the total size of volume group. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + snapCount: + description: SnapCount denotes number of snapshots in volume group. + format: int32 + type: integer + uuid: + description: UUID denotes a unique identity of a lvm volume group. + minLength: 1 + type: string + required: + - allocationPolicy + - free + - lvCount + - maxLv + - maxPv + - metadataCount + - metadataFree + - metadataSize + - metadataUsedCount + - missingPvCount + - name + - permissions + - pvCount + - size + - snapCount + - uuid + type: object + type: array + required: + - volumeGroups + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/kubezero-storage/charts/lvm-localpv/crds/lvmsnapshot.yaml b/charts/kubezero-storage/charts/lvm-localpv/crds/lvmsnapshot.yaml new file mode 100644 index 00000000..2dc5164b --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/crds/lvmsnapshot.yaml @@ -0,0 +1,85 @@ + + +############################################## +########### ############ +########### LVMSnapshot CRD ############ +########### ############ +############################################## + +# LVMSnapshot CRD is autogenerated via `make manifests` command. +# Do the modification in the code and run the `make manifests` command +# to generate the CRD definition + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + creationTimestamp: null + name: lvmsnapshots.local.openebs.io +spec: + group: local.openebs.io + names: + kind: LVMSnapshot + listKind: LVMSnapshotList + plural: lvmsnapshots + singular: lvmsnapshot + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: LVMSnapshot represents an LVM Snapshot of the lvm volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: LVMSnapshotSpec defines LVMSnapshot spec + properties: + ownerNodeID: + description: OwnerNodeID is the Node ID where the volume group is + present which is where the snapshot has been provisioned. OwnerNodeID + can not be edited after the snapshot has been provisioned. + minLength: 1 + type: string + snapSize: + description: SnapSize specifies the space reserved for the snapshot + type: string + volGroup: + description: VolGroup specifies the name of the volume group where + the snapshot has been created. + type: string + required: + - ownerNodeID + - volGroup + type: object + status: + description: SnapStatus string that reflects if the snapshot was created + successfully + properties: + state: + type: string + type: object + required: + - spec + - status + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/kubezero-storage/charts/lvm-localpv/crds/lvmvolume.yaml b/charts/kubezero-storage/charts/lvm-localpv/crds/lvmvolume.yaml new file mode 100644 index 00000000..61b78125 --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/crds/lvmvolume.yaml @@ -0,0 +1,153 @@ + + +############################################## +########### ############ +########### LVMVolume CRD ############ +########### ############ +############################################## + +# LVMVolume CRD is autogenerated via `make manifests` command. +# Do the modification in the code and run the `make manifests` command +# to generate the CRD definition + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + creationTimestamp: null + name: lvmvolumes.local.openebs.io +spec: + group: local.openebs.io + names: + kind: LVMVolume + listKind: LVMVolumeList + plural: lvmvolumes + shortNames: + - lvmvol + singular: lvmvolume + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: volume group where the volume is created + jsonPath: .spec.volGroup + name: VolGroup + type: string + - description: Node where the volume is created + jsonPath: .spec.ownerNodeID + name: Node + type: string + - description: Size of the volume + jsonPath: .spec.capacity + name: Size + type: string + - description: Status of the volume + jsonPath: .status.state + name: Status + type: string + - description: Age of the volume + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: LVMVolume represents a LVM based volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VolumeInfo defines LVM info + properties: + capacity: + description: Capacity of the volume + minLength: 1 + type: string + ownerNodeID: + description: OwnerNodeID is the Node ID where the volume group is + present which is where the volume has been provisioned. OwnerNodeID + can not be edited after the volume has been provisioned. + minLength: 1 + type: string + shared: + description: Shared specifies whether the volume can be shared among + multiple pods. If it is not set to "yes", then the LVM LocalPV Driver + will not allow the volumes to be mounted by more than one pods. + enum: + - "yes" + - "no" + type: string + thinProvision: + description: ThinProvision specifies whether logical volumes can be + thinly provisioned. If it is set to "yes", then the LVM LocalPV + Driver will create thinProvision i.e. logical volumes that are larger + than the available extents. + enum: + - "yes" + - "no" + type: string + vgPattern: + description: VgPattern specifies the regex to choose volume groups + where volume needs to be created. + type: string + volGroup: + description: VolGroup specifies the name of the volume group where + the volume has been created. + type: string + required: + - capacity + - ownerNodeID + - vgPattern + - volGroup + type: object + status: + description: VolStatus string that specifies the current state of the + volume provisioning request. + properties: + error: + description: Error denotes the error occurred during provisioning/expanding + a volume. Error field should only be set when State becomes Failed. + properties: + code: + description: VolumeErrorCode represents the error code to represent + specific class of errors. + type: string + message: + type: string + type: object + state: + description: State specifies the current state of the volume provisioning + request. The state "Pending" means that the volume creation request + has not processed yet. The state "Ready" means that the volume has + been created and it is ready for the use. "Failed" means that volume + provisioning has been failed and will not be retried by node agent + controller. + enum: + - Pending + - Ready + - Failed + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/kubezero-storage/charts/lvm-localpv/templates/NOTES.txt b/charts/kubezero-storage/charts/lvm-localpv/templates/NOTES.txt new file mode 100644 index 00000000..cc5aaf67 --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/templates/NOTES.txt @@ -0,0 +1,5 @@ +The OpenEBS LVM LocalPV has been installed. Check its status by running: +$ kubectl get pods -n {{ .Release.Namespace }} -l role=openebs-lvm + +For more information, visit our Slack at https://openebs.io/community or view +the documentation online at http://docs.openebs.io/. diff --git a/charts/kubezero-storage/charts/lvm-localpv/templates/_helpers.tpl b/charts/kubezero-storage/charts/lvm-localpv/templates/_helpers.tpl new file mode 100644 index 00000000..6251921d --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/templates/_helpers.tpl @@ -0,0 +1,138 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "lvmlocalpv.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified localpv provisioner name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "lvmlocalpv.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "lvmlocalpv.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{/* +Create the name of the service account for controller +*/}} +{{- define "lvmlocalpv.lvmController.serviceAccountName" -}} +{{- if .Values.serviceAccount.lvmController.create }} +{{- default (include "lvmlocalpv.fullname" .) .Values.serviceAccount.lvmController.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.lvmController.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "lvmlocalpv.lvmNode.serviceAccountName" -}} +{{- if .Values.serviceAccount.lvmNode.create }} +{{- default (include "lvmlocalpv.fullname" .) .Values.serviceAccount.lvmNode.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.lvmNode.name }} +{{- end -}} +{{- end -}} + +{{/* +Define meta labels for openebs lvm-localpv components +*/}} +{{- define "lvmlocalpv.common.metaLabels" -}} +chart: {{ template "lvmlocalpv.chart" . }} +heritage: {{ .Release.Service }} +openebs.io/version: {{ .Values.release.version | quote }} +role: {{ .Values.role | quote }} +{{- end -}} + +{{/* +Create match labels for openebs lvm-localpv controller +*/}} +{{- define "lvmlocalpv.lvmController.matchLabels" -}} +app: {{ .Values.lvmController.componentName | quote }} +release: {{ .Release.Name }} +component: {{ .Values.lvmController.componentName | quote }} +{{- end -}} + +{{/* +Create component labels for lvmlocalpv controller +*/}} +{{- define "lvmlocalpv.lvmController.componentLabels" -}} +openebs.io/component-name: {{ .Values.lvmController.componentName | quote }} +{{- end -}} + + +{{/* +Create labels for openebs lvm-localpv controller +*/}} +{{- define "lvmlocalpv.lvmController.labels" -}} +{{ include "lvmlocalpv.common.metaLabels" . }} +{{ include "lvmlocalpv.lvmController.matchLabels" . }} +{{ include "lvmlocalpv.lvmController.componentLabels" . }} +{{- end -}} + +{{/* +Create match labels for openebs lvm-localpv node daemon +*/}} +{{- define "lvmlocalpv.lvmNode.matchLabels" -}} +name: {{ .Values.lvmNode.componentName | quote }} +release: {{ .Release.Name }} +{{- end -}} + +{{/* +Create component labels openebs lvm-localpv node daemon +*/}} +{{- define "lvmlocalpv.lvmNode.componentLabels" -}} +openebs.io/component-name: {{ .Values.lvmNode.componentName | quote }} +{{- end -}} + + +{{/* +Create labels for openebs lvm-localpv node daemon +*/}} +{{- define "lvmlocalpv.lvmNode.labels" -}} +{{ include "lvmlocalpv.common.metaLabels" . }} +{{ include "lvmlocalpv.lvmNode.matchLabels" . }} +{{ include "lvmlocalpv.lvmNode.componentLabels" . }} +{{- end -}} + +{{/* +Create the name of the priority class for csi node plugin +*/}} +{{- define "lvmlocalpv.lvmNode.priorityClassName" -}} +{{- if .Values.lvmNode.priorityClass.create }} +{{- printf "%s-%s" .Release.Name .Values.lvmNode.priorityClass.name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s" .Values.lvmNode.priorityClass.name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} + +{{/* +Create the name of the priority class for csi controller plugin +*/}} +{{- define "lvmlocalpv.lvmController.priorityClassName" -}} +{{- if .Values.lvmController.priorityClass.create }} +{{- printf "%s-%s" .Release.Name .Values.lvmController.priorityClass.name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s" .Values.lvmController.priorityClass.name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} diff --git a/charts/kubezero-storage/charts/lvm-localpv/templates/csidriver.yaml b/charts/kubezero-storage/charts/lvm-localpv/templates/csidriver.yaml new file mode 100644 index 00000000..5eeac542 --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/templates/csidriver.yaml @@ -0,0 +1,10 @@ +# Create the CSI Driver object +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: local.csi.openebs.io +spec: + # do not require volumeattachment + attachRequired: false + podInfoOnMount: true + storageCapacity: {{ .Values.storageCapacity }} diff --git a/charts/kubezero-storage/charts/lvm-localpv/templates/lvm-controller.yaml b/charts/kubezero-storage/charts/lvm-localpv/templates/lvm-controller.yaml new file mode 100644 index 00000000..50a99d9c --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/templates/lvm-controller.yaml @@ -0,0 +1,147 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "lvmlocalpv.fullname" . }}-controller + {{- with .Values.lvmController.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "lvmlocalpv.lvmController.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "lvmlocalpv.lvmController.matchLabels" . | nindent 6 }} + serviceName: "{{ .Values.lvmController.serviceName }}" + replicas: {{ .Values.lvmController.replicas }} + template: + metadata: + {{- with .Values.lvmController.podAnnotations }} + annotations: {{ toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "lvmlocalpv.lvmController.labels" . | nindent 8 }} + {{- with .Values.lvmController.podLabels}} + {{ toYaml . | nindent 8 }} + {{- end}} + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - openebs-lvm-controller + topologyKey: "kubernetes.io/hostname" +{{- if .Values.lvmController.priorityClass.create }} + priorityClassName: {{ template "lvmlocalpv.lvmController.priorityClassName" . }} +{{- end }} + serviceAccount: {{ .Values.serviceAccount.lvmController.name }} + containers: + - name: {{ .Values.lvmController.resizer.name }} + image: "{{ .Values.lvmController.resizer.image.registry }}{{ .Values.lvmController.resizer.image.repository }}:{{ .Values.lvmController.resizer.image.tag }}" + args: + - "--v={{ .Values.lvmController.logLevel }}" + - "--csi-address=$(ADDRESS)" + {{- if gt (int .Values.lvmController.replicas) 1 }} + - "--leader-election" + {{- end }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: {{ .Values.lvmController.resizer.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + {{- toYaml .Values.lvmController.resources | nindent 12 }} + - name: {{ .Values.lvmController.snapshotter.name }} + image: "{{ .Values.lvmController.snapshotter.image.registry }}{{ .Values.lvmController.snapshotter.image.repository }}:{{ .Values.lvmController.snapshotter.image.tag }}" + imagePullPolicy: {{ .Values.lvmController.snapshotter.image.pullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + {{- if gt (int .Values.lvmController.replicas) 1 }} + - "--leader-election" + {{- end }} + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + {{- toYaml .Values.lvmController.resources | nindent 12 }} + - name: {{ .Values.lvmController.provisioner.name }} + image: "{{ .Values.lvmController.provisioner.image.registry }}{{ .Values.lvmController.provisioner.image.repository }}:{{ .Values.lvmController.provisioner.image.tag }}" + imagePullPolicy: {{ .Values.lvmController.provisioner.image.pullPolicy }} + args: + - "--csi-address=$(ADDRESS)" + - "--v={{ .Values.lvmController.logLevel }}" + - "--feature-gates=Topology=true" + - "--strict-topology" + {{- if gt (int .Values.lvmController.replicas) 1 }} + - "--leader-election" + {{- end }} + - "--enable-capacity={{ .Values.storageCapacity }}" + - "--extra-create-metadata=true" + - "--default-fstype=ext4" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + {{- toYaml .Values.lvmController.resources | nindent 12 }} + - name: {{ .Values.lvmPlugin.name }} + image: "{{ .Values.lvmPlugin.image.registry }}{{ .Values.lvmPlugin.image.repository }}:{{ .Values.lvmPlugin.image.tag }}" + imagePullPolicy: {{ .Values.lvmPlugin.image.pullPolicy }} + env: + - name: OPENEBS_CONTROLLER_DRIVER + value: controller + - name: OPENEBS_CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: LVM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPENEBS_IO_INSTALLER_TYPE + value: "lvm-localpv-helm" + - name: OPENEBS_IO_ENABLE_ANALYTICS + value: "{{ .Values.analytics.enabled }}" + args : + - "--endpoint=$(OPENEBS_CSI_ENDPOINT)" + - "--plugin=$(OPENEBS_CONTROLLER_DRIVER)" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + {{- toYaml .Values.lvmController.resources | nindent 12 }} + volumes: + - name: socket-dir + emptyDir: {} +{{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 2 }} +{{- end }} +{{- if .Values.lvmController.nodeSelector }} + nodeSelector: +{{ toYaml .Values.lvmController.nodeSelector | indent 8 }} +{{- end }} +{{- if .Values.lvmController.securityContext }} + securityContext: +{{ toYaml .Values.lvmController.securityContext | indent 8 }} +{{- end }} +{{- if .Values.lvmController.tolerations }} + tolerations: +{{ toYaml .Values.lvmController.tolerations | indent 8 }} +{{- end }} diff --git a/charts/kubezero-storage/charts/lvm-localpv/templates/lvm-node-service.yaml b/charts/kubezero-storage/charts/lvm-localpv/templates/lvm-node-service.yaml new file mode 100644 index 00000000..34c1977e --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/templates/lvm-node-service.yaml @@ -0,0 +1,18 @@ +{{- if .Values.lvmPlugin.metricsPort }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "lvmlocalpv.fullname" . }}-node-service + labels: + {{- include "lvmlocalpv.lvmNode.labels" . | nindent 4 }} +spec: + clusterIP: None + ports: + - name: metrics + port: {{ .Values.lvmPlugin.metricsPort }} + targetPort: {{ .Values.lvmPlugin.metricsPort }} + selector: + {{- with .Values.lvmNode.podLabels }} + {{ toYaml . }} + {{- end }} +{{- end }} diff --git a/charts/kubezero-storage/charts/lvm-localpv/templates/lvm-node.yaml b/charts/kubezero-storage/charts/lvm-localpv/templates/lvm-node.yaml new file mode 100644 index 00000000..9c104113 --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/templates/lvm-node.yaml @@ -0,0 +1,156 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ template "lvmlocalpv.fullname" . }}-node + {{- with .Values.lvmNode.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "lvmlocalpv.lvmNode.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "lvmlocalpv.lvmNode.matchLabels" . | nindent 6 }} + updateStrategy: + rollingUpdate: + maxUnavailable: 100% + type: RollingUpdate + template: + metadata: + {{- with .Values.lvmNode.podAnnotations }} + annotations: {{ toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "lvmlocalpv.lvmNode.labels" . | nindent 8 }} + {{- with .Values.lvmNode.podLabels}} + {{ toYaml . | nindent 8 }} + {{- end}} + spec: +{{- if .Values.lvmNode.priorityClass.create }} + priorityClassName: {{ template "lvmlocalpv.lvmNode.priorityClassName" . }} +{{- end }} + serviceAccount: {{ .Values.serviceAccount.lvmNode.name }} + hostNetwork: true + containers: + - name: {{ .Values.lvmNode.driverRegistrar.name }} + image: "{{ .Values.lvmNode.driverRegistrar.image.registry }}{{ .Values.lvmNode.driverRegistrar.image.repository }}:{{ .Values.lvmNode.driverRegistrar.image.tag }}" + imagePullPolicy: {{ .Values.lvmNode.driverRegistrar.image.pullPolicy }} + args: + - "--v={{ .Values.lvmNode.logLevel }}" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/lvm-localpv /registration/lvm-localpv-reg.sock"] + env: + - name: ADDRESS + value: /plugin/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: {{ .Values.lvmNode.kubeletDir }}plugins/lvm-localpv/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_DRIVER + value: openebs-lvm + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: registration-dir + mountPath: /registration + resources: + {{- toYaml .Values.lvmNode.resources | nindent 12 }} + - name: {{ .Values.lvmPlugin.name }} + securityContext: + privileged: true + allowPrivilegeEscalation: true + image: "{{ .Values.lvmPlugin.image.registry }}{{ .Values.lvmPlugin.image.repository }}:{{ .Values.lvmPlugin.image.tag }}" + imagePullPolicy: {{ .Values.lvmPlugin.image.pullPolicy }} + args: + - "--nodeid=$(OPENEBS_NODE_ID)" + - "--endpoint=$(OPENEBS_CSI_ENDPOINT)" + - "--plugin=$(OPENEBS_NODE_DRIVER)" + {{- if .Values.lvmPlugin.ioLimits.enabled }} + - "--setiolimits" + - "--container-runtime=$(CONTAINER_RUNTIME)" + - "--riops-per-gb=$(RIOPS_PER_GB)" + - "--wiops-per-gb=$(WIOPS_PER_GB)" + {{- end }} + {{- if .Values.lvmPlugin.metricsPort }} + - "--listen-address=$(METRICS_LISTEN_ADDRESS)" + {{- end }} + env: + - name: OPENEBS_NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPENEBS_CSI_ENDPOINT + value: unix:///plugin/csi.sock + - name: OPENEBS_NODE_DRIVER + value: agent + - name: LVM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.lvmPlugin.ioLimits.enabled }} + - name: CONTAINER_RUNTIME + value: {{ .Values.lvmPlugin.ioLimits.containerRuntime }} + - name: RIOPS_PER_GB + value: {{ .Values.lvmPlugin.ioLimits.readIopsPerGB }} + - name: WIOPS_PER_GB + value: {{ .Values.lvmPlugin.ioLimits.writeIopsPerGB }} + {{- end }} + {{- if .Values.lvmPlugin.metricsPort }} + - name: METRICS_LISTEN_ADDRESS + value: :{{ .Values.lvmPlugin.metricsPort }} + {{- end }} + {{- if .Values.lvmPlugin.allowedTopologies }} + - name: ALLOWED_TOPOLOGIES + value: {{ .Values.lvmPlugin.allowedTopologies }} + {{- end }} + volumeMounts: + - name: plugin-dir + mountPath: /plugin + - name: device-dir + mountPath: /dev + - name: pods-mount-dir + mountPath: {{ .Values.lvmNode.kubeletDir }} + # needed so that any mounts setup inside this container are + # propagated back to the host machine. + mountPropagation: "Bidirectional" + resources: + {{- toYaml .Values.lvmNode.resources | nindent 12 }} + volumes: + - name: device-dir + hostPath: + path: /dev + type: Directory + - name: registration-dir + hostPath: + path: {{ .Values.lvmNode.kubeletDir }}plugins_registry/ + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: {{ .Values.lvmNode.kubeletDir }}plugins/lvm-localpv/ + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: {{ .Values.lvmNode.kubeletDir }} + type: Directory +{{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 2 }} +{{- end }} +{{- if .Values.lvmNode.nodeSelector }} + nodeSelector: +{{ toYaml .Values.lvmNode.nodeSelector | indent 8 }} +{{- end }} +{{- if .Values.lvmNode.securityContext }} + securityContext: +{{ toYaml .Values.lvmNode.securityContext | indent 8 }} +{{- end }} +{{- if .Values.lvmNode.tolerations }} + tolerations: +{{ toYaml .Values.lvmNode.tolerations | indent 8 }} +{{- end }} diff --git a/charts/kubezero-storage/charts/lvm-localpv/templates/priority-class.yaml b/charts/kubezero-storage/charts/lvm-localpv/templates/priority-class.yaml new file mode 100644 index 00000000..44552ab3 --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/templates/priority-class.yaml @@ -0,0 +1,19 @@ +{{- if .Values.lvmController.priorityClass.create }} +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: {{ template "lvmlocalpv.lvmController.priorityClassName" . }} +value: 900000000 +globalDefault: false +description: "This priority class should be used for the CStor CSI driver controller deployment only." +{{- end }} +--- +{{- if .Values.lvmNode.priorityClass.create }} +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: {{ template "lvmlocalpv.lvmNode.priorityClassName" . }} +value: 900001000 +globalDefault: false +description: "This priority class should be used for the CStor CSI driver node deployment only." +{{- end }} diff --git a/charts/kubezero-storage/charts/lvm-localpv/templates/psp.yaml b/charts/kubezero-storage/charts/lvm-localpv/templates/psp.yaml new file mode 100644 index 00000000..7472d7c6 --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/templates/psp.yaml @@ -0,0 +1,24 @@ +{{- if .Values.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: openebs-lvm-node-psp + labels: + {{- include "lvmlocalpv.lvmNode.labels" . | nindent 4 }} +spec: + privileged: true + allowPrivilegeEscalation: true + allowedCapabilities: ['*'] + volumes: ['*'] + hostNetwork: true + hostIPC: true + hostPID: true + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' +{{- end }} diff --git a/charts/kubezero-storage/charts/lvm-localpv/templates/rbac.yaml b/charts/kubezero-storage/charts/lvm-localpv/templates/rbac.yaml new file mode 100644 index 00000000..5f54948c --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/templates/rbac.yaml @@ -0,0 +1,197 @@ +{{- if .Values.serviceAccount.lvmController.create -}} +kind: ServiceAccount +apiVersion: v1 +metadata: + name: {{ .Values.serviceAccount.lvmController.name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "lvmlocalpv.lvmController.labels" . | nindent 4 }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-lvm-provisioner-role + labels: + {{- include "lvmlocalpv.lvmController.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes", "services"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [ "storage.k8s.io" ] + resources: [ "csistoragecapacities"] + verbs: ["*"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["local.openebs.io"] + resources: ["lvmvolumes", "lvmsnapshots", "lvmnodes"] + verbs: ["*"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-lvm-provisioner-binding + labels: + {{- include "lvmlocalpv.lvmController.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.lvmController.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: openebs-lvm-provisioner-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-lvm-snapshotter-role + labels: + {{- include "lvmlocalpv.lvmController.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-lvm-snapshotter-binding + labels: + {{- include "lvmlocalpv.lvmController.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.lvmController.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: openebs-lvm-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- +{{- end }} +{{- if .Values.serviceAccount.lvmNode.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.lvmNode.name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "lvmlocalpv.lvmNode.labels" . | nindent 4 }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-lvm-driver-registrar-role + labels: + {{- include "lvmlocalpv.lvmNode.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumes", "nodes", "services"] + verbs: ["get", "list"] + - apiGroups: ["local.openebs.io"] + resources: ["lvmvolumes", "lvmsnapshots", "lvmnodes"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-lvm-driver-registrar-binding + labels: + {{- include "lvmlocalpv.lvmNode.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.lvmNode.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: openebs-lvm-driver-registrar-role + apiGroup: rbac.authorization.k8s.io + +{{- if .Values.rbac.pspEnabled }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: openebs-lvm-node-role + labels: + {{- include "lvmlocalpv.lvmNode.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - openebs-lvm-node-psp +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: openebs-lvm-node-binding + labels: + {{- include "lvmlocalpv.lvmNode.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: openebs-lvm-node-role +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.lvmNode.name }} + namespace: {{ $.Release.Namespace }} +{{- end }} +{{- end }} diff --git a/charts/kubezero-storage/charts/lvm-localpv/values.yaml b/charts/kubezero-storage/charts/lvm-localpv/values.yaml new file mode 100644 index 00000000..e664eed8 --- /dev/null +++ b/charts/kubezero-storage/charts/lvm-localpv/values.yaml @@ -0,0 +1,170 @@ +# Default values for openebs-lvmlocalpv. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +release: + version: "0.8.3" + +imagePullSecrets: +# - name: "image-pull-secret" + +# enable storage capacity tracking feature +# Ref: https://kubernetes:io/docs/concepts/storage/storage-capacity +storageCapacity: true + +rbac: + # rbac.pspEnabled: `true` if PodSecurityPolicy resources should be created + pspEnabled: false + +# lvmNode contains the configurables for +# the lvm node daemonset +lvmNode: + componentName: openebs-lvm-node + driverRegistrar: + name: "csi-node-driver-registrar" + image: + # Make sure that registry name end with a '/'. + # For example : k8s.gcr.io/ is a correct value here and quay.io is incorrect + registry: k8s.gcr.io/ + repository: sig-storage/csi-node-driver-registrar + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: v2.3.0 + updateStrategy: + type: RollingUpdate + annotations: {} + podAnnotations: {} + # This can be configured to run on various different k8s distributions like + # microk8s where kubelet dir is different + kubeletDir: "/var/lib/kubelet/" + resources: {} +# limits: +# cpu: 10m +# memory: 32Mi +# requests: +# cpu: 10m +# memory: 32Mi + ## Labels to be added to openebs-lvm node pods + podLabels: + app: openebs-lvm-node + nodeSelector: {} + tolerations: [] + securityContext: {} + labels: {} + priorityClass: + create: true + name: lvm-localpv-csi-node-critical + logLevel: 5 + + +# lvmController contains the configurables for +# the lvm controller statefulset +lvmController: + componentName: openebs-lvm-controller + replicas: 1 + serviceName: openebs-lvm + logLevel: 5 + resizer: + name: "csi-resizer" + image: + # Make sure that registry name end with a '/'. + # For example : k8s.gcr.io/ is a correct value here and quay.io is incorrect + registry: k8s.gcr.io/ + repository: sig-storage/csi-resizer + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: v1.2.0 + snapshotter: + name: "csi-snapshotter" + image: + # Make sure that registry name end with a '/'. + # For example : k8s.gcr.io/ is a correct value here and quay.io is incorrect + registry: k8s.gcr.io/ + repository: sig-storage/csi-snapshotter + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: v4.0.0 + snapshotController: + name: "snapshot-controller" + image: + # Make sure that registry name end with a '/'. + # For example : k8s.gcr.io/ is a correct value here and quay.io is incorrect + registry: k8s.gcr.io/ + repository: sig-storage/snapshot-controller + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: v4.0.0 + provisioner: + name: "csi-provisioner" + image: + # Make sure that registry name end with a '/'. + # For example : k8s.gcr.io/ is a correct value here and quay.io is incorrect + registry: k8s.gcr.io/ + repository: sig-storage/csi-provisioner + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: v3.0.0 + updateStrategy: + type: RollingUpdate + annotations: {} + podAnnotations: {} + resources: {} +# limits: +# cpu: 10m +# memory: 32Mi +# requests: +# cpu: 10m +# memory: 32Mi + ## Labels to be added to openebs-lvm controller pods + podLabels: + name: openebs-lvm-controller + nodeSelector: {} + tolerations: [] + securityContext: {} + priorityClass: + create: true + name: lvm-localpv-csi-controller-critical + +# lvmPlugin is the common csi container used by the +# controller statefulset and node daemonset +lvmPlugin: + name: "openebs-lvm-plugin" + image: + # Make sure that registry name end with a '/'. + # For example : k8s.gcr.io/ is a correct value here and quay.io is incorrect + registry: + repository: openebs/lvm-driver + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: 0.8.3 + ioLimits: + enabled: false + containerRuntime: containerd + readIopsPerGB: "" + writeIopsPerGB: "" + # The TCP port number used for exposing lvm-metrics. + # If not set, service will not be created to expose metrics endpoint to serviceMonitor and listen-address flag will not be set. + metricsPort: 9500 + # Comma seperated list of k8s worker node topologies + allowedTopologies: "kubernetes.io/hostname," + +role: openebs-lvm + +crd: + enableInstall: true + +serviceAccount: + lvmController: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: openebs-lvm-controller-sa + lvmNode: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: openebs-lvm-node-sa + +analytics: + enabled: true diff --git a/charts/kubezero-storage/crds/volumesnapshotclasses.yaml b/charts/kubezero-storage/crds/volumesnapshotclasses.yaml new file mode 100644 index 00000000..afd637b1 --- /dev/null +++ b/charts/kubezero-storage/crds/volumesnapshotclasses.yaml @@ -0,0 +1,112 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: false + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + diff --git a/charts/kubezero-storage/crds/volumesnapshotcontents.yaml b/charts/kubezero-storage/crds/volumesnapshotcontents.yaml new file mode 100644 index 00000000..c57a2e4a --- /dev/null +++ b/charts/kubezero-storage/crds/volumesnapshotcontents.yaml @@ -0,0 +1,291 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + oneOf: + - required: ["snapshotHandle"] + - required: ["volumeHandle"] + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/kubezero-storage/crds/volumesnapshots.yaml b/charts/kubezero-storage/crds/volumesnapshots.yaml new file mode 100644 index 00000000..13d89414 --- /dev/null +++ b/charts/kubezero-storage/crds/volumesnapshots.yaml @@ -0,0 +1,225 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + oneOf: + - required: ["persistentVolumeClaimName"] + - required: ["volumeSnapshotContentName"] + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/kubezero-storage/lvm.patch b/charts/kubezero-storage/lvm.patch new file mode 100644 index 00000000..086c782f --- /dev/null +++ b/charts/kubezero-storage/lvm.patch @@ -0,0 +1,20 @@ +diff -tuNr charts/lvm-localpv.orig/templates/lvm-controller.yaml charts/lvm-localpv/templates/lvm-controller.yaml +--- charts/lvm-localpv.orig/templates/lvm-controller.yaml 2022-01-11 13:47:28.717133358 +0100 ++++ charts/lvm-localpv/templates/lvm-controller.yaml 2022-01-11 13:49:49.160473040 +0100 +@@ -72,16 +72,6 @@ + mountPath: /var/lib/csi/sockets/pluginproxy/ + resources: + {{- toYaml .Values.lvmController.resources | nindent 12 }} +- - name: {{ .Values.lvmController.snapshotController.name }} +- image: "{{ .Values.lvmController.snapshotController.image.registry }}{{ .Values.lvmController.snapshotController.image.repository }}:{{ .Values.lvmController.snapshotController.image.tag }}" +- args: +- - "--v={{ .Values.lvmController.logLevel }}" +- {{- if gt (int .Values.lvmController.replicas) 1 }} +- - "--leader-election=true" +- {{- end }} +- imagePullPolicy: {{ .Values.lvmController.snapshotController.image.pullPolicy }} +- resources: +- {{- toYaml .Values.lvmController.resources | nindent 12 }} + - name: {{ .Values.lvmController.provisioner.name }} + image: "{{ .Values.lvmController.provisioner.image.registry }}{{ .Values.lvmController.provisioner.image.repository }}:{{ .Values.lvmController.provisioner.image.tag }}" + imagePullPolicy: {{ .Values.lvmController.provisioner.image.pullPolicy }} diff --git a/charts/kubezero-storage/templates/ebs/volumesnapshotclass.yaml b/charts/kubezero-storage/templates/ebs/volumesnapshotclass.yaml new file mode 100644 index 00000000..bc8dcc82 --- /dev/null +++ b/charts/kubezero-storage/templates/ebs/volumesnapshotclass.yaml @@ -0,0 +1,12 @@ +{{- if index .Values "aws-ebs-csi-driver" "enabled" }} +kind: VolumeSnapshotClass +apiVersion: snapshot.storage.k8s.io/v1 +metadata: + name: csi-aws-ebs + #annotations: + # snapshot.storage.kubernetes.io/is-default-class: "true" + labels: + {{- include "kubezero-lib.labels" . | nindent 4 }} +deletionPolicy: Delete +driver: ebs.csi.aws.com +{{- end }} diff --git a/charts/kubezero-storage/templates/efs_pvs.yaml b/charts/kubezero-storage/templates/efs/pv.yaml similarity index 100% rename from charts/kubezero-storage/templates/efs_pvs.yaml rename to charts/kubezero-storage/templates/efs/pv.yaml diff --git a/charts/kubezero-storage/templates/lvm-storageclass.yaml b/charts/kubezero-storage/templates/lvm/storageclass.yaml similarity index 88% rename from charts/kubezero-storage/templates/lvm-storageclass.yaml rename to charts/kubezero-storage/templates/lvm/storageclass.yaml index d55dcd57..db006f0f 100644 --- a/charts/kubezero-storage/templates/lvm-storageclass.yaml +++ b/charts/kubezero-storage/templates/lvm/storageclass.yaml @@ -3,6 +3,8 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: openebs-lvm + labels: + {{- include "kubezero-lib.labels" . | nindent 4 }} {{- if index .Values "lvm-localpv" "storageClass" "default" }} annotations: storageclass.kubernetes.io/is-default-class: "true" diff --git a/charts/kubezero-storage/templates/lvm/volumesnapshotclass.yaml b/charts/kubezero-storage/templates/lvm/volumesnapshotclass.yaml new file mode 100644 index 00000000..cdaf9cd3 --- /dev/null +++ b/charts/kubezero-storage/templates/lvm/volumesnapshotclass.yaml @@ -0,0 +1,12 @@ +{{- if index .Values "lvm-localpv" "enabled" }} +kind: VolumeSnapshotClass +apiVersion: snapshot.storage.k8s.io/v1 +metadata: + name: lvmpv-snapclass + #annotations: + # snapshot.storage.kubernetes.io/is-default-class: "true" + labels: + {{- include "kubezero-lib.labels" . | nindent 4 }} +driver: local.csi.openebs.io +deletionPolicy: Delete +{{- end }} diff --git a/charts/kubezero-storage/templates/snapshot-controller/controller.yaml b/charts/kubezero-storage/templates/snapshot-controller/controller.yaml new file mode 100644 index 00000000..456b3105 --- /dev/null +++ b/charts/kubezero-storage/templates/snapshot-controller/controller.yaml @@ -0,0 +1,40 @@ +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: snapshot-controller + namespace: kube-system + labels: + {{- include "kubezero-lib.labels" . | nindent 4 }} +spec: + serviceName: "snapshot-controller" + replicas: {{ .Values.snapshotController.replicas }} + selector: + matchLabels: + app: snapshot-controller + template: + metadata: + labels: + app: snapshot-controller + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - snapshot-controller + topologyKey: "kubernetes.io/hostname" + serviceAccount: snapshot-controller + containers: + - name: snapshot-controller + image: k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0 + args: + - "--v={{ .Values.snapshotController.logLevel }}" + {{- if gt (int .Values.snapshotController.replicas) 1 }} + - "--leader-election=true" + {{- end }} + imagePullPolicy: IfNotPresent + resources: + {{- toYaml .Values.snapshotController.resources | nindent 12 }} diff --git a/charts/kubezero-storage/templates/snapshot-controller/rbac.yaml b/charts/kubezero-storage/templates/snapshot-controller/rbac.yaml new file mode 100644 index 00000000..c3354135 --- /dev/null +++ b/charts/kubezero-storage/templates/snapshot-controller/rbac.yaml @@ -0,0 +1,86 @@ +# RBAC file for the snapshot controller. +# +# The snapshot controller implements the control loop for CSI snapshot functionality. +# It should be installed as part of the base Kubernetes distribution in an appropriate +# namespace for components implementing base system functionality. For installing with +# Vanilla Kubernetes, kube-system makes sense for the namespace. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: snapshot-controller + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # rename if there are conflicts + name: snapshot-controller-runner +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-role +subjects: + - kind: ServiceAccount + name: snapshot-controller + namespace: kube-system +roleRef: + kind: ClusterRole + # change the name also here if the ClusterRole gets renamed + name: snapshot-controller-runner + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: kube-system + name: snapshot-controller-leaderelection +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: snapshot-controller-leaderelection + namespace: kube-system +subjects: + - kind: ServiceAccount + name: snapshot-controller + namespace: kube-system +roleRef: + kind: Role + name: snapshot-controller-leaderelection + apiGroup: rbac.authorization.k8s.io + diff --git a/charts/kubezero-storage/update.sh b/charts/kubezero-storage/update.sh index 0695fdfb..04f21e5f 100755 --- a/charts/kubezero-storage/update.sh +++ b/charts/kubezero-storage/update.sh @@ -7,14 +7,20 @@ helm pull fairwinds-stable/gemini --untar --untardir charts # Patch to run gemini on controller nodes patch -p0 -i gemini.patch --no-backup-if-mismatch +### openEBS +VERSION=$(yq eval '.dependencies[] | select(.name=="lvm-localpv") | .version' Chart.yaml) +helm repo add openebs-lvmlocalpv https://openebs.github.io/lvm-localpv || true +rm -rf charts/lvm-localpv +helm pull openebs-lvmlocalpv/lvm-localpv --version $VERSION --untar --untardir charts +mv charts/lvm-localpv/crds/volumesnapshot* crds +patch -i lvm.patch -p0 --no-backup-if-mismatch + ### EBS VERSION=$(yq eval '.dependencies[] | select(.name=="aws-ebs-csi-driver") | .version' Chart.yaml) - rm -rf charts/aws-ebs-csi-driver curl -L -s -o - https://github.com/kubernetes-sigs/aws-ebs-csi-driver/releases/download/helm-chart-aws-ebs-csi-driver-${VERSION}/aws-ebs-csi-driver-${VERSION}.tgz | tar xfz - -C charts patch -i ebs.patch -p0 --no-backup-if-mismatch - ### EFS VERSION=$(yq eval '.dependencies[] | select(.name=="aws-efs-csi-driver") | .version' Chart.yaml) rm -rf charts/aws-efs-csi-driver diff --git a/charts/kubezero-storage/values.yaml b/charts/kubezero-storage/values.yaml index 6ca8792b..a696994f 100644 --- a/charts/kubezero-storage/values.yaml +++ b/charts/kubezero-storage/values.yaml @@ -1,3 +1,16 @@ +snapshotController: + enabled: true + replicas: 1 + logLevel: 2 + + resources: + requests: + memory: 16Mi + cpu: 20m + limits: + memory: 64Mi + cpu: 100m + lvm-localpv: enabled: false